From 1fb47cfe9a14b8a52096d536141dd86754075ce2 Mon Sep 17 00:00:00 2001 From: owinter Date: Thu, 21 Dec 2023 17:46:17 +0100 Subject: [PATCH 1/4] fix aws download bug --- one/remote/aws.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/one/remote/aws.py b/one/remote/aws.py index 0c9b41dc..adbaa92f 100644 --- a/one/remote/aws.py +++ b/one/remote/aws.py @@ -281,6 +281,12 @@ def s3_download_folder(source, destination, s3=None, bucket_name=S3_BUCKET_IBL, local_files = [] objects = s3.Bucket(name=bucket_name).objects.filter(Prefix=source) for obj_summary in filter(lambda x: not is_folder(x), objects): + # we can only filter an object collection by prefix, so we need to make sure the file + # is in the subpath of the source folder + # for example, if source is '/toto/tata' and obj_summary.key is '/toto/tata_alaternate/titi.txt' + # we need to exclude it + if not Path(source) in Path(obj_summary.key).parents: + continue local_file = Path(destination).joinpath(Path(obj_summary.key).relative_to(source)) lf = s3_download_file(obj_summary.key, local_file, s3=s3, bucket_name=bucket_name, overwrite=overwrite) From 00998ec061d73a5cc165511a973ef286fe6245c0 Mon Sep 17 00:00:00 2001 From: owinter Date: Thu, 21 Dec 2023 17:50:19 +0100 Subject: [PATCH 2/4] flake --- one/remote/aws.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/one/remote/aws.py b/one/remote/aws.py index adbaa92f..1b6015d8 100644 --- a/one/remote/aws.py +++ b/one/remote/aws.py @@ -283,8 +283,8 @@ def s3_download_folder(source, destination, s3=None, bucket_name=S3_BUCKET_IBL, for obj_summary in filter(lambda x: not is_folder(x), objects): # we can only filter an object collection by prefix, so we need to make sure the file # is in the subpath of the source folder - # for example, if source is '/toto/tata' and obj_summary.key is '/toto/tata_alaternate/titi.txt' - # we need to exclude it + # for example, if source is '/toto/tata' and obj_summary.key is + # '/toto/tata_alaternate/titi.txt', we need to exclude it if not Path(source) in Path(obj_summary.key).parents: continue local_file = Path(destination).joinpath(Path(obj_summary.key).relative_to(source)) From 144293f0a8cd5835aa38d2f3a3178e7675695ff2 Mon Sep 17 00:00:00 2001 From: owinter Date: Thu, 21 Dec 2023 19:33:12 +0100 Subject: [PATCH 3/4] fix test to work with newly released openalyx --- one/__init__.py | 2 +- one/tests/test_one.py | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/one/__init__.py b/one/__init__.py index 083e474f..946822fe 100644 --- a/one/__init__.py +++ b/one/__init__.py @@ -1,2 +1,2 @@ """The Open Neurophysiology Environment (ONE) API.""" -__version__ = '2.5.0' +__version__ = '2.5.1' diff --git a/one/tests/test_one.py b/one/tests/test_one.py index 746c7ebb..36c9743d 100644 --- a/one/tests/test_one.py +++ b/one/tests/test_one.py @@ -1251,7 +1251,7 @@ def test_list_datasets(self): self.one._cache['datasets'] = self.one._cache['datasets'].iloc[0:0].copy() dsets = self.one.list_datasets(self.eid, details=True, query_type='remote') - self.assertEqual(166, len(dsets)) + self.assertEqual(171, len(dsets)) # this may change after a BWM release or patch # Test missing eid dsets = self.one.list_datasets('FMR019/2021-03-18/008', details=True, query_type='remote') @@ -1267,7 +1267,7 @@ def test_list_datasets(self): # Test details=False, with eid dsets = self.one.list_datasets(self.eid, details=False, query_type='remote') self.assertIsInstance(dsets, list) - self.assertEqual(166, len(dsets)) + self.assertEqual(171, len(dsets)) # this may change after a BWM release or patch # Test with other filters dsets = self.one.list_datasets(self.eid, collection='*probe*', filename='*channels*', @@ -1558,7 +1558,10 @@ def test_download_aws(self): # Test behaviour when dataset not remotely accessible dsets = dsets[:1].copy() rec = self.one.alyx.rest('datasets', 'read', id=dsets.index[0]) - rec['file_records'][-1]['exists'] = False # Set AWS file record to non-existent + # need to find the index of matching aws repo, this is not constant accross releases + iaws = list(map(lambda x: x['data_repository'].startswith('aws'), + rec['file_records'])).index(True) + rec['file_records'][iaws]['exists'] = False # Set AWS file record to non-existent with mock.patch('one.remote.aws.get_s3_from_alyx', return_value=(None, None)), \ mock.patch.object(self.one.alyx, 'rest', return_value=[rec]), \ self.assertLogs('one.api', logging.DEBUG) as log: From bf7468dbb9024c2053977c03137c02961d50fcd0 Mon Sep 17 00:00:00 2001 From: owinter Date: Thu, 21 Dec 2023 19:44:52 +0100 Subject: [PATCH 4/4] flake and configure .flake8 --- .flake8 | 2 +- one/tests/test_one.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.flake8 b/.flake8 index 2b368e15..ecc7e06a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] max-line-length = 99 -ignore = W504, W503, E266 +ignore = W504, W503, E266, D, BLK exclude = .git, __pycache__, diff --git a/one/tests/test_one.py b/one/tests/test_one.py index 36c9743d..90202c88 100644 --- a/one/tests/test_one.py +++ b/one/tests/test_one.py @@ -1267,7 +1267,7 @@ def test_list_datasets(self): # Test details=False, with eid dsets = self.one.list_datasets(self.eid, details=False, query_type='remote') self.assertIsInstance(dsets, list) - self.assertEqual(171, len(dsets)) # this may change after a BWM release or patch + self.assertEqual(171, len(dsets)) # this may change after a BWM release or patch # Test with other filters dsets = self.one.list_datasets(self.eid, collection='*probe*', filename='*channels*',