diff --git a/.flake8 b/.flake8 index 2b368e15..ecc7e06a 100644 --- a/.flake8 +++ b/.flake8 @@ -1,6 +1,6 @@ [flake8] max-line-length = 99 -ignore = W504, W503, E266 +ignore = W504, W503, E266, D, BLK exclude = .git, __pycache__, diff --git a/one/__init__.py b/one/__init__.py index 083e474f..946822fe 100644 --- a/one/__init__.py +++ b/one/__init__.py @@ -1,2 +1,2 @@ """The Open Neurophysiology Environment (ONE) API.""" -__version__ = '2.5.0' +__version__ = '2.5.1' diff --git a/one/remote/aws.py b/one/remote/aws.py index 0c9b41dc..1b6015d8 100644 --- a/one/remote/aws.py +++ b/one/remote/aws.py @@ -281,6 +281,12 @@ def s3_download_folder(source, destination, s3=None, bucket_name=S3_BUCKET_IBL, local_files = [] objects = s3.Bucket(name=bucket_name).objects.filter(Prefix=source) for obj_summary in filter(lambda x: not is_folder(x), objects): + # we can only filter an object collection by prefix, so we need to make sure the file + # is in the subpath of the source folder + # for example, if source is '/toto/tata' and obj_summary.key is + # '/toto/tata_alaternate/titi.txt', we need to exclude it + if not Path(source) in Path(obj_summary.key).parents: + continue local_file = Path(destination).joinpath(Path(obj_summary.key).relative_to(source)) lf = s3_download_file(obj_summary.key, local_file, s3=s3, bucket_name=bucket_name, overwrite=overwrite) diff --git a/one/tests/test_one.py b/one/tests/test_one.py index 746c7ebb..90202c88 100644 --- a/one/tests/test_one.py +++ b/one/tests/test_one.py @@ -1251,7 +1251,7 @@ def test_list_datasets(self): self.one._cache['datasets'] = self.one._cache['datasets'].iloc[0:0].copy() dsets = self.one.list_datasets(self.eid, details=True, query_type='remote') - self.assertEqual(166, len(dsets)) + self.assertEqual(171, len(dsets)) # this may change after a BWM release or patch # Test missing eid dsets = self.one.list_datasets('FMR019/2021-03-18/008', details=True, query_type='remote') @@ -1267,7 +1267,7 @@ def test_list_datasets(self): # Test details=False, with eid dsets = self.one.list_datasets(self.eid, details=False, query_type='remote') self.assertIsInstance(dsets, list) - self.assertEqual(166, len(dsets)) + self.assertEqual(171, len(dsets)) # this may change after a BWM release or patch # Test with other filters dsets = self.one.list_datasets(self.eid, collection='*probe*', filename='*channels*', @@ -1558,7 +1558,10 @@ def test_download_aws(self): # Test behaviour when dataset not remotely accessible dsets = dsets[:1].copy() rec = self.one.alyx.rest('datasets', 'read', id=dsets.index[0]) - rec['file_records'][-1]['exists'] = False # Set AWS file record to non-existent + # need to find the index of matching aws repo, this is not constant accross releases + iaws = list(map(lambda x: x['data_repository'].startswith('aws'), + rec['file_records'])).index(True) + rec['file_records'][iaws]['exists'] = False # Set AWS file record to non-existent with mock.patch('one.remote.aws.get_s3_from_alyx', return_value=(None, None)), \ mock.patch.object(self.one.alyx, 'rest', return_value=[rec]), \ self.assertLogs('one.api', logging.DEBUG) as log: