Merge pull request #104 from int-brain-lab/bugfix_aws

fix aws download bug

.flake8

@@ -1,6 +1,6 @@
 [flake8]
 max-line-length = 99
-ignore = W504, W503, E266
+ignore = W504, W503, E266, D, BLK
 exclude =
     .git,
     __pycache__,

one/__init__.py

@@ -1,2 +1,2 @@
 """The Open Neurophysiology Environment (ONE) API."""
-__version__ = '2.5.0'
+__version__ = '2.5.1'

one/remote/aws.py

@@ -281,6 +281,12 @@ def s3_download_folder(source, destination, s3=None, bucket_name=S3_BUCKET_IBL,
     local_files = []
     objects = s3.Bucket(name=bucket_name).objects.filter(Prefix=source)
     for obj_summary in filter(lambda x: not is_folder(x), objects):
+        # we can only filter an object collection by prefix, so we need to make sure the file
+        # is in the subpath of the source folder
+        # for example, if source is '/toto/tata' and obj_summary.key is
+        # '/toto/tata_alaternate/titi.txt', we need to exclude it
+        if not Path(source) in Path(obj_summary.key).parents:
+            continue
         local_file = Path(destination).joinpath(Path(obj_summary.key).relative_to(source))
         lf = s3_download_file(obj_summary.key, local_file, s3=s3, bucket_name=bucket_name,
                               overwrite=overwrite)

one/tests/test_one.py

@@ -1251,7 +1251,7 @@ def test_list_datasets(self):
         self.one._cache['datasets'] = self.one._cache['datasets'].iloc[0:0].copy()
 
         dsets = self.one.list_datasets(self.eid, details=True, query_type='remote')
-        self.assertEqual(166, len(dsets))
+        self.assertEqual(171, len(dsets))  # this may change after a BWM release or patch
 
         # Test missing eid
         dsets = self.one.list_datasets('FMR019/2021-03-18/008', details=True, query_type='remote')
@@ -1267,7 +1267,7 @@ def test_list_datasets(self):
         # Test details=False, with eid
         dsets = self.one.list_datasets(self.eid, details=False, query_type='remote')
         self.assertIsInstance(dsets, list)
-        self.assertEqual(166, len(dsets))
+        self.assertEqual(171, len(dsets))  # this may change after a BWM release or patch
 
         # Test with other filters
         dsets = self.one.list_datasets(self.eid, collection='*probe*', filename='*channels*',
@@ -1558,7 +1558,10 @@ def test_download_aws(self):
         # Test behaviour when dataset not remotely accessible
         dsets = dsets[:1].copy()
         rec = self.one.alyx.rest('datasets', 'read', id=dsets.index[0])
-        rec['file_records'][-1]['exists'] = False  # Set AWS file record to non-existent
+        # need to find the index of matching aws repo, this is not constant accross releases
+        iaws = list(map(lambda x: x['data_repository'].startswith('aws'),
+                        rec['file_records'])).index(True)
+        rec['file_records'][iaws]['exists'] = False  # Set AWS file record to non-existent
         with mock.patch('one.remote.aws.get_s3_from_alyx', return_value=(None, None)), \
                 mock.patch.object(self.one.alyx, 'rest', return_value=[rec]), \
                 self.assertLogs('one.api', logging.DEBUG) as log: