Olive

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
-## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.5.2]
+## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.6.0]
+
+### Modified
+
+- one.load_dataset
+  - add an option to skip computing hash for existing files when loading datasets `check_hash=False`
+  - check filesize before computing hash for performance
+
+### Added
+
+- one.remote.aws.get_s3_allen() convenience function to interact with Allen Institute S3 bucket for atlases
+
+## [2.5.2]
 
 ### Modified
 
@@ -11,7 +23,7 @@
 
 - exclude irrelevant s3 objects with source name in key, e.g. for foo/bar exclude foo/bar_baz/ key
 
-## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.5.0]
+## [2.5.0]
 
 ### Added
 

one/api.py

@@ -523,7 +523,7 @@ def sort_fcn(itm):
         else:
             return eids
 
-    def _check_filesystem(self, datasets, offline=None, update_exists=True):
+    def _check_filesystem(self, datasets, offline=None, update_exists=True, check_hash=True):
         """Update the local filesystem for the given datasets.
 
         Given a set of datasets, check whether records correctly reflect the filesystem.
@@ -579,15 +579,15 @@ def _check_filesystem(self, datasets, offline=None, update_exists=True):
             if file.exists():
                 # Check if there's a hash mismatch
                 # If so, add this index to list of datasets that need downloading
-                if rec['hash'] is not None:
+                if rec['file_size'] and file.stat().st_size != rec['file_size']:
+                    _logger.warning('local file size mismatch on dataset: %s',
+                                    PurePosixPath(rec.session_path, rec.rel_path))
+                    indices_to_download.append(i)
+                elif check_hash and rec['hash'] is not None:
                     if hashfile.md5(file) != rec['hash']:
                         _logger.warning('local md5 mismatch on dataset: %s',
                                         PurePosixPath(rec.session_path, rec.rel_path))
                         indices_to_download.append(i)
-                elif rec['file_size'] and file.stat().st_size != rec['file_size']:
-                    _logger.warning('local file size mismatch on dataset: %s',
-                                    PurePosixPath(rec.session_path, rec.rel_path))
-                    indices_to_download.append(i)
                 files.append(file)  # File exists so add to file list
             else:
                 # File doesn't exist so add None to output file list

one/remote/aws.py

@@ -168,6 +168,23 @@ def get_s3_public():
     return s3, S3_BUCKET_IBL
 
 
+def get_s3_allen():
+    """
+    Retrieve the Allen public S3 service resource.
+
+    Returns
+    -------
+    s3.ServiceResource
+        An S3 ServiceResource instance with the provided.
+    str
+        The name of the S3 bucket.
+    """
+    S3_BUCKET_ALLEN = 'allen-brain-cell-atlas'
+    session = boto3.Session(region_name='us-west-2')
+    s3 = session.resource('s3', config=Config(signature_version=UNSIGNED))
+    return s3, S3_BUCKET_ALLEN
+
+
 def get_s3_from_alyx(alyx, repo_name=REPO_DEFAULT):
     """
     Create an S3 resource instance using credentials from an Alyx data repository.

one/tests/remote/test_aws.py

@@ -143,6 +143,20 @@ def test_url2uri(self):
         uri, loc = aws.url2uri(url, return_location=True)
         self.assertEqual(loc, 'eu-east-1')
 
+    @mock.patch('boto3.Session')
+    def test_get_ibl_s3(self, session_mock):
+        s3, bucket = aws.get_s3_public()
+        resource = session_mock().resource
+        self.assertIs(s3, resource())
+        self.assertEqual(bucket, 'ibl-brain-wide-map-public')
+
+    @mock.patch('boto3.Session')
+    def test_get_allen_s3(self, session_mock):
+        s3, bucket = aws.get_s3_allen()
+        resource = session_mock().resource
+        self.assertIs(s3, resource())
+        self.assertEqual(bucket, 'allen-brain-cell-atlas')
+
 
 if __name__ == '__main__':
     unittest.main(exit=False)

one/webclient.py

@@ -615,6 +615,7 @@ def _generic_request(self, reqfunction, rest_query, data=None, files=None):
                 message = json.loads(r.text)
                 message.pop('status_code', None)  # Get status code from response object instead
                 message = message.get('detail') or message  # Get details if available
+                _logger.error(message)
             except json.decoder.JSONDecodeError:
                 message = r.text
             raise requests.HTTPError(r.status_code, rest_query, message, response=r)