Skip to content

Commit

Permalink
add changelog and allen s3 bucket convenience functions
Browse files Browse the repository at this point in the history
  • Loading branch information
oliche committed Jan 10, 2024
1 parent 78765f6 commit 9b23f15
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 2 deletions.
16 changes: 14 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,17 @@
# Changelog
## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.5.2]
## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.6.0]

### Modified

- one.load_dataset
- add an option to skip computing hash for existing files when loading datasets `check_hash=False`
- check filesize before computing hash for performance

### Added

- one.remote.aws.get_s3_allen() convenience function to interact with Allen Institute S3 bucket for atlases

## [2.5.2]

### Modified

Expand All @@ -11,7 +23,7 @@

- exclude irrelevant s3 objects with source name in key, e.g. for foo/bar exclude foo/bar_baz/ key

## [Latest](https://github.com/int-brain-lab/ONE/commits/main) [2.5.0]
## [2.5.0]

### Added

Expand Down
17 changes: 17 additions & 0 deletions one/remote/aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,23 @@ def get_s3_public():
return s3, S3_BUCKET_IBL


def get_s3_allen():
"""
Retrieve the Allen public S3 service resource.
Returns
-------
s3.ServiceResource
An S3 ServiceResource instance with the provided.
str
The name of the S3 bucket.
"""
S3_BUCKET_ALLEN = 'allen-brain-cell-atlas'
session = boto3.Session(region_name='us-west-2')
s3 = session.resource('s3', config=Config(signature_version=UNSIGNED))
return s3, S3_BUCKET_ALLEN


def get_s3_from_alyx(alyx, repo_name=REPO_DEFAULT):
"""
Create an S3 resource instance using credentials from an Alyx data repository.
Expand Down
14 changes: 14 additions & 0 deletions one/tests/remote/test_aws.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,20 @@ def test_url2uri(self):
uri, loc = aws.url2uri(url, return_location=True)
self.assertEqual(loc, 'eu-east-1')

@mock.patch('boto3.Session')
def test_get_ibl_s3(self, session_mock):
s3, bucket = aws.get_s3_public()
resource = session_mock().resource
self.assertIs(s3, resource())
self.assertEqual(bucket, 'ibl-brain-wide-map-public')

@mock.patch('boto3.Session')
def test_get_allen_s3(self, session_mock):
s3, bucket = aws.get_s3_allen()
resource = session_mock().resource
self.assertIs(s3, resource())
self.assertEqual(bucket, 'allen-brain-cell-atlas')


if __name__ == '__main__':
unittest.main(exit=False)

0 comments on commit 9b23f15

Please sign in to comment.