From 9a735e555db3d2ac8dedad95937f02a38ddf74d8 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Thu, 17 Jan 2019 15:09:49 -0500 Subject: [PATCH 1/5] fix typo in CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 37da3eb..ff7aeae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,5 +24,5 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. Initial Release [Unreleased]: https://github.com/sat-utils/sat-stac/compare/master...develop -[v0.1.1]: https://github.com/sat-utils/sat-api/compare/0.1.0...v0.1.1 -[v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0 \ No newline at end of file +[v0.1.1]: https://github.com/sat-utils/sat-stac/compare/0.1.0...v0.1.1 +[v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0 From e3ec7316f1d433918e33f87fadcf13a328e0efdb Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Thu, 14 Feb 2019 14:09:32 -0500 Subject: [PATCH 2/5] fix typo in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0bd9bed..1a899e6 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ pip install . The latest version of sat-stac is 0.1.1, which uses the STAC spec v0.6.0. To install other versions of sat-stac, install the matching version of sat-stac. ```bash -pip install satstac==0.1.0 +pip install sat-stac==0.1.0 ``` The table below shows the corresponding versions between sat-stac and STAC: From 8b449d4c45cc449e1ad4c42465cb40fa3ca12228 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Thu, 14 Feb 2019 14:17:40 -0500 Subject: [PATCH 3/5] add requestor_pays keyword --- satstac/item.py | 4 ++-- satstac/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/satstac/item.py b/satstac/item.py index 526fbfe..a1c9b82 100644 --- a/satstac/item.py +++ b/satstac/item.py @@ -123,7 +123,7 @@ def substitute(self, string): subs[key] = self[key.replace('_colon_', ':')] return Template(string).substitute(**subs) - def download(self, key, overwrite=False, path='', filename='${id}'): + def download(self, key, overwrite=False, path='', filename='${id}', requestor_pays=False): """ Download this key (e.g., a band, or metadata file) from the scene """ asset = self.asset(key) if asset is None: @@ -137,7 +137,7 @@ def download(self, key, overwrite=False, path='', filename='${id}'): ext = os.path.splitext(asset['href'])[1] fout = os.path.join(_path, fname + '_' + key + ext) if not os.path.exists(fout) or overwrite: - _filename = utils.download_file(asset['href'], filename=fout) + _filename = utils.download_file(asset['href'], filename=fout, requestor_pays=requestor_pays) else: _filename = fout except Exception as e: diff --git a/satstac/utils.py b/satstac/utils.py index d6e0328..39a713a 100644 --- a/satstac/utils.py +++ b/satstac/utils.py @@ -52,14 +52,14 @@ def dict_merge(dct, merge_dct, add_keys=True): return dct -def download_file(url, filename=None): +def download_file(url, filename=None, requestor_pays=False): """ Download a file as filename """ filename = os.path.basename(url) if filename is None else filename logger.info('Downloading %s as %s' % (url, filename)) headers = {} # check if on s3, if so try to sign it if 's3.amazonaws.com' in url: - signed_url, signed_headers = get_s3_signed_url(url) + signed_url, signed_headers = get_s3_signed_url(url, requestor_pays=requestor_pays) resp = requests.get(signed_url, headers=signed_headers, stream=True) if resp.status_code != 200: resp = requests.get(url, headers=headers, stream=True) From 778b73512a4e40bbe1a4b0648f8227f2e16d0712 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Thu, 14 Feb 2019 14:53:08 -0500 Subject: [PATCH 4/5] add download_assets functions + tests --- satstac/item.py | 9 +++++++++ satstac/items.py | 8 ++++++++ test/test_item.py | 7 +++++++ test/test_items.py | 24 ++++++++++++++++++++---- 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/satstac/item.py b/satstac/item.py index a1c9b82..672d806 100644 --- a/satstac/item.py +++ b/satstac/item.py @@ -123,6 +123,15 @@ def substitute(self, string): subs[key] = self[key.replace('_colon_', ':')] return Template(string).substitute(**subs) + def download_assets(self, keys=None, **kwargs): + """ Download multiple assets """ + if keys is None: + keys = self.data['assets'].keys() + filenames = [] + for key in keys: + filenames.append(self.download(key, **kwargs)) + return filenames + def download(self, key, overwrite=False, path='', filename='${id}', requestor_pays=False): """ Download this key (e.g., a band, or metadata file) from the scene """ asset = self.asset(key) diff --git a/satstac/items.py b/satstac/items.py index ec9506b..79e60ad 100644 --- a/satstac/items.py +++ b/satstac/items.py @@ -118,6 +118,14 @@ def filter(self, key, values): items += list(filter(lambda x: x[key] == val, self._items)) self._items = items + def download_assets(self, *args, **kwargs): + filenames = [] + for i in self._items: + fnames = i.download_assets(*args, **kwargs) + if len(fnames) > 0: + filenames.append(fnames) + return filenames + def download(self, *args, **kwargs): """ Download all Items """ dls = [] diff --git a/test/test_item.py b/test/test_item.py index 5181583..c032d0b 100644 --- a/test/test_item.py +++ b/test/test_item.py @@ -97,6 +97,13 @@ def test_download(self): fname = item.download(key='MTL', path=self.path) assert(os.path.exists(fname)) + def test_download_assets(self): + """ Retrieve multiple data files """ + item = Item.open(self.filename) + fnames = item.download_assets(keys=['MTL', 'ANG'], path=self.path) + for f in fnames: + assert(os.path.exists(f)) + def test_download_nonexist(self): """ Test downloading of non-existent file """ item = Item.open(self.filename) diff --git a/test/test_items.py b/test/test_items.py index f20a504..1b0b497 100644 --- a/test/test_items.py +++ b/test/test_items.py @@ -2,12 +2,21 @@ import unittest from satstac import Items, Item +from shutil import rmtree testpath = os.path.dirname(__file__) class Test(unittest.TestCase): + path = os.path.join(testpath, 'test-item') + + @classmethod + def tearDownClass(cls): + """ Remove test files """ + if os.path.exists(cls.path): + rmtree(cls.path) + def load_items(self): return Items.load(os.path.join(testpath, 'items.json')) @@ -89,14 +98,21 @@ def test_filter(self): items.filter('eo:cloud_cover', [100]) assert(len(items) == 1) + def test_download_assets(self): + """ Download multiple assets from all items """ + items = self.load_items() + filenames = items.download_assets(keys=['MTL', 'ANG'], path=self.path) + assert(len(filenames) == 2) + for fnames in filenames: + assert(len(fnames) == 2) + for f in fnames: + assert(os.path.exists(f)) + def test_download(self): """ Download a data file from all items """ items = self.load_items() - fnames = items.download(key='MTL') + fnames = items.download(key='MTL', path=self.path) assert(len(fnames) == 2) for f in fnames: assert(os.path.exists(f)) - os.remove(f) - assert(not os.path.exists(f)) - #shutil.rmtree(os.path.join(testpath, 'landsat-8-l1')) From 0f4e5cec54ff2f3d9758771cbd1390acabb3a4a6 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Thu, 14 Feb 2019 14:53:39 -0500 Subject: [PATCH 5/5] bump version and update CHANGELOG --- CHANGELOG.md | 9 ++++++++- satstac/version.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff7aeae..29f1f0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v0.1.2] - 2019-02-14 + +### Added +- Item.download_assets() and Items.download_assets() works as download() except accepts a list of keys rather than a single key. Passing in `None` for keys will download all assets. +- requestor_pays keyword option added to Item.download() (and Items.download()). Defaults to False. Use it to acknowledge paying egress costs when downloading data from a Reqeuestor Pays bucket (e.g., Sentinel-2). If the bucket is requestor pays and this is not set to True an AccessDenied error message will occur. + ## [v0.1.1] - 2019-01-15 ### Added @@ -15,7 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Python 3 only. With Python 2.7 going unsupported in 2020 the time has come to stop supporting 2.7. There are too many additions in Python3 that continue to make backward compatability with Python 2.7 more difficult. In the case of this release the addition of caching using `functools` made sat-stac incompatible with Python 2.7. -- More lenient version requirements for `requests` (now <=2.19.1). Otherwise can cause dependency incompatibility problems in some cases. +- More lenient version requirements for `requests` (now >=2.19.1). Otherwise can cause dependency incompatibility problems in some cases. - Behavior of `path` and `filename` keyword arguments to Collection.add_item() has changed slightly. The components of `path` are now exclusively used to generate sub-catalogs, while `filename` is the relative filename (which could include a further subdirectory) from the last sub-catalog (it's parent). Before, it was assumed that Item files were always in a single subdirectory under it's parent catalog. - Tutorials updated @@ -24,5 +30,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. Initial Release [Unreleased]: https://github.com/sat-utils/sat-stac/compare/master...develop +[v0.1.2]: https://github.com/sat-utils/sat-stac/compare/0.1.1...v0.1.2 [v0.1.1]: https://github.com/sat-utils/sat-stac/compare/0.1.0...v0.1.1 [v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0 diff --git a/satstac/version.py b/satstac/version.py index df9144c..10939f0 100644 --- a/satstac/version.py +++ b/satstac/version.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = '0.1.2'