diff --git a/CHANGELOG.md b/CHANGELOG.md index 37da3eb..29f1f0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +## [v0.1.2] - 2019-02-14 + +### Added +- Item.download_assets() and Items.download_assets() works as download() except accepts a list of keys rather than a single key. Passing in `None` for keys will download all assets. +- requestor_pays keyword option added to Item.download() (and Items.download()). Defaults to False. Use it to acknowledge paying egress costs when downloading data from a Reqeuestor Pays bucket (e.g., Sentinel-2). If the bucket is requestor pays and this is not set to True an AccessDenied error message will occur. + ## [v0.1.1] - 2019-01-15 ### Added @@ -15,7 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Python 3 only. With Python 2.7 going unsupported in 2020 the time has come to stop supporting 2.7. There are too many additions in Python3 that continue to make backward compatability with Python 2.7 more difficult. In the case of this release the addition of caching using `functools` made sat-stac incompatible with Python 2.7. -- More lenient version requirements for `requests` (now <=2.19.1). Otherwise can cause dependency incompatibility problems in some cases. +- More lenient version requirements for `requests` (now >=2.19.1). Otherwise can cause dependency incompatibility problems in some cases. - Behavior of `path` and `filename` keyword arguments to Collection.add_item() has changed slightly. The components of `path` are now exclusively used to generate sub-catalogs, while `filename` is the relative filename (which could include a further subdirectory) from the last sub-catalog (it's parent). Before, it was assumed that Item files were always in a single subdirectory under it's parent catalog. - Tutorials updated @@ -24,5 +30,6 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. Initial Release [Unreleased]: https://github.com/sat-utils/sat-stac/compare/master...develop -[v0.1.1]: https://github.com/sat-utils/sat-api/compare/0.1.0...v0.1.1 -[v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0 \ No newline at end of file +[v0.1.2]: https://github.com/sat-utils/sat-stac/compare/0.1.1...v0.1.2 +[v0.1.1]: https://github.com/sat-utils/sat-stac/compare/0.1.0...v0.1.1 +[v0.1.0]: https://github.com/sat-utils/sat-stac/tree/0.1.0 diff --git a/README.md b/README.md index 0bd9bed..1a899e6 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ $ pip install . The latest version of sat-stac is 0.1.1, which uses the STAC spec v0.6.0. To install other versions of sat-stac, install the matching version of sat-stac. ```bash -pip install satstac==0.1.0 +pip install sat-stac==0.1.0 ``` The table below shows the corresponding versions between sat-stac and STAC: diff --git a/satstac/item.py b/satstac/item.py index 526fbfe..672d806 100644 --- a/satstac/item.py +++ b/satstac/item.py @@ -123,7 +123,16 @@ def substitute(self, string): subs[key] = self[key.replace('_colon_', ':')] return Template(string).substitute(**subs) - def download(self, key, overwrite=False, path='', filename='${id}'): + def download_assets(self, keys=None, **kwargs): + """ Download multiple assets """ + if keys is None: + keys = self.data['assets'].keys() + filenames = [] + for key in keys: + filenames.append(self.download(key, **kwargs)) + return filenames + + def download(self, key, overwrite=False, path='', filename='${id}', requestor_pays=False): """ Download this key (e.g., a band, or metadata file) from the scene """ asset = self.asset(key) if asset is None: @@ -137,7 +146,7 @@ def download(self, key, overwrite=False, path='', filename='${id}'): ext = os.path.splitext(asset['href'])[1] fout = os.path.join(_path, fname + '_' + key + ext) if not os.path.exists(fout) or overwrite: - _filename = utils.download_file(asset['href'], filename=fout) + _filename = utils.download_file(asset['href'], filename=fout, requestor_pays=requestor_pays) else: _filename = fout except Exception as e: diff --git a/satstac/items.py b/satstac/items.py index ec9506b..79e60ad 100644 --- a/satstac/items.py +++ b/satstac/items.py @@ -118,6 +118,14 @@ def filter(self, key, values): items += list(filter(lambda x: x[key] == val, self._items)) self._items = items + def download_assets(self, *args, **kwargs): + filenames = [] + for i in self._items: + fnames = i.download_assets(*args, **kwargs) + if len(fnames) > 0: + filenames.append(fnames) + return filenames + def download(self, *args, **kwargs): """ Download all Items """ dls = [] diff --git a/satstac/utils.py b/satstac/utils.py index d6e0328..39a713a 100644 --- a/satstac/utils.py +++ b/satstac/utils.py @@ -52,14 +52,14 @@ def dict_merge(dct, merge_dct, add_keys=True): return dct -def download_file(url, filename=None): +def download_file(url, filename=None, requestor_pays=False): """ Download a file as filename """ filename = os.path.basename(url) if filename is None else filename logger.info('Downloading %s as %s' % (url, filename)) headers = {} # check if on s3, if so try to sign it if 's3.amazonaws.com' in url: - signed_url, signed_headers = get_s3_signed_url(url) + signed_url, signed_headers = get_s3_signed_url(url, requestor_pays=requestor_pays) resp = requests.get(signed_url, headers=signed_headers, stream=True) if resp.status_code != 200: resp = requests.get(url, headers=headers, stream=True) diff --git a/satstac/version.py b/satstac/version.py index df9144c..10939f0 100644 --- a/satstac/version.py +++ b/satstac/version.py @@ -1 +1 @@ -__version__ = '0.1.1' +__version__ = '0.1.2' diff --git a/test/test_item.py b/test/test_item.py index 5181583..c032d0b 100644 --- a/test/test_item.py +++ b/test/test_item.py @@ -97,6 +97,13 @@ def test_download(self): fname = item.download(key='MTL', path=self.path) assert(os.path.exists(fname)) + def test_download_assets(self): + """ Retrieve multiple data files """ + item = Item.open(self.filename) + fnames = item.download_assets(keys=['MTL', 'ANG'], path=self.path) + for f in fnames: + assert(os.path.exists(f)) + def test_download_nonexist(self): """ Test downloading of non-existent file """ item = Item.open(self.filename) diff --git a/test/test_items.py b/test/test_items.py index f20a504..1b0b497 100644 --- a/test/test_items.py +++ b/test/test_items.py @@ -2,12 +2,21 @@ import unittest from satstac import Items, Item +from shutil import rmtree testpath = os.path.dirname(__file__) class Test(unittest.TestCase): + path = os.path.join(testpath, 'test-item') + + @classmethod + def tearDownClass(cls): + """ Remove test files """ + if os.path.exists(cls.path): + rmtree(cls.path) + def load_items(self): return Items.load(os.path.join(testpath, 'items.json')) @@ -89,14 +98,21 @@ def test_filter(self): items.filter('eo:cloud_cover', [100]) assert(len(items) == 1) + def test_download_assets(self): + """ Download multiple assets from all items """ + items = self.load_items() + filenames = items.download_assets(keys=['MTL', 'ANG'], path=self.path) + assert(len(filenames) == 2) + for fnames in filenames: + assert(len(fnames) == 2) + for f in fnames: + assert(os.path.exists(f)) + def test_download(self): """ Download a data file from all items """ items = self.load_items() - fnames = items.download(key='MTL') + fnames = items.download(key='MTL', path=self.path) assert(len(fnames) == 2) for f in fnames: assert(os.path.exists(f)) - os.remove(f) - assert(not os.path.exists(f)) - #shutil.rmtree(os.path.join(testpath, 'landsat-8-l1'))