From 055ecde26fa09f8faadb99c331a70c5fdc3e0294 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 17:04:19 -0400 Subject: [PATCH 01/15] rename Thing.publish() to Thing.add_self, and add some log statements --- CHANGELOG.md | 4 ++++ satstac/thing.py | 36 +++++++++++++++++++++++------------- 2 files changed, 27 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 147c6c7..a3355a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] +### Changed +- Thing.publish() function now called Thing.add_self(), same arguments + + ## [v0.1.3] - 2019-05-04 ### Added diff --git a/satstac/thing.py b/satstac/thing.py index 1c9efa1..fa11446 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -2,6 +2,7 @@ import os import requests +from logging import getLogger try: # Python 3 from urllib.parse import urljoin @@ -12,6 +13,9 @@ from .utils import mkdirp, get_s3_signed_url +logger = getLogger(__name__) + + class STACError(Exception): pass @@ -42,7 +46,7 @@ def open_remote(self, url, headers={}): @classmethod def open(cls, filename): """ Open an existing JSON data file """ - # TODO - open remote URLs + logger.debug('Opening %s' % filename) if filename[0:5] == 'https': try: dat = cls.open_remote(filename) @@ -113,6 +117,19 @@ def add_link(self, rel, link, type=None, title=None): l['title'] = title self.data['links'].append(l) + def add_self(self, endpoint, root): + """ Update self link with endpoint """ + if self.filename is None: + raise STACError('No filename, use save_as() before publishing') + # keep everything except self and root + links = [l for l in self.data['links'] if l['rel'] not in ['self', 'root']] + to_item = os.path.relpath(self.filename, os.path.dirname(root)) + to_root = os.path.relpath(root, os.path.dirname(self.filename)) + links.insert(0, {'rel': 'root', 'href': to_root}) + links.insert(0, {'rel': 'self', 'href': os.path.join(endpoint, to_item)}) + self.data['links'] = links + self.save() + def clean_hierarchy(self): """ Clean links of self, parent, and child links (for moving and publishing) """ rels = ['self', 'root', 'parent', 'child', 'collection', 'item'] @@ -131,6 +148,7 @@ def save(self): """ Write a catalog file """ if self.filename is None: raise STACError('No filename, use save_as()') + logger.debug('Saving %s as %s' % (self.id, self.filename)) fname = self.filename if self.filename[0:5] == 'https': # use signed URL @@ -156,15 +174,7 @@ def save_as(self, filename): self.save() return self - def publish(self, endpoint, root): - """ Update self link with endpoint """ - if self.filename is None: - raise STACError('No filename, use save_as() before publishing') - # keep everything except self and root - links = [l for l in self.data['links'] if l['rel'] not in ['self', 'root']] - to_item = os.path.relpath(self.filename, os.path.dirname(root)) - to_root = os.path.relpath(root, os.path.dirname(self.filename)) - links.insert(0, {'rel': 'root', 'href': to_root}) - links.insert(0, {'rel': 'self', 'href': os.path.join(endpoint, to_item)}) - self.data['links'] = links - self.save() + def publish(self, *args, **kwargs): + """ DEPRECATED - use add_self() """ + logger.warning('Thing.publish() is deprecated, use Thing.add_self() instead') + self.add_self(*args, **kwargs) \ No newline at end of file From bf7e77d60e02e06e8d55fb43b99a949b72f4ec78 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 17:05:23 -0400 Subject: [PATCH 02/15] bump version to a 0.2 alpha for STAC 0.7.0 --- README.md | 1 + satstac/version.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2da4ccb..08cd6d9 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ The table below shows the corresponding versions between sat-stac and STAC: | sat-stac | STAC | | -------- | ---- | | 0.1.x | 0.6.x | +| 0.2.x | 0.7.x | ## Tutorials diff --git a/satstac/version.py b/satstac/version.py index 8ce9b36..f2303e6 100644 --- a/satstac/version.py +++ b/satstac/version.py @@ -1 +1 @@ -__version__ = '0.1.3' +__version__ = '0.2.0a1' From 087ac539fb1e6c2b08367d26b6b7c9eecb0d2f3c Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 18:32:14 -0400 Subject: [PATCH 03/15] simplify cli for just creating catalogs - remove option for providing endpoint (b/c self links no longer required) --- satstac/cli.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/satstac/cli.py b/satstac/cli.py index 2b80441..389f055 100644 --- a/satstac/cli.py +++ b/satstac/cli.py @@ -28,17 +28,7 @@ def parse_args(args): parser.add_argument('id', help='ID of the new catalog') parser.add_argument('description', help='Description of new catalog') parser.add_argument('--filename', help='Filename of catalog', default='catalog.json') - group = parser.add_argument_group('root catalog options (mutually exclusive)') - group = group.add_mutually_exclusive_group(required=True) - group.add_argument('--root', help='Filename to existing root catalog', default=None) - group.add_argument('--endpoint', help='Endpoint for this new root catalog', default=None) - - # command 2 - h = 'Update entire catalog with a new endpoint (update self links)' - parser = subparsers.add_parser('publish', parents=[pparser], help=h, formatter_class=dhf) - parser.add_argument('root', help='Filename to existing root catalog') - parser.add_argument('endpoint', help='New endpoint') - # parser.add_argument() + parser.add_argument('--root', help='Filename to existing root catalog', default=None) # turn Namespace into dictinary parsed_args = vars(parser0.parse_args(args)) @@ -59,9 +49,6 @@ def cli(): else: cat = Catalog.create(id=args['id'], description=args['description'], root=args['endpoint']) cat.save_as(args['filename']) - elif cmd == 'publish': - cat = Catalog.open(args['root']) - cat.publish(args['endpoint']) if __name__ == "__main__": From 1150cfc211d9c1fda6991e5457123b4c326864fb Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 18:33:35 -0400 Subject: [PATCH 04/15] remove publish function deprecation --- satstac/thing.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/satstac/thing.py b/satstac/thing.py index fa11446..09f2a2c 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -120,7 +120,7 @@ def add_link(self, rel, link, type=None, title=None): def add_self(self, endpoint, root): """ Update self link with endpoint """ if self.filename is None: - raise STACError('No filename, use save_as() before publishing') + raise STACError('No filename, use save_as() first') # keep everything except self and root links = [l for l in self.data['links'] if l['rel'] not in ['self', 'root']] to_item = os.path.relpath(self.filename, os.path.dirname(root)) @@ -172,9 +172,4 @@ def save_as(self, filename): self.add_link('self', os.path.join(self._root, os.path.basename(filename))) self.add_link('root', './%s' % os.path.basename(filename)) self.save() - return self - - def publish(self, *args, **kwargs): - """ DEPRECATED - use add_self() """ - logger.warning('Thing.publish() is deprecated, use Thing.add_self() instead') - self.add_self(*args, **kwargs) \ No newline at end of file + return self \ No newline at end of file From f7a5957de8ac1dde28de0abd5b626054c7bd405e Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 20:09:16 -0400 Subject: [PATCH 05/15] use yield from (python 3 only) --- satstac/catalog.py | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/satstac/catalog.py b/satstac/catalog.py index 011c474..359c196 100644 --- a/satstac/catalog.py +++ b/satstac/catalog.py @@ -24,11 +24,13 @@ def description(self): return self.data.get('description', '') @classmethod - def create(cls, id='stac-catalog', description='A STAC Catalog', root=None, **kwargs): + def create(cls, id='stac-catalog', title='A STAC Catalog', + description='A STAC Catalog', root=None, **kwargs): """ Create new catalog """ kwargs.update({ 'id': id, 'stac_version': STAC_VERSION, + 'title': title, 'description': description, 'links': [] }) @@ -45,11 +47,7 @@ def catalogs(self): for cat in self.children(): for subcat in cat.children(): yield subcat - # Python 2 - for x in subcat.catalogs(): - yield x - # Python 3.3+ - # yield from subcat.catalogs() + yield from subcat.catalogs() yield cat def collections(self): @@ -59,22 +57,14 @@ def collections(self): yield Collection.open(cat.filename) # TODO - keep going? if other Collections can appear below a Collection else: - # Python 2 - for x in cat.collections(): - yield x - # Python 3.3+ - # yield from cat.collections() + yield from cat.collections() def items(self): """ Recursively get all items within this Catalog """ for item in self.links('item'): yield Item.open(item) for child in self.children(): - # Python 2 - for x in child.items(): - yield x - # Python 3.3+ - # yield from child.items() + yield from child.items() def add_catalog(self, catalog): """ Add a catalog to this catalog """ From f5aa6657e842a5856f4d9d71930017b9fec92313 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 20:14:42 -0400 Subject: [PATCH 06/15] remove _root property from Thing --- satstac/catalog.py | 1 - satstac/thing.py | 5 ----- 2 files changed, 6 deletions(-) diff --git a/satstac/catalog.py b/satstac/catalog.py index 359c196..f350bcd 100644 --- a/satstac/catalog.py +++ b/satstac/catalog.py @@ -11,7 +11,6 @@ class Catalog(Thing): def __init__(self, data, root=None, **kwargs): """ Initialize a catalog with a catalog file """ super(Catalog, self).__init__(data, **kwargs) - self._root = root @property def stac_version(self): diff --git a/satstac/thing.py b/satstac/thing.py index 09f2a2c..b465a3f 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -28,7 +28,6 @@ def __init__(self, data, filename=None): self.data = data if 'links' not in self.data.keys(): self.data['links'] = [] - self._root = None def __repr__(self): return self.id @@ -167,9 +166,5 @@ def save(self): def save_as(self, filename): """ Write a catalog file to a new file """ self.filename = filename - # TODO - if this is a root then add root link and self links to itself - if self._root is not None: - self.add_link('self', os.path.join(self._root, os.path.basename(filename))) - self.add_link('root', './%s' % os.path.basename(filename)) self.save() return self \ No newline at end of file From faa221ca6fe2cd85d19c8e544973a75c799ee599 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Fri, 5 Jul 2019 20:16:15 -0400 Subject: [PATCH 07/15] combine save_as() into save() with filename keyword --- satstac/thing.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/satstac/thing.py b/satstac/thing.py index b465a3f..b8de3a3 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -143,10 +143,12 @@ def __getitem__(self, key): props = self.data.get('properties', {}) return props.get(key, None) - def save(self): + def save(self, filename=None): """ Write a catalog file """ + if filename is not None: + self.filename = filename if self.filename is None: - raise STACError('No filename, use save_as()') + raise STACError('No filename provided, specify with filename keyword') logger.debug('Saving %s as %s' % (self.id, self.filename)) fname = self.filename if self.filename[0:5] == 'https': @@ -161,10 +163,4 @@ def save(self): mkdirp(os.path.dirname(fname)) with open(fname, 'w') as f: f.write(json.dumps(self.data)) - return self - - def save_as(self, filename): - """ Write a catalog file to a new file """ - self.filename = filename - self.save() return self \ No newline at end of file From 5db94730da4558df8c8b3fb71a4b0ad089e182a5 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Sun, 14 Jul 2019 23:48:42 -0400 Subject: [PATCH 08/15] update root and parent logic --- CHANGELOG.md | 2 +- satstac/thing.py | 23 +++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a3355a4..5873b0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Thing.publish() function now called Thing.add_self(), same arguments - +- Thing.root() and Thing.parent() functions now return `None` if no root or parent (rather than an empty list). If more than one root or parent then an error will now be thrown. ## [v0.1.3] - 2019-05-04 diff --git a/satstac/thing.py b/satstac/thing.py index b8de3a3..68ebdd2 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -3,12 +3,7 @@ import requests from logging import getLogger -try: - # Python 3 - from urllib.parse import urljoin -except ImportError: - # Python 2 - from urlparse import urljoin +from urllib.parse import urljoin from .version import __version__ from .utils import mkdirp, get_s3_signed_url @@ -26,6 +21,8 @@ def __init__(self, data, filename=None): """ Initialize a new class with a dictionary """ self.filename = filename self.data = data + if 'id' not in data: + raise STACError('ID is required') if 'links' not in self.data.keys(): self.data['links'] = [] @@ -96,12 +93,22 @@ def links(self, rel=None): def root(self): """ Get root link """ links = self.links('root') - return self.open(links[0]) if len(links) == 1 else [] + if len(links) == 1: + return self.open(links[0]) + elif len(links) == 0: + return None + else: + raise STACError('More than one root provided') def parent(self): """ Get parent link """ links = self.links('parent') - return self.open(links[0]) if len(links) == 1 else [] + if len(links) == 1: + return self.open(links[0]) + elif len(links) == 0: + return None + else: + raise STACError('More than one parent provided') def add_link(self, rel, link, type=None, title=None): """ Add a new link """ From 721c690110fcb5132f63f025cd810a003c00b25d Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Sun, 14 Jul 2019 23:49:14 -0400 Subject: [PATCH 09/15] some test updates --- test/test_catalog.py | 2 +- test/test_collection.py | 4 ++-- test/test_thing.py | 44 +++++++++++++++++++++++++++++++---------- 3 files changed, 37 insertions(+), 13 deletions(-) diff --git a/test/test_catalog.py b/test/test_catalog.py index 447d5f7..809d20e 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -78,7 +78,7 @@ def test_get_items(self): assert(len(items) == 2) def test_add_catalog(self): - cat = Catalog.create(root='http://my.cat').save_as(os.path.join(self.path, 'catalog.json')) + cat = Catalog.create(root='http://my.cat').save(os.path.join(self.path, 'catalog.json')) col = Catalog.open(os.path.join(testpath, 'catalog/eo/landsat-8-l1/catalog.json')) cat.add_catalog(col) child = [c for c in cat.children()][0] diff --git a/test/test_collection.py b/test/test_collection.py index d87ea0a..62874b5 100644 --- a/test/test_collection.py +++ b/test/test_collection.py @@ -44,7 +44,7 @@ def test_title(self): assert(len(cat.properties)) def test_add_item(self): - cat = Catalog.create(root='http://my.cat').save_as(os.path.join(self.path, 'catalog.json')) + cat = Catalog.create(root='http://my.cat').save(os.path.join(self.path, 'catalog.json')) col = Collection.open(os.path.join(testpath, 'catalog/eo/landsat-8-l1/catalog.json')) cat.add_catalog(col) item = Item.open(os.path.join(testpath, 'catalog/eo/landsat-8-l1/item.json')) @@ -58,7 +58,7 @@ def test_add_item_without_saving(self): col.add_item(item) def test_add_item_with_subcatalogs(self): - cat = Catalog.create(root='http://my.cat').save_as(os.path.join(self.path, 'test_subcatalogs.json')) + cat = Catalog.create(root='http://my.cat').save(os.path.join(self.path, 'test_subcatalogs.json')) col = Collection.open(os.path.join(testpath, 'catalog/eo/landsat-8-l1/catalog.json')) cat.add_catalog(col) item = Item.open(os.path.join(testpath, 'catalog/eo/landsat-8-l1/item.json')) diff --git a/test/test_thing.py b/test/test_thing.py index 1a3a5d6..4b909cb 100644 --- a/test/test_thing.py +++ b/test/test_thing.py @@ -23,7 +23,13 @@ def get_thing(self): """ Configure testing class """ with open(self.fname) as f: data = json.loads(f.read()) - return Thing(data) + thing = Thing(data) + thing.clean_hierarchy() + return thing + + def test_init_error(self): + with self.assertRaises(STACError): + Thing({}) def test_init(self): thing1 = self.get_thing() @@ -36,7 +42,8 @@ def test_init(self): assert(thing2.links() == []) with self.assertRaises(STACError): thing2.save() - print(thing1) + assert(thing2.id == str(thing2)) + #import pdb; pdb.set_trace() def test_open(self): thing1 = self.get_thing() @@ -46,6 +53,11 @@ def test_open(self): os.path.basename(thing1.links()[0]) == os.path.basename(thing2.links()[0]) ) + assert(thing2.path == os.path.dirname(self.fname)) + + def test_open_missing(self): + with self.assertRaises(STACError): + thing = Thing.open('nosuchfile.json') def test_open_remote(self): thing = Thing.open('https://landsat-stac.s3.amazonaws.com/catalog.json') @@ -72,9 +84,21 @@ def test_get_links(self): def test_add_link(self): thing = self.get_thing() thing.add_link('testlink', 'bobloblaw', type='text/plain', title='BobLoblaw') + # try adding it again, should not add it if rel and href the same + thing.add_link('testlink', 'bobloblaw', type='text/plain', title='BobLoblaw') assert(len(thing.links()) == 4) + assert(len(thing.links('testlink')) == 1) assert(thing.links('testlink')[0] == 'bobloblaw') + def test_get_root(self): + thing = self.get_thing() + assert(thing.root() == None) + thing.add_link('root', 'catalog.json', title='root1') + assert(thing.root()['title'] == 'root1') + thing.add_link('root', 'catalog;json', title='root2') + with self.assertRaises(STACError): + roots = thing.root() + def test_clean_hierarchy(self): thing = self.get_thing() thing.add_link('testlink', 'bobloblaw') @@ -90,30 +114,30 @@ def test_save(self): thing = Thing.open(self.fname) thing.save() fout = os.path.join(self.path, 'test-save.json') - thing.save_as(fout) + thing.save(fout) assert(os.path.exists(fout)) def test_save_remote_with_signed_url(self): thing = Thing.open(self.fname) - thing.save_as('https://landsat-stac.s3.amazonaws.com/test/thing.json') + thing.save('https://landsat-stac.s3.amazonaws.com/test/thing.json') def test_save_remote_with_bad_signed_url(self): envs = dict(os.environ) thing = Thing.open(self.fname) os.environ['AWS_BUCKET_REGION'] = 'us-east-1' with self.assertRaises(STACError): - thing.save_as('https://landsat-stac.s3.amazonaws.com/test/thing.json') + thing.save('https://landsat-stac.s3.amazonaws.com/test/thing.json') os.environ.clear() os.environ.update(envs) - def test_publish(self): + def test_add_self(self): thing = self.get_thing() fout = os.path.join(self.path, 'test-save.json') - thing.save_as(fout) - thing.publish('https://my.cat', root=fout) + thing.save(fout) + thing.add_self('https://my.cat', root=fout) assert(thing.links('self')[0] == 'https://my.cat/test-save.json') - def test_publish_without_saving(self): + def test_add_self_without_saving(self): thing = self.get_thing() with self.assertRaises(STACError): - thing.publish('https://my.cat', root=None) \ No newline at end of file + thing.add_self('https://my.cat', root=None) \ No newline at end of file From e641a892d5bd93772523fad2d1aff615cded742c Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 15 Jul 2019 22:57:09 -0400 Subject: [PATCH 10/15] updated Thing class --- CHANGELOG.md | 1 + satstac/thing.py | 36 ++++++++++++++++++------------------ test/test_thing.py | 36 ++++++++++++++++++++++++------------ 3 files changed, 43 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5873b0a..453a6c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed - Thing.publish() function now called Thing.add_self(), same arguments - Thing.root() and Thing.parent() functions now return `None` if no root or parent (rather than an empty list). If more than one root or parent then an error will now be thrown. +- Internal JSON data now stored in variable called `_data` rather than `data` ## [v0.1.3] - 2019-05-04 diff --git a/satstac/thing.py b/satstac/thing.py index 68ebdd2..2be9cfb 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -20,11 +20,11 @@ class Thing(object): def __init__(self, data, filename=None): """ Initialize a new class with a dictionary """ self.filename = filename - self.data = data + self._data = data if 'id' not in data: raise STACError('ID is required') - if 'links' not in self.data.keys(): - self.data['links'] = [] + if 'links' not in self._data.keys(): + self._data['links'] = [] def __repr__(self): return self.id @@ -58,10 +58,15 @@ def open(cls, filename): raise STACError('%s does not exist locally' % filename) return cls(dat, filename=filename) + def __getitem__(self, key): + """ Get key from properties """ + props = self._data.get('properties', {}) + return props.get(key, None) + @property def id(self): """ Return id of this entity """ - return self.data['id'] + return self._data['id'] @property def path(self): @@ -70,7 +75,7 @@ def path(self): def links(self, rel=None): """ Get links for specific rel type """ - links = self.data.get('links', []) + links = self._data.get('links', []) if rel is not None: links = [l for l in links if l.get('rel') == rel] links = [l['href'] for l in links] @@ -113,7 +118,7 @@ def parent(self): def add_link(self, rel, link, type=None, title=None): """ Add a new link """ # if this link already exists do not add it - for l in self.data['links']: + for l in self._data['links']: if l['rel'] == rel and l['href'] == link: return l = {'rel': rel, 'href': link} @@ -121,34 +126,29 @@ def add_link(self, rel, link, type=None, title=None): l['type'] = type if title is not None: l['title'] = title - self.data['links'].append(l) + self._data['links'].append(l) def add_self(self, endpoint, root): """ Update self link with endpoint """ if self.filename is None: raise STACError('No filename, use save_as() first') # keep everything except self and root - links = [l for l in self.data['links'] if l['rel'] not in ['self', 'root']] + links = [l for l in self._data['links'] if l['rel'] not in ['self', 'root']] to_item = os.path.relpath(self.filename, os.path.dirname(root)) to_root = os.path.relpath(root, os.path.dirname(self.filename)) links.insert(0, {'rel': 'root', 'href': to_root}) links.insert(0, {'rel': 'self', 'href': os.path.join(endpoint, to_item)}) - self.data['links'] = links + self._data['links'] = links self.save() def clean_hierarchy(self): """ Clean links of self, parent, and child links (for moving and publishing) """ rels = ['self', 'root', 'parent', 'child', 'collection', 'item'] links = [] - for l in self.data['links']: + for l in self._data['links']: if l['rel'] not in rels: links.append(l) - self.data['links'] = links - - def __getitem__(self, key): - """ Get key from properties """ - props = self.data.get('properties', {}) - return props.get(key, None) + self._data['links'] = links def save(self, filename=None): """ Write a catalog file """ @@ -162,12 +162,12 @@ def save(self, filename=None): # use signed URL signed_url, signed_headers = get_s3_signed_url(self.filename, rtype='PUT', public=True, content_type='application/json') - resp = requests.put(signed_url, data=json.dumps(self.data), headers=signed_headers) + resp = requests.put(signed_url, data=json.dumps(self._data), headers=signed_headers) if resp.status_code != 200: raise STACError('Unable to save file to %s: %s' % (self.filename, resp.text)) else: # local file save mkdirp(os.path.dirname(fname)) with open(fname, 'w') as f: - f.write(json.dumps(self.data)) + f.write(json.dumps(self._data)) return self \ No newline at end of file diff --git a/test/test_thing.py b/test/test_thing.py index 4b909cb..bd02bfe 100644 --- a/test/test_thing.py +++ b/test/test_thing.py @@ -21,11 +21,7 @@ def tearDownClass(cls): def get_thing(self): """ Configure testing class """ - with open(self.fname) as f: - data = json.loads(f.read()) - thing = Thing(data) - thing.clean_hierarchy() - return thing + return Thing.open(self.fname) def test_init_error(self): with self.assertRaises(STACError): @@ -43,7 +39,6 @@ def test_init(self): with self.assertRaises(STACError): thing2.save() assert(thing2.id == str(thing2)) - #import pdb; pdb.set_trace() def test_open(self): thing1 = self.get_thing() @@ -83,21 +78,37 @@ def test_get_links(self): def test_add_link(self): thing = self.get_thing() + thing.clean_hierarchy() + thing.filename = None thing.add_link('testlink', 'bobloblaw', type='text/plain', title='BobLoblaw') + assert(len(thing.links()) == 1) # try adding it again, should not add it if rel and href the same thing.add_link('testlink', 'bobloblaw', type='text/plain', title='BobLoblaw') - assert(len(thing.links()) == 4) + assert(len(thing.links()) == 1) assert(len(thing.links('testlink')) == 1) assert(thing.links('testlink')[0] == 'bobloblaw') def test_get_root(self): thing = self.get_thing() - assert(thing.root() == None) - thing.add_link('root', 'catalog.json', title='root1') - assert(thing.root()['title'] == 'root1') - thing.add_link('root', 'catalog;json', title='root2') + root = thing.root() + assert(root.filename == os.path.join(testpath, 'catalog/catalog.json')) + thing.add_link('root', 'root', title='root2') + with self.assertRaises(STACError): + thing.root() + thing.clean_hierarchy() + root = thing.root() + assert(root is None) + + def test_get_parent(self): + thing = Thing.open(os.path.join(testpath, 'catalog/eo/catalog.json')) + parent = thing.parent() + assert(parent.filename == os.path.join(testpath, 'catalog/catalog.json')) + thing.add_link('parent', 'catalog.json', title='parent2') with self.assertRaises(STACError): - roots = thing.root() + thing.parent() + thing.clean_hierarchy() + parent = thing.parent() + assert(parent is None) def test_clean_hierarchy(self): thing = self.get_thing() @@ -139,5 +150,6 @@ def test_add_self(self): def test_add_self_without_saving(self): thing = self.get_thing() + thing.filename = None with self.assertRaises(STACError): thing.add_self('https://my.cat', root=None) \ No newline at end of file From d6d57da7b5d9730d788342b6bce6b1883068c1d1 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Mon, 15 Jul 2019 23:43:58 -0400 Subject: [PATCH 11/15] remove self links and all related functionality (publish), self links not recommended for static catalogs --- CHANGELOG.md | 2 +- satstac/catalog.py | 33 ++++++--------------------------- satstac/cli.py | 4 ++-- satstac/collection.py | 19 +++++++++---------- satstac/item.py | 10 +++++----- satstac/items.py | 4 ++-- satstac/thing.py | 17 +++-------------- test/test_catalog.py | 10 +--------- test/test_cli.py | 16 +++------------- test/test_item.py | 10 +++++----- test/test_thing.py | 27 +++++++-------------------- 11 files changed, 44 insertions(+), 108 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 453a6c8..f7739e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Changed -- Thing.publish() function now called Thing.add_self(), same arguments +- Thing.publish() removed. Self links are not used at all (and not recommended for static catalogs) - Thing.root() and Thing.parent() functions now return `None` if no root or parent (rather than an empty list). If more than one root or parent then an error will now be thrown. - Internal JSON data now stored in variable called `_data` rather than `data` diff --git a/satstac/catalog.py b/satstac/catalog.py index f350bcd..eb5325d 100644 --- a/satstac/catalog.py +++ b/satstac/catalog.py @@ -15,12 +15,12 @@ def __init__(self, data, root=None, **kwargs): @property def stac_version(self): """ Get the STAC version of this catalog """ - return self.data['stac_version'] + return self._data['stac_version'] @property def description(self): """ Get catalog description """ - return self.data.get('description', '') + return self._data.get('description', '') @classmethod def create(cls, id='stac-catalog', title='A STAC Catalog', @@ -52,7 +52,7 @@ def catalogs(self): def collections(self): """ Recursively get all collections within this Catalog """ for cat in self.children(): - if 'extent' in cat.data.keys(): + if 'extent' in cat._data.keys(): yield Collection.open(cat.filename) # TODO - keep going? if other Collections can appear below a Collection else: @@ -73,40 +73,19 @@ def add_catalog(self, catalog): child_link = '%s/catalog.json' % catalog.id child_fname = os.path.join(self.path, child_link) child_path = os.path.dirname(child_fname) - root_link = self.links('root')[0] + root_links = self.links('root') + root_link = root_links[0] if len(root_links) > 0 else self.filename root_path = os.path.dirname(root_link) self.add_link('child', child_link) self.save() # strip self, parent, child links from catalog and add new links catalog.clean_hierarchy() - catalog.add_link('self', os.path.join(self.endpoint(), os.path.relpath(child_fname, root_path))) catalog.add_link('root', os.path.relpath(root_link, child_path)) catalog.add_link('parent', os.path.relpath(self.filename, child_path)) # create catalog file - catalog.save_as(child_fname) + catalog.save(filename=child_fname) return self - def endpoint(self): - """ Get endpoint URL to the root catalog """ - return os.path.dirname(self.root().links('self')[0]) - - def publish(self, endpoint, root=None): - """ Update all self links throughout catalog to use new endpoint """ - # we don't use the catalogs and items functions as we'd have to go - # through the tree twice, once for catalogs and once for items - # update myself - if root is None: - root = self.filename - super(Catalog, self).publish(endpoint, root=root) - # update direct items - for link in self.links('item'): - item = Item.open(link) - item.publish(endpoint, root=root) - # follow children - for cat in self.children(): - cat.publish(endpoint, root=root) - - # import and end of module prevents problems with circular dependencies. # Catalogs use Items and Items use Collections (which are Catalogs) diff --git a/satstac/cli.py b/satstac/cli.py index 389f055..91d58fc 100644 --- a/satstac/cli.py +++ b/satstac/cli.py @@ -47,8 +47,8 @@ def cli(): cat = Catalog.create(id=args['id'], description=args['description']) root.add_catalog(cat) else: - cat = Catalog.create(id=args['id'], description=args['description'], root=args['endpoint']) - cat.save_as(args['filename']) + cat = Catalog.create(id=args['id'], description=args['description']) + cat.save(filename=args['filename']) if __name__ == "__main__": diff --git a/satstac/collection.py b/satstac/collection.py index 452c237..dd8c5e6 100644 --- a/satstac/collection.py +++ b/satstac/collection.py @@ -22,32 +22,32 @@ def __init__(self, *args, **kwargs): @property def title(self): - return self.data.get('title', '') + return self._data.get('title', '') @property def keywords(self): - return self.data.get('keywords', []) + return self._data.get('keywords', []) @property def version(self): - return self.data.get('version', '') + return self._data.get('version', '') @property def license(self): - return self.data.get('license') + return self._data.get('license') @property def providers(self): - return self.data.get('providers', []) + return self._data.get('providers', []) @property def extent(self): - return self.data.get('extent') + return self._data.get('extent') @property def properties(self): """ Get dictionary of properties """ - return self.data.get('properties', {}) + return self._data.get('properties', {}) @functools.lru_cache() def parent_catalog(self, path): @@ -63,7 +63,7 @@ def parent_catalog(self, path): except STACError as err: # create a new sub-catalog subcat = self.create(id=d, description='%s catalog' % var_names[i]) - subcat.save_as(fname) + subcat.save(filename=fname) # add the sub-catalog to this catalog cat.add_catalog(subcat) cat = subcat @@ -88,13 +88,12 @@ def add_item(self, item, path='', filename='${id}'): # create links from item item.clean_hierarchy() - item.add_link('self', os.path.join(self.endpoint(), os.path.relpath(item_fname, root_path))) item.add_link('root', os.path.relpath(root_link, item_path)) item.add_link('parent', os.path.relpath(parent.filename, item_path)) # this assumes the item has been added to a Collection, not a Catalog item.add_link('collection', os.path.relpath(self.filename, item_path)) # save item - item.save_as(item_fname) + item.save(filename=item_fname) logger.debug('Added %s in %s seconds' % (item.filename, datetime.now()-start)) return self diff --git a/satstac/item.py b/satstac/item.py index 96753dd..cbd7094 100644 --- a/satstac/item.py +++ b/satstac/item.py @@ -47,7 +47,7 @@ def eobands(self): @property def properties(self): """ Get dictionary of properties """ - return self.data.get('properties', {}) + return self._data.get('properties', {}) def __getitem__(self, key): """ Get key from properties """ @@ -68,17 +68,17 @@ def datetime(self): @property def geometry(self): - return self.data['geometry'] + return self._data['geometry'] @property def bbox(self): """ Get bounding box of scene """ - return self.data['bbox'] + return self._data['bbox'] @property def assets(self): """ Return dictionary of assets """ - return self.data.get('assets', {}) + return self._data.get('assets', {}) @property def assets_by_common_name(self): @@ -126,7 +126,7 @@ def substitute(self, string): def download_assets(self, keys=None, **kwargs): """ Download multiple assets """ if keys is None: - keys = self.data['assets'].keys() + keys = self._data['assets'].keys() filenames = [] for key in keys: filenames.append(self.download(key, **kwargs)) diff --git a/satstac/items.py b/satstac/items.py index b0d4c1e..6ce1d0e 100644 --- a/satstac/items.py +++ b/satstac/items.py @@ -109,11 +109,11 @@ def save(self, filename): def geojson(self): """ Get Items as GeoJSON FeatureCollection """ - features = [s.data for s in self._items] + features = [s._data for s in self._items] geoj = { 'type': 'FeatureCollection', 'features': features, - 'collections': [c.data for c in self._collections], + 'collections': [c._data for c in self._collections], } if self._search is not None: geoj['search'] = self._search diff --git a/satstac/thing.py b/satstac/thing.py index 2be9cfb..f4b4f73 100644 --- a/satstac/thing.py +++ b/satstac/thing.py @@ -84,7 +84,7 @@ def links(self, rel=None): for l in links: if os.path.isabs(l) or l[0:4] == 'http': # if absolute or https - link = l + link = l else: # relative path if self.filename[0:4] == 'http': @@ -101,7 +101,8 @@ def root(self): if len(links) == 1: return self.open(links[0]) elif len(links) == 0: - return None + # i'm the root of myself + return self else: raise STACError('More than one root provided') @@ -128,18 +129,6 @@ def add_link(self, rel, link, type=None, title=None): l['title'] = title self._data['links'].append(l) - def add_self(self, endpoint, root): - """ Update self link with endpoint """ - if self.filename is None: - raise STACError('No filename, use save_as() first') - # keep everything except self and root - links = [l for l in self._data['links'] if l['rel'] not in ['self', 'root']] - to_item = os.path.relpath(self.filename, os.path.dirname(root)) - to_root = os.path.relpath(root, os.path.dirname(self.filename)) - links.insert(0, {'rel': 'root', 'href': to_root}) - links.insert(0, {'rel': 'self', 'href': os.path.join(endpoint, to_item)}) - self._data['links'] = links - self.save() def clean_hierarchy(self): """ Clean links of self, parent, and child links (for moving and publishing) """ diff --git a/test/test_catalog.py b/test/test_catalog.py index 809d20e..d6effa2 100644 --- a/test/test_catalog.py +++ b/test/test_catalog.py @@ -38,7 +38,7 @@ def test_init(self): def test_open(self): """ Initialize Catalog with a file """ cat = self.get_catalog() - assert(len(cat.data.keys()) == 4) + assert(len(cat._data.keys()) == 4) assert(cat.id == 'stac') assert(len(cat.links())==3) @@ -88,11 +88,3 @@ def test_add_catalog_without_saving(self): cat = Catalog.create() with self.assertRaises(STACError): cat.add_catalog({}) - - def test_publish(self): - path = os.path.join(self.path, 'test_publish') - shutil.copytree(os.path.join(testpath, 'catalog'), path) - cat = Catalog.open(os.path.join(path, 'catalog.json')) - cat.publish('https://my.cat') - item = Item.open(os.path.join(path, 'eo/landsat-8-l1/item.json')) - assert(item.links('self')[0] == 'https://my.cat/eo/landsat-8-l1/item.json') \ No newline at end of file diff --git a/test/test_cli.py b/test/test_cli.py index e96d199..1fd5a24 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -21,14 +21,13 @@ def test_parse_no_args(self): parse_args(['-h']) def test_parse_args(self): - input = "create testid 'this is a description' --endpoint 'https://my.cat'" + input = "create testid 'this is a description'" args = parse_args(split(input)) - assert(len(args) == 7) - assert(args['endpoint'] == 'https://my.cat') + assert(len(args) == 6) assert(args['id'] == 'testid') def test_cli_create(self): - input = "sat-stac create cat 'this is a description' --endpoint 'https://my.cat'" + input = "sat-stac create cat 'this is a description'" sys.argv = split(input) cli() assert(os.path.exists('catalog.json')) @@ -38,12 +37,3 @@ def test_cli_create(self): assert(os.path.exists('subcat/catalog.json')) rmtree('subcat') os.remove('catalog.json') - - def test_cli_publish(self): - cat = Catalog.create(root='https://my.cat').save_as('catalog.json') - input = "sat-stac publish catalog.json https://my.kitten" - sys.argv = split(input) - cli() - cat = Catalog.open('catalog.json') - assert(cat.links('self')[0] == 'https://my.kitten/catalog.json') - os.remove('catalog.json') \ No newline at end of file diff --git a/test/test_item.py b/test/test_item.py index c032d0b..9a303c6 100644 --- a/test/test_item.py +++ b/test/test_item.py @@ -45,9 +45,9 @@ def test_open(self): item = Item.open(self.filename) dt, tm = item.properties['datetime'].split('T') assert(str(item.date) == dt) - assert(item.id == item.data['id']) - assert(item.geometry == item.data['geometry']) - assert(str(item) == item.data['id']) + assert(item.id == item._data['id']) + assert(item.geometry == item._data['geometry']) + assert(str(item) == item._data['id']) assert(len(item.bbox) == 4) #assert(list(item.keys()) == ['id', 'collection', 'datetime', 'eo:platform']) @@ -60,13 +60,13 @@ def test_open_with_collection(self): def test_class_properties(self): """ Test the property functions of the Item class """ item = Item.open(self.filename) - l = os.path.join(os.path.dirname(item.filename), item.data['links'][0]['href']) + l = os.path.join(os.path.dirname(item.filename), item._data['links'][0]['href']) assert(os.path.abspath(item.links()[0]) == os.path.abspath(l)) def test_assets(self): """ Get assets for download """ item = Item.open(self.filename) - href = item.data['assets']['B1']['href'] + href = item._data['assets']['B1']['href'] assert(item.assets['B1']['href'] == href) assert(item.asset('B1')['href'] == href) assert(item.asset('coastal')['href'] == href) diff --git a/test/test_thing.py b/test/test_thing.py index bd02bfe..655c33d 100644 --- a/test/test_thing.py +++ b/test/test_thing.py @@ -32,7 +32,7 @@ def test_init(self): assert(thing1.id == 'stac') assert(len(thing1.links()) == 3) assert(len(thing1.links('self')) == 1) - data = thing1.data + data = thing1._data del data['links'] thing2 = Thing(data) assert(thing2.links() == []) @@ -57,7 +57,7 @@ def test_open_missing(self): def test_open_remote(self): thing = Thing.open('https://landsat-stac.s3.amazonaws.com/catalog.json') assert(thing.id == 'landsat-stac') - assert(len(thing.data['links']) == 3) + assert(len(thing._data['links']) == 3) def test_open_missing_remote(self): with self.assertRaises(STACError): @@ -65,10 +65,10 @@ def test_open_missing_remote(self): def test_thing(self): thing = self.get_thing() - assert('id' in thing.data.keys()) - assert('links' in thing.data.keys()) - del thing.data['links'] - assert('links' not in thing.data.keys()) + assert('id' in thing._data.keys()) + assert('links' in thing._data.keys()) + del thing._data['links'] + assert('links' not in thing._data.keys()) assert(thing.links() == []) def test_get_links(self): @@ -97,7 +97,7 @@ def test_get_root(self): thing.root() thing.clean_hierarchy() root = thing.root() - assert(root is None) + assert(root == thing) def test_get_parent(self): thing = Thing.open(os.path.join(testpath, 'catalog/eo/catalog.json')) @@ -140,16 +140,3 @@ def test_save_remote_with_bad_signed_url(self): thing.save('https://landsat-stac.s3.amazonaws.com/test/thing.json') os.environ.clear() os.environ.update(envs) - - def test_add_self(self): - thing = self.get_thing() - fout = os.path.join(self.path, 'test-save.json') - thing.save(fout) - thing.add_self('https://my.cat', root=fout) - assert(thing.links('self')[0] == 'https://my.cat/test-save.json') - - def test_add_self_without_saving(self): - thing = self.get_thing() - thing.filename = None - with self.assertRaises(STACError): - thing.add_self('https://my.cat', root=None) \ No newline at end of file From 49ab75d50b22870ccbb2a78199ca70b3256b6d81 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Tue, 16 Jul 2019 00:19:04 -0400 Subject: [PATCH 12/15] add add_collection function to Catalog --- satstac/catalog.py | 8 ++++++-- test/catalog/eo/landsat-8-l1/item.json | 2 +- test/catalog/eo/sentinel-2-l1c/sentinel-2a/item.json | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/satstac/catalog.py b/satstac/catalog.py index eb5325d..cea1932 100644 --- a/satstac/catalog.py +++ b/satstac/catalog.py @@ -65,12 +65,12 @@ def items(self): for child in self.children(): yield from child.items() - def add_catalog(self, catalog): + def add_catalog(self, catalog, basename='catalog'): """ Add a catalog to this catalog """ if self.filename is None: raise STACError('Save catalog before adding sub-catalogs') # add new catalog child link - child_link = '%s/catalog.json' % catalog.id + child_link = '%s/%s.json' % (catalog.id, basename) child_fname = os.path.join(self.path, child_link) child_path = os.path.dirname(child_fname) root_links = self.links('root') @@ -86,6 +86,10 @@ def add_catalog(self, catalog): catalog.save(filename=child_fname) return self + def add_collection(self, catalog, basename='collection'): + """ Add a collection to this catalog """ + return self.add_catalog(catalog, basename=basename) + # import and end of module prevents problems with circular dependencies. # Catalogs use Items and Items use Collections (which are Catalogs) diff --git a/test/catalog/eo/landsat-8-l1/item.json b/test/catalog/eo/landsat-8-l1/item.json index 0b0d0be..9ea46dc 100644 --- a/test/catalog/eo/landsat-8-l1/item.json +++ b/test/catalog/eo/landsat-8-l1/item.json @@ -1,5 +1,6 @@ { "id": "LC08_L1GT_120046_20181012_20181012_01_RT", + "collection": "landsat-8-l1", "bbox": [ 114.89853, 19.54292, @@ -34,7 +35,6 @@ ] }, "properties": { - "collection": "landsat-8-l1", "datetime": "2018-10-12T02:40:06.547Z", "eo:cloud_cover": 91, "eo:sun_azimuth": 141.68039757, diff --git a/test/catalog/eo/sentinel-2-l1c/sentinel-2a/item.json b/test/catalog/eo/sentinel-2-l1c/sentinel-2a/item.json index 50da45c..e6f0f6d 100644 --- a/test/catalog/eo/sentinel-2-l1c/sentinel-2a/item.json +++ b/test/catalog/eo/sentinel-2-l1c/sentinel-2a/item.json @@ -1,5 +1,6 @@ { "id": "L1C_T53MNQ_A017245_20181011T011722", + "collection": "sentinel-2-l1c", "bbox": [ 135.6392640640852, -5.5163474919383235, @@ -38,7 +39,6 @@ ] }, "properties": { - "collection": "sentinel-2-l1c", "datetime": "2018-10-11T01:17:22.460Z", "eo:platform": "Sentinel-2A", "eo:cloud_cover": 21, From 0d389b5fec3c4a3e4a36409ee50bc98966e90772 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Tue, 16 Jul 2019 00:19:23 -0400 Subject: [PATCH 13/15] bump version --- satstac/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satstac/version.py b/satstac/version.py index f2303e6..7fd229a 100644 --- a/satstac/version.py +++ b/satstac/version.py @@ -1 +1 @@ -__version__ = '0.2.0a1' +__version__ = '0.2.0' From 24a96390d1ab078e6f9caa7bc6c8c67330dddcb1 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Tue, 16 Jul 2019 00:23:21 -0400 Subject: [PATCH 14/15] updated notebook tutorials --- tutorial-1.ipynb | 61 +++++++++++++++++------------------ tutorial-2.ipynb | 84 ++++++++++++++++++++++++++++++++++++------------ 2 files changed, 93 insertions(+), 52 deletions(-) diff --git a/tutorial-1.ipynb b/tutorial-1.ipynb index 1b88d6c..4088be4 100644 --- a/tutorial-1.ipynb +++ b/tutorial-1.ipynb @@ -16,7 +16,6 @@ " - Adding collections to catalogs\n", " - Adding items to collections\n", "- Views (sub-catalogs)\n", - "- Publishing catalogs\n", "\n", "\n", "The examples here use the [test catalog in the sat-stac repo](https://github.com/developmentseed/sat-stac/tree/master/test/catalog). The directory structure of the test catalog looks like the following, where the catalog.json files under the landsat-8-l1 and sentinel-2-l1c are `Collection`s, the rest of the catalog.json files are simple `Catalog`s, and the item.json files are `Item`s.\n", @@ -51,7 +50,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": true + }, "outputs": [], "source": [ "from satstac import Catalog\n", @@ -81,7 +82,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -121,7 +124,9 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -153,7 +158,9 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -205,7 +212,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -242,7 +251,9 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -276,7 +287,9 @@ { "cell_type": "code", "execution_count": 7, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -315,7 +328,9 @@ { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -359,7 +374,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -424,7 +441,9 @@ { "cell_type": "code", "execution_count": 10, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -474,26 +493,6 @@ " ├── catalog.json\n", "```" ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Publishing a catalog\n", - "\n", - "The STAC spec allows for all of the hierarchical links to be stored as relative paths, except for self which must be an absolute path. However, when creating a Catalog that is going to be moved elsewhere, absolute paths do not yet make sense, so sat-stac keeps self links as relative.\n", - "\n", - "The Catalog can be published with a new endpoint by calling publish with the new root link. This is the absolute link to the root catalog. The publish() function will traverse the tree and update all of the self links in every Catalog, Collection, and Item to be an absolute path using the provided root link." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [], - "source": [ - "mycat.publish('https://my.other.cat')" - ] } ], "metadata": { diff --git a/tutorial-2.ipynb b/tutorial-2.ipynb index a3db153..ffb37c6 100644 --- a/tutorial-2.ipynb +++ b/tutorial-2.ipynb @@ -34,7 +34,9 @@ { "cell_type": "code", "execution_count": 1, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -71,7 +73,9 @@ { "cell_type": "code", "execution_count": 2, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -100,7 +104,9 @@ { "cell_type": "code", "execution_count": 3, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -134,7 +140,9 @@ { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -166,7 +174,9 @@ { "cell_type": "code", "execution_count": 5, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -194,7 +204,9 @@ { "cell_type": "code", "execution_count": 6, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -222,7 +234,9 @@ { "cell_type": "code", "execution_count": 7, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -257,7 +271,9 @@ { "cell_type": "code", "execution_count": 8, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -283,7 +299,9 @@ { "cell_type": "code", "execution_count": 9, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -310,7 +328,9 @@ { "cell_type": "code", "execution_count": 10, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -336,7 +356,9 @@ { "cell_type": "code", "execution_count": 11, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -369,7 +391,9 @@ { "cell_type": "code", "execution_count": 12, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -397,7 +421,9 @@ { "cell_type": "code", "execution_count": 13, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -415,7 +441,9 @@ { "cell_type": "code", "execution_count": 14, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -471,7 +499,9 @@ { "cell_type": "code", "execution_count": 15, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -510,7 +540,9 @@ { "cell_type": "code", "execution_count": 16, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -531,7 +563,9 @@ { "cell_type": "code", "execution_count": 17, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -562,7 +596,9 @@ { "cell_type": "code", "execution_count": 18, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -589,7 +625,9 @@ { "cell_type": "code", "execution_count": 19, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -626,7 +664,9 @@ { "cell_type": "code", "execution_count": 20, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", @@ -644,7 +684,9 @@ { "cell_type": "code", "execution_count": 21, - "metadata": {}, + "metadata": { + "collapsed": false + }, "outputs": [ { "name": "stdout", From 64b977ca65c1e2ed27b4486560bbf9dab33e4393 Mon Sep 17 00:00:00 2001 From: Matthew Hanson Date: Tue, 16 Jul 2019 00:25:52 -0400 Subject: [PATCH 15/15] remove testing of Python 3.5 and 3.6 --- .circleci/config.yml | 69 +++++--------------------------------------- 1 file changed, 7 insertions(+), 62 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f6768dd..c24608b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,55 +25,6 @@ references: jobs: - build_and_test_35: - docker: - - image: circleci/python:3.5 - steps: - - *restore_repo - - checkout - - *save_repo - - restore_cache: - keys: - - v1-dependencies35-{{ checksum "requirements.txt"}} - - v1-dependencies35 - - run: | - pip install virtualenv - virtualenv ~/venv35 - . ~/venv35/bin/activate - pip install -r requirements.txt - pip install -r requirements-dev.txt - pip install . - cd test - pytest -v --cov satstac --cov-report term-missing - - save_cache: - key: v1-dependencies35-{{ checksum "requirements.txt"}} - paths: - - ~/venv35 - - build_and_test_36: - docker: - - image: circleci/python:3.6 - steps: - - *restore_repo - - checkout - - *save_repo - - restore_cache: - keys: - - v1-dependencies36-{{ checksum "requirements.txt"}} - - v1-dependencies36 - - run: | - python3 -m venv ~/venv36 - . ~/venv36/bin/activate - pip install -r requirements.txt - pip install -r requirements-dev.txt - pip install . - cd test - pytest -v --cov satstac --cov-report term-missing - - save_cache: - key: v1-dependencies36-{{ checksum "requirements.txt"}} - paths: - - ~/venv36 - build_and_test_37: docker: - image: circleci/python:3.7 @@ -101,17 +52,17 @@ jobs: deploy: docker: - - image: circleci/python:3.6 + - image: circleci/python:3.7 steps: - *restore_repo - restore_cache: keys: - - v1-dependencies36-{{ checksum "requirements.txt"}} - - v1-dependencies36 + - v1-dependencies37-{{ checksum "requirements.txt"}} + - v1-dependencies37 - run: name: Deploy command: | - . ~/venv36/bin/activate + . ~/venv37/bin/activate mkdir -p ~/.ssh ssh-keyscan github.com >> ~/.ssh/known_hosts pip install twine @@ -124,18 +75,12 @@ jobs: workflows: version: 2 - build_test_35: - jobs: - - build_and_test_35 - build_test_36: + build_test_37: jobs: - - build_and_test_36 + - build_and_test_37 - deploy: requires: - - build_and_test_36 + - build_and_test_37 filters: branches: only: master - build_test_37: - jobs: - - build_and_test_37