Merge pull request #90 from NASA-IMPACT/feature/agnostic-default-disc…

…overy

Make default asset name agnostic to raster vs vector in discovery

dags/veda_data_pipeline/utils/s3_discovery.py

@@ -114,7 +114,7 @@ def construct_single_asset_items(discovered_files: List[str]) -> dict:
         item = {
             "item_id": filename,
             "assets": {
-                "cog_default": {
+                "default": {
                     "title": "Default COG Layer",
                     "description": "Cloud optimized default layer to display on map",
                     "href": f"{prefix}/{filename}",
@@ -215,12 +215,6 @@ def s3_discovery_handler(event, chunk_size=2800, role_arn=None, bucket_output=No
     for item in items_with_assets:
         item["item_id"] = id_template.format(item["item_id"])
 
-    if dry_run:
-        print(f"-DRYRUN- Discovered {len(items_with_assets)} items")
-        for idx in range(0, min(10, len(items_with_assets))):
-            print("-DRYRUN- Example item")
-            print(json.dumps(items_with_assets[idx]))
-
     item_count = 0
     for item in items_with_assets:
         item_count += 1
@@ -240,6 +234,10 @@ def s3_discovery_handler(event, chunk_size=2800, role_arn=None, bucket_output=No
             **date_fields,
         }
 
+        if dry_run and item_count < 10:
+            print("-DRYRUN- Example item")
+            print(json.dumps(file_obj))
+
         payload["objects"].append(file_obj)
         if records == chunk_size:
             out_keys.append(generate_payload(s3_prefix_key=key, payload=payload))

docker_tasks/build_stac/utils/stac.py

@@ -94,6 +94,9 @@ def generate_stac(event: events.RegexEvent) -> pystac.Item:
         for asset_name, asset_definition in event.assets.items():
             with rasterio.open(asset_definition["href"]) as src:
                 media_type = stac.get_media_type(src)
+            # The default asset name for cogs is "cog_default", so we need to intercept 'default'
+            if asset_name == "default":
+                asset_name = "cog_default"
             assets[asset_name] = pystac.Asset(
                 title=asset_definition["title"],
                 description=asset_definition["description"],

docker_tasks/vector_ingest/handler.py

@@ -274,7 +274,7 @@ def handler(event, context):
     s3_objects = event_received["objects"]
     status = list()
     for s3_object in s3_objects:
-        href = s3_object["s3_filename"]
+        href = s3_object["assets"]["default"]["href"]
         collection = s3_object["collection"]
         downloaded_filepath = download_file(href)
         print(f"[ DOWNLOAD FILEPATH ]: {downloaded_filepath}")