diff --git a/dags/veda_data_pipeline/utils/s3_discovery.py b/dags/veda_data_pipeline/utils/s3_discovery.py index a89777b7..3952804a 100644 --- a/dags/veda_data_pipeline/utils/s3_discovery.py +++ b/dags/veda_data_pipeline/utils/s3_discovery.py @@ -114,7 +114,7 @@ def construct_single_asset_items(discovered_files: List[str]) -> dict: item = { "item_id": filename, "assets": { - "cog_default": { + "default": { "title": "Default COG Layer", "description": "Cloud optimized default layer to display on map", "href": f"{prefix}/{filename}", @@ -215,12 +215,6 @@ def s3_discovery_handler(event, chunk_size=2800, role_arn=None, bucket_output=No for item in items_with_assets: item["item_id"] = id_template.format(item["item_id"]) - if dry_run: - print(f"-DRYRUN- Discovered {len(items_with_assets)} items") - for idx in range(0, min(10, len(items_with_assets))): - print("-DRYRUN- Example item") - print(json.dumps(items_with_assets[idx])) - item_count = 0 for item in items_with_assets: item_count += 1 @@ -240,6 +234,10 @@ def s3_discovery_handler(event, chunk_size=2800, role_arn=None, bucket_output=No **date_fields, } + if dry_run and item_count < 10: + print("-DRYRUN- Example item") + print(json.dumps(file_obj)) + payload["objects"].append(file_obj) if records == chunk_size: out_keys.append(generate_payload(s3_prefix_key=key, payload=payload)) diff --git a/docker_tasks/build_stac/utils/stac.py b/docker_tasks/build_stac/utils/stac.py index 2de0a09c..70c393ba 100644 --- a/docker_tasks/build_stac/utils/stac.py +++ b/docker_tasks/build_stac/utils/stac.py @@ -94,6 +94,9 @@ def generate_stac(event: events.RegexEvent) -> pystac.Item: for asset_name, asset_definition in event.assets.items(): with rasterio.open(asset_definition["href"]) as src: media_type = stac.get_media_type(src) + # The default asset name for cogs is "cog_default", so we need to intercept 'default' + if asset_name == "default": + asset_name = "cog_default" assets[asset_name] = pystac.Asset( title=asset_definition["title"], description=asset_definition["description"], diff --git a/docker_tasks/vector_ingest/handler.py b/docker_tasks/vector_ingest/handler.py index 52a496a9..0ca3b1a7 100644 --- a/docker_tasks/vector_ingest/handler.py +++ b/docker_tasks/vector_ingest/handler.py @@ -274,7 +274,7 @@ def handler(event, context): s3_objects = event_received["objects"] status = list() for s3_object in s3_objects: - href = s3_object["s3_filename"] + href = s3_object["assets"]["default"]["href"] collection = s3_object["collection"] downloaded_filepath = download_file(href) print(f"[ DOWNLOAD FILEPATH ]: {downloaded_filepath}")