From d23734f682cd71ae3345417d755a505591867949 Mon Sep 17 00:00:00 2001 From: Marc-Philipp Esser Date: Fri, 6 Oct 2023 23:04:50 +0200 Subject: [PATCH] rename file / flow and update descriptions --- ...pi_gcs.py => ingest_photos_expanded_napi_bigquery.py} | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) rename src/prefect/{ingest_photos_expanded_napi_gcs.py => ingest_photos_expanded_napi_bigquery.py} (97%) diff --git a/src/prefect/ingest_photos_expanded_napi_gcs.py b/src/prefect/ingest_photos_expanded_napi_bigquery.py similarity index 97% rename from src/prefect/ingest_photos_expanded_napi_gcs.py rename to src/prefect/ingest_photos_expanded_napi_bigquery.py index 9df7407..29e9e4e 100644 --- a/src/prefect/ingest_photos_expanded_napi_gcs.py +++ b/src/prefect/ingest_photos_expanded_napi_bigquery.py @@ -1,4 +1,5 @@ -""" Flow to request https://unsplash.com/napi/photos/ Endpoint (Backend API)""" +""" Flow to request https://unsplash.com/napi/photos/ Endpoint (Backend API) +to receive editorial metadata and store it in Bigquery """ import asyncio import datetime @@ -112,10 +113,10 @@ def write_request_log_to_bigquery( @flow(timeout_seconds=120) # Main Flow (1st level) # Main Flow (1st level) @timer -def ingest_photos_expanded_napi_gcs( +def ingest_photos_expanded_napi_bigquery( gcp_credential_block_name: str, batch_size: int = 30, total_record_size: int = 300 ): - """Flow to load Editorial photos from Unsplash and store them in a Google Cloud Storage Bucket""" + """Flow to load editorial photo metadata from Unsplash and store them in Bigquery""" logger = get_run_logger() @@ -247,7 +248,7 @@ def ingest_photos_expanded_napi_gcs( if __name__ == "__main__": # @see https://github.com/PrefectHQ/prefect/pull/8983 faulthandler.dump_traceback_later(60) - ingest_photos_expanded_napi_gcs( + ingest_photos_expanded_napi_bigquery( gcp_credential_block_name="unsplash-photo-trends-deployment-sa", batch_size=30, total_record_size=300,