From 60568148b7922a9b2ba46d4e7b17c60cc68e5f5a Mon Sep 17 00:00:00 2001 From: Marc-Philipp Esser Date: Sun, 17 Sep 2023 20:29:24 +0200 Subject: [PATCH] increase interval to every 15 minutes, but collect 450 instead of 300 photo's metadata in one run --- make/03_deploy_flow.mk | 2 +- src/prefect/ingest_photos_gcs.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/make/03_deploy_flow.mk b/make/03_deploy_flow.mk index 3f0cf8a..27d5431 100644 --- a/make/03_deploy_flow.mk +++ b/make/03_deploy_flow.mk @@ -64,7 +64,7 @@ deploy-ingest-photos-gcs: ## Deploy Ingest Topic GCS Flow as Google Cloud Run --output deployments/ingest-topics-gcs-${ENV}-deployment.yaml \ --pool ${ENV}-cloud-run-push-work-pool \ --params='{"gcp_credential_block_name": "unsplash-photo-trends-deployment-sa"}' \ - --cron "*/12 * * * *" \ + --cron "*/15 * * * *" \ --timezone 'Europe/Berlin' \ --apply diff --git a/src/prefect/ingest_photos_gcs.py b/src/prefect/ingest_photos_gcs.py index 662a894..c228b2c 100644 --- a/src/prefect/ingest_photos_gcs.py +++ b/src/prefect/ingest_photos_gcs.py @@ -191,10 +191,10 @@ def ingest_photos_gcs(gcp_credential_block_name: str): ) break - # Break the Loop if 300 images have been collected to avoid "OSError: [Errno 24] Too many open files" - if number_stored_images == 300: + # Break the Loop if 450 images have been collected to avoid "OSError: [Errno 24] Too many open files" + if number_stored_images == 450: logger.info( - "Downloaded metadata for 300 Editorial images of Unsplash platform" + "Downloaded metadata for 450 Editorial images of Unsplash platform" ) break