move deployment of 'ingest_topics_gcs.py' / 'ingest_monthly_platform_…

…stats.py' / 'healthcheck.py' to prefect.yaml
m-p-esser · Oct 16, 2023 · 20e2543 · 20e2543
1 parent 3be1548
commit 20e2543
Show file tree

Hide file tree

Showing 2 changed files with 61 additions and 71 deletions.
diff --git a/make/03_deploy_flow.mk b/make/03_deploy_flow.mk
@@ -10,53 +10,15 @@ push-prefect-runner-image: ## Deploy Prefect Runner Image (generic Prefect image
 		-f images/prefect_runner/Dockerfile .
 	docker push $(GCP_DEFAULT_REGION)-docker.pkg.dev/$(GCP_PROJECT_ID)/prefect-${ENV}/prefect:$(PREFECT_VERSION)-python$(PYTHON_VERSION)
 
-# @see https://crontab.guru/ for how to write cron expressions
-.PHONY: deploy-ingest-topics-gcs
-deploy-ingest-topics-gcs: ## Deploy Ingest Topic GCS Flow as Google Cloud Run
+.PHONY: deployment-preperations
+deployment-preperations: ## Preperation steps before deploying
 	make env-init
-	make push-prefect-runner-image
-	prefect deployment build src/prefect/ingest_topics_gcs.py:ingest_topics_gcs \
-		--name ingest-topics-gcs-${ENV} \
-		--infra-block cloud-run-job/${GCP_PROJECT_ID}-google-cloud-run-${ENV} \
-		--storage-block github/${GCP_PROJECT_ID}-github-${ENV} \
-		--output deployments/ingest-topics-gcs-${ENV}-deployment.yaml \
-		--pool ${ENV}-cloud-run-push-work-pool \
-		--cron "0 9 * * *" \
-		--timezone 'Europe/Berlin' \
-		--apply
-
-.PHONY: deploy-healthcheck
-deploy-healthcheck: ## Deploy Healtcheck Flow as Google Cloud Run
-	make env-init
-	make push-prefect-runner-image
-	prefect deployment build src/prefect/healthcheck.py:healthcheck \
-		--name healthcheck-${ENV} \
-		--infra-block cloud-run-job/${GCP_PROJECT_ID}-google-cloud-run-${ENV} \
-		--storage-block github/${GCP_PROJECT_ID}-github-${ENV} \
-		--output deployments/healthcheck-${ENV}-deployment.yaml \
-		--pool ${ENV}-cloud-run-push-work-pool \
-		--cron "0 8 * * *" \
-		--timezone 'Europe/Berlin' \
-		--apply
-
-.PHONY: deploy-ingest-monthly-platform-stats-gcs
-deploy-ingest-monthly-platform-stats-gcs: ## Deploy Monthly Platform Stats GCS Flow as Google Cloud Run
-	make env-init
-	make push-prefect-runner-image
-	prefect deployment build src/prefect/ingest_monthly_platform_stats_gcs.py:ingest_monthly_platform_stats_gcs \
-		--name ingest-monthly-platform-stats-gcs-${ENV} \
-		--infra-block cloud-run-job/${GCP_PROJECT_ID}-google-cloud-run-${ENV} \
-		--storage-block github/${GCP_PROJECT_ID}-github-${ENV} \
-		--output deployments/ingest-monthly-platform-stats-gcs-${ENV}-deployment.yaml \
-		--pool ${ENV}-cloud-run-push-work-pool \
-		--cron "0 8 * * *" \
-		--timezone 'Europe/Berlin' \
-		--apply
+	poetry export -o requirements.txt --without-hashes --without-urls --without=dev,test
 
-.PHONY: deploy-ingest-photos-gcs
-deploy-ingest-photos-gcs: ## Deploy Ingest Photos GCS Flow to Vertex AI
-	make env-init
-	prefect deploy -n ingest-photos-gcs-${ENV}
+.PHONY: deploy-flow
+deploy-flow: ## Deploy any flow
+	make deployment-preperations
+	prefect deploy
 
 .PHONY: deploy-ingest-photos-napi-gcs
 deploy-ingest-photos-napi-gcs: ## Deploy Ingest Photos NAPI GCS Flow as Google Cloud Run

diff --git a/prefect.yaml b/prefect.yaml
@@ -4,41 +4,69 @@
 
 # Generic metadata about this project
 name: unsplash-photo-trends
-prefect-version: 2.13.0
+prefect-version: "{{ prefect.variables.prefect_version }}"
 
 # build section allows you to manage and build docker images
 build:
-- prefect_docker.deployments.steps.build_docker_image:
-    requires: prefect-docker>=0.3.1
-    id: build-image
-    dockerfile: auto
-    image_name: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-test/prefect
-    tag: 2.13.0-python3.10
+    - prefect.deployments.steps.run_shell_script:
+        id: get-env
+        script: "bash env_init.sh"
+        directory: src/scripts
+    - prefect_docker.deployments.steps.build_docker_image:
+        id: build-image
+        requires: prefect-docker>=0.3.1
+        dockerfile: auto
+        image_name: "{{ prefect.variables.gcp_default_region }}-docker.pkg.dev/{{ prefect.variables.gcp_project_id }}/prefect-{{ get-env.stdout }}/prefect"
+        tag: "{{ prefect.variables.prefect_version }}-python{{ prefect.variables.python_version }}"
 
 # push section allows you to manage if and how this project is uploaded to remote locations
 push:
-- prefect_docker.deployments.steps.push_docker_image:
-    requires: prefect-docker>=0.3.1
-    image_name: '{{ build-image.image_name }}'
-    tag: '{{ build-image.tag }}'
+    - prefect_docker.deployments.steps.push_docker_image:
+        requires: prefect-docker>=0.3.1
+        image_name: "{{ build-image.image_name }}"
+        tag: "{{ build-image.tag }}"
 
 # pull section allows you to provide instructions for cloning this project in remote locations
 pull:
-- prefect.deployments.steps.set_working_directory:
-    directory: /opt/prefect/unsplash-photo-trends
+    - prefect.deployments.steps.set_working_directory:
+        directory: "/opt/prefect/{{ prefect.variables.gcp_project_id }}"
+    - prefect.deployments.steps.pip_install_requirements:
+        requirements_file: requirements.txt
+
+# @see https://docs.prefect.io/2.13.7/guides/prefect-deploy/?h=prefect.yaml#reusing-configuration-across-deployments
+definitions:
+    work_pools:
+        dev-cloud_run_work_pool: &dev_cloud_run_work_pool 
+            name: "dev-cloud-run-push-work-pool"
+            work_queue_name: null
+            job_variables:
+                image: "{{ build-image.image }}"
+        test-cloud_run_work_pool: &test_cloud_run_work_pool 
+            name: "test-cloud-run-push-work-pool"
+            work_queue_name: null
+            job_variables:
+                image: "{{ build-image.image }}"
+        prod-cloud_run_work_pool: &prod_cloud_run_work_pool 
+            name: "prod-cloud-run-push-work-pool"
+            work_queue_name: null
+            job_variables:
+                image: "{{ build-image.image }}"
 
 # the deployments section allows you to provide configuration for deploying flows
 deployments:
-- name: ingest-photos-gcs-test
-  version: null
-  tags: []
-  description: Flow to download photos from unsplash and store them in Google Cloud
-    Storage Bucket
-  entrypoint: src/prefect/ingest_photos_gcs.py:ingest_photos_gcs
-  parameters: {}
-  work_pool:
-    name: test-docker-container-pool
-    work_queue_name: null
-    job_variables:
-      image: '{{ build-image.image }}'
-  schedule: null
+    - name: "healthcheck-test"
+      entrypoint: src/prefect/healthcheck.py:healthcheck
+      work_pool: *test_cloud_run_work_pool
+      schedule: null
+    - name: "ingest-monthly-platform-stats-gcs-test"
+      entrypoint: src/prefect/ingest_monthly_platform_stats_gcs.py:ingest_monthly_platform_stats_gcs
+      work_pool: *test_cloud_run_work_pool
+      schedule:
+        cron: "0 8 * * *"
+        timezone: 'Europe/Berlin'
+    - name: "ingest-topics-gcs-test"
+      entrypoint: src/prefect/ingest_topics_gcs.py:ingest_topics_gcs
+      work_pool: *test_cloud_run_work_pool
+      schedule:
+        cron: "0 9 * * *"
+        timezone: 'Europe/Berlin'