diff --git a/deployments/.gitkeep b/deployments/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/deployments/ingest-topics-gcs-prod-deployment.yaml b/deployments/ingest-topics-gcs-prod-deployment.yaml deleted file mode 100644 index a767287..0000000 --- a/deployments/ingest-topics-gcs-prod-deployment.yaml +++ /dev/null @@ -1,90 +0,0 @@ -### -### A complete description of a Prefect Deployment for flow 'ingest-photos-gcs' -### -name: ingest-photos-gcs-prod -description: Flow to load Editorial photos from Unsplash and store them in a Google - Cloud Storage Bucket -version: 7d6af57bd87a168021c84e1cf32fbea5 -# The work queue that will handle this deployment's runs -work_queue_name: default -work_pool_name: prod-cloud-run-push-work-pool -tags: [] -parameters: - gcp_credential_block_name: unsplash-photo-trends-deployment-sa -schedule: - cron: '*/15 * * * *' - timezone: Europe/Berlin - day_or: true -is_schedule_active: true -infra_overrides: {} - -### -### DO NOT EDIT BELOW THIS LINE -### -flow_name: ingest-photos-gcs -manifest_path: null -infrastructure: - type: cloud-run-job - env: {} - labels: {} - name: null - command: null - image: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-prod/prefect:2.13.0-python3.10 - region: europe-west3 - credentials: - service_account_file: null - service_account_info: - type: '**********' - auth_uri: '**********' - client_id: '**********' - token_uri: '**********' - project_id: '**********' - private_key: '**********' - client_email: '**********' - private_key_id: '**********' - universe_domain: '**********' - client_x509_cert_url: '**********' - auth_provider_x509_cert_url: '**********' - project: unsplash-photo-trends - _service_account_email: deployment-sa@unsplash-photo-trends.iam.gserviceaccount.com - _block_document_id: 136bddbf-b6de-4f26-94d7-f0656118bb85 - _block_document_name: unsplash-photo-trends-deployment-sa - _is_anonymous: false - block_type_slug: gcp-credentials - cpu: null - memory: null - memory_unit: null - vpc_connector_name: null - args: null - keep_job: false - timeout: 600 - _block_document_id: b9cd562f-48ed-4821-9da5-fda25e8ee86a - _block_document_name: unsplash-photo-trends-google-cloud-run-prod - _is_anonymous: false - block_type_slug: cloud-run-job - _block_type_slug: cloud-run-job -storage: - repository: https://github.com/m-p-esser/unsplash_photo_trends.git - reference: master - access_token: null - include_git_objects: true - _block_document_id: 2a80a779-ad1d-4f39-9f2c-52989870aba6 - _block_document_name: unsplash-photo-trends-github-prod - _is_anonymous: false - block_type_slug: github - _block_type_slug: github -path: '' -entrypoint: src/prefect/ingest_photos_gcs.py:ingest_photos_gcs -parameter_openapi_schema: - title: Parameters - type: object - properties: - gcp_credential_block_name: - title: gcp_credential_block_name - position: 0 - type: string - required: - - gcp_credential_block_name - definitions: null -timestamp: '2023-09-17T18:30:42.000914+00:00' -triggers: [] diff --git a/deployments/ingest-topics-gcs-test-deployment.yaml b/deployments/ingest-topics-gcs-test-deployment.yaml deleted file mode 100644 index c1c1943..0000000 --- a/deployments/ingest-topics-gcs-test-deployment.yaml +++ /dev/null @@ -1,87 +0,0 @@ -### -### A complete description of a Prefect Deployment for flow 'ingest-photos-gcs' -### -name: ingest-photos-gcs-test -description: Flow to load Editorial photos from Unsplash and store them in a Google - Cloud Storage Bucket -version: 7d6af57bd87a168021c84e1cf32fbea5 -# The work queue that will handle this deployment's runs -work_queue_name: default -work_pool_name: test-cloud-run-push-work-pool -tags: [] -parameters: - gcp_credential_block_name: unsplash-photo-trends-deployment-sa -schedule: null -is_schedule_active: true -infra_overrides: {} - -### -### DO NOT EDIT BELOW THIS LINE -### -flow_name: ingest-photos-gcs -manifest_path: null -infrastructure: - type: cloud-run-job - env: {} - labels: {} - name: null - command: null - image: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-test/prefect:2.13.0-python3.10 - region: europe-west3 - credentials: - service_account_file: null - service_account_info: - type: '**********' - auth_uri: '**********' - client_id: '**********' - token_uri: '**********' - project_id: '**********' - private_key: '**********' - client_email: '**********' - private_key_id: '**********' - universe_domain: '**********' - client_x509_cert_url: '**********' - auth_provider_x509_cert_url: '**********' - project: unsplash-photo-trends - _service_account_email: deployment-sa@unsplash-photo-trends.iam.gserviceaccount.com - _block_document_id: 136bddbf-b6de-4f26-94d7-f0656118bb85 - _block_document_name: unsplash-photo-trends-deployment-sa - _is_anonymous: false - block_type_slug: gcp-credentials - cpu: null - memory: null - memory_unit: null - vpc_connector_name: null - args: null - keep_job: false - timeout: 600 - _block_document_id: 67c66d21-0817-4432-8007-75346ec972c7 - _block_document_name: unsplash-photo-trends-google-cloud-run-test - _is_anonymous: false - block_type_slug: cloud-run-job - _block_type_slug: cloud-run-job -storage: - repository: https://github.com/m-p-esser/unsplash_photo_trends.git - reference: test - access_token: null - include_git_objects: true - _block_document_id: fcbcf700-cd0a-4f2d-a500-a6ce2904b235 - _block_document_name: unsplash-photo-trends-github-test - _is_anonymous: false - block_type_slug: github - _block_type_slug: github -path: '' -entrypoint: src/prefect/ingest_photos_gcs.py:ingest_photos_gcs -parameter_openapi_schema: - title: Parameters - type: object - properties: - gcp_credential_block_name: - title: gcp_credential_block_name - position: 0 - type: string - required: - - gcp_credential_block_name - definitions: null -timestamp: '2023-09-16T12:10:44.659713+00:00' -triggers: [] diff --git a/deployments/sync-gcs-to-bigquery-deployment.yaml b/deployments/sync-gcs-to-bigquery-deployment.yaml deleted file mode 100644 index a4965d7..0000000 --- a/deployments/sync-gcs-to-bigquery-deployment.yaml +++ /dev/null @@ -1,103 +0,0 @@ -### -### A complete description of a Prefect Deployment for flow 'sync-gcs-to-bigquery' -### -name: sync-gcs-to-bigquery -description: Sync Google Cloud Storage with Bigquery Table using Push pattern -version: 7d6af57bd87a168021c84e1cf32fbea5 -# The work queue that will handle this deployment's runs -work_queue_name: default -work_pool_name: test-cloud-run-push-work-pool -tags: [] -parameters: - table_name: topics - source_uri: gs://unsplash-topics-test/*.parquet - file_format: PARQUET -schedule: null -is_schedule_active: true -infra_overrides: {} - -### -### DO NOT EDIT BELOW THIS LINE -### -flow_name: sync-gcs-to-bigquery -manifest_path: null -infrastructure: - type: cloud-run-job - env: {} - labels: {} - name: null - command: null - image: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-test/prefect:2.13.0-python3.10 - region: europe-west3 - credentials: - service_account_file: null - service_account_info: - type: '**********' - auth_uri: '**********' - client_id: '**********' - token_uri: '**********' - project_id: '**********' - private_key: '**********' - client_email: '**********' - private_key_id: '**********' - universe_domain: '**********' - client_x509_cert_url: '**********' - auth_provider_x509_cert_url: '**********' - project: unsplash-photo-trends - _service_account_email: deployment-sa@unsplash-photo-trends.iam.gserviceaccount.com - _block_document_id: 136bddbf-b6de-4f26-94d7-f0656118bb85 - _block_document_name: unsplash-photo-trends-deployment-sa - _is_anonymous: false - block_type_slug: gcp-credentials - cpu: null - memory: null - memory_unit: null - vpc_connector_name: null - args: null - keep_job: false - timeout: 600 - _block_document_id: 67c66d21-0817-4432-8007-75346ec972c7 - _block_document_name: unsplash-photo-trends-google-cloud-run-test - _is_anonymous: false - block_type_slug: cloud-run-job - _block_type_slug: cloud-run-job -storage: - repository: https://github.com/m-p-esser/unsplash_photo_trends.git - reference: test - access_token: null - include_git_objects: true - _block_document_id: fcbcf700-cd0a-4f2d-a500-a6ce2904b235 - _block_document_name: unsplash-photo-trends-github-test - _is_anonymous: false - block_type_slug: github - _block_type_slug: github -path: '' -entrypoint: src/prefect/sync_gcs_to_bigquery.py:sync_gcs_to_bigquery -parameter_openapi_schema: - title: Parameters - type: object - properties: - table_name: - title: table_name - position: 0 - type: string - source_uri: - title: source_uri - position: 1 - type: string - file_format: - title: file_format - position: 2 - type: string - env: - title: env - default: dev - position: 3 - type: string - required: - - table_name - - source_uri - - file_format - definitions: null -timestamp: '2023-09-10T11:16:09.026346+00:00' -triggers: [] diff --git a/deployments/sync-topics-gcs-to-bigquery-dev-deployment.yaml b/deployments/sync-topics-gcs-to-bigquery-dev-deployment.yaml deleted file mode 100644 index 2171d2b..0000000 --- a/deployments/sync-topics-gcs-to-bigquery-dev-deployment.yaml +++ /dev/null @@ -1,95 +0,0 @@ -### -### A complete description of a Prefect Deployment for flow 'sync-topics-gcs-to-bigquery' -### -name: sync-topics-gcs-to-bigquery-dev -description: 'Sync Google Cloud Storage with Bigquery Table using Push pattern ' -version: 7d6af57bd87a168021c84e1cf32fbea5 -# The work queue that will handle this deployment's runs -work_queue_name: default -work_pool_name: dev-cloud-run-push-work-pool -tags: [] -parameters: {} -schedule: null -is_schedule_active: true -infra_overrides: {} - -### -### DO NOT EDIT BELOW THIS LINE -### -flow_name: sync-topics-gcs-to-bigquery -manifest_path: null -infrastructure: - type: cloud-run-job - env: {} - labels: {} - name: null - command: null - image: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-dev/prefect:2.13.0-python3.10 - region: europe-west3 - credentials: - service_account_file: null - service_account_info: - type: '**********' - auth_uri: '**********' - client_id: '**********' - token_uri: '**********' - project_id: '**********' - private_key: '**********' - client_email: '**********' - private_key_id: '**********' - universe_domain: '**********' - client_x509_cert_url: '**********' - auth_provider_x509_cert_url: '**********' - project: unsplash-photo-trends - _service_account_email: deployment-sa@unsplash-photo-trends.iam.gserviceaccount.com - _block_document_id: 136bddbf-b6de-4f26-94d7-f0656118bb85 - _block_document_name: unsplash-photo-trends-deployment-sa - _is_anonymous: false - block_type_slug: gcp-credentials - cpu: null - memory: null - memory_unit: null - vpc_connector_name: null - args: null - keep_job: false - timeout: 600 - _block_document_id: bf8acf5e-56a9-4aac-9146-e07d3d16d957 - _block_document_name: unsplash-photo-trends-google-cloud-run-dev - _is_anonymous: false - block_type_slug: cloud-run-job - _block_type_slug: cloud-run-job -storage: - repository: https://github.com/m-p-esser/unsplash_photo_trends.git - reference: develop - access_token: null - include_git_objects: true - _block_document_id: 6d99d82b-f9b7-41d2-8e5e-e074f9a43493 - _block_document_name: unsplash-photo-trends-github-dev - _is_anonymous: false - block_type_slug: github - _block_type_slug: github -path: '' -entrypoint: src/prefect/sync_topics_gcs_to_bigquery.py:sync_topics_gcs_to_bigquery -parameter_openapi_schema: - title: Parameters - type: object - properties: - table_name: - title: table_name - position: 0 - type: string - source_uri: - title: source_uri - position: 1 - type: string - file_format: - title: file_format - position: 2 - type: string - required: - - table_name - - source_uri - - file_format - definitions: null -timestamp: '2023-09-09T16:54:10.529405+00:00' -triggers: [] diff --git a/deployments/sync-topics-gcs-to-bigquery-prod-deployment.yaml b/deployments/sync-topics-gcs-to-bigquery-prod-deployment.yaml deleted file mode 100644 index 69315c5..0000000 --- a/deployments/sync-topics-gcs-to-bigquery-prod-deployment.yaml +++ /dev/null @@ -1,98 +0,0 @@ -### -### A complete description of a Prefect Deployment for flow 'sync-topics-gcs-to-bigquery' -### -name: sync-topics-gcs-to-bigquery -description: Sync Google Cloud Storage with Bigquery Table using Push pattern -version: 7d6af57bd87a168021c84e1cf32fbea5 -# The work queue that will handle this deployment's runs -work_queue_name: default -work_pool_name: prod-cloud-run-push-work-pool -tags: [] -parameters: - table_name: topics - source_uri: gs://unsplash-topics-prod/*.parquet - file_format: PARQUET -schedule: null -is_schedule_active: true -infra_overrides: {} - -### -### DO NOT EDIT BELOW THIS LINE -### -flow_name: sync-topics-gcs-to-bigquery -manifest_path: null -infrastructure: - type: cloud-run-job - env: {} - labels: {} - name: null - command: null - image: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-prod/prefect:2.13.0-python3.10 - region: europe-west3 - credentials: - service_account_file: null - service_account_info: - type: '**********' - auth_uri: '**********' - client_id: '**********' - token_uri: '**********' - project_id: '**********' - private_key: '**********' - client_email: '**********' - private_key_id: '**********' - universe_domain: '**********' - client_x509_cert_url: '**********' - auth_provider_x509_cert_url: '**********' - project: unsplash-photo-trends - _service_account_email: deployment-sa@unsplash-photo-trends.iam.gserviceaccount.com - _block_document_id: 136bddbf-b6de-4f26-94d7-f0656118bb85 - _block_document_name: unsplash-photo-trends-deployment-sa - _is_anonymous: false - block_type_slug: gcp-credentials - cpu: null - memory: null - memory_unit: null - vpc_connector_name: null - args: null - keep_job: false - timeout: 600 - _block_document_id: b9cd562f-48ed-4821-9da5-fda25e8ee86a - _block_document_name: unsplash-photo-trends-google-cloud-run-prod - _is_anonymous: false - block_type_slug: cloud-run-job - _block_type_slug: cloud-run-job -storage: - repository: https://github.com/m-p-esser/unsplash_photo_trends.git - reference: master - access_token: null - include_git_objects: true - _block_document_id: 2a80a779-ad1d-4f39-9f2c-52989870aba6 - _block_document_name: unsplash-photo-trends-github-prod - _is_anonymous: false - block_type_slug: github - _block_type_slug: github -path: '' -entrypoint: src/prefect/sync_topics_gcs_to_bigquery.py:sync_topics_gcs_to_bigquery -parameter_openapi_schema: - title: Parameters - type: object - properties: - table_name: - title: table_name - position: 0 - type: string - source_uri: - title: source_uri - position: 1 - type: string - file_format: - title: file_format - position: 2 - type: string - required: - - table_name - - source_uri - - file_format - definitions: null -timestamp: '2023-09-09T17:38:29.866765+00:00' -triggers: [] diff --git a/deployments/sync-topics-gcs-to-bigquery-test-deployment.yaml b/deployments/sync-topics-gcs-to-bigquery-test-deployment.yaml deleted file mode 100644 index fcd1544..0000000 --- a/deployments/sync-topics-gcs-to-bigquery-test-deployment.yaml +++ /dev/null @@ -1,98 +0,0 @@ -### -### A complete description of a Prefect Deployment for flow 'sync-topics-gcs-to-bigquery' -### -name: sync-topics-gcs-to-bigquery -description: Sync Google Cloud Storage with Bigquery Table using Push pattern -version: 7d6af57bd87a168021c84e1cf32fbea5 -# The work queue that will handle this deployment's runs -work_queue_name: default -work_pool_name: test-cloud-run-push-work-pool -tags: [] -parameters: - table_name: topics - source_uri: gs://unsplash-topics-test/*.parquet - file_format: PARQUET -schedule: null -is_schedule_active: null -infra_overrides: {} - -### -### DO NOT EDIT BELOW THIS LINE -### -flow_name: sync-topics-gcs-to-bigquery -manifest_path: null -infrastructure: - type: cloud-run-job - env: {} - labels: {} - name: null - command: null - image: europe-west3-docker.pkg.dev/unsplash-photo-trends/prefect-test/prefect:2.13.0-python3.10 - region: europe-west3 - credentials: - service_account_file: null - service_account_info: - type: '**********' - auth_uri: '**********' - client_id: '**********' - token_uri: '**********' - project_id: '**********' - private_key: '**********' - client_email: '**********' - private_key_id: '**********' - universe_domain: '**********' - client_x509_cert_url: '**********' - auth_provider_x509_cert_url: '**********' - project: unsplash-photo-trends - _service_account_email: deployment-sa@unsplash-photo-trends.iam.gserviceaccount.com - _block_document_id: 136bddbf-b6de-4f26-94d7-f0656118bb85 - _block_document_name: unsplash-photo-trends-deployment-sa - _is_anonymous: false - block_type_slug: gcp-credentials - cpu: null - memory: null - memory_unit: null - vpc_connector_name: null - args: null - keep_job: false - timeout: 600 - _block_document_id: 67c66d21-0817-4432-8007-75346ec972c7 - _block_document_name: unsplash-photo-trends-google-cloud-run-test - _is_anonymous: false - block_type_slug: cloud-run-job - _block_type_slug: cloud-run-job -storage: - repository: https://github.com/m-p-esser/unsplash_photo_trends.git - reference: test - access_token: null - include_git_objects: true - _block_document_id: fcbcf700-cd0a-4f2d-a500-a6ce2904b235 - _block_document_name: unsplash-photo-trends-github-test - _is_anonymous: false - block_type_slug: github - _block_type_slug: github -path: '' -entrypoint: src/prefect/sync_topics_gcs_to_bigquery.py:sync_topics_gcs_to_bigquery -parameter_openapi_schema: - title: Parameters - type: object - properties: - table_name: - title: table_name - position: 0 - type: string - source_uri: - title: source_uri - position: 1 - type: string - file_format: - title: file_format - position: 2 - type: string - required: - - table_name - - source_uri - - file_format - definitions: null -timestamp: '2023-09-09T17:24:02.808105+00:00' -triggers: []