Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docs: Add Workflows snippet #550

Merged
merged 15 commits into from
Jan 17, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/CODEOWNERS
Validating CODEOWNERS rules …
Original file line number Diff line number Diff line change
Expand Up @@ -20,3 +20,4 @@
/traffic_director/ @terraform-google-modules/dee-infra @terraform-google-modules/terraform-samples-reviewers
/vertex_ai/ @terraform-google-modules/dee-data-ai @terraform-google-modules/terraform-samples-reviewers
/vpc/ @terraform-google-modules/dee-infra @terraform-google-modules/terraform-samples-reviewers
/workflows/ @terraform-google-modules/torus-dpe @terraform-google-modules/terraform-samples-reviewers
139 changes: 139 additions & 0 deletions workflows/cloud_run_job/main.tf
Original file line number Diff line number Diff line change
@@ -0,0 +1,139 @@
/**
* Copyright 2023 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
Create a Cloud Run job that processes data files in a Cloud Storage bucket.
See https://cloud.google.com/workflows/docs/tutorials/execute-cloud-run-jobs
before running the code snippet.
*/

# [START workflows_terraform_create_bucket]
# Create a Cloud Storage bucket
resource "google_storage_bucket" "default" {
name = "input-PROJECT_ID"
glasnt marked this conversation as resolved.
Show resolved Hide resolved
location = "us-central1"
storage_class = "STANDARD"
force_destroy = false
uniform_bucket_level_access = true
}
# [END workflows_terraform_create_bucket]

# [START workflows_terraform_create_ar_repo]
# Create an Artifact Registry repository
resource "google_artifact_registry_repository" "default" {
location = "us-central1"
repository_id = "REPOSITORY"
format = "docker"
}
# [END workflows_terraform_create_ar_repo]

# [START workflows_terraform_create_run_job]
# Create a Cloud Run job
resource "google_cloud_run_v2_job" "default" {
name = "parallel-job"
location = "us-central1"

template {
task_count = 10
template {
containers {
image = "us-central1-docker.pkg.dev/PROJECT_ID/REPOSITORY/parallel-job:latest"
glasnt marked this conversation as resolved.
Show resolved Hide resolved
command = ["python"]
args = ["process.py"]
env {
name = "INPUT_BUCKET"
value = "input-PROJECT_ID"
glasnt marked this conversation as resolved.
Show resolved Hide resolved
}
env {
name = "INPUT_FILE"
value = "input_file.txt"
}
}
}
}
}
# [END workflows_terraform_create_run_job]

# [START workflows_terraform_create_workflow]
# Create a workflow
resource "google_workflows_workflow" "default" {
name = "cloud-run-job-workflow"
region = "us-central1"
description = "Workflow that routes a Cloud Storage event and executes a Cloud Run job"

# Note that $$ is needed for Terraform
source_contents = <<EOF
main:
params: [event]
steps:
- init:
assign:
- project_id: $${sys.get_env("GOOGLE_CLOUD_PROJECT_ID")}
- event_bucket: $${event.data.bucket}
- event_file: $${event.data.name}
- target_bucket: $${"input-" + project_id}
- job_name: parallel-job
- job_location: us-central1
- check_input_file:
switch:
- condition: $${event_bucket == target_bucket}
next: run_job
- condition: true
next: end
- run_job:
call: googleapis.run.v1.namespaces.jobs.run
args:
name: $${"namespaces/" + project_id + "/jobs/" + job_name}
location: $${job_location}
body:
overrides:
containerOverrides:
env:
- name: INPUT_BUCKET
value: $${event_bucket}
- name: INPUT_FILE
value: $${event_file}
result: job_execution
- finish:
return: $${job_execution}
EOF
}
# [END workflows_terraform_create_workflow]

# [START workflows_terraform_create_trigger]
# Create an Eventarc trigger that routes Cloud Storage events to Workflows
resource "google_eventarc_trigger" "default" {
name = "cloud-run-job-trigger"
location = google_workflows_workflow.default.region

# Capture objects changed in the bucket
matching_criteria {
attribute = "type"
value = "google.cloud.storage.object.v1.finalized"
}
matching_criteria {
attribute = "bucket"
value = google_storage_bucket.default.name
}

# Send events to Workflows
destination {
workflow = google_workflows_workflow.default.id
}

service_account = "SERVICE_ACCOUNT_NAME@PROJECT_ID.iam.gserviceaccount.com"
glasnt marked this conversation as resolved.
Show resolved Hide resolved

}
# [END workflows_terraform_create_trigger]