From 05209897f1d6fe5755e9d62219d63fa1b9a0beb0 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Tue, 26 Mar 2024 15:41:52 -0400 Subject: [PATCH 01/11] Enable cloudresourcemanager in tests --- test/setup/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/test/setup/main.tf b/test/setup/main.tf index f17a408..7f9bebe 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -29,6 +29,7 @@ module "project" { "artifactregistry.googleapis.com", "cloudbuild.googleapis.com", "cloudfunctions.googleapis.com", + "cloudresourcemanager.googleapis.com", "documentai.googleapis.com", "eventarc.googleapis.com", "firestore.googleapis.com", From 8bad02e36d797219af75c0778714245fa38dd463 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Tue, 26 Mar 2024 17:21:23 -0400 Subject: [PATCH 02/11] Add serviceusage.serviceUsageAdmin role to tests --- test/setup/iam.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 8041497..7abb18e 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -21,6 +21,7 @@ locals { "roles/documentai.editor", "roles/firebase.admin", "roles/iam.serviceAccountUser", + "roles/serviceusage.serviceUsageAdmin", ] } From c7421888e1fdfdea2a3d854bb81245daaf2a5dff Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Tue, 26 Mar 2024 18:01:10 -0400 Subject: [PATCH 03/11] Add roles/iam.serviceAccountAdmin to tests --- test/setup/iam.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 7abb18e..a8dfc3e 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -20,6 +20,7 @@ locals { "roles/artifactregistry.admin", "roles/documentai.editor", "roles/firebase.admin", + "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", "roles/serviceusage.serviceUsageAdmin", ] From 5c51c2d9523566cf8c92d9d2e27cfacb412bb04e Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Tue, 26 Mar 2024 19:23:45 -0400 Subject: [PATCH 04/11] Add roles/resourcemanager.projectIamAdmin to tests --- test/setup/iam.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/test/setup/iam.tf b/test/setup/iam.tf index a8dfc3e..191c873 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -22,6 +22,7 @@ locals { "roles/firebase.admin", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", + "roles/resourcemanager.projectIamAdmin" "roles/serviceusage.serviceUsageAdmin", ] } From 365e2de239784cb38b7e72f13a87dd71182793fc Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Tue, 26 Mar 2024 19:24:02 -0400 Subject: [PATCH 05/11] Add missing comma --- test/setup/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/setup/iam.tf b/test/setup/iam.tf index 191c873..2cdf36c 100644 --- a/test/setup/iam.tf +++ b/test/setup/iam.tf @@ -22,7 +22,7 @@ locals { "roles/firebase.admin", "roles/iam.serviceAccountAdmin", "roles/iam.serviceAccountUser", - "roles/resourcemanager.projectIamAdmin" + "roles/resourcemanager.projectIamAdmin", "roles/serviceusage.serviceUsageAdmin", ] } From da6f7a45a11b903c1864ccf0906ba5202b9563c4 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Wed, 27 Mar 2024 10:15:17 -0400 Subject: [PATCH 06/11] Use variable for initial-index.json This addresses the following error seen in the simple_example test: Error: open /workspace/examples/simple_example/initial-index.json: no such file or directory --- examples/simple_example/main.tf | 1 + main.tf | 2 +- variables.tf | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index aad8120..f85846a 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -18,4 +18,5 @@ module "simple" { source = "../../" project_id = var.project_id webhook_path = abspath("../../webhook") + initial_index_json_path = abspath("../../intial-index.json") } diff --git a/main.tf b/main.tf index 43079f7..d2e3db2 100644 --- a/main.tf +++ b/main.tf @@ -260,5 +260,5 @@ resource "google_vertex_ai_index_endpoint" "docs" { resource "google_storage_bucket_object" "index_initial" { bucket = google_storage_bucket.main.name name = "vector-search-index/initial.json" - source = abspath("initial-index.json") + source = var.initial_index_json_path } diff --git a/variables.tf b/variables.tf index 1783048..6fad9bb 100644 --- a/variables.tf +++ b/variables.tf @@ -56,6 +56,13 @@ variable "webhook_path" { default = "webhook" } +# Used for testing. +variable "initial_index_json_path" { + description = "Path to the initial (placeholder) Vector Search index" + type = string + default = abspath("initial-index.json") +} + variable "unique_names" { description = "Whether to use unique names for resources" type = bool From 648d2a093bbf7cf576bc7cf17b9209537e74a3ec Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Wed, 27 Mar 2024 10:24:25 -0400 Subject: [PATCH 07/11] Run make docker_generate_docs --- metadata.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metadata.yaml b/metadata.yaml index 0b9fd7d..d3f9935 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -33,8 +33,8 @@ spec: detailed: This solution showcases how to extract question & answer pairs out of documents using Generative AI. It provides an end-to-end demonstration of QA extraction and fine-tuning of a large language model (LLM) on Vertex AI. Along the way, the solution utilizes Document AI Character Recognition (OCR), Firestore, Vector Search, Vertex AI Studio, and Cloud Functions. icon: assets/icon.png deploymentDuration: - configurationSecs: 120 - deploymentSecs: 360 + configurationSecs: "120" + deploymentSecs: "360" costEstimate: description: Cost Details url: https://cloud.google.com/products/calculator-legacy#id=2ed62cc0-d0ab-4bff-918a-d5cb297b4201 @@ -83,7 +83,6 @@ spec: - name: project_id description: The Google Cloud project ID to deploy to varType: string - defaultValue: null required: true - name: region description: The Google Cloud region to deploy to @@ -124,12 +123,16 @@ spec: - roles/artifactregistry.admin - roles/documentai.editor - roles/firebase.admin + - roles/iam.serviceAccountAdmin - roles/iam.serviceAccountUser + - roles/resourcemanager.projectIamAdmin + - roles/serviceusage.serviceUsageAdmin services: - aiplatform.googleapis.com - artifactregistry.googleapis.com - cloudbuild.googleapis.com - cloudfunctions.googleapis.com + - cloudresourcemanager.googleapis.com - documentai.googleapis.com - eventarc.googleapis.com - firestore.googleapis.com From c41c147c4160dae590f845b7c572aae7148cbda8 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Wed, 27 Mar 2024 10:37:00 -0400 Subject: [PATCH 08/11] Run terraform fmt --- examples/simple_example/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index f85846a..0c53099 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -15,8 +15,8 @@ */ module "simple" { - source = "../../" - project_id = var.project_id - webhook_path = abspath("../../webhook") + source = "../../" + project_id = var.project_id + webhook_path = abspath("../../webhook") initial_index_json_path = abspath("../../intial-index.json") } From c83a1662340d1e13540d0e0d991eb909faf9f504 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Wed, 27 Mar 2024 10:38:57 -0400 Subject: [PATCH 09/11] Avoid using abs() in variable default --- variables.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/variables.tf b/variables.tf index 6fad9bb..885bab9 100644 --- a/variables.tf +++ b/variables.tf @@ -60,7 +60,7 @@ variable "webhook_path" { variable "initial_index_json_path" { description = "Path to the initial (placeholder) Vector Search index" type = string - default = abspath("initial-index.json") + default = "initial-index.json" } variable "unique_names" { From 2d5808f2f4cdd73a80d225d888ed41b69d857321 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Wed, 27 Mar 2024 10:53:38 -0400 Subject: [PATCH 10/11] Run make docker_generate_docs --- README.md | 1 + metadata.display.yaml | 3 +++ metadata.yaml | 4 ++++ 3 files changed, 8 insertions(+) diff --git a/README.md b/README.md index e4b4561..495fcf5 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Deployment: 6 mins | disable\_services\_on\_destroy | Whether project services will be disabled when the resources are destroyed. | `bool` | `false` | no | | documentai\_location | Document AI location, see https://cloud.google.com/document-ai/docs/regions | `string` | `"us"` | no | | firestore\_location | Firestore location, see https://firebase.google.com/docs/firestore/locations | `string` | `"nam5"` | no | +| initial\_index\_json\_path | Path to the initial (placeholder) Vector Search index | `string` | `"initial-index.json"` | no | | project\_id | The Google Cloud project ID to deploy to | `string` | n/a | yes | | region | The Google Cloud region to deploy to | `string` | `"us-central1"` | no | | unique\_names | Whether to use unique names for resources | `bool` | `false` | no | diff --git a/metadata.display.yaml b/metadata.display.yaml index 818dc7c..a92c3eb 100644 --- a/metadata.display.yaml +++ b/metadata.display.yaml @@ -37,6 +37,9 @@ spec: firestore_location: name: firestore_location title: Firestore Location + initial_index_json_path: + name: initial_index_json_path + title: Initial Index Json Path project_id: name: project_id title: Project Id diff --git a/metadata.yaml b/metadata.yaml index d3f9935..6b7fd2f 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -80,6 +80,10 @@ spec: description: Firestore location, see https://firebase.google.com/docs/firestore/locations varType: string defaultValue: nam5 + - name: initial_index_json_path + description: Path to the initial (placeholder) Vector Search index + varType: string + defaultValue: initial-index.json - name: project_id description: The Google Cloud project ID to deploy to varType: string From de51883018d7832a0a7fd7e4f9cf1255158f18d4 Mon Sep 17 00:00:00 2001 From: Nim Jayawardena Date: Wed, 27 Mar 2024 10:57:35 -0400 Subject: [PATCH 11/11] Fix typo in "initial-index.json" --- examples/simple_example/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/simple_example/main.tf b/examples/simple_example/main.tf index 0c53099..a3b33cf 100644 --- a/examples/simple_example/main.tf +++ b/examples/simple_example/main.tf @@ -18,5 +18,5 @@ module "simple" { source = "../../" project_id = var.project_id webhook_path = abspath("../../webhook") - initial_index_json_path = abspath("../../intial-index.json") + initial_index_json_path = abspath("../../initial-index.json") }