From e5a924b5dd187c5966daa7c200e773f767802ede Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Fri, 24 Nov 2023 16:40:07 -0800 Subject: [PATCH 1/6] feat(vertexai): Support google_vertex_ai_feature_group --- mmv1/products/vertexai/FeatureGroup.yaml | 104 ++++++++++++++++++ ...ai_feature_group_ignore_description.go.erb | 17 +++ .../examples/vertex_ai_feature_group.tf.erb | 48 ++++++++ 3 files changed, 169 insertions(+) create mode 100644 mmv1/products/vertexai/FeatureGroup.yaml create mode 100644 mmv1/templates/terraform/custom_flatten/vertex_ai_feature_group_ignore_description.go.erb create mode 100644 mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb diff --git a/mmv1/products/vertexai/FeatureGroup.yaml b/mmv1/products/vertexai/FeatureGroup.yaml new file mode 100644 index 000000000000..1d918d41fe26 --- /dev/null +++ b/mmv1/products/vertexai/FeatureGroup.yaml @@ -0,0 +1,104 @@ +# Copyright 2023 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Resource +name: FeatureGroup +base_url: projects/{{project}}/locations/{{region}}/featureGroups +create_url: projects/{{project}}/locations/{{region}}/featureGroups?featureGroupId={{name}} +self_link: projects/{{project}}/locations/{{region}}/featureGroups/{{name}} +update_verb: :PATCH +update_mask: true +create_verb: :POST +references: !ruby/object:Api::Resource::ReferenceLinks + api: https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featureGroups +async: !ruby/object:Api::OpAsync + actions: + - create + - delete + - update + operation: !ruby/object:Api::OpAsync::Operation + path: 'name' + base_url: '{{op_id}}' + wait_ms: 1000 + result: !ruby/object:Api::OpAsync::Result + path: 'response' + resource_inside_response: true + status: !ruby/object:Api::OpAsync::Status + path: 'done' + complete: true + allowed: + - true + - false + error: !ruby/object:Api::OpAsync::Error + path: 'error' + message: 'message' +description: Vertex AI Feature Group. +autogen_async: false +examples: + - !ruby/object:Provider::Terraform::Examples + name: vertex_ai_feature_group + primary_resource_id: feature_group + ignore_read_extra: + - description + vars: + job_id: job_load +parameters: + - !ruby/object:Api::Type::String + name: region + description: The region of feature group. eg us-central1 + url_param_only: true + immutable: true +properties: + - !ruby/object:Api::Type::String + name: name + description: The resource name of the Feature Group. + custom_flatten: templates/terraform/custom_flatten/name_from_self_link.erb + - !ruby/object:Api::Type::String + name: createTime + output: true + description: The timestamp of when the FeatureGroup was created in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. + - !ruby/object:Api::Type::String + name: updateTime + output: true + description: The timestamp of when the FeatureGroup was last updated in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. + - !ruby/object:Api::Type::String + name: etag + description: Used to perform consistent read-modify-write updates. + output: true + ignore_read: true + - !ruby/object:Api::Type::KeyValueLabels + name: labels + description: The labels with user-defined metadata to organize your FeatureGroup. + - !ruby/object:Api::Type::String + name: description + description: The description of the FeatureGroup. + # When API starts returning description, this line should be removed. + custom_flatten: templates/terraform/custom_flatten/vertex_ai_feature_group_ignore_description.go.erb + - !ruby/object:Api::Type::NestedObject + name: bigQuery + description: Indicates that features for this group come from BigQuery Table/View. By default treats the source as a sparse time series source, which is required to have an entityId and a feature_timestamp column in the source. + properties: + - !ruby/object:Api::Type::NestedObject + name: bigQuerySource + required: true + immutable: true + description: The BigQuery source URI that points to either a BigQuery Table or View. + properties: + - !ruby/object:Api::Type::String + name: 'inputUri' + required: true + description: 'BigQuery URI to a table, up to 2000 characters long. For example: `bq://projectId.bqDatasetId.bqTableId.`' + - !ruby/object:Api::Type::Array + name: entityIdColumns + description: Columns to construct entityId / row keys. Currently only supports 1 entity_id_column. If not provided defaults to entityId. + item_type: Api::Type::String diff --git a/mmv1/templates/terraform/custom_flatten/vertex_ai_feature_group_ignore_description.go.erb b/mmv1/templates/terraform/custom_flatten/vertex_ai_feature_group_ignore_description.go.erb new file mode 100644 index 000000000000..61f2aa0d8915 --- /dev/null +++ b/mmv1/templates/terraform/custom_flatten/vertex_ai_feature_group_ignore_description.go.erb @@ -0,0 +1,17 @@ +<%# The license inside this block applies to this file. + # Copyright 2023 Google Inc. + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. +-%> +func flatten<%= prefix -%><%= titlelize_property(property) -%>(v interface{}, d *schema.ResourceData, config *transport_tpg.Config) interface{} { + return d.Get("description") +} diff --git a/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb new file mode 100644 index 000000000000..84f29c2dea9d --- /dev/null +++ b/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb @@ -0,0 +1,48 @@ + resource "google_vertex_ai_feature_group" "<%= ctx[:primary_resource_id] %>" { + name = "sample_feature_group_movie_reviews" + description = "A sample feature group" + region = "us-central1" + labels = { + label-one = "value-one" + } + big_query { + big_query_source { + # The source table must have a column named 'feature_timestamp' of type TIMESTAMP. + input_uri = "bq://${google_bigquery_table.sample_table.project}.${google_bigquery_table.sample_table.dataset_id}.${google_bigquery_table.sample_table.table_id}" + } + entity_id_columns = ["movie_id"] + } +} + +resource "google_bigquery_dataset" "sample_dataset" { + dataset_id = "<%= ctx[:vars]['job_id'] %>_dataset" + friendly_name = "test" + description = "This is a test description" + location = "US" +} + +resource "google_bigquery_table" "sample_table" { + deletion_protection = false + dataset_id = google_bigquery_dataset.sample_dataset.dataset_id + table_id = "<%= ctx[:vars]['job_id'] %>_table" + + schema = < Date: Sat, 25 Nov 2023 08:10:18 -0800 Subject: [PATCH 2/6] test: add an acc test to check updatable fields of feature group --- .../resource_vertex_ai_feature_group_test.go | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go new file mode 100644 index 000000000000..01204e813b54 --- /dev/null +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go @@ -0,0 +1,150 @@ +package vertexai_test + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + + "github.com/hashicorp/terraform-provider-google/google/acctest" +) + +func TestAccVertexAIFeatureGroup_updated(t *testing.T) { + t.Parallel() + + context := map[string]interface{}{ + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckVertexAIFeatureGroupDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccVertexAIFeatureGroup_basic(context), + }, + { + ResourceName: "google_vertex_ai_feature_group.feature_group", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"etag", "region", "description", "labels", "terraform_labels"}, + }, + { + Config: testAccVertexAIFeatureGroup_updated(context), + }, + { + ResourceName: "google_vertex_ai_feature_group.feature_group", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"etag", "region", "description", "labels", "terraform_labels"}, + }, + }, + }) +} + +func testAccVertexAIFeatureGroup_basic(context map[string]interface{}) string { + return acctest.Nprintf(` + resource "google_vertex_ai_feature_group" "feature_group" { + name = "sample_feature_group_movie_reviews" + description = "A sample feature group" + region = "us-central1" + labels = { + label-one = "value-one" + } + big_query { + big_query_source { + # The source table must have a column named 'feature_timestamp' of type TIMESTAMP. + input_uri = "bq://${google_bigquery_table.sample_table.project}.${google_bigquery_table.sample_table.dataset_id}.${google_bigquery_table.sample_table.table_id}" + } + entity_id_columns = ["movie_id"] + } +} + +resource "google_bigquery_dataset" "sample_dataset" { + dataset_id = "tf_test_job_load%{random_suffix}_dataset" + friendly_name = "test" + description = "This is a test description" + location = "US" +} + +resource "google_bigquery_table" "sample_table" { + deletion_protection = false + dataset_id = google_bigquery_dataset.sample_dataset.dataset_id + table_id = "tf_test_job_load%{random_suffix}_table" + + schema = < Date: Mon, 11 Dec 2023 22:15:56 -0800 Subject: [PATCH 3/6] feat: update code based on feedback --- mmv1/products/vertexai/FeatureGroup.yaml | 1 + .../examples/vertex_ai_feature_group.tf.erb | 11 +++------- .../resource_vertex_ai_feature_group_test.go | 22 +++++-------------- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/mmv1/products/vertexai/FeatureGroup.yaml b/mmv1/products/vertexai/FeatureGroup.yaml index 1d918d41fe26..c44edc24e871 100644 --- a/mmv1/products/vertexai/FeatureGroup.yaml +++ b/mmv1/products/vertexai/FeatureGroup.yaml @@ -52,6 +52,7 @@ examples: - description vars: job_id: job_load + feature_group_name: example_feature_group parameters: - !ruby/object:Api::Type::String name: region diff --git a/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb b/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb index 84f29c2dea9d..8bab10127d28 100644 --- a/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb +++ b/mmv1/templates/terraform/examples/vertex_ai_feature_group.tf.erb @@ -1,5 +1,5 @@ resource "google_vertex_ai_feature_group" "<%= ctx[:primary_resource_id] %>" { - name = "sample_feature_group_movie_reviews" + name = "<%= ctx[:vars]['feature_group_name'] %>" description = "A sample feature group" region = "us-central1" labels = { @@ -10,7 +10,7 @@ # The source table must have a column named 'feature_timestamp' of type TIMESTAMP. input_uri = "bq://${google_bigquery_table.sample_table.project}.${google_bigquery_table.sample_table.dataset_id}.${google_bigquery_table.sample_table.table_id}" } - entity_id_columns = ["movie_id"] + entity_id_columns = ["feature_id"] } } @@ -29,12 +29,7 @@ resource "google_bigquery_table" "sample_table" { schema = < Date: Thu, 14 Dec 2023 14:29:31 -0800 Subject: [PATCH 4/6] Apply suggestions from code review Co-authored-by: Nick Elliot --- mmv1/products/vertexai/FeatureGroup.yaml | 4 +++- .../vertexai/resource_vertex_ai_feature_group_test.go | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/mmv1/products/vertexai/FeatureGroup.yaml b/mmv1/products/vertexai/FeatureGroup.yaml index c44edc24e871..a66578f4ea88 100644 --- a/mmv1/products/vertexai/FeatureGroup.yaml +++ b/mmv1/products/vertexai/FeatureGroup.yaml @@ -20,7 +20,9 @@ update_verb: :PATCH update_mask: true create_verb: :POST references: !ruby/object:Api::Resource::ReferenceLinks - api: https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featureGroups + guides: + 'Creating a Feature Group': 'https://cloud.google.com/vertex-ai/docs/featurestore/latest/create-featuregroup' + api: 'https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featureGroups' async: !ruby/object:Api::OpAsync actions: - create diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go index 96be364ed381..d2007e4002a0 100644 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go @@ -53,7 +53,6 @@ func testAccVertexAIFeatureGroup_basic(context map[string]interface{}) string { } big_query { big_query_source { - # The source table must have a column named 'feature_timestamp' of type TIMESTAMP. input_uri = "bq://${google_bigquery_table.sample_table.project}.${google_bigquery_table.sample_table.dataset_id}.${google_bigquery_table.sample_table.table_id}" } entity_id_columns = ["feature_id"] From c3d18fa15814846d341109b5894d752a7e68c833 Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Thu, 14 Dec 2023 14:30:27 -0800 Subject: [PATCH 5/6] Apply suggestions from code review Co-authored-by: Nick Elliot --- .../services/vertexai/resource_vertex_ai_feature_group_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go index d2007e4002a0..a31bb11f5bee 100644 --- a/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go +++ b/mmv1/third_party/terraform/services/vertexai/resource_vertex_ai_feature_group_test.go @@ -101,7 +101,6 @@ func testAccVertexAIFeatureGroup_updated(context map[string]interface{}) string } big_query { big_query_source { - # The source table must have a column named 'feature_timestamp' of type TIMESTAMP. input_uri = "bq://${google_bigquery_table.sample_table.project}.${google_bigquery_table.sample_table.dataset_id}.${google_bigquery_table.sample_table.table_id}" } entity_id_columns = ["feature_id"] From cff94b6dcb2ab313566f712468d7661e4a96b958 Mon Sep 17 00:00:00 2001 From: Shotaro Kohama Date: Thu, 14 Dec 2023 14:36:11 -0800 Subject: [PATCH 6/6] format yaml based on yamllint --- mmv1/products/vertexai/FeatureGroup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/products/vertexai/FeatureGroup.yaml b/mmv1/products/vertexai/FeatureGroup.yaml index a66578f4ea88..22ca5e4df9d9 100644 --- a/mmv1/products/vertexai/FeatureGroup.yaml +++ b/mmv1/products/vertexai/FeatureGroup.yaml @@ -20,7 +20,7 @@ update_verb: :PATCH update_mask: true create_verb: :POST references: !ruby/object:Api::Resource::ReferenceLinks - guides: + guides: 'Creating a Feature Group': 'https://cloud.google.com/vertex-ai/docs/featurestore/latest/create-featuregroup' api: 'https://cloud.google.com/vertex-ai/docs/reference/rest/v1/projects.locations.featureGroups' async: !ruby/object:Api::OpAsync