Merge pull request #9110 from GoogleCloudPlatform/FEATURE-BRANCH-majo…

…r-release-5.0.0 (#6395)

Signed-off-by: Modular Magician <[email protected]>

.changelog/9110.txt

@@ -0,0 +1,171 @@
+```release-note:breaking-change
+bigquery: added more input validations for BigQuery table schema
+```
+```release-note:breaking-change
+firebase: changed `deletion_policy` default to `DELETE` for `google_firebase_web_app`.
+```
+```release-note:breaking-change
+cloudrunv2: Removed deprecated fields `startup_probe` and `liveness_probe` from `google_cloud_run_v2_job` resource.
+```
+```release-note:breaking-change
+cloudrunv2: Removed deprecated field `liveness_probe.tcp_socket` from `google_cloud_run_v2_service` resource. 
+```
+```release-note:bug
+bigquery: fixed view and materialized view creation when schema is specified
+```
+```release-note:bug
+compute: used APIs default value for field `enable_endpoint_independent_mapping` in resource `google_compute_router_nat`
+```
+```release-note:breaking-change
+dataplex: removed `data_profile_result` and `data_quality_result` from `google_dataplex_scan`
+```
+```release-note:breaking-change
+bigquery: made `routine_type` required for `google_bigquery_routine`
+```
+```release-note:bug
+compute: added default value to `metric.filter` in the resource `google_compute_autoscaler` (beta)
+```
+```release-note:deprecation
+privateca: removed deprecated fields `configValues`, `pemCertificates`
+```
+```release-note:breaking-change
+gameservices: Remove Terraform support for `gameservices`
+```
+```release-note:bug
+sql: fixed diffs when re-ordering existing `database_flags`
+```
+```release-note:breaking-change
+logging: made `growth_factor`, `num_finite_buckets`, and `scale` required for `google_logging_metric`
+```
+```release-note:breaking-change
+compute: removed default value for `rule.rate_limit_options.encorce_on_key` on resource `google_compute_security_policy`
+```
+```release-note:note
+provider: some provider default values are now shown at plan-time
+```
+```release-note:deprecation
+cloudiot: deprecated resource  `google_cloudiot_registry`
+```
+```release-note:deprecation
+cloudiot:  deprecated resource `google_cloudiot_device`
+```
+```release-note:deprecation
+cloudiot: deprecated resource `google_cloudiot_registry_iam_*`
+```
+```release-note:deprecation
+cloudiot:  deprecated datasource `google_cloudiot_registry_iam_policy`
+```
+```release-note:enhancement
+provider: added provider default labels
+```
+```release-note:breaking-change
+logging: changed the default value of `unique_writer_identity` from `false` to `true` in `google_logging_project_sink`.
+```
+```release-note:breaking-change
+accesscontextmanager: changed multiple array fields to sets where appropriate to prevent duplicates and fix diffs caused by server side reordering.
+```
+```release-note:breaking-change
+servicenetworking: used Create instead of Patch to create `google_service_networking_connection`
+```
+```release-note:breaking-change
+firebase: removed `google_firebase_project_location`
+```
+```release-note:breaking-change
+provider: data sources now return errors on 404s when applicable instead of silently failing
+```
+```release-note:breaking-change
+cloudfunction2: made `location` required on `google_cloudfunctions2_function`
+```
+```release-note:breaking-change
+cloudrunv2: transitioned `volumes.cloud_sql_instance.instances` to SET from ARRAY for `google_cloud_run_v2_service`
+```
+```release-note:breaking-change
+secretmanager: removed `automatic` field in `google_secret_manager_secret` resource
+```
+```release-note:breaking-change
+container: removed `enable_binary_authorization` in `google_container_cluster`
+```
+```release-note:breaking-change
+container: removed the behaviour that `google_container_cluster` will delete the cluster if it's created in an error state. Instead, it will mark the cluster as tainted, allowing manual inspection and intervention. To proceed with deletion, run another `terraform apply`.
+```
+```release-note:bug
+compute: removed the default value for field `reconcile_connections ` in resource `google_compute_service_attachment`, the field will now default to a value returned by the API when not set in configuration
+```
+```release-note:breaking-change
+container: removed default value in `network_policy.provider` in `google_container_cluster`
+```
+```release-note:breaking-change
+container: removed default for `logging_variant` in `google_container_node_pool`
+```
+```release-note:breaking-change
+container: changed `management.auto_repair` and `management.auto_upgrade` defaults to true in `google_container_node_pool`
+```
+```release-note:breaking-change
+servicenetworking: used the `deleteConnection` method to delete the resource `google_service_networking_connection`
+```
+```release-note:bug
+provider: fixed a bug where labels/annotations field not exists in GA for some resources
+```
+```release-note:breaking-change
+provider: Empty strings in the provider configuration block will no longer be ignored when configuring the provider
+```
+```release-note:breaking-change
+looker: removed `LOOKER_MODELER` as a possible value in `google_looker_instance. platform_edition`
+```
+```release-note:breaking-change
+container: reworked the `taint` field in `google_container_cluster` and `google_container_node_pool` to only manage a subset of taint keys based on those already in state. Most existing resources are unaffected, unless they use `sandbox_config`- see upgrade guide for details.
+```
+```release-note:enhancement
+container: added the `effective_taints` attribute to `google_container_cluster` and `google_container_node_pool`, outputting all known taint values
+```
+```release-note:bug
+`dataflow`: fixed permadiff when SdkPipeline values are supplied via parameters. 
+```
+```release-note:bug
+`dataflow`: fixed max_workers read value permanently displaying as 0.
+```
+```release-note:bug
+`dataflow`: fixed issue causing error message when max_workers and num_workers were supplied via parameters.
+```
+```release-note:breaking-change
+provider: added provider-level validation so these fields are not set as empty strings in a user's config: `credentials`, `access_token`, `impersonate_service_account`, `project`, `billing_project`, `region`, `zone`
+```
+```release-note:breaking-change
+provider: fixed many import functions throughout the provider that matched a subset of the provided input when possible. Now, the GCP resource id supplied to "terraform import" must match exactly.
+```
+```release-note:breaking-change
+compute: retyped `consumer_accept_lists` to a SET from an ARRAY type for `google_compute_service_attachment
+```
+```release-note:breaking-change
+monitoring: made `labels` immutable in `google_monitoring_metric_descriptor`
+```
+```release-note:bug
+monitoring: fixed an issue where `metadata` was not able to be updated in `google_monitoring_metric_descriptor`
+```
+```release-note:breaking-change
+firebase: made `google_firebase_rules.release` immutable
+```
+```release-note:enhancement
+containeraws: added `binary_authorization` to `google_container_aws_cluster`
+```
+```release-note:enhancement
+containeraws: added `update_settings` to `google_container_aws_node_pool`
+```
+```release-note:breaking-change
+compute: `size` in `google_compute_node_group` is now an output only field. 
+```
+```release-note:enhancement
+compute: `google_compute_node_group` made mutable
+```
+```release-note:note
+compute: `google_compute_node_group` made to require one of `initial_size` or `autoscaling_policy` fields configured upon resource creation
+```
+```release-note:enhancement
+baremetal: make delete a noop for the resource `google_bare_metal_admin_cluster` to better align with actual behavior
+```
+```release-note:breaking-change
+container: `google_container_cluster` now has `deletion_protection` enabled to `true` by default. When enabled, this field prevents Terraform from deleting the resource.
+```
+```release-note:breaking-change
+monitoring: fixed perma-diffs in `google_monitoring_dashboard.dashboard_json` by suppressing values returned by the API that are not in configuration
+```

.teamcity/components/generated/services.kt

@@ -161,11 +161,6 @@ var services = mapOf(
         "displayName" to "Cloudids",
         "path" to "./google-beta/services/cloudids"
     ),
-    "cloudiot" to mapOf(
-        "name" to "cloudiot",
-        "displayName" to "Cloudiot",
-        "path" to "./google-beta/services/cloudiot"
-    ),
     "cloudrun" to mapOf(
         "name" to "cloudrun",
         "displayName" to "Cloudrun",
@@ -371,11 +366,6 @@ var services = mapOf(
         "displayName" to "Firestore",
         "path" to "./google-beta/services/firestore"
     ),
-    "gameservices" to mapOf(
-        "name" to "gameservices",
-        "displayName" to "Gameservices",
-        "path" to "./google-beta/services/gameservices"
-    ),
     "gkebackup" to mapOf(
         "name" to "gkebackup",
         "displayName" to "Gkebackup",

go.mod

@@ -3,7 +3,7 @@ go 1.19
 
 require (
 	cloud.google.com/go/bigtable v1.19.0
-	github.com/GoogleCloudPlatform/declarative-resource-client-library v1.51.0
+	github.com/GoogleCloudPlatform/declarative-resource-client-library v1.52.0
 	github.com/apparentlymart/go-cidr v1.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dnaeon/go-vcr v1.0.1

go.sum

@@ -17,6 +17,8 @@ cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHS
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/GoogleCloudPlatform/declarative-resource-client-library v1.51.0 h1:YhWTPhOf6gVpA9mSfnLOuL8Y6j8W5pzmHE7flXjTke4=
 github.com/GoogleCloudPlatform/declarative-resource-client-library v1.51.0/go.mod h1:pL2Qt5HT+x6xrTd806oMiM3awW6kNIXB/iiuClz6m6k=
+github.com/GoogleCloudPlatform/declarative-resource-client-library v1.52.0 h1:KswxXF4E5iWv2ggktqv265zOvwmXA3mgma3UQfYA4tU=
+github.com/GoogleCloudPlatform/declarative-resource-client-library v1.52.0/go.mod h1:pL2Qt5HT+x6xrTd806oMiM3awW6kNIXB/iiuClz6m6k=
 github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
 github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk=
 github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0=
@@ -428,5 +430,3 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE=
-github.com/GoogleCloudPlatform/declarative-resource-client-library v1.51.0 h1:YhWTPhOf6gVpA9mSfnLOuL8Y6j8W5pzmHE7flXjTke4=
-github.com/GoogleCloudPlatform/declarative-resource-client-library v1.51.0/go.mod h1:pL2Qt5HT+x6xrTd806oMiM3awW6kNIXB/iiuClz6m6k=

google-beta/acctest/bootstrap_test_utils.go

@@ -15,6 +15,7 @@ import (
 	tpgcompute "github.com/hashicorp/terraform-provider-google-beta/google-beta/services/compute"
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/services/privateca"
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/services/resourcemanager"
+	tpgservicenetworking "github.com/hashicorp/terraform-provider-google-beta/google-beta/services/servicenetworking"
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/services/sql"
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgiamresource"
 	"github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgresource"
@@ -25,6 +26,7 @@ import (
 	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
 	iam "google.golang.org/api/iam/v1"
 	"google.golang.org/api/iamcredentials/v1"
+	"google.golang.org/api/servicenetworking/v1"
 	"google.golang.org/api/serviceusage/v1"
 	sqladmin "google.golang.org/api/sqladmin/v1beta4"
 )
@@ -356,6 +358,135 @@ func BootstrapSharedTestNetwork(t *testing.T, testId string) string {
 	return network.Name
 }
 
+const SharedTestGlobalAddressPrefix = "tf-bootstrap-addr-"
+
+func BootstrapSharedTestGlobalAddress(t *testing.T, testId, networkId string) string {
+	project := envvar.GetTestProjectFromEnv()
+	addressName := SharedTestGlobalAddressPrefix + testId
+
+	config := BootstrapConfig(t)
+	if config == nil {
+		return ""
+	}
+
+	log.Printf("[DEBUG] Getting shared test global address %q", addressName)
+	_, err := config.NewComputeClient(config.UserAgent).GlobalAddresses.Get(project, addressName).Do()
+	if err != nil && transport_tpg.IsGoogleApiErrorWithCode(err, 404) {
+		log.Printf("[DEBUG] Global address %q not found, bootstrapping", addressName)
+		url := fmt.Sprintf("%sprojects/%s/global/addresses", config.ComputeBasePath, project)
+		netObj := map[string]interface{}{
+			"name":          addressName,
+			"address_type":  "INTERNAL",
+			"purpose":       "VPC_PEERING",
+			"prefix_length": 16,
+			"network":       networkId,
+		}
+
+		res, err := transport_tpg.SendRequest(transport_tpg.SendRequestOptions{
+			Config:    config,
+			Method:    "POST",
+			Project:   project,
+			RawURL:    url,
+			UserAgent: config.UserAgent,
+			Body:      netObj,
+			Timeout:   4 * time.Minute,
+		})
+		if err != nil {
+			t.Fatalf("Error bootstrapping shared test global address %q: %s", addressName, err)
+		}
+
+		log.Printf("[DEBUG] Waiting for global address creation to finish")
+		err = tpgcompute.ComputeOperationWaitTime(config, res, project, "Error bootstrapping shared test global address", config.UserAgent, 4*time.Minute)
+		if err != nil {
+			t.Fatalf("Error bootstrapping shared test global address %q: %s", addressName, err)
+		}
+	}
+
+	address, err := config.NewComputeClient(config.UserAgent).GlobalAddresses.Get(project, addressName).Do()
+	if err != nil {
+		t.Errorf("Error getting shared test global address %q: %s", addressName, err)
+	}
+	if address == nil {
+		t.Fatalf("Error getting shared test global address %q: is nil", addressName)
+	}
+	return address.Name
+}
+
+// BootstrapSharedServiceNetworkingConnection will create a shared network
+// if it hasn't been created in the test project, a global address
+// if it hasn't been created in the test project, and a service networking connection
+// if it hasn't been created in the test project.
+//
+// BootstrapSharedServiceNetworkingConnection returns a persistent compute network name
+// for a test or set of tests.
+//
+// To delete a service networking conneciton, all of the service instances that use that connection
+// must be deleted first. After the service instances are deleted, some service producers delay the deletion
+// utnil a waiting period has passed. For example, after four days that you delete a SQL instance,
+// the service networking connection can be deleted.
+// That is the reason to use the shared service networking connection for thest resources.
+// https://cloud.google.com/vpc/docs/configure-private-services-access#removing-connection
+//
+// testId specifies the test for which a shared network and a gobal address are used/initialized.
+func BootstrapSharedServiceNetworkingConnection(t *testing.T, testId string) string {
+	parentService := "services/servicenetworking.googleapis.com"
+	project := envvar.GetTestProjectFromEnv()
+	projectNumber := envvar.GetTestProjectNumberFromEnv()
+
+	config := BootstrapConfig(t)
+	if config == nil {
+		return ""
+	}
+
+	networkName := BootstrapSharedTestNetwork(t, testId)
+	networkId := fmt.Sprintf("projects/%v/global/networks/%v", projectNumber, networkName)
+	globalAddressName := BootstrapSharedTestGlobalAddress(t, testId, networkId)
+
+	readCall := config.NewServiceNetworkingClient(config.UserAgent).Services.Connections.List(parentService).Network(networkId)
+	if config.UserProjectOverride {
+		readCall.Header().Add("X-Goog-User-Project", project)
+	}
+	response, err := readCall.Do()
+	if err != nil {
+		t.Errorf("Error getting shared test service networking connection: %s", err)
+	}
+
+	var connection *servicenetworking.Connection
+	for _, c := range response.Connections {
+		if c.Network == networkId {
+			connection = c
+			break
+		}
+	}
+
+	if connection == nil {
+		log.Printf("[DEBUG] Service networking connection not found, bootstrapping")
+
+		connection := &servicenetworking.Connection{
+			Network:               networkId,
+			ReservedPeeringRanges: []string{globalAddressName},
+		}
+
+		createCall := config.NewServiceNetworkingClient(config.UserAgent).Services.Connections.Create(parentService, connection)
+		if config.UserProjectOverride {
+			createCall.Header().Add("X-Goog-User-Project", project)
+		}
+		op, err := createCall.Do()
+		if err != nil {
+			t.Fatalf("Error bootstrapping shared test service networking connection: %s", err)
+		}
+
+		log.Printf("[DEBUG] Waiting for service networking connection creation to finish")
+		if err := tpgservicenetworking.ServiceNetworkingOperationWaitTime(config, op, "Create Service Networking Connection", config.UserAgent, project, 4*time.Minute); err != nil {
+			t.Fatalf("Error bootstrapping shared test service networking connection: %s", err)
+		}
+	}
+
+	log.Printf("[DEBUG] Getting shared test service networking connection")
+
+	return networkName
+}
+
 var SharedServicePerimeterProjectPrefix = "tf-bootstrap-sp-"
 
 func BootstrapServicePerimeterProjects(t *testing.T, desiredProjects int) []*cloudresourcemanager.Project {

google-beta/acctest/test_utils.go

@@ -50,6 +50,12 @@ func CheckDataSourceStateMatchesResourceStateWithIgnores(dataSourceName, resourc
 			if _, ok := ignoreFields[k]; ok {
 				continue
 			}
+			if _, ok := ignoreFields["labels.%"]; ok && strings.HasPrefix(k, "labels.") {
+				continue
+			}
+			if _, ok := ignoreFields["terraform_labels.%"]; ok && strings.HasPrefix(k, "terraform_labels.") {
+				continue
+			}
 			if k == "%" {
 				continue
 			}