From ed6ae5a806f6bb7a6c82c025e8b224195764700c Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Tue, 1 Oct 2024 20:05:14 +0200 Subject: [PATCH 1/3] bug: Surface bug - scraping lists of references does not work Scraping does not handle `TupleConsExpr` when parsing example HCL code from Terraform documentation. Example 1): ```hcl require_attestations_by = [google_binary_authorization_attestor.attestor.name] ``` Does not add: ``` cluster_admission_rules.require_attestations_by: google_binary_authorization_attestor.attestor.name ``` As it should since the reference is contained in a _list_. 1) https://github.com/hashicorp/terraform-provider-google/blob/v5.39.0/website/docs/r/binary_authorization_policy.html.markdown?plain=1#L49 Signed-off-by: Rickard von Essen --- pkg/registry/testdata/gcp/pm.yaml | 172 ++++++++++++ ...ty_service_connection_policy.html.markdown | 261 ++++++++++++++++++ 2 files changed, 433 insertions(+) create mode 100644 pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown diff --git a/pkg/registry/testdata/gcp/pm.yaml b/pkg/registry/testdata/gcp/pm.yaml index c0cdfa9c..0af786b2 100644 --- a/pkg/registry/testdata/gcp/pm.yaml +++ b/pkg/registry/testdata/gcp/pm.yaml @@ -991,3 +991,175 @@ resources: - (Optional) The custom object to return when a requested resource is not found. importStatements: [] + google_network_connectivity_service_connection_policy: + subCategory: Network Connectivity + description: Manage Service Connection Policies. + name: google_network_connectivity_service_connection_policy + # title: google_network_connectivity_service_connection_policy + title: "" + examples: + - name: default + manifest: |- + { + "description": "my basic service connection policy", + "location": "us-central1", + "name": "my-network-connectivity-policy", + "network": "${google_compute_network.producer_net.id}", + "psc_config": [ + { + "limit": 2, + "subnetworks": [ + "${google_compute_subnetwork.producer_subnet.id}" + ] + } + ], + "service_class": "my-basic-service-class" + } + references: + network: google_compute_network.producer_net.id + dependencies: + google_compute_network.producer_net: |- + { + "auto_create_subnetworks": false, + "name": "producer-net" + } + google_compute_subnetwork.producer_subnet: |- + { + "ip_cidr_range": "10.0.0.0/16", + "name": "producer-subnet", + "network": "${google_compute_network.producer_net.id}", + "region": "us-central1" + } + argumentDocs: + create: '- Default is 30 minutes.' + create_time: |- + - + The timestamp when the resource was created. + delete: '- Default is 30 minutes.' + description: |- + - + (Optional) + Free-text description of the resource. + effective_labels: for all of the labels present on the resource. + error.code: |- + - + (Optional) + The status code, which should be an enum value of [google.rpc.Code][]. + error.details: |- + - + (Output) + A list of messages that carry the error details. + error.message: |- + - + (Optional) + A developer-facing error message. + error_info.domain: |- + - + (Optional) + The logical grouping to which the "reason" belongs. + error_info.metadata: |- + - + (Optional) + Additional structured details about this error. + error_info.reason: |- + - + (Optional) + The reason of the error. + etag: |- + - + The etag is computed by the server, and may be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. + id: '- an identifier for the resource with format projects/{{project}}/locations/{{location}}/serviceConnectionPolicies/{{name}}' + infrastructure: |- + - + The type of underlying resources used to create the connection. + labels: |- + - + (Optional) + User-defined labels. + location: |- + - + (Required) + The location of the ServiceConnectionPolicy. + service_class: |- + - + (Required) + The service class identifier for which this ServiceConnectionPolicy is for. The service class identifier is a unique, symbolic representation of a ServiceClass. + It is provided by the Service Producer. Google services have a prefix of gcp. For example, gcp-cloud-sql. 3rd party services do not. For example, test-service-a3dfcx. + network: |- + - + (Required) + The resource path of the consumer network. Example: - projects/{projectNumOrId}/global/networks/{resourceId}. + name: |- + - + (Required) + The name of a ServiceConnectionPolicy. Format: projects/{project}/locations/{location}/serviceConnectionPolicies/{service_connection_policy} See: https://google.aip.dev/122#fields-representing-resource-names + project: |- + - (Optional) The ID of the project in which the resource belongs. + If it is not provided, the provider project is used. + psc_config: |- + - + (Optional) + Configuration used for Private Service Connect connections. Used when Infrastructure is PSC. + Structure is documented below. + psc_config.limit: |- + - + (Optional) + Max number of PSC connections for this policy. + psc_config.subnetworks: |- + - + (Required) + IDs of the subnetworks or fully qualified identifiers for the subnetworks + psc_connections: |- + - + Information about each Private Service Connect connection. + Structure is documented below. + psc_connections.consumer_address: |- + - + (Optional) + The resource reference of the consumer address. + psc_connections.consumer_forwarding_rule: |- + - + (Optional) + The resource reference of the PSC Forwarding Rule within the consumer VPC. + psc_connections.consumer_target_project: |- + - + (Optional) + The project where the PSC connection is created. + psc_connections.error: |- + - + (Optional) + The most recent error during operating this connection. + Structure is documented below. + psc_connections.error_info: |- + - + (Optional) + The error info for the latest error during operating this connection. + Structure is documented below. + psc_connections.error_type: |- + - + (Optional) + The error type indicates whether the error is consumer facing, producer + facing or system internal. + Possible values are: CONNECTION_ERROR_TYPE_UNSPECIFIED, ERROR_INTERNAL, ERROR_CONSUMER_SIDE, ERROR_PRODUCER_SIDE. + psc_connections.gce_operation: |- + - + (Optional) + The last Compute Engine operation to setup PSC connection. + psc_connections.psc_connection_id: |- + - + (Optional) + The PSC connection id of the PSC forwarding rule. + psc_connections.state: |- + - + (Optional) + The state of the PSC connection. + Possible values are: STATE_UNSPECIFIED, ACTIVE, CREATING, DELETING, FAILED. + terraform_labels: |- + - + The combination of labels configured directly on the resource + and default labels configured on the provider. + update: '- Default is 30 minutes.' + update_time: |- + - + The timestamp when the resource was updated. + importStatements: [] diff --git a/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown b/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown new file mode 100644 index 00000000..7df34357 --- /dev/null +++ b/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown @@ -0,0 +1,261 @@ +--- +# ---------------------------------------------------------------------------- +# +# *** AUTO GENERATED CODE *** Type: MMv1 *** +# +# ---------------------------------------------------------------------------- +# +# This file is automatically generated by Magic Modules and manual +# changes will be clobbered when the file is regenerated. +# +# Please read more about how to change this file in +# .github/CONTRIBUTING.md. +# +# ---------------------------------------------------------------------------- +subcategory: "Network Connectivity" +description: |- + Manage Service Connection Policies. +--- + +# google_network_connectivity_service_connection_policy + +Manage Service Connection Policies. + + +To get more information about ServiceConnectionPolicy, see: + +* [API documentation](https://cloud.google.com/secure-web-proxy/docs/reference/networkconnectivity/rest/v1/projects.locations.networkConnectionPolicies) +* How-to Guides + * [About Service Connection Policies](https://cloud.google.com/vpc/docs/about-service-connection-policies#service-policies) + +
+ + Open in Cloud Shell + +
+## Example Usage - Network Connectivity Policy Basic + + +```hcl +resource "google_compute_network" "producer_net" { + name = "producer-net" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "producer_subnet" { + name = "producer-subnet" + ip_cidr_range = "10.0.0.0/16" + region = "us-central1" + network = google_compute_network.producer_net.id +} + +resource "google_network_connectivity_service_connection_policy" "default" { + name = "my-network-connectivity-policy" + location = "us-central1" + service_class = "my-basic-service-class" + description = "my basic service connection policy" + network = google_compute_network.producer_net.id + psc_config { + subnetworks = [google_compute_subnetwork.producer_subnet.id] + limit = 2 + } +} +``` + +## Argument Reference + +The following arguments are supported: + + +* `service_class` - + (Required) + The service class identifier for which this ServiceConnectionPolicy is for. The service class identifier is a unique, symbolic representation of a ServiceClass. + It is provided by the Service Producer. Google services have a prefix of gcp. For example, gcp-cloud-sql. 3rd party services do not. For example, test-service-a3dfcx. + +* `network` - + (Required) + The resource path of the consumer network. Example: - projects/{projectNumOrId}/global/networks/{resourceId}. + +* `name` - + (Required) + The name of a ServiceConnectionPolicy. Format: projects/{project}/locations/{location}/serviceConnectionPolicies/{service_connection_policy} See: https://google.aip.dev/122#fields-representing-resource-names + +* `location` - + (Required) + The location of the ServiceConnectionPolicy. + + +- - - + + +* `description` - + (Optional) + Free-text description of the resource. + +* `psc_config` - + (Optional) + Configuration used for Private Service Connect connections. Used when Infrastructure is PSC. + Structure is [documented below](#nested_psc_config). + +* `labels` - + (Optional) + User-defined labels. + + **Note**: This field is non-authoritative, and will only manage the labels present in your configuration. + Please refer to the field `effective_labels` for all of the labels present on the resource. + +* `project` - (Optional) The ID of the project in which the resource belongs. + If it is not provided, the provider project is used. + + +The `psc_config` block supports: + +* `subnetworks` - + (Required) + IDs of the subnetworks or fully qualified identifiers for the subnetworks + +* `limit` - + (Optional) + Max number of PSC connections for this policy. + +## Attributes Reference + +In addition to the arguments listed above, the following computed attributes are exported: + +* `id` - an identifier for the resource with format `projects/{{project}}/locations/{{location}}/serviceConnectionPolicies/{{name}}` + +* `create_time` - + The timestamp when the resource was created. + +* `update_time` - + The timestamp when the resource was updated. + +* `etag` - + The etag is computed by the server, and may be sent on update and delete requests to ensure the client has an up-to-date value before proceeding. + +* `psc_connections` - + Information about each Private Service Connect connection. + Structure is [documented below](#nested_psc_connections). + +* `infrastructure` - + The type of underlying resources used to create the connection. + +* `terraform_labels` - + The combination of labels configured directly on the resource + and default labels configured on the provider. + +* `effective_labels` - + All of labels (key/value pairs) present on the resource in GCP, including the labels configured through Terraform, other clients and services. + + +The `psc_connections` block contains: + +* `state` - + (Optional) + The state of the PSC connection. + Possible values are: `STATE_UNSPECIFIED`, `ACTIVE`, `CREATING`, `DELETING`, `FAILED`. + +* `consumer_forwarding_rule` - + (Optional) + The resource reference of the PSC Forwarding Rule within the consumer VPC. + +* `consumer_address` - + (Optional) + The resource reference of the consumer address. + +* `error_type` - + (Optional) + The error type indicates whether the error is consumer facing, producer + facing or system internal. + Possible values are: `CONNECTION_ERROR_TYPE_UNSPECIFIED`, `ERROR_INTERNAL`, `ERROR_CONSUMER_SIDE`, `ERROR_PRODUCER_SIDE`. + +* `error` - + (Optional) + The most recent error during operating this connection. + Structure is [documented below](#nested_error). + +* `gce_operation` - + (Optional) + The last Compute Engine operation to setup PSC connection. + +* `consumer_target_project` - + (Optional) + The project where the PSC connection is created. + +* `psc_connection_id` - + (Optional) + The PSC connection id of the PSC forwarding rule. + +* `error_info` - + (Optional) + The error info for the latest error during operating this connection. + Structure is [documented below](#nested_error_info). + + +The `error` block supports: + +* `message` - + (Optional) + A developer-facing error message. + +* `code` - + (Optional) + The status code, which should be an enum value of [google.rpc.Code][]. + +* `details` - + (Output) + A list of messages that carry the error details. + +The `error_info` block supports: + +* `reason` - + (Optional) + The reason of the error. + +* `domain` - + (Optional) + The logical grouping to which the "reason" belongs. + +* `metadata` - + (Optional) + Additional structured details about this error. + +## Timeouts + +This resource provides the following +[Timeouts](https://developer.hashicorp.com/terraform/plugin/sdkv2/resources/retries-and-customizable-timeouts) configuration options: + +- `create` - Default is 30 minutes. +- `update` - Default is 30 minutes. +- `delete` - Default is 30 minutes. + +## Import + + +ServiceConnectionPolicy can be imported using any of these accepted formats: + +* `projects/{{project}}/locations/{{location}}/serviceConnectionPolicies/{{name}}` +* `{{project}}/{{location}}/{{name}}` +* `{{location}}/{{name}}` + + +In Terraform v1.5.0 and later, use an [`import` block](https://developer.hashicorp.com/terraform/language/import) to import ServiceConnectionPolicy using one of the formats above. For example: + +```tf +import { + id = "projects/{{project}}/locations/{{location}}/serviceConnectionPolicies/{{name}}" + to = google_network_connectivity_service_connection_policy.default +} +``` + +When using the [`terraform import` command](https://developer.hashicorp.com/terraform/cli/commands/import), ServiceConnectionPolicy can be imported using one of the formats above. For example: + +``` +$ terraform import google_network_connectivity_service_connection_policy.default projects/{{project}}/locations/{{location}}/serviceConnectionPolicies/{{name}} +$ terraform import google_network_connectivity_service_connection_policy.default {{project}}/{{location}}/{{name}} +$ terraform import google_network_connectivity_service_connection_policy.default {{location}}/{{name}} +``` + +## User Project Overrides + +This resource supports [User Project Overrides](https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#user_project_override). From 6dae02b730edf87410692399f7c07623b74c4f01 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Wed, 2 Oct 2024 14:49:43 +0200 Subject: [PATCH 2/3] Fix scraping Refs from attributes containing lists This correctly parses references contained in lists and adds them to the map of references. Example 1): ```hcl require_attestations_by = [google_binary_authorization_attestor.attestor.name] ``` Correctly generates a ref and building `provider-upjet-gcp` with this change produces the expected `examples-generated/binaryauthorization/v1beta2/policy.yaml` with this diff compared to without this change. ``` @@ -14,8 +14,8 @@ spec: - cluster: us-central1-a.prod-cluster enforcementMode: ENFORCED_BLOCK_AND_AUDIT_LOG evaluationMode: REQUIRE_ATTESTATION - requireAttestationsBy: - - ${google_binary_authorization_attestor.attestor.name} + requireAttestationsByRefs: + - name: attestor defaultAdmissionRule: - enforcementMode: ENFORCED_BLOCK_AND_AUDIT_LOG evaluationMode: ALWAYS_ALLOW ``` 1) https://github.com/hashicorp/terraform-provider-google/blob/v5.39.0/website/docs/r/binary_authorization_policy.html.markdown?plain=1#L49 Signed-off-by: Rickard von Essen --- pkg/registry/meta.go | 13 ++++++++++++- pkg/registry/testdata/gcp/pm.yaml | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/registry/meta.go b/pkg/registry/meta.go index 94033526..aea3d2a0 100644 --- a/pkg/registry/meta.go +++ b/pkg/registry/meta.go @@ -116,9 +116,20 @@ func (r *Resource) findReferences(parentPath string, file *hcl.File, b *hclsynta return refs, nil } for name, attr := range b.Body.Attributes { + if name == "depends_on" { + continue + } e, ok := attr.Expr.(*hclsyntax.ScopeTraversalExpr) if !ok { - continue + t, ok := attr.Expr.(*hclsyntax.TupleConsExpr) + if ok && len(t.Exprs) > 0 { + e, ok = t.Exprs[0].(*hclsyntax.ScopeTraversalExpr) + if !ok { + continue + } + } else { + continue + } } refName := name if parentPath != "" { diff --git a/pkg/registry/testdata/gcp/pm.yaml b/pkg/registry/testdata/gcp/pm.yaml index 0af786b2..54ad68e0 100644 --- a/pkg/registry/testdata/gcp/pm.yaml +++ b/pkg/registry/testdata/gcp/pm.yaml @@ -1017,6 +1017,7 @@ resources: } references: network: google_compute_network.producer_net.id + psc_config.subnetworks: google_compute_subnetwork.producer_subnet.id dependencies: google_compute_network.producer_net: |- { From 1ae4c81e89c0f761b4f5abbf31f3767716c4d668 Mon Sep 17 00:00:00 2001 From: Rickard von Essen Date: Mon, 7 Oct 2024 09:09:43 +0200 Subject: [PATCH 3/3] Add license header Signed-off-by: Rickard von Essen --- ...ork_connectivity_service_connection_policy.html.markdown | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown b/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown index 7df34357..71544180 100644 --- a/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown +++ b/pkg/registry/testdata/gcp/r/network_connectivity_service_connection_policy.html.markdown @@ -1,3 +1,9 @@ + + --- # ---------------------------------------------------------------------------- #