From dc5355bf2db3d432f205ad221500064ca4e83f02 Mon Sep 17 00:00:00 2001 From: Gemma Hou Date: Mon, 28 Oct 2024 17:02:47 +0000 Subject: [PATCH] fix dataproccluster dynamic test --- config/tests/samples/create/harness.go | 2 +- .../dataproccluster/_vcr_cassettes/nontf.yaml | 6250 +++++++++-------- .../dataproccluster/_vcr_cassettes/tf.yaml | 572 +- .../v1beta1/dataproccluster/create.yaml | 2 + .../v1beta1/dataproccluster/dependencies.yaml | 12 + .../v1beta1/dataproccluster/update.yaml | 2 + 6 files changed, 3731 insertions(+), 3109 deletions(-) diff --git a/config/tests/samples/create/harness.go b/config/tests/samples/create/harness.go index 9180d45f0a..58a4a17a9c 100644 --- a/config/tests/samples/create/harness.go +++ b/config/tests/samples/create/harness.go @@ -423,7 +423,7 @@ func NewHarness(ctx context.Context, t *testing.T, opts ...HarnessOption) *Harne h.Project = testgcp.GCPProject{ ProjectID: "example-project", - ProjectNumber: 12345678, + ProjectNumber: 123456789, } testgcp.TestDependentOrgProjectID.Set("example-project-01") testgcp.TestDependentFolderProjectID.Set("example-project-02") diff --git a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/nontf.yaml b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/nontf.yaml index 7d66331bcf..8ae24731da 100644 --- a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/nontf.yaml +++ b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/nontf.yaml @@ -15,3168 +15,3204 @@ --- version: 2 interactions: - - id: 0 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 2004177118677253372;o=0 - status: 404 Not Found - code: 404 - duration: 324.086636ms - - id: 1 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 4661834806583333198;o=0 - status: 404 Not Found - code: 404 - duration: 107.893734ms - - id: 2 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 7020507199461256248;o=0 - status: 404 Not Found - code: 404 - duration: 97.294265ms - - id: 3 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 237 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: '{"basicAlgorithm":{"yarnConfig":{"gracefulDecommissionTimeout":"30s","scaleDownFactor":0.5,"scaleUpFactor":0.5}},"id":"dataprocautoscalingpolicy-1tzjbegzhqu4o","secondaryWorkerConfig":{"maxInstances":2},"workerConfig":{"maxInstances":2}}' - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies?alt=json - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 3851570124521108812;o=0 - status: 200 OK - code: 200 - duration: 173.556676ms - - id: 4 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 13760009607435117398;o=0 - status: 200 OK - code: 200 - duration: 132.461793ms - - id: 5 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 160310122617543993;o=0 - status: 200 OK - code: 200 - duration: 126.01213ms - - id: 6 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 994173531859678876;o=0 - status: 200 OK - code: 200 - duration: 138.125391ms - - id: 7 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 16239676685968320814;o=0 - status: 404 Not Found - code: 404 - duration: 258.006025ms - - id: 8 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 10952091519720914537;o=0 - status: 200 OK - code: 200 - duration: 134.82503ms - - id: 9 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 1631361065317769988;o=0 - status: 404 Not Found - code: 404 - duration: 91.884457ms - - id: 10 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 2112474367824215308;o=0 - status: 200 OK - code: 200 - duration: 267.391196ms - - id: 11 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 6807397707200473596;o=0 - status: 404 Not Found - code: 404 - duration: 132.406152ms - - id: 12 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 402 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: '{"clusterName":"dataproccluster-1tzjbegzhqu4o","config":{"autoscalingConfig":{"policyUri":"projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o"},"masterConfig":{"diskConfig":{"bootDiskType":"pd-standard"},"machineTypeUri":"n2-standard-2","numInstances":1}},"labels":{"cnrm-test":"true","label-one":"value-one","managed-by-cnrm":"true"}}' - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters?alt=json - method: POST - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "name": "projects/example-project/regions/us-central1/operations/33f9666b-0543-4332-9e2f-71f8e29be9c2", - "metadata": { - "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "status": { - "state": "PENDING", - "innerState": "PENDING", - "stateStartTime": "2024-04-25T01:31:21.683829Z" - }, - "operationType": "CREATE", - "description": "Create cluster with 2 workers", - "warnings": [ - "No image specified. Using the default image version. It is recommended to select a specific image version in production, as the default image version may change at any time.", - "Permissions are missing for the default service account '123456789-compute@developer.gserviceaccount.com', missing permissions: [storage.buckets.get, storage.objects.create, storage.objects.delete, storage.objects.get, storage.objects.list, storage.objects.update] on the project 'projects/example-project'. This usually happens when a custom resource (ex: custom staging bucket) or a user-managed VM Service account has been provided and the default/user-managed service account hasn't been granted enough permissions on the resource. See https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#VM_service_account.", - "The firewall rules for specified network or subnetwork would allow ingress traffic from 0.0.0.0/0, which could be a security risk." - ] - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 15481174575836554123;o=0 - status: 200 OK - code: 200 - duration: 1.931809573s - - id: 13 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/operations/33f9666b-0543-4332-9e2f-71f8e29be9c2?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "name": "projects/example-project/regions/us-central1/operations/33f9666b-0543-4332-9e2f-71f8e29be9c2", - "metadata": { - "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "status": { - "state": "DONE", - "innerState": "DONE", - "stateStartTime": "2024-04-25T01:34:06.218855Z" - }, - "statusHistory": [ - { - "state": "PENDING", - "stateStartTime": "2024-04-25T01:31:21.683829Z" + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 5618223841483454790;o=0 + status: 404 Not Found + code: 404 + duration: 381.477483ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 13057095382519866344;o=0 + status: 404 Not Found + code: 404 + duration: 117.571971ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 1277035720498282288;o=0 + status: 404 Not Found + code: 404 + duration: 119.644051ms + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 237 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: '{"basicAlgorithm":{"yarnConfig":{"gracefulDecommissionTimeout":"30s","scaleDownFactor":0.5,"scaleUpFactor":0.5}},"id":"dataprocautoscalingpolicy-1tzjbegzhqu4o","secondaryWorkerConfig":{"maxInstances":2},"workerConfig":{"maxInstances":2}}' + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies?alt=json + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" }, - { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:31:21.776515Z" + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - ], - "operationType": "CREATE", - "description": "Create cluster with 2 workers", - "warnings": [ - "No image specified. Using the default image version. It is recommended to select a specific image version in production, as the default image version may change at any time.", - "Permissions are missing for the default service account '123456789-compute@developer.gserviceaccount.com', missing permissions: [storage.buckets.get, storage.objects.create, storage.objects.delete, storage.objects.get, storage.objects.list, storage.objects.update] on the project 'projects/example-project'. This usually happens when a custom resource (ex: custom staging bucket) or a user-managed VM Service account has been provided and the default/user-managed service account hasn't been granted enough permissions on the resource. See https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/service-accounts#VM_service_account.", - "The firewall rules for specified network or subnetwork would allow ingress traffic from 0.0.0.0/0, which could be a security risk." - ] - }, - "done": true, - "response": { - "@type": "type.googleapis.com/google.cloud.dataproc.v1.Cluster", - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 5867195220251431215;o=0 + status: 200 OK + code: 200 + duration: 154.16758ms + - id: 4 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "masterConfig": { - "numInstances": 1, - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 3528243094837927669;o=0 + status: 200 OK + code: 200 + duration: 140.232711ms + - id: 5 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" + "cooldownPeriod": "120s" }, "workerConfig": { - "numInstances": 2, - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 16685578279426277178;o=0 + status: 200 OK + code: 200 + duration: 125.385212ms + - id: 6 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" + "cooldownPeriod": "120s" }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" - } + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 10690004360792237805;o=0 + status: 200 OK + code: 200 + duration: 121.350741ms + - id: 7 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 14715905865185975816;o=0 + status: 404 Not Found + code: 404 + duration: 256.415162ms + - id: 8 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "endpointConfig": {} - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } } - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 566658776741266600;o=0 - status: 200 OK - code: 200 - duration: 79.524553ms - - id: 14 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 10544954321015139542;o=0 + status: 200 OK + code: 200 + duration: 107.7366ms + - id: 9 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 13403542243666207470;o=0 + status: 200 OK + code: 200 + duration: 109.298411ms + - id: 10 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 688882900701730189;o=0 + status: 404 Not Found + code: 404 + duration: 201.504082ms + - id: 11 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 7279129670724302616;o=0 + status: 404 Not Found + code: 404 + duration: 88.242171ms + - id: 12 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 430 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: '{"clusterName":"dataproccluster-1tzjbegzhqu4o","config":{"autoscalingConfig":{"policyUri":"projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o"},"gceClusterConfig":{"internalIpOnly":false},"masterConfig":{"diskConfig":{"bootDiskType":"pd-standard"},"machineTypeUri":"n2-standard-2","numInstances":1}},"labels":{"cnrm-test":"true","label-one":"value-one","managed-by-cnrm":"true"}}' + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters?alt=json + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "name": "projects/example-project/regions/us-central1/operations/015cc4b7-ed16-4029-aa54-6ad7e16b3f45", + "metadata": { + "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "status": { + "state": "PENDING", + "innerState": "PENDING", + "stateStartTime": "2024-10-28T01:29:50.586606Z" + }, + "operationType": "CREATE", + "description": "Create cluster with 2 workers", + "warnings": [ + "The firewall rules for specified network or subnetwork would allow ingress traffic from 0.0.0.0/0, which could be a security risk.", + "The specified custom staging bucket 'dataproc-staging-us-central1-123456789-pxeyagwz' is not using uniform bucket level access IAM configuration. It is recommended to update bucket to enable the same. See https://cloud.google.com/storage/docs/uniform-bucket-level-access.", + "No image specified. Using the default image version. It is recommended to select a specific image version in production, as the default image version may change at any time." + ] + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 318595821380288450;o=0 - status: 200 OK - code: 200 - duration: 91.675974ms - - id: 15 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 1747190019451247620;o=0 + status: 200 OK + code: 200 + duration: 1.211920356s + - id: 13 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/operations/015cc4b7-ed16-4029-aa54-6ad7e16b3f45?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "name": "projects/example-project/regions/us-central1/operations/015cc4b7-ed16-4029-aa54-6ad7e16b3f45", + "metadata": { + "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "status": { + "state": "DONE", + "innerState": "DONE", + "stateStartTime": "2024-10-28T01:32:58.714884Z" + }, + "statusHistory": [ + { + "state": "PENDING", + "stateStartTime": "2024-10-28T01:29:50.586606Z" + }, + { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:29:50.662066Z" + } + ], + "operationType": "CREATE", + "description": "Create cluster with 2 workers", + "warnings": [ + "The firewall rules for specified network or subnetwork would allow ingress traffic from 0.0.0.0/0, which could be a security risk.", + "The specified custom staging bucket 'dataproc-staging-us-central1-123456789-pxeyagwz' is not using uniform bucket level access IAM configuration. It is recommended to update bucket to enable the same. See https://cloud.google.com/storage/docs/uniform-bucket-level-access.", + "No image specified. Using the default image version. It is recommended to select a specific image version in production, as the default image version may change at any time." + ] + }, + "done": true, + "response": { + "@type": "type.googleapis.com/google.cloud.dataproc.v1.Cluster", + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 15872740506406845630;o=0 + status: 200 OK + code: 200 + duration: 108.757768ms + - id: 14 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 4496562470519862565;o=0 + status: 200 OK + code: 200 + duration: 114.603759ms + - id: 15 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } + ], + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 13690223853732693091;o=0 + status: 200 OK + code: 200 + duration: 104.251749ms + - id: 16 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" + }, + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } + ], + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 3410303408321141704;o=0 - status: 200 OK - code: 200 - duration: 86.153734ms - - id: 16 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 12172642657078526664;o=0 + status: 200 OK + code: 200 + duration: 117.617758ms + - id: 17 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" + }, + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 13136269263152684257;o=0 + status: 200 OK + code: 200 + duration: 118.193607ms + - id: 18 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" + }, + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 10565258142610643259;o=0 + status: 200 OK + code: 200 + duration: 107.147098ms + - id: 19 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" + }, + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 13428624793011693073;o=0 + status: 200 OK + code: 200 + duration: 116.909138ms + - id: 20 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 1361166820241806023;o=0 + status: 200 OK + code: 200 + duration: 99.278919ms + - id: 21 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 3277729569353771814;o=0 - status: 200 OK - code: 200 - duration: 101.195669ms - - id: 17 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 12999337409893835647;o=0 + status: 200 OK + code: 200 + duration: 107.803358ms + - id: 22 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" + }, + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" + } + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 14311833596317602763;o=0 + status: 200 OK + code: 200 + duration: 107.209108ms + - id: 23 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 9030808438420408533;o=0 + status: 400 Bad Request + code: 400 + duration: 113.867939ms + - id: 24 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "projectId": "example-project", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "config": { + "configBucket": "dataproc-staging-us-central1-123456789-pxeyagwz", + "tempBucket": "dataproc-temp-us-central1-123456789-jcn7juyd", + "gceClusterConfig": { + "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f", + "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", + "serviceAccountScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "internalIpOnly": false, + "shieldedInstanceConfig": { + "enableSecureBoot": true, + "enableVtpm": true, + "enableIntegrityMonitoring": true + } + }, + "masterConfig": { + "numInstances": 1, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-m" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-2", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "workerConfig": { + "numInstances": 2, + "instanceNames": [ + "dataproccluster-1tzjbegzhqu4o-w-0", + "dataproccluster-1tzjbegzhqu4o-w-1" + ], + "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-2-deb12-20241018-045100-rc01", + "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-f/machineTypes/n2-standard-4", + "diskConfig": { + "bootDiskSizeGb": 1000, + "bootDiskType": "pd-standard" + }, + "minCpuPlatform": "AUTOMATIC", + "preemptibility": "NON_PREEMPTIBLE" + }, + "softwareConfig": { + "imageVersion": "2.2.38-debian12", + "properties": { + "capacity-scheduler:yarn.scheduler.capacity.resource-calculator": "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator", + "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", + "core:fs.gs.block.size": "134217728", + "core:fs.gs.metadata.cache.enable": "false", + "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", + "distcp:mapreduce.map.java.opts": "-Xmx768m", + "distcp:mapreduce.map.memory.mb": "1024", + "distcp:mapreduce.reduce.java.opts": "-Xmx768m", + "distcp:mapreduce.reduce.memory.mb": "1024", + "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", + "hdfs:dfs.datanode.address": "0.0.0.0:9866", + "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", + "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", + "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", + "hdfs:dfs.namenode.handler.count": "20", + "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", + "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", + "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", + "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", + "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", + "hdfs:dfs.namenode.service.handler.count": "10", + "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", + "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", + "mapred:mapreduce.job.maps": "21", + "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", + "mapred:mapreduce.job.reduces": "7", + "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", + "mapred:mapreduce.map.cpu.vcores": "1", + "mapred:mapreduce.map.java.opts": "-Xmx2708m", + "mapred:mapreduce.map.maxattempts": "10", + "mapred:mapreduce.map.memory.mb": "3386", + "mapred:mapreduce.reduce.cpu.vcores": "1", + "mapred:mapreduce.reduce.java.opts": "-Xmx2708m", + "mapred:mapreduce.reduce.maxattempts": "10", + "mapred:mapreduce.reduce.memory.mb": "3386", + "mapred:mapreduce.task.io.sort.mb": "256", + "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2708m", + "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", + "mapred:yarn.app.mapreduce.am.resource.mb": "3386", + "spark-env:SPARK_DAEMON_MEMORY": "2048m", + "spark:spark.decommission.enabled": "true", + "spark:spark.driver.maxResultSize": "1024m", + "spark:spark.driver.memory": "2048m", + "spark:spark.executor.cores": "2", + "spark:spark.executor.instances": "2", + "spark:spark.executor.memory": "6157m", + "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", + "spark:spark.plugins.defaultList": "com.google.cloud.dataproc.DataprocSparkPlugin", + "spark:spark.scheduler.mode": "FAIR", + "spark:spark.sql.cbo.enabled": "true", + "spark:spark.sql.optimizer.runtime.bloomFilter.join.pattern.enabled": "true", + "spark:spark.stage.maxConsecutiveAttempts": "10", + "spark:spark.storage.decommission.enabled": "true", + "spark:spark.storage.decommission.rddBlocks.enabled": "true", + "spark:spark.storage.decommission.shuffleBlocks.enabled": "true", + "spark:spark.task.maxFailures": "10", + "spark:spark.ui.port": "0", + "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", + "spark:spark.yarn.am.memory": "640m", + "spark:spark.yarn.executor.failuresValidityInterval": "1h", + "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1638", + "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", + "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", + "yarn:yarn.nodemanager.address": "0.0.0.0:8026", + "yarn:yarn.nodemanager.resource.cpu-vcores": "4", + "yarn:yarn.nodemanager.resource.memory-mb": "13544", + "yarn:yarn.resourcemanager.am.max-attempts": "10", + "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", + "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", + "yarn:yarn.scheduler.maximum-allocation-mb": "13544", + "yarn:yarn.scheduler.minimum-allocation-mb": "1" + } + }, + "autoscalingConfig": { + "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" + }, + "endpointConfig": {} + }, + "status": { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:32:58.687694Z" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "statusHistory": [ + { + "state": "CREATING", + "stateStartTime": "2024-10-28T01:29:50.584476Z" + } + ], + "labels": { + "cnrm-test": "true", + "label-one": "value-one", + "managed-by-cnrm": "true", + "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", + "goog-dataproc-cluster-uuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "goog-dataproc-location": "us-central1", + "goog-dataproc-autozone": "enabled" } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 17502296957390318111;o=0 + status: 200 OK + code: 200 + duration: 90.508758ms + - id: 25 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "name": "projects/example-project/regions/us-central1/operations/689d9685-9797-4885-8292-f7e8fcdceff1", + "metadata": { + "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "status": { + "state": "PENDING", + "innerState": "PENDING", + "stateStartTime": "2024-10-28T01:33:23.817502Z" + }, + "operationType": "DELETE", + "description": "Delete cluster" + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 12819965534160796748;o=0 - status: 200 OK - code: 200 - duration: 100.339995ms - - id: 18 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 10336882479990121854;o=0 + status: 200 OK + code: 200 + duration: 135.627027ms + - id: 26 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 1711738177287926831;o=0 + status: 200 OK + code: 200 + duration: 98.891777ms + - id: 27 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 3053704855704828705;o=0 - status: 200 OK - code: 200 - duration: 96.974837ms - - id: 19 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 13769975990933878578;o=0 + status: 200 OK + code: 200 + duration: 110.035388ms + - id: 28 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 12412145843749627013;o=0 + status: 400 Bad Request + code: 400 + duration: 101.942558ms + - id: 29 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 14603862722005783189;o=0 + status: 200 OK + code: 200 + duration: 71.382628ms + - id: 30 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 3938541435418355328;o=0 - status: 200 OK - code: 200 - duration: 85.516409ms - - id: 20 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 14187382414638272851;o=0 - status: 200 OK - code: 200 - duration: 114.520989ms - - id: 21 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 4801995640850414283;o=0 - status: 200 OK - code: 200 - duration: 94.620808ms - - id: 22 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 18132455668500357778;o=0 + status: 200 OK + code: 200 + duration: 61.573809ms + - id: 31 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 5547739367565113412;o=0 + status: 400 Bad Request + code: 400 + duration: 137.573827ms + - id: 32 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 16488533378824085177;o=0 + status: 200 OK + code: 200 + duration: 60.734829ms + - id: 33 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 14409635342884362411;o=0 - status: 200 OK - code: 200 - duration: 92.166959ms - - id: 23 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: DELETE - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 2747887024980340183;o=0 - status: 400 Bad Request - code: 400 - duration: 94.779073ms - - id: 24 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "projectId": "example-project", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "config": { - "configBucket": "dataproc-staging-us-central1-123456789-elcrnd0z", - "tempBucket": "dataproc-temp-us-central1-123456789-cfkxv0rd", - "gceClusterConfig": { - "zoneUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c", - "networkUri": "https://www.googleapis.com/compute/v1/projects/example-project/global/networks/default", - "serviceAccountScopes": [ - "https://www.googleapis.com/auth/bigquery", - "https://www.googleapis.com/auth/bigtable.admin.table", - "https://www.googleapis.com/auth/bigtable.data", - "https://www.googleapis.com/auth/cloud.useraccounts.readonly", - "https://www.googleapis.com/auth/devstorage.full_control", - "https://www.googleapis.com/auth/devstorage.read_write", - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring.write" - ], - "internalIpOnly": false - }, - "masterConfig": { - "numInstances": 1, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-m" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n2-standard-2", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 4534970759181093779;o=0 + status: 200 OK + code: 200 + duration: 60.928219ms + - id: 34 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 16255333858687324115;o=0 + status: 400 Bad Request + code: 400 + duration: 77.175439ms + - id: 35 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "workerConfig": { - "numInstances": 2, - "instanceNames": [ - "dataproccluster-1tzjbegzhqu4o-w-0", - "dataproccluster-1tzjbegzhqu4o-w-1" - ], - "imageUri": "https://www.googleapis.com/compute/v1/projects/cloud-dataproc/global/images/dataproc-2-0-deb10-20240418-165100-rc01", - "machineTypeUri": "https://www.googleapis.com/compute/v1/projects/example-project/zones/us-central1-c/machineTypes/n1-standard-4", - "diskConfig": { - "bootDiskSizeGb": 1000, - "bootDiskType": "pd-standard" + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 }, - "minCpuPlatform": "AUTOMATIC", - "preemptibility": "NON_PREEMPTIBLE" - }, - "softwareConfig": { - "imageVersion": "2.0.99-debian10", - "properties": { - "capacity-scheduler:yarn.scheduler.capacity.root.default.ordering-policy": "fair", - "core:fs.gs.block.size": "134217728", - "core:fs.gs.metadata.cache.enable": "false", - "core:hadoop.ssl.enabled.protocols": "TLSv1,TLSv1.1,TLSv1.2", - "distcp:mapreduce.map.java.opts": "-Xmx768m", - "distcp:mapreduce.map.memory.mb": "1024", - "distcp:mapreduce.reduce.java.opts": "-Xmx768m", - "distcp:mapreduce.reduce.memory.mb": "1024", - "hadoop-env:HADOOP_DATANODE_OPTS": "-Xmx512m", - "hdfs:dfs.datanode.address": "0.0.0.0:9866", - "hdfs:dfs.datanode.http.address": "0.0.0.0:9864", - "hdfs:dfs.datanode.https.address": "0.0.0.0:9865", - "hdfs:dfs.datanode.ipc.address": "0.0.0.0:9867", - "hdfs:dfs.namenode.handler.count": "20", - "hdfs:dfs.namenode.http-address": "0.0.0.0:9870", - "hdfs:dfs.namenode.https-address": "0.0.0.0:9871", - "hdfs:dfs.namenode.lifeline.rpc-address": "dataproccluster-1tzjbegzhqu4o-m:8050", - "hdfs:dfs.namenode.secondary.http-address": "0.0.0.0:9868", - "hdfs:dfs.namenode.secondary.https-address": "0.0.0.0:9869", - "hdfs:dfs.namenode.service.handler.count": "10", - "hdfs:dfs.namenode.servicerpc-address": "dataproccluster-1tzjbegzhqu4o-m:8051", - "hive:hive.fetch.task.conversion": "none", - "mapred-env:HADOOP_JOB_HISTORYSERVER_HEAPSIZE": "2048", - "mapred:mapreduce.job.maps": "21", - "mapred:mapreduce.job.reduce.slowstart.completedmaps": "0.95", - "mapred:mapreduce.job.reduces": "7", - "mapred:mapreduce.jobhistory.recovery.store.class": "org.apache.hadoop.mapreduce.v2.hs.HistoryServerLeveldbStateStoreService", - "mapred:mapreduce.map.cpu.vcores": "1", - "mapred:mapreduce.map.java.opts": "-Xmx2524m", - "mapred:mapreduce.map.maxattempts": "10", - "mapred:mapreduce.map.memory.mb": "3156", - "mapred:mapreduce.reduce.cpu.vcores": "1", - "mapred:mapreduce.reduce.java.opts": "-Xmx2524m", - "mapred:mapreduce.reduce.maxattempts": "10", - "mapred:mapreduce.reduce.memory.mb": "3156", - "mapred:mapreduce.task.io.sort.mb": "256", - "mapred:yarn.app.mapreduce.am.command-opts": "-Xmx2524m", - "mapred:yarn.app.mapreduce.am.resource.cpu-vcores": "1", - "mapred:yarn.app.mapreduce.am.resource.mb": "3156", - "spark-env:SPARK_DAEMON_MEMORY": "2048m", - "spark:spark.driver.maxResultSize": "1024m", - "spark:spark.driver.memory": "2048m", - "spark:spark.executor.cores": "2", - "spark:spark.executor.instances": "2", - "spark:spark.executor.memory": "5739m", - "spark:spark.executorEnv.OPENBLAS_NUM_THREADS": "1", - "spark:spark.scheduler.mode": "FAIR", - "spark:spark.sql.cbo.enabled": "true", - "spark:spark.stage.maxConsecutiveAttempts": "10", - "spark:spark.task.maxFailures": "10", - "spark:spark.ui.port": "0", - "spark:spark.yarn.am.attemptFailuresValidityInterval": "1h", - "spark:spark.yarn.am.memory": "640m", - "spark:spark.yarn.executor.failuresValidityInterval": "1h", - "yarn-env:YARN_NODEMANAGER_HEAPSIZE": "1536", - "yarn-env:YARN_RESOURCEMANAGER_HEAPSIZE": "2048", - "yarn-env:YARN_TIMELINESERVER_HEAPSIZE": "2048", - "yarn:yarn.nodemanager.address": "0.0.0.0:8026", - "yarn:yarn.nodemanager.resource.cpu-vcores": "4", - "yarn:yarn.nodemanager.resource.memory-mb": "12624", - "yarn:yarn.resourcemanager.am.max-attempts": "10", - "yarn:yarn.resourcemanager.decommissioning-nodes-watcher.decommission-if-no-shuffle-data": "true", - "yarn:yarn.resourcemanager.nodemanager-graceful-decommission-timeout-secs": "86400", - "yarn:yarn.scheduler.maximum-allocation-mb": "12624", - "yarn:yarn.scheduler.minimum-allocation-mb": "1" + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 } - }, - "autoscalingConfig": { - "policyUri": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o" - }, - "endpointConfig": {} - }, - "status": { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:06.199568Z" - }, - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "statusHistory": [ + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 9742170361156239144;o=0 + status: 200 OK + code: 200 + duration: 81.274288ms + - id: 36 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | { - "state": "CREATING", - "stateStartTime": "2024-04-25T01:31:21.680923Z" + "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", + "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", + "basicAlgorithm": { + "yarnConfig": { + "scaleUpFactor": 0.5, + "scaleDownFactor": 0.5, + "gracefulDecommissionTimeout": "30s" + }, + "cooldownPeriod": "120s" + }, + "workerConfig": { + "minInstances": 2, + "maxInstances": 2, + "weight": 1 + }, + "secondaryWorkerConfig": { + "maxInstances": 2, + "weight": 1 + } } - ], - "labels": { - "cnrm-test": "true", - "label-one": "value-one", - "managed-by-cnrm": "true", - "goog-dataproc-cluster-name": "dataproccluster-1tzjbegzhqu4o", - "goog-dataproc-cluster-uuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "goog-dataproc-location": "us-central1", - "goog-dataproc-autozone": "enabled" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 14266580221142144205;o=0 - status: 200 OK - code: 200 - duration: 109.084272ms - - id: 25 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: DELETE - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "name": "projects/example-project/regions/us-central1/operations/2d18fc11-952a-4739-8e23-5f048d2c2197", - "metadata": { - "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "status": { - "state": "PENDING", - "innerState": "PENDING", - "stateStartTime": "2024-04-25T01:34:19.662252Z" - }, - "operationType": "DELETE", - "description": "Delete cluster" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 8354986735620490426;o=0 - status: 200 OK - code: 200 - duration: 120.396491ms - - id: 26 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 16445970022736299428;o=0 - status: 200 OK - code: 200 - duration: 87.712096ms - - id: 27 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 13790945477808153341;o=0 - status: 200 OK - code: 200 - duration: 100.384364ms - - id: 28 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: DELETE - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 11445395100403553023;o=0 - status: 400 Bad Request - code: 400 - duration: 111.177988ms - - id: 29 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 11486176903866129474;o=0 - status: 200 OK - code: 200 - duration: 87.359734ms - - id: 30 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 14951153689767845789;o=0 - status: 200 OK - code: 200 - duration: 61.29692ms - - id: 31 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: DELETE - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 7399387131384158069;o=0 - status: 400 Bad Request - code: 400 - duration: 90.780332ms - - id: 32 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 16318353591610638230;o=0 - status: 200 OK - code: 200 - duration: 89.603009ms - - id: 33 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 1503367818740386004;o=0 - status: 200 OK - code: 200 - duration: 82.03603ms - - id: 34 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: DELETE - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 1836519392141680417;o=0 - status: 400 Bad Request - code: 400 - duration: 116.700032ms - - id: 35 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/operations/2d18fc11-952a-4739-8e23-5f048d2c2197?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "name": "projects/example-project/regions/us-central1/operations/2d18fc11-952a-4739-8e23-5f048d2c2197", - "metadata": { - "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", - "clusterName": "dataproccluster-1tzjbegzhqu4o", - "clusterUuid": "f41bb5c6-51c0-47e6-9fd3-db5a769e4fef", - "status": { - "state": "DONE", - "innerState": "DONE", - "stateStartTime": "2024-04-25T01:34:48.919322Z" - }, - "statusHistory": [ - { - "state": "PENDING", - "stateStartTime": "2024-04-25T01:34:19.662252Z" + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 5307945365680548114;o=0 + status: 200 OK + code: 200 + duration: 79.361478ms + - id: 37 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: DELETE + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + {} + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 5258230204992342792;o=0 + status: 200 OK + code: 200 + duration: 123.125287ms + - id: 38 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 8072791253634638270;o=0 + status: 404 Not Found + code: 404 + duration: 132.291707ms + - id: 39 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/operations/689d9685-9797-4885-8292-f7e8fcdceff1?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: | + { + "name": "projects/example-project/regions/us-central1/operations/689d9685-9797-4885-8292-f7e8fcdceff1", + "metadata": { + "@type": "type.googleapis.com/google.cloud.dataproc.v1.ClusterOperationMetadata", + "clusterName": "dataproccluster-1tzjbegzhqu4o", + "clusterUuid": "db2201b2-81ad-49b1-a339-3610a8c5c985", + "status": { + "state": "DONE", + "innerState": "DONE", + "stateStartTime": "2024-10-28T01:34:21.620825Z" + }, + "statusHistory": [ + { + "state": "PENDING", + "stateStartTime": "2024-10-28T01:33:23.817502Z" + }, + { + "state": "RUNNING", + "stateStartTime": "2024-10-28T01:33:23.841230Z" + } + ], + "operationType": "DELETE", + "description": "Delete cluster" }, - { - "state": "RUNNING", - "stateStartTime": "2024-04-25T01:34:19.701628Z" + "done": true, + "response": { + "@type": "type.googleapis.com/google.protobuf.Empty" } - ], - "operationType": "DELETE", - "description": "Delete cluster" - }, - "done": true, - "response": { - "@type": "type.googleapis.com/google.protobuf.Empty" - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 3042995276759990123;o=0 - status: 200 OK - code: 200 - duration: 71.76235ms - - id: 36 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 17926891054458102466;o=0 - status: 404 Not Found - code: 404 - duration: 82.476738ms - - id: 37 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 15082492347416549992;o=0 - status: 200 OK - code: 200 - duration: 93.860133ms - - id: 38 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - { - "id": "dataprocautoscalingpolicy-1tzjbegzhqu4o", - "name": "projects/example-project/regions/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o", - "basicAlgorithm": { - "yarnConfig": { - "scaleUpFactor": 0.5, - "scaleDownFactor": 0.5, - "gracefulDecommissionTimeout": "30s" - }, - "cooldownPeriod": "120s" - }, - "workerConfig": { - "minInstances": 2, - "maxInstances": 2, - "weight": 1 - }, - "secondaryWorkerConfig": { - "maxInstances": 2, - "weight": 1 - } - } - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 5146908352174520698;o=0 - status: 200 OK - code: 200 - duration: 100.590837ms - - id: 39 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: DELETE - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: -1 - uncompressed: true - body: | - {} - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 9305004769843711074;o=0 - status: 200 OK - code: 200 - duration: 123.579557ms - - id: 40 - request: - proto: HTTP/1.1 - proto_major: 1 - proto_minor: 1 - content_length: 0 - transfer_encoding: [] - trailer: {} - host: dataproc.googleapis.com - remote_addr: "" - request_uri: "" - body: "" - form: {} - headers: - Content-Type: - - application/json - url: https://dataproc.googleapis.com/v1/projects/example-project/locations/us-central1/autoscalingPolicies/dataprocautoscalingpolicy-1tzjbegzhqu4o?alt=json - method: GET - response: - proto: HTTP/2.0 - proto_major: 2 - proto_minor: 0 - transfer_encoding: [] - trailer: {} - content_length: 0 - uncompressed: true - body: fake error message - headers: - Content-Type: - - application/json; charset=UTF-8 - X-Debug-Tracking-Id: - - 5695112999535016577;o=0 - status: 404 Not Found - code: 404 - duration: 89.587335ms \ No newline at end of file + } + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 5321835317763586282;o=0 + status: 200 OK + code: 200 + duration: 75.925258ms + - id: 40 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 0 + transfer_encoding: [] + trailer: {} + host: dataproc.googleapis.com + remote_addr: "" + request_uri: "" + body: "" + form: {} + headers: + Content-Type: + - application/json + url: https://dataproc.googleapis.com/v1/projects/example-project/regions/us-central1/clusters/dataproccluster-1tzjbegzhqu4o?alt=json + method: GET + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: 0 + uncompressed: true + body: fake error message + headers: + Content-Type: + - application/json; charset=UTF-8 + X-Debug-Tracking-Id: + - 11692399059878954729;o=0 + status: 404 Not Found + code: 404 + duration: 87.215608ms diff --git a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/tf.yaml b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/tf.yaml index da46f9fa04..e0d19338f2 100644 --- a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/tf.yaml +++ b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/_vcr_cassettes/tf.yaml @@ -14,4 +14,574 @@ --- version: 2 -interactions: [] +interactions: + - id: 0 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlfzVzdKQ=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=186 + status: 200 OK + code: 200 + duration: 246.461042ms + - id: 1 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlfzVzdKQ=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=140 + status: 200 OK + code: 200 + duration: 141.299071ms + - id: 2 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlfzVzdKQ=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=140 + status: 200 OK + code: 200 + duration: 141.928691ms + - id: 3 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 7814 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"policy":{"bindings":[{"members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"],"role":"roles/alloydb.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"],"role":"roles/appengine.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"],"role":"roles/artifactregistry.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"],"role":"roles/bigqueryconnection.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"],"role":"roles/binaryauthorization.serviceAgent"},{"members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"],"role":"roles/cloudbuild.builds.builder"},{"members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"],"role":"roles/cloudbuild.serviceAgent"},{"members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"],"role":"roles/cloudfunctions.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"],"role":"roles/cloudscheduler.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/compute.orgFirewallPolicyAdmin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/compute.securityAdmin"},{"members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"],"role":"roles/compute.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/compute.storageAdmin"},{"members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"],"role":"roles/container.serviceAgent"},{"members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"],"role":"roles/containeranalysis.ServiceAgent"},{"members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"],"role":"roles/containerregistry.ServiceAgent"},{"members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"],"role":"roles/dataflow.serviceAgent"},{"members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"],"role":"roles/dataproc.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/dataproc.worker"},{"members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"],"role":"roles/edgecontainer.clusterServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"],"role":"roles/edgecontainer.serviceAgent"},{"members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"],"role":"roles/editor"},{"members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"],"role":"roles/firestore.serviceAgent"},{"members":["user:yuhou@google.com"],"role":"roles/gkehub.admin"},{"members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"],"role":"roles/gkehub.serviceAgent"},{"members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/gkemulticloud.admin"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.containerServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.nodePoolMachineServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/iam.securityAdmin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/iam.serviceAccountUser"},{"members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"],"role":"roles/logging.serviceAgent"},{"members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"],"role":"roles/memcache.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"],"role":"roles/monitoring.notificationServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"],"role":"roles/multiclustermetering.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"],"role":"roles/networkactions.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"],"role":"roles/networkconnectivity.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/owner"},{"members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"],"role":"roles/pubsub.serviceAgent"},{"members":["user:yuhou@google.com"],"role":"roles/resourcemanager.organizationAdmin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/resourcemanager.projectIamAdmin"},{"members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"],"role":"roles/run.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"],"role":"roles/serviceextensions.serviceAgent"},{"members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"],"role":"roles/servicenetworking.serviceAgent"},{"members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"],"role":"roles/sourcerepo.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/storage.admin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/storage.objectAdmin"},{"members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"],"role":"roles/storageinsights.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"],"role":"roles/vpcaccess.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"],"role":"roles/websecurityscanner.serviceAgent"}],"etag":"BwYlfzVzdKQ=","version":3},"updateMask":"bindings,etag,auditConfigs"} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:setIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=1056 + status: 200 OK + code: 200 + duration: 1.057496236s + - id: 4 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=171 + status: 200 OK + code: 200 + duration: 173.021151ms + - id: 5 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=186 + status: 200 OK + code: 200 + duration: 187.707031ms + - id: 6 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=120 + status: 200 OK + code: 200 + duration: 121.98488ms + - id: 7 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=186 + status: 200 OK + code: 200 + duration: 188.219872ms + - id: 8 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=161 + status: 200 OK + code: 200 + duration: 168.031477ms + - id: 9 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf2e1rv0=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/dataproc.worker","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=128 + status: 200 OK + code: 200 + duration: 130.182388ms + - id: 10 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 7701 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"policy":{"bindings":[{"members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"],"role":"roles/alloydb.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"],"role":"roles/appengine.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"],"role":"roles/artifactregistry.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"],"role":"roles/bigqueryconnection.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"],"role":"roles/binaryauthorization.serviceAgent"},{"members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"],"role":"roles/cloudbuild.builds.builder"},{"members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"],"role":"roles/cloudbuild.serviceAgent"},{"members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"],"role":"roles/cloudfunctions.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"],"role":"roles/cloudscheduler.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/compute.orgFirewallPolicyAdmin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/compute.securityAdmin"},{"members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"],"role":"roles/compute.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/compute.storageAdmin"},{"members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"],"role":"roles/container.serviceAgent"},{"members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"],"role":"roles/containeranalysis.ServiceAgent"},{"members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"],"role":"roles/containerregistry.ServiceAgent"},{"members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"],"role":"roles/dataflow.serviceAgent"},{"members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"],"role":"roles/dataproc.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"],"role":"roles/edgecontainer.clusterServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"],"role":"roles/edgecontainer.serviceAgent"},{"members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"],"role":"roles/editor"},{"members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"],"role":"roles/firestore.serviceAgent"},{"members":["user:yuhou@google.com"],"role":"roles/gkehub.admin"},{"members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"],"role":"roles/gkehub.serviceAgent"},{"members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/gkemulticloud.admin"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.containerServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.nodePoolMachineServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"],"role":"roles/gkemulticloud.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/iam.securityAdmin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/iam.serviceAccountUser"},{"members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"],"role":"roles/logging.serviceAgent"},{"members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"],"role":"roles/memcache.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"],"role":"roles/monitoring.notificationServiceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"],"role":"roles/multiclustermetering.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"],"role":"roles/networkactions.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"],"role":"roles/networkconnectivity.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/owner"},{"members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"],"role":"roles/pubsub.serviceAgent"},{"members":["user:yuhou@google.com"],"role":"roles/resourcemanager.organizationAdmin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/resourcemanager.projectIamAdmin"},{"members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"],"role":"roles/run.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"],"role":"roles/serviceextensions.serviceAgent"},{"members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"],"role":"roles/servicenetworking.serviceAgent"},{"members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"],"role":"roles/sourcerepo.serviceAgent"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"],"role":"roles/storage.admin"},{"members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"],"role":"roles/storage.objectAdmin"},{"members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"],"role":"roles/storageinsights.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"],"role":"roles/vpcaccess.serviceAgent"},{"members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"],"role":"roles/websecurityscanner.serviceAgent"}],"etag":"BwYlf2e1rv0=","version":1},"updateMask":"bindings,etag,auditConfigs"} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:setIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf3SbITc=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=1018 + status: 200 OK + code: 200 + duration: 1.021250312s + - id: 11 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf3SbITc=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=230 + status: 200 OK + code: 200 + duration: 231.784246ms + - id: 12 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf3SbITc=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=181 + status: 200 OK + code: 200 + duration: 182.368187ms + - id: 13 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf3SbITc=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=186 + status: 200 OK + code: 200 + duration: 187.925697ms + - id: 14 + request: + proto: HTTP/1.1 + proto_major: 1 + proto_minor: 1 + content_length: 41 + transfer_encoding: [] + trailer: {} + host: cloudresourcemanager.googleapis.com + remote_addr: "" + request_uri: "" + body: | + {"options":{"requestedPolicyVersion":3}} + form: {} + headers: + Content-Type: + - application/json + X-Goog-Api-Client: + - gl-go/1.23.0--20240626-RC01 cl/646990413 +5a18e79687 X:fieldtrack,boringcrypto gdcl/0.203.0 + url: https://cloudresourcemanager.googleapis.com/v1/projects/example-project:getIamPolicy?alt=json&prettyPrint=false + method: POST + response: + proto: HTTP/2.0 + proto_major: 2 + proto_minor: 0 + transfer_encoding: [] + trailer: {} + content_length: -1 + uncompressed: true + body: '{"version":1,"etag":"BwYlf3SbITc=","bindings":[{"role":"roles/alloydb.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-alloydb.iam.gserviceaccount.com"]},{"role":"roles/appengine.serviceAgent","members":["serviceAccount:service-123456789@gcp-gae-service.iam.gserviceaccount.com"]},{"role":"roles/artifactregistry.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-artifactregistry.iam.gserviceaccount.com"]},{"role":"roles/bigqueryconnection.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-bigqueryconnection.iam.gserviceaccount.com"]},{"role":"roles/binaryauthorization.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-binaryauthorization.iam.gserviceaccount.com"]},{"role":"roles/cloudbuild.builds.builder","members":["serviceAccount:123456789@cloudbuild.gserviceaccount.com"]},{"role":"roles/cloudbuild.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudbuild.iam.gserviceaccount.com"]},{"role":"roles/cloudfunctions.serviceAgent","members":["serviceAccount:service-123456789@gcf-admin-robot.iam.gserviceaccount.com"]},{"role":"roles/cloudscheduler.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-cloudscheduler.iam.gserviceaccount.com"]},{"role":"roles/compute.orgFirewallPolicyAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/compute.serviceAgent","members":["serviceAccount:service-123456789@compute-system.iam.gserviceaccount.com"]},{"role":"roles/compute.storageAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/container.serviceAgent","members":["serviceAccount:service-123456789@container-engine-robot.iam.gserviceaccount.com"]},{"role":"roles/containeranalysis.ServiceAgent","members":["serviceAccount:service-123456789@container-analysis.iam.gserviceaccount.com"]},{"role":"roles/containerregistry.ServiceAgent","members":["serviceAccount:service-123456789@containerregistry.iam.gserviceaccount.com"]},{"role":"roles/dataflow.serviceAgent","members":["serviceAccount:service-123456789@dataflow-service-producer-prod.iam.gserviceaccount.com"]},{"role":"roles/dataproc.serviceAgent","members":["serviceAccount:service-123456789@dataproc-accounts.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.clusterServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainercluster.iam.gserviceaccount.com"]},{"role":"roles/edgecontainer.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-edgecontainer.iam.gserviceaccount.com"]},{"role":"roles/editor","members":["serviceAccount:123456789@cloudservices.gserviceaccount.com","serviceAccount:gsa-1aup5dgbfrmww@example-project.iam.gserviceaccount.com","serviceAccount:gsa-2nclohxazas3twtuouba@example-project.iam.gserviceaccount.com","serviceAccount:gsa-3amnof2ufoel7ay@example-project.iam.gserviceaccount.com","serviceAccount:gsa-65djglyugqvdpkdgr2qa@example-project.iam.gserviceaccount.com","serviceAccount:gsa-6hb42jo3co6ednn5nirq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-iuujpj65h3tatn3e2nma@example-project.iam.gserviceaccount.com","serviceAccount:gsa-ivn4bj7jzbyhyzxtb47a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-nvujpctp2qjt7aqz7a7a@example-project.iam.gserviceaccount.com","serviceAccount:gsa-rl6lulbe7pa4jfmylsdq@example-project.iam.gserviceaccount.com","serviceAccount:gsa-waq4m5xsz26evt34kzyq@example-project.iam.gserviceaccount.com"]},{"role":"roles/firestore.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-firestore.iam.gserviceaccount.com"]},{"role":"roles/gkehub.admin","members":["user:yuhou@google.com"]},{"role":"roles/gkehub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkehub.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.admin","members":["serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/gkemulticloud.containerServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcontainer.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.controlPlaneMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudcpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.nodePoolMachineServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloudnpmachine.iam.gserviceaccount.com"]},{"role":"roles/gkemulticloud.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-gkemulticloud.iam.gserviceaccount.com"]},{"role":"roles/iam.securityAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/iam.serviceAccountUser","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/logging.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-logging.iam.gserviceaccount.com"]},{"role":"roles/memcache.serviceAgent","members":["serviceAccount:service-123456789@cloud-memcache-sa.iam.gserviceaccount.com"]},{"role":"roles/monitoring.notificationServiceAgent","members":["serviceAccount:service-123456789@gcp-sa-monitoring-notification.iam.gserviceaccount.com"]},{"role":"roles/multiclustermetering.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-mcmetering.iam.gserviceaccount.com"]},{"role":"roles/networkactions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkactions.iam.gserviceaccount.com"]},{"role":"roles/networkconnectivity.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-networkconnectivity.iam.gserviceaccount.com"]},{"role":"roles/owner","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/pubsub.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-pubsub.iam.gserviceaccount.com"]},{"role":"roles/resourcemanager.organizationAdmin","members":["user:yuhou@google.com"]},{"role":"roles/resourcemanager.projectIamAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","serviceAccount:cnrm-system@example-project.iam.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/run.serviceAgent","members":["serviceAccount:service-123456789@serverless-robot-prod.iam.gserviceaccount.com"]},{"role":"roles/serviceextensions.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-dep.iam.gserviceaccount.com"]},{"role":"roles/servicenetworking.serviceAgent","members":["serviceAccount:service-123456789@service-networking.iam.gserviceaccount.com"]},{"role":"roles/sourcerepo.serviceAgent","members":["serviceAccount:service-123456789@sourcerepo-service-accounts.iam.gserviceaccount.com"]},{"role":"roles/storage.admin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com","user:yuhou@google.com"]},{"role":"roles/storage.objectAdmin","members":["serviceAccount:123456789-compute@developer.gserviceaccount.com"]},{"role":"roles/storageinsights.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-storageinsights.iam.gserviceaccount.com"]},{"role":"roles/vpcaccess.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-vpcaccess.iam.gserviceaccount.com"]},{"role":"roles/websecurityscanner.serviceAgent","members":["serviceAccount:service-123456789@gcp-sa-websecurityscanner.iam.gserviceaccount.com"]}]}' + headers: + Content-Type: + - application/json; charset=UTF-8 + Server-Timing: + - gfet4t7; dur=128 + status: 200 OK + code: 200 + duration: 129.585908ms diff --git a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/create.yaml b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/create.yaml index 1c5687957b..b191c02d9e 100644 --- a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/create.yaml +++ b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/create.yaml @@ -26,6 +26,8 @@ spec: autoscalingConfig: policyRef: name: dataprocautoscalingpolicy-${uniqueId} + gceClusterConfig: + internalIPOnly: false masterConfig: diskConfig: bootDiskType: pd-standard diff --git a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/dependencies.yaml b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/dependencies.yaml index cd0785dea9..e31c690588 100644 --- a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/dependencies.yaml +++ b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/dependencies.yaml @@ -12,6 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +apiVersion: iam.cnrm.cloud.google.com/v1beta1 +kind: IAMPolicyMember +metadata: + name: iampolicymember-${uniqueId} +spec: + member: serviceAccount:${projectNumber}-compute@developer.gserviceaccount.com + role: roles/dataproc.worker # required by default service account to access Dataproc data plane operations + resourceRef: + apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 + kind: Project + external: projects/${projectId} +--- apiVersion: dataproc.cnrm.cloud.google.com/v1beta1 kind: DataprocAutoscalingPolicy metadata: diff --git a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/update.yaml b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/update.yaml index 5ae1c5fa57..dea13653df 100644 --- a/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/update.yaml +++ b/pkg/test/resourcefixture/testdata/basic/dataproc/v1beta1/dataproccluster/update.yaml @@ -26,6 +26,8 @@ spec: autoscalingConfig: policyRef: name: dataprocautoscalingpolicy-${uniqueId} + gceClusterConfig: + internalIPOnly: false masterConfig: diskConfig: bootDiskType: pd-standard