Sync Feature branch major release 5.0.0 into main

.ci/gcb-generate-diffs-new.yml

@@ -202,45 +202,45 @@ steps:
           - "17" # Build step           # remove after 08/2023
           - terraform-google-conversion # remove after 08/2023
 
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: tgc-test-integration-0.12.31
-      entrypoint: '/workspace/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh'
-      allowFailure: true
-      secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
-      env:
-        - TERRAFORM_VERSION=0.12.31
-        - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
-        - TEST_FOLDER_ID=$_VALIDATOR_TEST_FOLDER
-        - TEST_ANCESTRY=$_VALIDATOR_TEST_ANCESTRY
-        - TEST_ORG_ID=$_VALIDATOR_TEST_ORG
-      args:
-          - $_PR_NUMBER
-          - $COMMIT_SHA
-          - $BUILD_ID
-          - $PROJECT_ID
-          - "18"  # Build step
-          - terraform-google-conversion
+    # - name: 'gcr.io/graphite-docker-images/go-plus'
+    #   id: tgc-test-integration-0.12.31
+    #   entrypoint: '/workspace/.ci/scripts/go-plus/terraform-validator-tester-integration/test_terraform_validator_integration.sh'
+    #   allowFailure: true
+    #   secretEnv: ["GITHUB_TOKEN"]
+    #   waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
+    #   env:
+    #     - TERRAFORM_VERSION=0.12.31
+    #     - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
+    #     - TEST_FOLDER_ID=$_VALIDATOR_TEST_FOLDER
+    #     - TEST_ANCESTRY=$_VALIDATOR_TEST_ANCESTRY
+    #     - TEST_ORG_ID=$_VALIDATOR_TEST_ORG
+    #   args:
+    #       - $_PR_NUMBER
+    #       - $COMMIT_SHA
+    #       - $BUILD_ID
+    #       - $PROJECT_ID
+    #       - "18"  # Build step
+    #       - terraform-google-conversion
 
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: tgc-test-integration-0.13.7
-      entrypoint: '/workspace/.ci/scripts/go-plus/tgc-tester-integration/test_tgc_integration.sh'
-      allowFailure: true
-      secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
-      env:
-        - TERRAFORM_VERSION=0.13.7
-        - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
-        - TEST_FOLDER_ID=$_VALIDATOR_TEST_FOLDER
-        - TEST_ANCESTRY=$_VALIDATOR_TEST_ANCESTRY
-        - TEST_ORG_ID=$_VALIDATOR_TEST_ORG
-      args:
-          - $_PR_NUMBER
-          - $COMMIT_SHA
-          - $BUILD_ID
-          - $PROJECT_ID
-          - "19"  # Build step
-          - terraform-google-conversion
+    # - name: 'gcr.io/graphite-docker-images/go-plus'
+    #   id: tgc-test-integration-0.13.7
+    #   entrypoint: '/workspace/.ci/scripts/go-plus/terraform-validator-tester-integration/test_terraform_validator_integration.sh'
+    #   allowFailure: true
+    #   secretEnv: ["GITHUB_TOKEN"]
+    #   waitFor: ["tpgb-head", "tpgb-base", "tgc-head", "tgc-base"]
+    #   env:
+    #     - TERRAFORM_VERSION=0.13.7
+    #     - TEST_PROJECT=$_VALIDATOR_TEST_PROJECT
+    #     - TEST_FOLDER_ID=$_VALIDATOR_TEST_FOLDER
+    #     - TEST_ANCESTRY=$_VALIDATOR_TEST_ANCESTRY
+    #     - TEST_ORG_ID=$_VALIDATOR_TEST_ORG
+    #   args:
+    #       - $_PR_NUMBER
+    #       - $COMMIT_SHA
+    #       - $BUILD_ID
+    #       - $PROJECT_ID
+    #       - "19"  # Build step
+    #       - terraform-google-conversion
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: tpgb-test

.ci/infra/terraform/main.tf

@@ -242,7 +242,6 @@ module "project-services" {
     "firebasestorage.googleapis.com",
     "firestore.googleapis.com",
     "firestorekeyvisualizer.googleapis.com",
-    "gameservices.googleapis.com",
     "gkebackup.googleapis.com",
     "gkeconnect.googleapis.com",
     "gkehub.googleapis.com",

mmv1/api/resource.rb

@@ -201,6 +201,10 @@ module Properties
       # mmv1/templates/terraform/state_migrations/
       # used for maintaining state stability with resources first provisioned on older api versions.
       attr_reader :schema_version
+      # From this schema version on, state_upgrader code is generated for the resource.
+      # When unset, state_upgrade_base_schema_version defauts to 0.
+      # Normally, it is not needed to be set.
+      attr_reader :state_upgrade_base_schema_version
       attr_reader :state_upgraders
       # This block inserts the named function and its attribute into the
       # resource schema -- the code for the migrate_state function must
@@ -217,6 +221,10 @@ module Properties
       # public ca external account keys
       attr_reader :skip_read
 
+      # Set to true for resources that wish to disable automatic generation of default provider
+      # value customdiff functions
+      attr_reader :skip_default_cdiff
+
       # This enables resources that get their project via a reference to a different resource
       # instead of a project field to use User Project Overrides
       attr_reader :supports_indirect_user_project_override
@@ -318,10 +326,12 @@ def validate
       check :error_retry_predicates, type: Array, item_type: String
       check :error_abort_predicates, type: Array, item_type: String
       check :schema_version, type: Integer
+      check :state_upgrade_base_schema_version, type: Integer, default: 0
       check :state_upgraders, type: :boolean, default: false
       check :migrate_state, type: String
       check :skip_delete, type: :boolean, default: false
       check :skip_read, type: :boolean, default: false
+      check :skip_default_cdiff, type: :boolean, default: false
       check :supports_indirect_user_project_override, type: :boolean, default: false
       check :legacy_long_form_project, type: :boolean, default: false
       check :read_error_transform, type: String
@@ -342,6 +352,10 @@ def all_properties
       ((@properties || []) + (@parameters || []))
     end
 
+    def properties_with_excluded
+      @properties || []
+    end
+
     def properties
       (@properties || []).reject(&:exclude)
     end
@@ -385,8 +399,13 @@ def all_resourcerefs
     # At Create, they have no value but they can just be read in anyways, and after a Read
     # they will need to be set in every Update.
     def settable_properties
-      all_user_properties.reject { |v| v.output && !v.is_a?(Api::Type::Fingerprint) }
-                         .reject(&:url_param_only)
+      props = all_user_properties.reject do |v|
+        v.output && !v.is_a?(Api::Type::Fingerprint) && !v.is_a?(Api::Type::KeyValueEffectiveLabels)
+      end
+      props = props.reject(&:url_param_only)
+      props.reject do |v|
+        v.is_a?(Api::Type::KeyValueLabels) || v.is_a?(Api::Type::KeyValueAnnotations)
+      end
     end
 
     # Properties that will be returned in the API body
@@ -439,6 +458,102 @@ def decoder?
       !@transport&.decoder.nil?
     end
 
+    def add_labels_related_fields(props, parent)
+      props.each do |p|
+        if p.is_a? Api::Type::KeyValueLabels
+          add_labels_fields(props, parent, p)
+        elsif p.is_a? Api::Type::KeyValueAnnotations
+          add_annotations_fields(props, parent, p)
+        elsif (p.is_a? Api::Type::NestedObject) && !p.all_properties.nil?
+          p.properties = add_labels_related_fields(p.all_properties, p)
+        end
+      end
+      props
+    end
+
+    def add_labels_fields(props, parent, labels)
+      # The effective_labels field is used to write to API, instead of the labels field.
+      labels.ignore_write = true
+      labels.description = "#{labels.description}\n\n#{get_labels_field_note(labels.name)}"
+
+      @custom_diff ||= []
+      if parent.nil? || parent.flatten_object
+        @custom_diff.append('tpgresource.SetLabelsDiff')
+      elsif parent.name == 'metadata'
+        @custom_diff.append('tpgresource.SetMetadataLabelsDiff')
+      end
+
+      props << build_terraform_labels_field('labels', labels.field_min_version)
+      props << build_effective_labels_field('labels', labels)
+    end
+
+    def add_annotations_fields(props, parent, annotations)
+      # The effective_annotations field is used to write to API,
+      # instead of the annotations field.
+      annotations.ignore_write = true
+
+      @custom_diff ||= []
+      if parent.nil?
+        @custom_diff.append('tpgresource.SetAnnotationsDiff')
+      elsif parent.name == 'metadata'
+        @custom_diff.append('tpgresource.SetMetadataAnnotationsDiff')
+      end
+
+      props << build_effective_labels_field('annotations', annotations)
+    end
+
+    def build_effective_labels_field(name, labels)
+      description = "All of #{name} (key/value pairs)\
+ present on the resource in GCP, including the #{name} configured through Terraform,\
+ other clients and services."
+
+      Api::Type::KeyValueEffectiveLabels.new(
+        name: "effective#{name.capitalize}",
+        output: true,
+        api_name: name,
+        description:,
+        min_version: labels.field_min_version,
+        update_verb: labels.update_verb,
+        update_url: labels.update_url,
+        immutable: labels.immutable
+      )
+    end
+
+    def build_terraform_labels_field(name, min_version)
+      description = "The combination of #{name} configured directly on the resource
+ and default #{name} configured on the provider."
+
+      Api::Type::KeyValueTerraformLabels.new(
+        name: "terraform#{name.capitalize}",
+        output: true,
+        api_name: name,
+        description:,
+        min_version:,
+        ignore_write: true
+      )
+    end
+
+    # Return labels fields that should be added to ImportStateVerifyIgnore
+    def ignore_read_labels_fields(props)
+      fields = []
+      props.each do |p|
+        if (p.is_a? Api::Type::KeyValueLabels) ||
+           (p.is_a? Api::Type::KeyValueTerraformLabels) ||
+           (p.is_a? Api::Type::KeyValueAnnotations)
+          fields << p.terraform_lineage
+        elsif (p.is_a? Api::Type::NestedObject) && !p.all_properties.nil?
+          fields.concat(ignore_read_labels_fields(p.all_properties))
+        end
+      end
+      fields
+    end
+
+    def get_labels_field_note(title)
+      "**Note**: This field is non-authoritative, and will only manage the #{title} present " \
+"in your configuration.
+Please refer to the field `effective_#{title}` for all of the #{title} present on the resource."
+    end
+
     # ====================
     # Version-related methods
     # ====================

mmv1/api/type.rb

@@ -23,7 +23,7 @@ module Fields
       include Api::Object::Named::Properties
 
       attr_reader :default_value
-      attr_reader :description
+      attr_accessor :description
       attr_reader :exclude
 
       # Add a deprecation message for a field that's been deprecated in the API
@@ -266,6 +266,14 @@ def lineage
       "#{__parent.lineage}.#{name&.underscore}"
     end
 
+    # Prints the access path of the field in the configration eg: metadata.0.labels
+    # The only intended purpose is to get the value of the labes field by calling d.Get().
+    def terraform_lineage
+      return name&.underscore if __parent.nil? || __parent.flatten_object
+
+      "#{__parent.terraform_lineage}.0.#{name&.underscore}"
+    end
+
     def to_json(opts = nil)
       # ignore fields that will contain references to parent resources and
       # those which will be added later
@@ -728,6 +736,8 @@ def properties
         @properties.reject(&:exclude)
       end
 
+      attr_writer :properties
+
       def nested_properties
         properties
       end
@@ -755,6 +765,64 @@ def exclude_if_not_in_version!(version)
     # simpler property to generate and means we can avoid conditional logic
     # in Map.
     class KeyValuePairs < Composite
+      # Ignore writing the "effective_labels" and "effective_annotations" fields to API.
+      attr_accessor :ignore_write
+
+      def initialize(name: nil, output: nil, api_name: nil, description: nil, min_version: nil,
+                     ignore_write: nil, update_verb: nil, update_url: nil, immutable: nil)
+        super()
+
+        @name = name
+        @output = output
+        @api_name = api_name
+        @description = description
+        @min_version = min_version
+        @ignore_write = ignore_write
+        @update_verb = update_verb
+        @update_url = update_url
+        @immutable = immutable
+      end
+
+      def validate
+        super
+        check :ignore_write, type: :boolean, default: false
+      end
+
+      def field_min_version
+        @min_version
+      end
+    end
+
+    # An array of string -> string key -> value pairs used specifically for the "labels" field.
+    # The field name with this type should be "labels" literally.
+    class KeyValueLabels < KeyValuePairs
+      def validate
+        super
+        return unless @name != 'labels'
+
+        raise "The field #{name} has the type KeyValueLabels, but the field name is not 'labels'!"
+      end
+    end
+
+    # An array of string -> string key -> value pairs used for the "terraform_labels" field.
+    class KeyValueTerraformLabels < KeyValuePairs
+    end
+
+    # An array of string -> string key -> value pairs used for the "effective_labels"
+    # and "effective_annotations" fields.
+    class KeyValueEffectiveLabels < KeyValuePairs
+    end
+
+    # An array of string -> string key -> value pairs used specifically for the "annotations" field.
+    # The field name with this type should be "annotations" literally.
+    class KeyValueAnnotations < KeyValuePairs
+      def validate
+        super
+        return unless @name != 'annotations'
+
+        raise "The field #{name} has the type KeyValueAnnotations,\
+ but the field name is not 'annotations'!"
+      end
     end
 
     # Map from string keys -> nested object entries

mmv1/compiler.rb

@@ -224,6 +224,9 @@
       end
       res_yaml = File.read(file_path)
       resource = Api::Compiler.new(res_yaml).run
+      resource.properties = resource.add_labels_related_fields(
+        resource.properties_with_excluded, nil
+      )
       resource.validate
       resources.push(resource)
     end