Refactor TeamCity Services Diff Check to run as one job (#10158)

.github/actions/build-downstream/action.yml

@@ -0,0 +1,105 @@
+name: build
+description: Generates given downstream Magic Modules Repository from `repo` input
+inputs:
+  repo:
+    description: "provider repo"
+    required: true
+  token:
+    description: "github token"
+    required: true
+
+runs:
+    using: "composite"
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@036ef458ddccddb148a2b9fb67e95a22fdbf728b # v1.160.0
+        with:
+          ruby-version: '3.1'
+
+      - name: Cache Bundler gems
+        uses: actions/cache@v3
+        with:
+          path: mmv1/vendor/bundle
+          key: ${{ runner.os }}-gems-${{ hashFiles('mmv1/**/Gemfile.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-gems-
+
+      - name: Install Ruby dependencies
+        shell: bash
+        run: |
+          bundle config path mmv1/vendor/bundle
+          bundle install
+        working-directory: mmv1
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: '^1.20'
+
+      # Cache Go modules
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+
+      - run: go install golang.org/x/tools/cmd/goimports@latest
+        shell: bash
+      - name: Build ${{ inputs.repo  }}
+        shell: bash
+        env:
+          BASE_BRANCH: ${{ github.event.pull_request.base.ref || github.ref_name }}
+          GITHUB_TOKEN: ${{ inputs.token }}
+        run: |
+          set -e
+          set -x
+          # Set GOPATH to a directory the runner user has access to
+          export GOPATH=~/go
+
+          function clone_repo() {
+            export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+            echo "OUTPUT_PATH=$OUTPUT_PATH" >> $GITHUB_ENV
+            GITHUB_PATH=https://x-access-token:[email protected]/$UPSTREAM_OWNER/$GH_REPO
+            mkdir -p "$(dirname $OUTPUT_PATH)"
+            git clone $GITHUB_PATH $OUTPUT_PATH --branch $BASE_BRANCH
+          }
+
+          GH_REPO="${{ inputs.repo  }}"
+          if [ "$GH_REPO" == "docs-examples" ] && [ "$BASE_BRANCH" == "main" ]; then
+              BASE_BRANCH="master"
+          fi
+
+          GITHUB_PATH=https://x-access-token:[email protected]/$UPSTREAM_OWNER/$GH_REPO
+
+          if [[ "$GH_REPO" == terraform-provider-google* ]]; then
+            UPSTREAM_OWNER=hashicorp
+            clone_repo
+          if [ "$GH_REPO" == "terraform-provider-google" ]; then
+              export VERSION=ga
+            else
+              export VERSION=beta
+            fi
+            make clean-provider
+            make provider
+          elif [ "$GH_REPO" == "terraform-google-conversion" ]; then
+            UPSTREAM_OWNER=GoogleCloudPlatform
+            clone_repo
+            make clean-tgc
+            make tgc
+          elif [ "$GH_REPO" == "docs-examples" ]; then
+            UPSTREAM_OWNER=terraform-google-modules
+            clone_repo
+            make tf-oics
+          else
+            echo "case not supported"
+            exit 1
+          fi
+
+          (current_dir=$(pwd) && cd $OUTPUT_PATH && zip -r "$current_dir/output.zip" .)

.github/workflows/teamcity-services-diff-check-weekly.yml

@@ -11,57 +11,37 @@ on:
     - cron:  '0 4 * * 1-2'
 
 jobs:
-  terraform-provider-google:
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google'
-
-  terraform-provider-google-beta:
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google-beta'
-
   teamcity-services-diff-check:
-    needs: [terraform-provider-google, terraform-provider-google-beta]
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: '^1.20'
-
-      - name: Download built artifacts - GA provider
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-terraform-provider-google
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
-      - name: Download built artifacts - Beta provider
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-terraform-provider-google-beta
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
-      - name: Check that new services have been added to the TeamCity configuration code
-        run: |
-          # Create lists of service packages in providers
-          ls provider/google/services > tools/teamcity-diff-check/services_ga.txt
-          ls provider/google-beta/services > tools/teamcity-diff-check/services_beta.txt
-
-          # Run tool to compare service packages in the providers vs those listed in TeamCity config files
-          cd tools/teamcity-diff-check
-          go run main.go -service_file=services_ga
-          go run main.go -service_file=services_beta
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      runs-on: ubuntu-22.04
+      steps:
+        - name: Checkout Repository
+          uses: actions/checkout@v4
+        - name: TeamCity Google Provider Generate
+          uses: ./.github/actions/build-downstream
+          with:
+            repo: 'terraform-provider-google'
+            token: '$GITHUB_TOKEN'
+        # The path where GA/Beta providers are generated is grabbed from the OUTPUT_PATH that's set in build_downstream.yaml
+        # export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+        # OUTPUT_PATH changes after each generate (GA/beta)
+        - name: Set GOOGLE_REPO_PATH to path where GA provider was generated
+          run: echo "GOOGLE_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+        - name: TeamCity Google Beta Provider Generate
+          uses: ./.github/actions/build-downstream
+          with:
+            repo: 'terraform-provider-google-beta'
+            token: '$GITHUB_TOKEN'
+        - name: Set GOOGLE_BETA_REPO_PATH to path where beta provider was generated
+          run: echo "GOOGLE_BETA_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+        - name: Check that new services have been added to the TeamCity configuration code
+          run: |
+              # Create lists of service packages in providers
+              ls ${{env.GOOGLE_REPO_PATH}}/google/services > tools/teamcity-diff-check/services_ga.txt
+              ls ${{env.GOOGLE_BETA_REPO_PATH}}/google-beta/services > tools/teamcity-diff-check/services_beta.txt
+    
+              # Run tool to compare service packages in the providers vs those listed in TeamCity config files
+              cd tools/teamcity-diff-check
+              go run main.go -service_file=services_ga
+              go run main.go -service_file=services_beta

.github/workflows/teamcity-services-diff-check.yml

@@ -9,10 +9,10 @@ on:
       - 'mmv1/third_party/terraform/.teamcity/components/inputs/services_beta.kt'
       - 'mmv1/products/**'
 jobs:
-  check-pr:
+  teamcity-services-diff-check:
+    env:
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     runs-on: ubuntu-22.04
-    outputs:
-      services: ${{steps.services.outputs.services}}
     steps:
       - name: Checkout Repository
         uses: actions/checkout@v4
@@ -24,61 +24,37 @@ jobs:
           newServices=$(($(git diff --name-only --diff-filter=A origin/main HEAD | grep -P "mmv1/products/.*/product.yaml" | wc -l)))
           echo "services=$newServices" >> "${GITHUB_OUTPUT}"
           if [ "$newServices" = "0" ];then
-          echo "No new service found." 
+          echo "No new service found."
           fi
-  terraform-provider-google:
-    if: ${{needs.check-pr.outputs.services != '0'}}
-    needs: check-pr
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google'
-
-  terraform-provider-google-beta:
-    if: ${{needs.check-pr.outputs.services != '0'}} 
-    needs: check-pr
-    uses: ./.github/workflows/build-downstream.yml
-    with:
-      repo: 'terraform-provider-google-beta'
-
-  teamcity-services-diff-check:
-    needs: [terraform-provider-google, terraform-provider-google-beta]
-    runs-on: ubuntu-22.04
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-
-      - name: Setup Go
-        uses: actions/setup-go@v4
+      - name: TeamCity Google Provider Generate
+        id: generate
+        if: ${{steps.services.outputs.services != '0'}}
+        uses: ./.github/actions/build-downstream
         with:
-          go-version: '^1.20'
-
-      - name: Download built artifacts - GA provider
-        uses: actions/download-artifact@v2
+          repo: 'terraform-provider-google'
+          token: '$GITHUB_TOKEN'
+        # The path where GA/Beta providers are generated is grabbed from the OUTPUT_PATH that's set in build_downstream.yaml
+        # export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO
+        # OUTPUT_PATH changes after each generate (GA/beta)
+      - name: Set GOOGLE_REPO_PATH to path where GA provider was generated
+        run: echo "GOOGLE_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+      - name: TeamCity Google Beta Provider Generate
+        if: steps.generate.outcome == 'success'
+        uses: ./.github/actions/build-downstream
         with:
-          name: artifact-terraform-provider-google
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
-      - name: Download built artifacts - Beta provider
-        uses: actions/download-artifact@v2
-        with:
-          name: artifact-terraform-provider-google-beta
-          path: artifacts
-
-      - name: Unzip the artifacts and delete the zip
-        run: |
-          unzip -o artifacts/output.zip -d ./provider
-          rm artifacts/output.zip
-
+          repo: 'terraform-provider-google-beta'
+          token: '$GITHUB_TOKEN'
+      - name: Set GOOGLE_BETA_REPO_PATH to path where beta provider was generated
+        run: echo "GOOGLE_BETA_REPO_PATH=${{ env.OUTPUT_PATH}}" >> $GITHUB_ENV
+      - name: Checkout Repository
+        if: steps.generate.outcome == 'success'
+        uses: actions/checkout@v4
       - name: Check that new services have been added to the TeamCity configuration code
+        if: steps.generate.outcome == 'success'
         run: |
           # Create lists of service packages in providers
-          ls provider/google/services > tools/teamcity-diff-check/services_ga.txt
-          ls provider/google-beta/services > tools/teamcity-diff-check/services_beta.txt
+          ls ${{env.GOOGLE_REPO_PATH}}/google/services > tools/teamcity-diff-check/services_ga.txt
+          ls ${{env.GOOGLE_BETA_REPO_PATH}}/google-beta/services > tools/teamcity-diff-check/services_beta.txt
 
           # Run tool to compare service packages in the providers vs those listed in TeamCity config files
           cd tools/teamcity-diff-check