diff --git a/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh b/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh index 88ac64f5be87..68d26e190dde 100755 --- a/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh +++ b/.ci/scripts/build-environment/downstream-builder/generate_downstream.sh @@ -82,9 +82,6 @@ clone_repo git config --local user.name "Modular Magician" git config --local user.email "magic-modules@google.com" -# MMv1 now lives inside a sub-folder -pushd mmv1 - if [ "$COMMAND" == "head" ]; then BRANCH=auto-pr-$REFERENCE COMMIT_MESSAGE="New generated code for MM PR $REFERENCE." @@ -101,67 +98,90 @@ elif [ "$COMMAND" == "downstream" ]; then COMMIT_MESSAGE="$ORIGINAL_MESSAGE$NEWLINE[upstream:$REFERENCE]" fi -if [ "$REPO" == "terraform" ]; then - pushd $LOCAL_PATH - go mod download - find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "CHANGELOG_v*.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name ".go-version" -not -name ".hashibot.hcl" -print0 | xargs -0 git rm - popd -fi +if [ "$COMMAND" == "base" ]; then + pushd mmv1 + if [ "$REPO" == "terraform" ]; then + pushd $LOCAL_PATH + go mod download + find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "CHANGELOG_v*.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name ".go-version" -not -name ".hashibot.hcl" -print0 | xargs -0 git rm + popd + fi -if [ "$REPO" == "terraform-google-conversion" ]; then - # Generate tfplan2cai - pushd $LOCAL_PATH - # clear out the templates as they are copied during - # generation from mmv1/third_party/validator/tests/data - rm -rf ./tfplan2cai/testdata/templates/ - rm -rf ./tfplan2cai/testdata/generatedconvert/ - rm -rf ./tfplan2cai/converters/google/provider - rm -rf ./tfplan2cai/converters/google/resources - find ./tfplan2cai/test/** -type f -exec git rm {} \; - popd + if [ "$REPO" == "terraform-google-conversion" ]; then + # Generate tfplan2cai + pushd $LOCAL_PATH + # clear out the templates as they are copied during + # generation from mmv1/third_party/validator/tests/data + rm -rf ./tfplan2cai/testdata/templates/ + rm -rf ./tfplan2cai/testdata/generatedconvert/ + rm -rf ./tfplan2cai/converters/google/provider + rm -rf ./tfplan2cai/converters/google/resources + find ./tfplan2cai/test/** -type f -exec git rm {} \; + popd - bundle exec compiler.rb -a -e terraform -f validator -o $LOCAL_PATH/tfplan2cai -v $VERSION + bundle exec compiler.rb -a -e terraform -f validator -o $LOCAL_PATH/tfplan2cai -v $VERSION - # Generate cai2hcl - pushd $LOCAL_PATH - rm -rf ./cai2hcl/* - popd + # Generate cai2hcl + pushd $LOCAL_PATH + rm -rf ./cai2hcl/* + popd - bundle exec compiler.rb -a -e terraform -f tgc_cai2hcl -o $LOCAL_PATH/cai2hcl -v $VERSION + bundle exec compiler.rb -a -e terraform -f tgc_cai2hcl -o $LOCAL_PATH/cai2hcl -v $VERSION - if [ "$COMMAND" == "downstream" ]; then - pushd $LOCAL_PATH - go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH - go mod tidy - set +e - make build - set -e - popd + if [ "$COMMAND" == "downstream" ]; then + pushd $LOCAL_PATH + go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH + go mod tidy + set +e + make build + set -e + popd + fi + elif [ "$REPO" == "tf-oics" ]; then + # use terraform generator with oics override + bundle exec compiler.rb -a -e terraform -f oics -o $LOCAL_PATH -v $VERSION + else + if [ "$REPO" == "terraform" ]; then + if [ "$VERSION" == "ga" ]; then + bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION --no-docs + bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v beta --no-code + else + bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION + fi + pushd ../ + make tpgtools OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION + + # Only generate TeamCity-related file for TPG and TPGB + if [ "$VERSION" == "ga" ] || [ "$VERSION" == "beta" ]; then + make teamcity-servicemap-generate OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION + fi + popd + fi fi -elif [ "$REPO" == "tf-oics" ]; then - # use terraform generator with oics override - bundle exec compiler.rb -a -e terraform -f oics -o $LOCAL_PATH -v $VERSION + popd else - if [ "$REPO" == "terraform" ]; then - if [ "$VERSION" == "ga" ]; then - bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION --no-docs - bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v beta --no-code - else - bundle exec compiler.rb -a -e $REPO -o $LOCAL_PATH -v $VERSION + if [ "$REPO" == "terraform-google-conversion" ]; then + make clean-tgc OUTPUT_PATH="$LOCAL_PATH" + make tgc OUTPUT_PATH="$LOCAL_PATH" + + if [ "$COMMAND" == "downstream" ]; then + pushd $LOCAL_PATH + go get -d github.com/hashicorp/terraform-provider-google-beta@$BASE_BRANCH + go mod tidy + set +e + make build + set -e + popd fi - pushd ../ - make tpgtools OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION - - # Only generate TeamCity-related file for TPG and TPGB - if [ "$VERSION" == "ga" ] || [ "$VERSION" == "beta" ]; then - make teamcity-servicemap-generate OUTPUT_PATH=$LOCAL_PATH VERSION=$VERSION - fi - popd + elif [ "$REPO" == "tf-oics" ]; then + # use terraform generator with oics override + make tf-oics OUTPUT_PATH="$LOCAL_PATH" + elif [ "$REPO" == "terraform" ]; then + make clean-provider OUTPUT_PATH="$LOCAL_PATH" + make provider OUTPUT_PATH="$LOCAL_PATH" VERSION=$VERSION fi fi -popd - pushd $LOCAL_PATH git config --local user.name "Modular Magician" @@ -222,5 +242,3 @@ if [ "$COMMITTED" == "true" ] && [ "$COMMAND" == "downstream" ]; then -d '{"merge_method": "squash"}' \ "https://api.github.com/repos/$UPSTREAM_OWNER/$GH_REPO/pulls/$NEW_PR_NUMBER/merge" fi - -popd diff --git a/.github/workflows/build-downstream.yml b/.github/workflows/build-downstream.yml new file mode 100644 index 000000000000..04312df88454 --- /dev/null +++ b/.github/workflows/build-downstream.yml @@ -0,0 +1,114 @@ +name: build +permissions: read-all + +on: + workflow_call: + inputs: + repo: + description: 'Repository to build' + required: true + type: string + +concurrency: + group: ${{ inputs.repo }}-${{ github.event_name == 'pull_request' && format('pr-{0}-', github.event.pull_request.number) || format('commit-{0}', github.sha) }} + cancel-in-progress: true + +jobs: + generate-repository: + runs-on: ubuntu-22.04 + env: + BASE_BRANCH: ${{ github.event.pull_request.base.ref || github.ref }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Checkout Repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Ruby + uses: ruby/setup-ruby@ec02537da5712d66d4d50a0f33b7eb52773b5ed1 + with: + ruby-version: '3.1' + + - name: Cache Bundler gems + uses: actions/cache@v2 + with: + path: mmv1/vendor/bundle + key: ${{ runner.os }}-gems-${{ hashFiles('mmv1/**/Gemfile.lock') }} + restore-keys: | + ${{ runner.os }}-gems- + + - name: Install Ruby dependencies + run: | + bundle config path mmv1/vendor/bundle + bundle install + working-directory: mmv1 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '^1.19' + + # Cache Go modules + - name: Cache Go modules + uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - run: go install golang.org/x/tools/cmd/goimports@latest + + - name: Build ${{ inputs.repo }} + run: | + set -e + set -x + # Set GOPATH to a directory the runner user has access to + export GOPATH=~/go + + function clone_repo() { + export OUTPUT_PATH=$GOPATH/src/github.com/$UPSTREAM_OWNER/$GH_REPO + GITHUB_PATH=https://x-access-token:$GITHUB_TOKEN@github.com/$UPSTREAM_OWNER/$GH_REPO + mkdir -p "$(dirname $OUTPUT_PATH)" + git clone $GITHUB_PATH $OUTPUT_PATH --branch $BASE_BRANCH + } + + GH_REPO="${{ inputs.repo }}" + if [ "$GH_REPO" == "docs-examples" ] && [ "$BASE_BRANCH" == "main" ]; then + BASE_BRANCH="master" + fi + + GITHUB_PATH=https://x-access-token:$GITHUB_TOKEN@github.com/$UPSTREAM_OWNER/$GH_REPO + + if [[ "$GH_REPO" == terraform-provider-google* ]]; then + UPSTREAM_OWNER=hashicorp + clone_repo + if [ "$GH_REPO" == "terraform-provider-google" ]; then + export VERSION=ga + else + export VERSION=beta + fi + make clean-provider + make provider + elif [ "$GH_REPO" == "terraform-google-conversion" ]; then + UPSTREAM_OWNER=GoogleCloudPlatform + clone_repo + make clean-tgc + make tgc + elif [ "$GH_REPO" == "docs-examples" ]; then + UPSTREAM_OWNER=terraform-google-modules + clone_repo + make tf-oics + else + echo "case not supported" + exit 1 + fi + + (current_dir=$(pwd) && cd $OUTPUT_PATH && zip -r "$current_dir/output.zip" .) + + - name: Upload built artifacts + uses: actions/upload-artifact@v2 + with: + name: artifact-${{ inputs.repo }} + path: output.zip \ No newline at end of file diff --git a/.github/workflows/downstreams.yml b/.github/workflows/downstreams.yml new file mode 100644 index 000000000000..98273ca7e18e --- /dev/null +++ b/.github/workflows/downstreams.yml @@ -0,0 +1,50 @@ +name: downstreams +permissions: read-all + +on: + push: + branches: + - main + - 'FEATURE-BRANCH-*' + +concurrency: + group: ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.pull_request.number) || format('commit-{0}', github.sha) }} + cancel-in-progress: true + +jobs: + + terraform-provider-google: + uses: ./.github/workflows/build-downstream.yml + with: + repo: 'terraform-provider-google' + terraform-provider-google-beta: + uses: ./.github/workflows/build-downstream.yml + with: + repo: 'terraform-provider-google-beta' + terraform-google-conversion: + uses: ./.github/workflows/build-downstream.yml + with: + repo: 'terraform-google-conversion' + docs-examples: + uses: ./.github/workflows/build-downstream.yml + with: + repo: 'docs-examples' + + unit-test-terraform-provider-google: + name: terraform-provider-google + needs: terraform-provider-google + uses: ./.github/workflows/unit-test-tpg.yml + with: + repo: 'terraform-provider-google' + + unit-test-terraform-provider-googele-beta: + name: terraform-provider-google-beta + needs: terraform-provider-google-beta + uses: ./.github/workflows/unit-test-tpg.yml + with: + repo: 'terraform-provider-google-beta' + + unit-test-terraform-google-conversion: + name: terraform-google-conversion + needs: [terraform-google-conversion, terraform-provider-google-beta] + uses: ./.github/workflows/unit-test-tgc.yml diff --git a/.github/workflows/unit-test-tgc.yml b/.github/workflows/unit-test-tgc.yml new file mode 100644 index 000000000000..e718ea8066f7 --- /dev/null +++ b/.github/workflows/unit-test-tgc.yml @@ -0,0 +1,55 @@ +name: unit-test-tgc +permissions: read-all + +on: + workflow_call: + +jobs: + compile-and-unit-tests: + runs-on: ubuntu-22.04 + steps: + - name: Download built artifacts + uses: actions/download-artifact@v2 + with: + name: artifact-terraform-google-conversion + path: artifacts-tgc + + - name: Download built artifacts + uses: actions/download-artifact@v2 + with: + name: artifact-terraform-provider-google-beta + path: artifacts-tpgb + + - name: Unzip the artifacts and delete the zip + run: | + unzip artifacts-tgc/output.zip -d ./tgc + unzip artifacts-tpgb/output.zip -d ./tpgb + rm artifacts-tgc/output.zip + rm artifacts-tpgb/output.zip + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '^1.19' + + - name: Cache Go modules and build cache + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + key: ${{ runner.os }}-test-terraform-google-conversion-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-test-terraform-google-conversion-${{ hashFiles('**/go.sum') }} + ${{ runner.os }}-test-terraform-google-conversion- + + - name: Build Terraform Google Conversion + run: | + cd tgc + go mod edit -replace=github.com/hashicorp/terraform-provider-google-beta=../tpgb + go mod tidy + make build + + - name: Run Unit Tests + run: | + cd tgc + make test \ No newline at end of file diff --git a/.github/workflows/unit-test-tpg.yml b/.github/workflows/unit-test-tpg.yml new file mode 100644 index 000000000000..4bfb4dd80d87 --- /dev/null +++ b/.github/workflows/unit-test-tpg.yml @@ -0,0 +1,58 @@ +name: unit-test-tpg +permissions: read-all + +on: + workflow_call: + inputs: + repo: + description: 'Repository for the matrix' + required: true + type: string + +jobs: + compile-and-unit-tests: + name: unit-test-${{ inputs.repo }} + runs-on: ubuntu-22.04 + steps: + - name: Download built artifacts + uses: actions/download-artifact@v2 + with: + name: artifact-${{ inputs.repo }} + path: artifacts + + - name: Unzip the artifacts and delete the zip + run: | + unzip artifacts/output.zip -d ./ + rm artifacts/output.zip + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: '^1.19' + + - name: Cache Go modules and build cache + uses: actions/cache@v3 + with: + path: | + ~/go/pkg/mod + ~/.cache/go-build + key: ${{ runner.os }}-test-${{ inputs.repo }}-${{hashFiles('go.sum','google-*/transport/**','google-*/tpgresource/**','google-*/acctest/**','google-*/envvar/**','google-*/sweeper/**','google-*/verify/**') }} + restore-keys: | + ${{ runner.os }}-test-${{ inputs.repo }}-${{ hashFiles('go.sum') }} + ${{ runner.os }}-test-${{ inputs.repo }}- + + - name: Build Provider + run: | + go build + + - name: Run Unit Tests + run: | + make testnolint + + - name: Lint Check + run: | + make lint + + - name: Documentation Check + run: | + make docscheck diff --git a/GNUmakefile b/GNUmakefile index 788043d693aa..f0929ae4f8a8 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -64,7 +64,12 @@ terraform build provider: mmv1: cd mmv1;\ bundle; \ - bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v $(VERSION) $(mmv1_compile); + if [ "$(VERSION)" = "ga" ]; then \ + bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v ga --no-docs $(mmv1_compile); \ + bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v beta --no-code $(mmv1_compile); \ + else \ + bundle exec compiler.rb -e terraform -o $(OUTPUT_PATH) -v $(VERSION) $(mmv1_compile); \ + fi tpgtools: make serialize @@ -77,10 +82,31 @@ teamcity-servicemap-generate: cd tools/teamcity-generator;\ go run . --output $(OUTPUT_PATH) --version $(VERSION) + +clean-provider: + cd $(OUTPUT_PATH);\ + go mod download;\ + find . -type f -not -wholename "./.git*" -not -wholename "./.changelog*" -not -name ".travis.yml" -not -name ".golangci.yml" -not -name "CHANGELOG.md" -not -name "CHANGELOG_v*.md" -not -name "GNUmakefile" -not -name "docscheck.sh" -not -name "LICENSE" -not -name "README.md" -not -wholename "./examples*" -not -name ".go-version" -not -name ".hashibot.hcl" -print0 | xargs -0 git rm > /dev/null + +clean-tgc: + cd $(OUTPUT_PATH);\ + rm -rf ./tfplan2cai/testdata/templates/;\ + rm -rf ./tfplan2cai/testdata/generatedconvert/;\ + rm -rf ./tfplan2cai/converters/google/provider;\ + rm -rf ./tfplan2cai/converters/google/resources;\ + rm -rf ./cai2hcl/*;\ + find ./tfplan2cai/test/** -type f -exec git rm {} \; > /dev/null;\ + tgc: cd mmv1;\ bundle;\ - bundle exec compiler -e terraform -f tgc -v beta -o $(OUTPUT_PATH) $(mmv1_compile);\ + bundle exec compiler -e terraform -f tgc -v beta -o $(OUTPUT_PATH)/tfplan2cai $(mmv1_compile);\ + bundle exec compiler -e terraform -f tgc_cai2hcl -v beta -o $(OUTPUT_PATH)/cai2hcl $(mmv1_compile);\ + +tf-oics: + cd mmv1;\ + bundle;\ + bundle exec compiler.rb -e terraform -f oics -o $(OUTPUT_PATH) $(mmv1_compile);\ test: cd mmv1; \