[Backport standard-runners]: Backports PRs #187 #188 #189 #190

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.19'
+  GO_VERSION: '1.21'
   GOLANGCI_VERSION: 'v1.54.2'
   DOCKER_BUILDX_VERSION: 'v0.8.2'
 

.github/workflows/pr-comment-trigger.yml

@@ -17,6 +17,11 @@ on:
         default: 'provider'
         required: false
         type: string
+      cleanup-disk:
+        description: "If set to true, an initial step will be run to reclaim some extra disk space for the uptest job in this workflow"
+        required: false
+        type: boolean
+        default: false
       update-test-parameter:
         description: 'Input parameter to use during update step. If this field
         is empty, then the update step will be skipped.
@@ -113,6 +118,7 @@ jobs:
     steps:
       - name: Cleanup Disk
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        if: ${{ inputs.cleanup-disk }}
         with:
           android: true
           dotnet: true

.github/workflows/promote.yaml

@@ -13,7 +13,7 @@ on:
 
 env:
   # Common versions
-  GO_VERSION: '1.19'
+  GO_VERSION: '1.21'
 
   # Common users. We can't run a step 'if secrets.XXX != ""' but we can run
   # a step 'if env.XXX' != ""', so we copy these to succinctly test whether

.github/workflows/provider-ci.yml

@@ -4,6 +4,7 @@ on:
   workflow_call:
     inputs:
       upjet-based-provider:
+        description: 'Should be set to true if the repo running the workflow is an upjet-based provider repo'
         required: false
         type: boolean
         default: true
@@ -12,11 +13,21 @@ on:
         default: '1.20'
         required: false
         type: string
+      cleanup-disk:
+        description: "If set to true, an initial step will be run to reclaim some extra disk space for the build/test jobs in this workflow"
+        required: false
+        type: boolean
+        default: false
       golangci-version:
         description: 'The version string to be used with the golangci-lint action'
         default: 'v1.55.2'
         required: false
         type: string
+      golangci-skip:
+        description: 'If set to true, the lint job will be skipped'
+        default: false
+        required: false
+        type: boolean
     secrets:
       UPBOUND_MARKETPLACE_PUSH_ROBOT_USR:
         required: true
@@ -75,9 +86,20 @@ jobs:
   lint:
     runs-on: ubuntu-22.04
     needs: detect-noop
-    if: needs.detect-noop.outputs.noop != 'true'
+    if: needs.detect-noop.outputs.noop != 'true' && !inputs.golangci-skip
 
     steps:
+      - name: Cleanup Disk
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        if: ${{ inputs.cleanup-disk }}
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          tool-cache: true
+          large-packages: false
+          swap-storage: false
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
         with:
@@ -131,6 +153,17 @@ jobs:
     if: needs.detect-noop.outputs.noop != 'true'
 
     steps:
+      - name: Cleanup Disk
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        if: ${{ inputs.cleanup-disk }}
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          tool-cache: true
+          large-packages: false
+          swap-storage: false
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
         with:
@@ -197,6 +230,17 @@ jobs:
     if: needs.detect-noop.outputs.noop != 'true'
 
     steps:
+      - name: Cleanup Disk
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        if: ${{ inputs.cleanup-disk }}
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          tool-cache: true
+          large-packages: false
+          swap-storage: false
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
         with:
@@ -248,6 +292,17 @@ jobs:
     if: needs.detect-noop.outputs.noop != 'true'
 
     steps:
+      - name: Cleanup Disk
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        if: ${{ inputs.cleanup-disk }}
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          tool-cache: true
+          large-packages: false
+          swap-storage: false
+
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
         with:
@@ -295,6 +350,7 @@ jobs:
     steps:
       - name: Cleanup Disk
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        if: ${{ inputs.cleanup-disk }}
         with:
           android: true
           dotnet: true

.github/workflows/provider-publish-service-artifacts.yml

@@ -48,7 +48,7 @@ env:
   # Common versions
   GOLANGCI_VERSION: 'v1.54.2'
   DOCKER_BUILDX_VERSION: 'v0.8.2'
-  UP_VERSION: 'v0.17.0'
+  UP_VERSION: 'v0.24.2'
 
   # Common users. We can't run a step 'if secrets.XXX != ""' but we can run a
   # step 'if env.XXX' != ""', so we copy these to succinctly test whether
@@ -87,11 +87,11 @@ jobs:
       - name: Login to Upbound
         if: env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR != ''
         env:
-          UP_PASSWORD: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}
+          UP_TOKEN: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}
         run: |
           curl -fsSLo /tmp/up --create-dirs 'https://cli.upbound.io/stable/${{ env.UP_VERSION }}/bin/linux_amd64/up' && \
           chmod +x /tmp/up && \
-          /tmp/up login -u ${{ env.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }}
+          /tmp/up login
 
       - name: Checkout
         uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

Makefile

@@ -17,12 +17,12 @@ S3_BUCKET ?= upbound.official-providers-ci.releases
 
 # ====================================================================================
 # Setup Go
-GO_REQUIRED_VERSION = 1.19
+GO_REQUIRED_VERSION = 1.21
 # GOLANGCILINT_VERSION is inherited from build submodule by default.
 # Uncomment below if you need to override the version.
 # GOLANGCILINT_VERSION ?= 1.54.0
 
-GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/uptest $(GO_PROJECT)/cmd/crddiff $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf $(GO_PROJECT)/cmd/linter/lint-provider-family
+GO_STATIC_PACKAGES = $(GO_PROJECT)/cmd/uptest $(GO_PROJECT)/cmd/crddiff $(GO_PROJECT)/cmd/buildtagger $(GO_PROJECT)/cmd/updoc $(GO_PROJECT)/cmd/ttr $(GO_PROJECT)/cmd/perf $(GO_PROJECT)/cmd/linter/lint-provider-family
 GO_LDFLAGS += -X $(GO_PROJECT)/internal/version.Version=$(VERSION)
 GO_SUBDIRS += cmd internal
 GO111MODULE = on

cmd/buildtagger/main.go

@@ -0,0 +1,145 @@
+// Copyright 2024 Upbound Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// main package for the buildtagger application which is used in the
+// official provider repositories to generate build tags for the
+// provider families. Each family's source modules can be tagged using
+// the buildtagger tool.
+package main
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+	"gopkg.in/alecthomas/kingpin.v2"
+)
+
+var (
+	app = kingpin.New("buildtagger", "A tool for generating build tags (constraints) for the source modules of the official provider families.").DefaultEnvars()
+)
+
+var (
+	parentDir = app.Flag("parent-dir", "Parent directory which will be recursively walked to find the Go source files whose relative path to this parent matches the specified regular expression. The files found will be tagged using the specified build tag.").Default("./").String()
+	regex     = app.Flag("regex", `The regular expression against which a discovered Go source file's relative path or name will be matched. This expression must contain one and only one group whose value will be substituted in the given tag format string. An example is "(.+)/.+/.+\.go"`).Default(".*").String()
+	tagFormat = app.Flag("tag-format", `A Printf format string to construct the build tag. An example is "(%s || all) && !ignore_autogenerated", where the "%s" format specifier can be replaced by a family resource provider group name.`+
+		`There should be a string format specifier for each of the capturing groups specified in the "regex".`).Default("!ignore_autogenerated").String()
+	mode       = app.Flag("mode", `If "file", the file name of the discovered Go source is matched against the given regular expression. If "dir", the relative path of the source file is matched.`).Default("dir").Enum("file", "dir")
+	deleteTags = app.Flag("delete", `If set, the build tags are removed from the discovered Go sources, instead of being added.`).Default("false").Bool()
+)
+
+// addOrUpdateBuildTag traverses directories from the parent,
+// updating or adding build tags in Go files. If a build tag already exists,
+// it's replaced with the computed tags.
+func addOrUpdateBuildTag(parent, regex, tagFormat, mode string, deleteTags bool) error {
+	re, err := regexp.Compile(regex)
+	kingpin.FatalIfError(err, "Failed to compile the given regular expression: %s", regex)
+	matched := false
+	err = filepath.Walk(parent, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if !info.IsDir() {
+			matchString, err := getMatchString(parent, path, mode)
+			if err != nil {
+				return err
+			}
+			matches := re.FindStringSubmatch(matchString)
+			if len(matches) == 0 {
+				return nil
+			}
+			matched = true
+			args := make([]any, len(matches)-1)
+			for i, a := range matches[1:] {
+				args[i] = a
+			}
+			tag := fmt.Sprintf("//go:build "+strings.TrimSpace(tagFormat), args...)
+			err = updateFileWithBuildTag(path, tag, deleteTags)
+			if err != nil {
+				return errors.Wrap(err, "failed to update the source file")
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	if matched {
+		return nil
+	}
+	return errors.Errorf("no Go source files under %s matched the regular expression %q", parent, regex)
+}
+
+func getMatchString(parent, path, mode string) (string, error) {
+	switch mode {
+	case "file":
+		// regex is matched against the filename
+		return filepath.Base(path), nil
+	case "dir":
+		// regex is matched against the relative path
+		matchString, err := filepath.Rel(parent, path)
+		return matchString, errors.Wrapf(err, "failed to determine the relative path of %s wrt to %s", path, parent)
+	default:
+		return "", errors.Errorf("unknown match mode %q", mode)
+	}
+}
+
+// updateFileWithBuildTag reads a Go file and updates or inserts the specified build tag.
+func updateFileWithBuildTag(filePath, buildTag string, deleteTag bool) error {
+	content, err := os.ReadFile(filepath.Clean(filePath))
+	if err != nil {
+		return errors.Wrapf(err, "failed to read the source file at path %s", filePath)
+	}
+
+	lines := strings.Split(string(content), "\n")
+	if len(lines) < 1 {
+		return nil
+	}
+	var updatedLines []string
+	index, tagExists, emptyLineFollows := getLineStartIndex(lines, deleteTag)
+	updatedLines = lines[index+1:]
+	if !deleteTag {
+		addedLines := [2]string{buildTag}
+		trimIndex := 2
+		if emptyLineFollows && tagExists {
+			trimIndex = 1
+		}
+		updatedLines = append(addedLines[:trimIndex], updatedLines...)
+	}
+	// Write the updated content back to the file
+	return errors.Wrapf(os.WriteFile(filePath, []byte(strings.Join(updatedLines, "\n")), 0600), "failed to write the source file at path %s", filePath)
+}
+
+func getLineStartIndex(lines []string, deleteTag bool) (int, bool, bool) {
+	index := -1
+	tagExists := false
+	if strings.HasPrefix(lines[0], "//go:build") {
+		tagExists = true
+		index++
+	}
+	emptyLineFollows := len(lines) > 1 && strings.TrimSpace(lines[1]) == ""
+	if deleteTag && emptyLineFollows && tagExists {
+		index++
+	}
+	return index, tagExists, emptyLineFollows
+}
+
+func main() {
+	kingpin.MustParse(app.Parse(os.Args[1:]))
+	kingpin.FatalIfError(addOrUpdateBuildTag(*parentDir, *regex, *tagFormat, *mode, *deleteTags), "Failed to run the buildtagger...")
+}

go.mod

@@ -1,6 +1,6 @@
 module github.com/upbound/uptest
 
-go 1.19
+go 1.21
 
 require (
 	cloud.google.com/go/storage v1.27.0