Merge pull request #1194 from ulucinar/linter-mem

Decrease linter's memory usage

.github/workflows/ci.yml

@@ -12,7 +12,80 @@ jobs:
   ci:
     uses: upbound/uptest/.github/workflows/provider-ci.yml@main
     with:
-      go-version: 1.21
+      go-version: "1.21"
+      golangci-skip: true # we will run the linter via "make lint"
     secrets:
       UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }}
       UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}
+
+  detect-noop:
+    runs-on: ubuntu-22.04
+    outputs:
+      noop: ${{ steps.noop.outputs.should_skip }}
+    steps:
+      - name: Detect No-op Changes
+        id: noop
+        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v5.3.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          paths_ignore: '["**.md", "**.png", "**.jpg"]'
+          do_not_skip: '["workflow_dispatch", "schedule", "push"]'
+
+  lint:
+    runs-on: ubuntu-22.04
+    needs: detect-noop
+    if: needs.detect-noop.outputs.noop != 'true'
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+        with:
+          submodules: true
+
+      - name: Setup Go
+        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
+        with:
+          go-version: "1.21"
+
+      - name: Find the Go Build Cache
+        id: go_cache
+        run: |
+          echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \
+          echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT && \
+          echo "analysis_cache=$HOME/.cache/golangci-lint" >> $GITHUB_OUTPUT && \
+          echo "analysis_cache_key=$(make go.lint.analysiskey)" >> $GITHUB_OUTPUT && \
+          echo "analysis_cache_key_int=$(make go.lint.analysiskey-interval)" >> $GITHUB_OUTPUT
+          
+
+      - name: Cache the Go Build Cache
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
+        with:
+          path: ${{ steps.go_cache.outputs.cache }}
+          key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-build-lint-
+
+      - name: Cache Go Dependencies
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
+        with:
+          path: ${{ steps.go_cache.outputs.mod_cache }}
+          key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-pkg-
+
+      - name: Cache Linter Analysis
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
+        id: cache-analysis
+        with:
+          path: ${{ steps.go_cache.outputs.analysis_cache }}
+          key: ${{ steps.go_cache.outputs.analysis_cache_key }}
+          restore-keys: |
+            ${{ steps.go_cache.outputs.analysis_cache_key_int }}
+
+      - name: Vendor Dependencies
+        run: make vendor vendor.check
+
+      - name: Lint
+        env:
+          GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }}
+          SKIP_LINTER_ANALYSIS: ${{ steps.cache-analysis.outputs.cache-hit }}
+          RUN_BUILDTAGGER: true
+        run: make lint

.golangci.yml

@@ -1,12 +1,11 @@
 run:
-  timeout: 20m
-
-  skip-files:
-  - "zz_.+\\.go$"
+  timeout: 60m
+  show-stats: true
 
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
   format: colored-line-number
+  print-linter-name: true
 
 linters-settings:
   errcheck:

Makefile

@@ -48,6 +48,15 @@ GO_REQUIRED_VERSION ?= 1.21
 # Uncomment below if you need to override the version.
 GOLANGCILINT_VERSION ?= 1.55.2
 
+RUN_BUILDTAGGER ?= true
+# if RUN_BUILDTAGGER is set to "true", we will use build constraints
+# and use the buildtagger tool to generate the build tags.
+ifeq ($(RUN_BUILDTAGGER),true)
+GO_LINT_ARGS ?= -v --build-tags all
+BUILDTAGGER_VERSION ?= v0.12.0-rc.0.28.gdc5d6f3
+BUILDTAGGER_DOWNLOAD_URL ?= https://s3.us-west-2.amazonaws.com/upbound.official-providers-ci.releases/main/$(BUILDTAGGER_VERSION)/bin/$(SAFEHOST_PLATFORM)/buildtagger
+endif
+
 # SUBPACKAGES ?= $(shell find cmd/provider -type d -maxdepth 1 -mindepth 1 | cut -d/ -f3)
 SUBPACKAGES ?= monolith
 GO_STATIC_PACKAGES ?= $(GO_PROJECT)/cmd/generator ${SUBPACKAGES:%=$(GO_PROJECT)/cmd/provider/%}
@@ -311,7 +320,14 @@ go.cachedir:
 go.mod.cachedir:
 	@go env GOMODCACHE
 
-.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA)
+go.lint.analysiskey-interval:
+	@# cache is invalidated at least every 7 days
+	@echo -n golangci-lint.cache-$$(( $$(date +%s) / (7 * 86400) ))-
+
+go.lint.analysiskey:
+	@echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1)
+
+.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir go.lint.analysiskey-interval go.lint.analysiskey run crds.clean $(TERRAFORM_PROVIDER_SCHEMA)
 
 build.init: kustomize-crds
 
@@ -325,7 +341,27 @@ kustomize-crds: output.init $(KUSTOMIZE) $(YQ)
 	XDG_CONFIG_HOME=$(PWD)/package $(KUSTOMIZE) build --enable-alpha-plugins $(OUTPUT_DIR)/package/kustomize -o $(OUTPUT_DIR)/package/crds.yaml || $(FAIL)
 	@$(OK) Kustomizing CRDs.
 
+.PHONY: kustomize-crds
+
 checkout-to-old-api:
 	CHECKOUT_RELEASE_VERSION=$(CHECKOUT_RELEASE_VERSION) hack/check-duplicate.sh
 
-.PHONY: kustomize-crds
+ifeq ($(RUN_BUILDTAGGER),true)
+lint.init: build-lint-cache
+lint.done: delete-build-tags
+
+build-lint-cache: $(GOLANGCILINT)
+	@$(INFO) Running golangci-lint with the analysis cache building phase.
+	@# we run the initial analysis cache build phase using the relatively
+	@# smaller API group "account", to keep the memory requirements at a
+	@# minimum.
+	@(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \
+	(([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \
+	[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags account,configregistry,configprovider,linter_run -v --concurrency 3 --disable-all --exclude '.*')) || $(FAIL)
+	@$(OK) Running golangci-lint with the analysis cache building phase.
+
+delete-build-tags:
+	@$(INFO) Untagging source files.
+	@EXTRA_BUILDTAGGER_ARGS="--delete" RESTORE_DEEPCOPY_TAGS="true" ./scripts/tag.sh || $(FAIL)
+	@$(OK) Untagging source files.
+endif

apis/linter_run.go

@@ -0,0 +1,17 @@
+//go:build linter_run
+
+// SPDX-FileCopyrightText: 2024 The Crossplane Authors <https://crossplane.io>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package apis
+
+import "k8s.io/apimachinery/pkg/runtime"
+
+// AddToSchemes may be used to add all resources defined in the project to a Scheme
+var AddToSchemes runtime.SchemeBuilder
+
+// AddToScheme adds all Resources to the Scheme
+func AddToScheme(s *runtime.Scheme) error {
+	panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`)
+}