Merge pull request #690 from ulucinar/fix-linter

Decrease linter's memory usage

.github/workflows/ci.yml

@@ -17,7 +17,91 @@ jobs:
     uses: upbound/uptest/.github/workflows/provider-ci.yml@standard-runners
     with:
       go-version: 1.21
+      golangci-skip: true # we will run the linter via "make lint"
       cleanup-disk: true
     secrets:
       UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }}
       UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}
+
+  detect-noop:
+    runs-on: ubuntu-22.04
+    outputs:
+      noop: ${{ steps.noop.outputs.should_skip }}
+    steps:
+      - name: Detect No-op Changes
+        id: noop
+        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v5.3.0
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          paths_ignore: '["**.md", "**.png", "**.jpg"]'
+          do_not_skip: '["workflow_dispatch", "schedule", "push"]'
+
+  lint:
+    runs-on: ubuntu-22.04
+    needs: detect-noop
+    if: needs.detect-noop.outputs.noop != 'true'
+
+    steps:
+      - name: Cleanup Disk
+        uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          tool-cache: true
+          large-packages: false
+          swap-storage: false
+
+      - name: Checkout
+        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
+        with:
+          submodules: true
+
+      - name: Setup Go
+        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
+        with:
+          go-version: "1.21"
+
+      - name: Find the Go Build Cache
+        id: go_cache
+        run: |
+          echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \
+          echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT && \
+          echo "analysis_cache=$HOME/.cache/golangci-lint" >> $GITHUB_OUTPUT && \
+          echo "analysis_cache_key=$(make go.lint.analysiskey)" >> $GITHUB_OUTPUT && \
+          echo "analysis_cache_key_int=$(make go.lint.analysiskey-interval)" >> $GITHUB_OUTPUT
+
+
+      - name: Cache the Go Build Cache
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
+        with:
+          path: ${{ steps.go_cache.outputs.cache }}
+          key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-build-lint-
+
+      - name: Cache Go Dependencies
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
+        with:
+          path: ${{ steps.go_cache.outputs.mod_cache }}
+          key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-pkg-
+
+      - name: Cache Linter Analysis
+        uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
+        id: cache-analysis
+        with:
+          path: ${{ steps.go_cache.outputs.analysis_cache }}
+          key: ${{ steps.go_cache.outputs.analysis_cache_key }}
+          restore-keys: |
+            ${{ steps.go_cache.outputs.analysis_cache_key_int }}
+
+      - name: Vendor Dependencies
+        run: make vendor vendor.check
+
+      - name: Lint
+        env:
+          GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }}
+          SKIP_LINTER_ANALYSIS: false
+          RUN_BUILDTAGGER: true
+          GOGC: "50"
+        run: make lint

.golangci.yml

@@ -3,14 +3,13 @@
 # SPDX-License-Identifier: CC0-1.0
 
 run:
-  deadline: 20m
-
-  skip-files:
-  - "zz_.*go$"
+  timeout: 90m
+  concurrency: 1
 
 output:
   # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
   format: colored-line-number
+  print-linter-name: true
 
 linters-settings:
   errcheck:
@@ -131,6 +130,11 @@ linters:
 issues:
   # Excluding configuration per-path and per-linter
   exclude-rules:
+    # some group names we have (like "hdinsight") make this linter
+    # unhappy, so just disable the "misspell" linter on generated files.
+    - path: zz_.+\.go$
+      linters:
+        - misspell
     # Exclude some linters from running on tests files.
     - path: _test(ing)?\.go
       linters:

Makefile

@@ -51,7 +51,16 @@ export GOPRIVATE = github.com/upbound/*
 GO_REQUIRED_VERSION ?= 1.21
 # GOLANGCILINT_VERSION is inherited from build submodule by default.
 # Uncomment below if you need to override the version.
-# GOLANGCILINT_VERSION ?= 1.54.0
+GOLANGCILINT_VERSION ?= 1.55.2
+
+RUN_BUILDTAGGER ?= true
+# if RUN_BUILDTAGGER is set to "true", we will use build constraints
+# and use the buildtagger tool to generate the build tags.
+ifeq ($(RUN_BUILDTAGGER),true)
+GO_LINT_ARGS ?= -v --build-tags all
+BUILDTAGGER_VERSION ?= v0.12.0-rc.0.28.gdc5d6f3
+BUILDTAGGER_DOWNLOAD_URL ?= https://s3.us-west-2.amazonaws.com/upbound.official-providers-ci.releases/main/$(BUILDTAGGER_VERSION)/bin/$(SAFEHOST_PLATFORM)/buildtagger
+endif
 
 # SUBPACKAGES ?= $(shell find cmd/provider -type d -maxdepth 1 -mindepth 1 | cut -d/ -f3)
 SUBPACKAGES ?= monolith
@@ -67,7 +76,7 @@ export SUBPACKAGES := $(SUBPACKAGES)
 # ====================================================================================
 # Setup Kubernetes tools
 
-KIND_VERSION = v0.15.0
+KIND_VERSION = v0.21.0
 UP_VERSION = v0.20.0
 UP_CHANNEL = stable
 UPTEST_VERSION = v0.11.1
@@ -304,7 +313,14 @@ go.cachedir:
 go.mod.cachedir:
 	@go env GOMODCACHE
 
-.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA)
+go.lint.analysiskey-interval:
+	@# cache is invalidated at least every 7 days
+	@echo -n golangci-lint.cache-$$(( $$(date +%s) / (7 * 86400) ))-
+
+go.lint.analysiskey:
+	@echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1)
+
+.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir go.lint.analysiskey-interval go.lint.analysiskey run crds.clean $(TERRAFORM_PROVIDER_SCHEMA)
 
 build.init: kustomize-crds
 
@@ -318,4 +334,24 @@ kustomize-crds: output.init $(KUSTOMIZE) $(YQ)
 	XDG_CONFIG_HOME=$(PWD)/package $(KUSTOMIZE) build --enable-alpha-plugins $(OUTPUT_DIR)/package/kustomize -o $(OUTPUT_DIR)/package/crds.yaml || $(FAIL)
 	@$(OK) Kustomizing CRDs.
 
-.PHONY: kustomize-crds
+.PHONY: kustomize-crds
+
+ifeq ($(RUN_BUILDTAGGER),true)
+lint.init: build-lint-cache
+lint.done: delete-build-tags
+
+build-lint-cache: $(GOLANGCILINT)
+	@$(INFO) Running golangci-lint with the analysis cache building phase.
+	@# we run the initial analysis cache build phase using the relatively
+	@# smaller API group "account", to keep the memory requirements at a
+	@# minimum.
+	@(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \
+	(([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \
+	[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags analysisservices,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL)
+	@$(OK) Running golangci-lint with the analysis cache building phase.
+
+delete-build-tags:
+	@$(INFO) Untagging source files.
+	@EXTRA_BUILDTAGGER_ARGS="--delete" RESTORE_DEEPCOPY_TAGS="true" ./scripts/tag.sh || $(FAIL)
+	@$(OK) Untagging source files.
+endif

apis/linter_run.go

@@ -0,0 +1,17 @@
+//go:build linter_run
+
+// SPDX-FileCopyrightText: 2024 The Crossplane Authors <https://crossplane.io>
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package apis
+
+import "k8s.io/apimachinery/pkg/runtime"
+
+// AddToSchemes may be used to add all resources defined in the project to a Scheme
+var AddToSchemes runtime.SchemeBuilder
+
+// AddToScheme adds all Resources to the Scheme
+func AddToScheme(s *runtime.Scheme) error {
+	panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`)
+}