Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Decrease linter's memory usage #690

Merged
merged 1 commit into from
Apr 2, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 84 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,91 @@ jobs:
uses: upbound/uptest/.github/workflows/provider-ci.yml@standard-runners
with:
go-version: 1.21
golangci-skip: true # we will run the linter via "make lint"
cleanup-disk: true
secrets:
UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }}
UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }}

detect-noop:
runs-on: ubuntu-22.04
outputs:
noop: ${{ steps.noop.outputs.should_skip }}
steps:
- name: Detect No-op Changes
id: noop
uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v5.3.0
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
paths_ignore: '["**.md", "**.png", "**.jpg"]'
do_not_skip: '["workflow_dispatch", "schedule", "push"]'

lint:
runs-on: ubuntu-22.04
needs: detect-noop
if: needs.detect-noop.outputs.noop != 'true'

steps:
- name: Cleanup Disk
uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
with:
android: true
dotnet: true
haskell: true
tool-cache: true
large-packages: false
swap-storage: false

- name: Checkout
uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
with:
submodules: true

- name: Setup Go
uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3
with:
go-version: "1.21"

- name: Find the Go Build Cache
id: go_cache
run: |
echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \
echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT && \
echo "analysis_cache=$HOME/.cache/golangci-lint" >> $GITHUB_OUTPUT && \
echo "analysis_cache_key=$(make go.lint.analysiskey)" >> $GITHUB_OUTPUT && \
echo "analysis_cache_key_int=$(make go.lint.analysiskey-interval)" >> $GITHUB_OUTPUT


- name: Cache the Go Build Cache
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
with:
path: ${{ steps.go_cache.outputs.cache }}
key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-build-lint-

- name: Cache Go Dependencies
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
with:
path: ${{ steps.go_cache.outputs.mod_cache }}
key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }}
restore-keys: ${{ runner.os }}-pkg-

- name: Cache Linter Analysis
uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3
id: cache-analysis
with:
path: ${{ steps.go_cache.outputs.analysis_cache }}
key: ${{ steps.go_cache.outputs.analysis_cache_key }}
restore-keys: |
${{ steps.go_cache.outputs.analysis_cache_key_int }}

- name: Vendor Dependencies
run: make vendor vendor.check

- name: Lint
env:
GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }}
SKIP_LINTER_ANALYSIS: false
RUN_BUILDTAGGER: true
GOGC: "50"
run: make lint
12 changes: 8 additions & 4 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,13 @@
# SPDX-License-Identifier: CC0-1.0

run:
deadline: 20m

skip-files:
- "zz_.*go$"
timeout: 90m
concurrency: 1

output:
# colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number"
format: colored-line-number
print-linter-name: true

linters-settings:
errcheck:
Expand Down Expand Up @@ -131,6 +130,11 @@ linters:
issues:
# Excluding configuration per-path and per-linter
exclude-rules:
# some group names we have (like "hdinsight") make this linter
# unhappy, so just disable the "misspell" linter on generated files.
- path: zz_.+\.go$
linters:
- misspell
# Exclude some linters from running on tests files.
- path: _test(ing)?\.go
linters:
Expand Down
44 changes: 40 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,16 @@ export GOPRIVATE = github.com/upbound/*
GO_REQUIRED_VERSION ?= 1.21
# GOLANGCILINT_VERSION is inherited from build submodule by default.
# Uncomment below if you need to override the version.
# GOLANGCILINT_VERSION ?= 1.54.0
GOLANGCILINT_VERSION ?= 1.55.2

RUN_BUILDTAGGER ?= true
# if RUN_BUILDTAGGER is set to "true", we will use build constraints
# and use the buildtagger tool to generate the build tags.
ifeq ($(RUN_BUILDTAGGER),true)
GO_LINT_ARGS ?= -v --build-tags all
BUILDTAGGER_VERSION ?= v0.12.0-rc.0.28.gdc5d6f3
BUILDTAGGER_DOWNLOAD_URL ?= https://s3.us-west-2.amazonaws.com/upbound.official-providers-ci.releases/main/$(BUILDTAGGER_VERSION)/bin/$(SAFEHOST_PLATFORM)/buildtagger
endif

# SUBPACKAGES ?= $(shell find cmd/provider -type d -maxdepth 1 -mindepth 1 | cut -d/ -f3)
SUBPACKAGES ?= monolith
Expand All @@ -67,7 +76,7 @@ export SUBPACKAGES := $(SUBPACKAGES)
# ====================================================================================
# Setup Kubernetes tools

KIND_VERSION = v0.15.0
KIND_VERSION = v0.21.0
UP_VERSION = v0.20.0
UP_CHANNEL = stable
UPTEST_VERSION = v0.11.1
Expand Down Expand Up @@ -304,7 +313,14 @@ go.cachedir:
go.mod.cachedir:
@go env GOMODCACHE

.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA)
go.lint.analysiskey-interval:
@# cache is invalidated at least every 7 days
@echo -n golangci-lint.cache-$$(( $$(date +%s) / (7 * 86400) ))-

go.lint.analysiskey:
@echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1)

.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir go.lint.analysiskey-interval go.lint.analysiskey run crds.clean $(TERRAFORM_PROVIDER_SCHEMA)

build.init: kustomize-crds

Expand All @@ -318,4 +334,24 @@ kustomize-crds: output.init $(KUSTOMIZE) $(YQ)
XDG_CONFIG_HOME=$(PWD)/package $(KUSTOMIZE) build --enable-alpha-plugins $(OUTPUT_DIR)/package/kustomize -o $(OUTPUT_DIR)/package/crds.yaml || $(FAIL)
@$(OK) Kustomizing CRDs.

.PHONY: kustomize-crds
.PHONY: kustomize-crds

ifeq ($(RUN_BUILDTAGGER),true)
lint.init: build-lint-cache
lint.done: delete-build-tags

build-lint-cache: $(GOLANGCILINT)
@$(INFO) Running golangci-lint with the analysis cache building phase.
@# we run the initial analysis cache build phase using the relatively
@# smaller API group "account", to keep the memory requirements at a
@# minimum.
@(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \
(([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \
[[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags analysisservices,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL)
@$(OK) Running golangci-lint with the analysis cache building phase.

delete-build-tags:
@$(INFO) Untagging source files.
@EXTRA_BUILDTAGGER_ARGS="--delete" RESTORE_DEEPCOPY_TAGS="true" ./scripts/tag.sh || $(FAIL)
@$(OK) Untagging source files.
endif
17 changes: 17 additions & 0 deletions apis/linter_run.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
//go:build linter_run

// SPDX-FileCopyrightText: 2024 The Crossplane Authors <https://crossplane.io>
//
// SPDX-License-Identifier: Apache-2.0

package apis

import "k8s.io/apimachinery/pkg/runtime"

// AddToSchemes may be used to add all resources defined in the project to a Scheme
var AddToSchemes runtime.SchemeBuilder

// AddToScheme adds all Resources to the Scheme
func AddToScheme(s *runtime.Scheme) error {
panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`)
}
Loading
Loading