From 21a938ac9c56e7378de7befd3c4a79976a2f6e91 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 15 Mar 2024 12:21:01 +0300 Subject: [PATCH 1/2] Decrease linter's memory usage Signed-off-by: Alper Rifat Ulucinar --- .github/workflows/ci.yml | 83 ++++++++++ Makefile | 44 +++++- apis/linter_run.go | 17 +++ config/provider.go | 321 +++++++++++---------------------------- config/registry.go | 177 +++++++++++++++++++++ scripts/tag.sh | 42 +++++ 6 files changed, 446 insertions(+), 238 deletions(-) create mode 100644 apis/linter_run.go create mode 100644 config/registry.go create mode 100755 scripts/tag.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b53f0e751..a432228f71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,90 @@ jobs: uses: upbound/uptest/.github/workflows/provider-ci.yml@standard-runners with: go-version: "1.21" + golangci-skip: true # we will run the linter via "make lint" cleanup-disk: true secrets: UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }} + + detect-noop: + runs-on: ubuntu-22.04 + outputs: + noop: ${{ steps.noop.outputs.should_skip }} + steps: + - name: Detect No-op Changes + id: noop + uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v5.3.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + paths_ignore: '["**.md", "**.png", "**.jpg"]' + do_not_skip: '["workflow_dispatch", "schedule", "push"]' + + lint: + runs-on: ubuntu-22.04 + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' + + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + android: true + dotnet: true + haskell: true + tool-cache: true + large-packages: false + swap-storage: false + + - name: Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + with: + submodules: true + + - name: Setup Go + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + with: + go-version: "1.21" + + - name: Find the Go Build Cache + id: go_cache + run: | + echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \ + echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT && \ + echo "analysis_cache=$HOME/.cache/golangci-lint" >> $GITHUB_OUTPUT && \ + echo "analysis_cache_key=$(make go.lint.analysiskey)" >> $GITHUB_OUTPUT && \ + echo "analysis_cache_key_int=$(make go.lint.analysiskey-interval)" >> $GITHUB_OUTPUT + + + - name: Cache the Go Build Cache + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 + with: + path: ${{ steps.go_cache.outputs.cache }} + key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-build-lint- + + - name: Cache Go Dependencies + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 + with: + path: ${{ steps.go_cache.outputs.mod_cache }} + key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-pkg- + + - name: Cache Linter Analysis + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 + id: cache-analysis + with: + path: ${{ steps.go_cache.outputs.analysis_cache }} + key: ${{ steps.go_cache.outputs.analysis_cache_key }} + restore-keys: | + ${{ steps.go_cache.outputs.analysis_cache_key_int }} + + - name: Vendor Dependencies + run: make vendor vendor.check + + - name: Lint + env: + GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }} + SKIP_LINTER_ANALYSIS: false + RUN_BUILDTAGGER: true + run: make lint diff --git a/Makefile b/Makefile index c506ea4e06..0f7be657d2 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,15 @@ GO_REQUIRED_VERSION ?= 1.21 # Uncomment below if you need to override the version. GOLANGCILINT_VERSION ?= 1.55.2 +RUN_BUILDTAGGER ?= true +# if RUN_BUILDTAGGER is set to "true", we will use build constraints +# and use the buildtagger tool to generate the build tags. +ifeq ($(RUN_BUILDTAGGER),true) +GO_LINT_ARGS ?= -v --build-tags all +BUILDTAGGER_VERSION ?= v0.12.0-rc.0.28.gdc5d6f3 +BUILDTAGGER_DOWNLOAD_URL ?= https://s3.us-west-2.amazonaws.com/upbound.official-providers-ci.releases/main/$(BUILDTAGGER_VERSION)/bin/$(SAFEHOST_PLATFORM)/buildtagger +endif + # SUBPACKAGES ?= $(shell find cmd/provider -type d -maxdepth 1 -mindepth 1 | cut -d/ -f3) SUBPACKAGES ?= monolith GO_STATIC_PACKAGES ?= $(GO_PROJECT)/cmd/generator ${SUBPACKAGES:%=$(GO_PROJECT)/cmd/provider/%} @@ -64,10 +73,10 @@ export SUBPACKAGES := $(SUBPACKAGES) # ==================================================================================== # Setup Kubernetes tools -KIND_VERSION = v0.15.0 +KIND_VERSION = v0.21.0 UP_VERSION = v0.20.0 UP_CHANNEL = stable -UPTEST_VERSION = v0.11.0 +UPTEST_VERSION = v0.11.1 UXP_VERSION = 1.14.6-up.1 export UP_VERSION := $(UP_VERSION) @@ -307,4 +316,33 @@ go.cachedir: go.mod.cachedir: @go env GOMODCACHE -.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) +go.lint.analysiskey-interval: + @# cache is invalidated at least every 7 days + @echo -n golangci-lint.cache-$$(( $$(date +%s) / (7 * 86400) ))- + +go.lint.analysiskey: + @echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1) + + + +.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir go.lint.analysiskey-interval go.lint.analysiskey run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) + +ifeq ($(RUN_BUILDTAGGER),true) +lint.init: build-lint-cache +lint.done: delete-build-tags + +build-lint-cache: $(GOLANGCILINT) + @$(INFO) Running golangci-lint with the analysis cache building phase. + @# we run the initial analysis cache build phase using the relatively + @# smaller API group "account", to keep the memory requirements at a + @# minimum. + @(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \ + (([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \ + [[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags account,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL) + @$(OK) Running golangci-lint with the analysis cache building phase. + +delete-build-tags: + @$(INFO) Untagging source files. + @EXTRA_BUILDTAGGER_ARGS="--delete" RESTORE_DEEPCOPY_TAGS="true" ./scripts/tag.sh || $(FAIL) + @$(OK) Untagging source files. +endif diff --git a/apis/linter_run.go b/apis/linter_run.go new file mode 100644 index 0000000000..e7ffddaf06 --- /dev/null +++ b/apis/linter_run.go @@ -0,0 +1,17 @@ +//go:build linter_run + +// SPDX-FileCopyrightText: 2024 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package apis + +import "k8s.io/apimachinery/pkg/runtime" + +// AddToSchemes may be used to add all resources defined in the project to a Scheme +var AddToSchemes runtime.SchemeBuilder + +// AddToScheme adds all Resources to the Scheme +func AddToScheme(s *runtime.Scheme) error { + panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`) +} diff --git a/config/provider.go b/config/provider.go index 479f9294e7..928adccd8b 100644 --- a/config/provider.go +++ b/config/provider.go @@ -5,21 +5,6 @@ Copyright 2021 Upbound Inc. package config import ( - "context" - "reflect" - "unsafe" - - // Note(ezgidemirel): we are importing this to embed provider schema document - _ "embed" - - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/registry/reference" - conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" - tfjson "github.com/hashicorp/terraform-json" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/hashicorp/terraform-provider-aws/xpprovider" - "github.com/pkg/errors" - "github.com/upbound/provider-aws/config/acm" "github.com/upbound/provider-aws/config/acmpca" "github.com/upbound/provider-aws/config/apigateway" @@ -103,227 +88,93 @@ import ( "github.com/upbound/provider-aws/config/sqs" "github.com/upbound/provider-aws/config/ssoadmin" "github.com/upbound/provider-aws/config/transfer" - "github.com/upbound/provider-aws/hack" -) - -var ( - //go:embed schema.json - providerSchema string - //go:embed provider-metadata.yaml - providerMetadata []byte + // Note(ezgidemirel): we are importing this to embed provider schema document + _ "embed" ) -var skipList = []string{ - "aws_waf_rule_group$", // Too big CRD schema - "aws_wafregional_rule_group$", // Too big CRD schema - "aws_mwaa_environment$", // See https://github.com/crossplane-contrib/terrajet/issues/100 - "aws_ecs_tag$", // tags are already managed by ecs resources. - "aws_alb$", // identical with aws_lb - "aws_alb_target_group_attachment$", // identical with aws_lb_target_group_attachment - "aws_iam_policy_attachment$", // identical with aws_iam_*_policy_attachment resources. - "aws_iam_group_policy$", // identical with aws_iam_*_policy_attachment resources. - "aws_iam_user_policy$", // identical with aws_iam_*_policy_attachment resources. - "aws_alb$", // identical with aws_lb. - "aws_alb_listener$", // identical with aws_lb_listener. - "aws_alb_target_group$", // identical with aws_lb_target_group. - "aws_alb_target_group_attachment$", // identical with aws_lb_target_group_attachment. - "aws_iot_authorizer$", // failure with unknown reason. - "aws_location_map$", // failure with unknown reason. - "aws_appflow_connector_profile$", // failure with unknown reason. - "aws_rds_reserved_instance", // Expense of testing -} - -// workaround for the TF AWS v4.67.0-based no-fork release: We would like to -// keep the types in the generated CRDs intact -// (prevent number->int type replacements). -func getProviderSchema(s string) (*schema.Provider, error) { - ps := tfjson.ProviderSchemas{} - if err := ps.UnmarshalJSON([]byte(s)); err != nil { - panic(err) - } - if len(ps.Schemas) != 1 { - return nil, errors.Errorf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas)) - } - var rs map[string]*tfjson.Schema - for _, v := range ps.Schemas { - rs = v.ResourceSchemas - break - } - return &schema.Provider{ - ResourcesMap: conversiontfjson.GetV2ResourceMap(rs), - }, nil -} - -// GetProvider returns the provider configuration. -// The `generationProvider` argument specifies whether the provider -// configuration is being read for the code generation pipelines. -// In that case, we will only use the JSON schema for generating -// the CRDs. -func GetProvider(ctx context.Context, generationProvider bool) (*config.Provider, *xpprovider.AWSClient, error) { - var p *schema.Provider - var err error - if generationProvider { - p, err = getProviderSchema(providerSchema) - } else { - p, err = xpprovider.GetProviderSchema(ctx) - } - if err != nil { - return nil, nil, errors.Wrapf(err, "cannot get the Terraform provider schema with generation mode set to %t", generationProvider) - } - // we set schema.Provider's meta to nil because p.Configure modifies - // a singleton pointer. This further assumes that the - // schema.Provider.Configure calls do not modify the global state - // represented by the config.Provider.TerraformProvider. - var awsClient *xpprovider.AWSClient - if !generationProvider { - // #nosec G103 - awsClient = (*xpprovider.AWSClient)(unsafe.Pointer(reflect.ValueOf(p.Meta()).Pointer())) - } - p.SetMeta(nil) - modulePath := "github.com/upbound/provider-aws" - pc := config.NewProvider([]byte(providerSchema), "aws", - modulePath, providerMetadata, - config.WithShortName("aws"), - config.WithRootGroup("aws.upbound.io"), - config.WithIncludeList(CLIReconciledResourceList()), - config.WithTerraformPluginSDKIncludeList(NoForkResourceList()), - config.WithReferenceInjectors([]config.ReferenceInjector{reference.NewInjector(modulePath)}), - config.WithSkipList(skipList), - config.WithFeaturesPackage("internal/features"), - config.WithMainTemplate(hack.MainTemplate), - config.WithTerraformProvider(p), - config.WithDefaultResourceOptions( - GroupKindOverrides(), - KindOverrides(), - RegionAddition(), - TagsAllRemoval(), - IdentifierAssignedByAWS(), - KnownReferencers(), - AddExternalTagsField(), - ResourceConfigurator(), - NamePrefixRemoval(), - DocumentationForTags(), - ), - ) - pc.BasePackages.ControllerMap["internal/controller/eks/clusterauth"] = "eks" - - for _, configure := range []func(provider *config.Provider){ - acm.Configure, - acmpca.Configure, - apigateway.Configure, - apigatewayv2.Configure, - apprunner.Configure, - appstream.Configure, - athena.Configure, - autoscaling.Configure, - backup.Configure, - cloudfront.Configure, - cloudsearch.Configure, - cloudwatch.Configure, - cloudwatchlogs.Configure, - cognitoidentity.Configure, - cognitoidp.Configure, - connect.Configure, - cur.Configure, - datasync.Configure, - dax.Configure, - devicefarm.Configure, - dms.Configure, - docdb.Configure, - dynamodb.Configure, - ebs.Configure, - ec2.Configure, - ecr.Configure, - ecrpublic.Configure, - ecs.Configure, - efs.Configure, - eks.Configure, - elasticache.Configure, - elb.Configure, - elbv2.Configure, - firehose.Configure, - gamelift.Configure, - globalaccelerator.Configure, - glue.Configure, - grafana.Configure, - iam.Configure, - kafka.Configure, - kinesis.Configure, - kinesisanalytics.Configure, - kinesisanalytics2.Configure, - kms.Configure, - lakeformation.Configure, - lambda.Configure, - licensemanager.Configure, - memorydb.Configure, - mq.Configure, - neptune.Configure, - networkfirewall.Configure, - opensearch.Configure, - ram.Configure, - rds.Configure, - redshift.Configure, - rolesanywhere.Configure, - route53.Configure, - route53resolver.Configure, - route53recoverycontrolconfig.Configure, - s3.Configure, - secretsmanager.Configure, - servicecatalog.Configure, - organization.Configure, - cloudwatchevents.Configure, - budgets.Configure, - servicediscovery.Configure, - sfn.Configure, - sns.Configure, - sqs.Configure, - transfer.Configure, - directconnect.Configure, - ds.Configure, - qldb.Configure, - fsx.Configure, - networkmanager.Configure, - opsworks.Configure, - sagemaker.Configure, - redshiftserverless.Configure, - kendra.Configure, - medialive.Configure, - ssoadmin.Configure, - identitystore.Configure, - iot.Configure, - } { - configure(pc) - } - - pc.ConfigureResources() - return pc, awsClient, nil -} - -// CLIReconciledResourceList returns the list of resources that have external -// name configured in ExternalNameConfigs table and to be reconciled under -// the TF CLI based architecture. -func CLIReconciledResourceList() []string { - l := make([]string, len(CLIReconciledExternalNameConfigs)) - i := 0 - for name := range CLIReconciledExternalNameConfigs { - // Expected format is regex, and we'd like to have exact matches. - l[i] = name + "$" - i++ - } - return l -} - -// NoForkResourceList returns the list of resources that have external -// name configured in ExternalNameConfigs table and to be reconciled under -// the no-fork architecture. -func NoForkResourceList() []string { - l := make([]string, len(NoForkExternalNameConfigs)) - i := 0 - for name := range NoForkExternalNameConfigs { - // Expected format is regex, and we'd like to have exact matches. - l[i] = name + "$" - i++ - } - return l +func init() { + ProviderConfiguration.AddConfig(acm.Configure) + ProviderConfiguration.AddConfig(acmpca.Configure) + ProviderConfiguration.AddConfig(apigateway.Configure) + ProviderConfiguration.AddConfig(apigatewayv2.Configure) + ProviderConfiguration.AddConfig(apprunner.Configure) + ProviderConfiguration.AddConfig(appstream.Configure) + ProviderConfiguration.AddConfig(athena.Configure) + ProviderConfiguration.AddConfig(autoscaling.Configure) + ProviderConfiguration.AddConfig(backup.Configure) + ProviderConfiguration.AddConfig(cloudfront.Configure) + ProviderConfiguration.AddConfig(cloudsearch.Configure) + ProviderConfiguration.AddConfig(cloudwatch.Configure) + ProviderConfiguration.AddConfig(cloudwatchlogs.Configure) + ProviderConfiguration.AddConfig(cognitoidentity.Configure) + ProviderConfiguration.AddConfig(cognitoidp.Configure) + ProviderConfiguration.AddConfig(connect.Configure) + ProviderConfiguration.AddConfig(cur.Configure) + ProviderConfiguration.AddConfig(datasync.Configure) + ProviderConfiguration.AddConfig(dax.Configure) + ProviderConfiguration.AddConfig(devicefarm.Configure) + ProviderConfiguration.AddConfig(dms.Configure) + ProviderConfiguration.AddConfig(docdb.Configure) + ProviderConfiguration.AddConfig(dynamodb.Configure) + ProviderConfiguration.AddConfig(ebs.Configure) + ProviderConfiguration.AddConfig(ec2.Configure) + ProviderConfiguration.AddConfig(ecr.Configure) + ProviderConfiguration.AddConfig(ecrpublic.Configure) + ProviderConfiguration.AddConfig(ecs.Configure) + ProviderConfiguration.AddConfig(efs.Configure) + ProviderConfiguration.AddConfig(eks.Configure) + ProviderConfiguration.AddConfig(elasticache.Configure) + ProviderConfiguration.AddConfig(elb.Configure) + ProviderConfiguration.AddConfig(elbv2.Configure) + ProviderConfiguration.AddConfig(firehose.Configure) + ProviderConfiguration.AddConfig(gamelift.Configure) + ProviderConfiguration.AddConfig(globalaccelerator.Configure) + ProviderConfiguration.AddConfig(glue.Configure) + ProviderConfiguration.AddConfig(grafana.Configure) + ProviderConfiguration.AddConfig(iam.Configure) + ProviderConfiguration.AddConfig(kafka.Configure) + ProviderConfiguration.AddConfig(kinesis.Configure) + ProviderConfiguration.AddConfig(kinesisanalytics.Configure) + ProviderConfiguration.AddConfig(kinesisanalytics2.Configure) + ProviderConfiguration.AddConfig(kms.Configure) + ProviderConfiguration.AddConfig(lakeformation.Configure) + ProviderConfiguration.AddConfig(lambda.Configure) + ProviderConfiguration.AddConfig(licensemanager.Configure) + ProviderConfiguration.AddConfig(memorydb.Configure) + ProviderConfiguration.AddConfig(mq.Configure) + ProviderConfiguration.AddConfig(neptune.Configure) + ProviderConfiguration.AddConfig(networkfirewall.Configure) + ProviderConfiguration.AddConfig(opensearch.Configure) + ProviderConfiguration.AddConfig(ram.Configure) + ProviderConfiguration.AddConfig(rds.Configure) + ProviderConfiguration.AddConfig(redshift.Configure) + ProviderConfiguration.AddConfig(rolesanywhere.Configure) + ProviderConfiguration.AddConfig(route53.Configure) + ProviderConfiguration.AddConfig(route53resolver.Configure) + ProviderConfiguration.AddConfig(route53recoverycontrolconfig.Configure) + ProviderConfiguration.AddConfig(s3.Configure) + ProviderConfiguration.AddConfig(secretsmanager.Configure) + ProviderConfiguration.AddConfig(servicecatalog.Configure) + ProviderConfiguration.AddConfig(organization.Configure) + ProviderConfiguration.AddConfig(cloudwatchevents.Configure) + ProviderConfiguration.AddConfig(budgets.Configure) + ProviderConfiguration.AddConfig(servicediscovery.Configure) + ProviderConfiguration.AddConfig(sfn.Configure) + ProviderConfiguration.AddConfig(sns.Configure) + ProviderConfiguration.AddConfig(sqs.Configure) + ProviderConfiguration.AddConfig(transfer.Configure) + ProviderConfiguration.AddConfig(directconnect.Configure) + ProviderConfiguration.AddConfig(ds.Configure) + ProviderConfiguration.AddConfig(qldb.Configure) + ProviderConfiguration.AddConfig(fsx.Configure) + ProviderConfiguration.AddConfig(networkmanager.Configure) + ProviderConfiguration.AddConfig(opsworks.Configure) + ProviderConfiguration.AddConfig(sagemaker.Configure) + ProviderConfiguration.AddConfig(redshiftserverless.Configure) + ProviderConfiguration.AddConfig(kendra.Configure) + ProviderConfiguration.AddConfig(medialive.Configure) + ProviderConfiguration.AddConfig(ssoadmin.Configure) + ProviderConfiguration.AddConfig(identitystore.Configure) + ProviderConfiguration.AddConfig(iot.Configure) } diff --git a/config/registry.go b/config/registry.go new file mode 100644 index 0000000000..f66d03a080 --- /dev/null +++ b/config/registry.go @@ -0,0 +1,177 @@ +// SPDX-FileCopyrightText: 2024 The Crossplane Authors +// +// SPDX-License-Identifier: CC0-1.0 + +package config + +import ( + "context" + // Note(ezgidemirel): we are importing this to embed provider schema document + _ "embed" + "reflect" + "unsafe" + + "github.com/crossplane/upjet/pkg/registry/reference" + conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" + tfjson "github.com/hashicorp/terraform-json" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-provider-aws/xpprovider" + "github.com/pkg/errors" + + "github.com/upbound/provider-aws/hack" + + "github.com/crossplane/upjet/pkg/config" +) + +var ( + //go:embed schema.json + providerSchema string + + //go:embed provider-metadata.yaml + providerMetadata []byte +) + +var skipList = []string{ + "aws_waf_rule_group$", // Too big CRD schema + "aws_wafregional_rule_group$", // Too big CRD schema + "aws_mwaa_environment$", // See https://github.com/crossplane-contrib/terrajet/issues/100 + "aws_ecs_tag$", // tags are already managed by ecs resources. + "aws_alb$", // identical with aws_lb + "aws_alb_target_group_attachment$", // identical with aws_lb_target_group_attachment + "aws_iam_policy_attachment$", // identical with aws_iam_*_policy_attachment resources. + "aws_iam_group_policy$", // identical with aws_iam_*_policy_attachment resources. + "aws_iam_user_policy$", // identical with aws_iam_*_policy_attachment resources. + "aws_alb$", // identical with aws_lb. + "aws_alb_listener$", // identical with aws_lb_listener. + "aws_alb_target_group$", // identical with aws_lb_target_group. + "aws_alb_target_group_attachment$", // identical with aws_lb_target_group_attachment. + "aws_iot_authorizer$", // failure with unknown reason. + "aws_location_map$", // failure with unknown reason. + "aws_appflow_connector_profile$", // failure with unknown reason. + "aws_rds_reserved_instance", // Expense of testing +} + +// workaround for the TF AWS v4.67.0-based no-fork release: We would like to +// keep the types in the generated CRDs intact +// (prevent number->int type replacements). +func getProviderSchema(s string) (*schema.Provider, error) { + ps := tfjson.ProviderSchemas{} + if err := ps.UnmarshalJSON([]byte(s)); err != nil { + panic(err) + } + if len(ps.Schemas) != 1 { + return nil, errors.Errorf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas)) + } + var rs map[string]*tfjson.Schema + for _, v := range ps.Schemas { + rs = v.ResourceSchemas + break + } + return &schema.Provider{ + ResourcesMap: conversiontfjson.GetV2ResourceMap(rs), + }, nil +} + +// GetProvider returns the provider configuration. +// The `generationProvider` argument specifies whether the provider +// configuration is being read for the code generation pipelines. +// In that case, we will only use the JSON schema for generating +// the CRDs. +func GetProvider(ctx context.Context, generationProvider bool) (*config.Provider, *xpprovider.AWSClient, error) { + var p *schema.Provider + var err error + if generationProvider { + p, err = getProviderSchema(providerSchema) + } else { + p, err = xpprovider.GetProviderSchema(ctx) + } + if err != nil { + return nil, nil, errors.Wrapf(err, "cannot get the Terraform provider schema with generation mode set to %t", generationProvider) + } + // we set schema.Provider's meta to nil because p.Configure modifies + // a singleton pointer. This further assumes that the + // schema.Provider.Configure calls do not modify the global state + // represented by the config.Provider.TerraformProvider. + var awsClient *xpprovider.AWSClient + if !generationProvider { + // #nosec G103 + awsClient = (*xpprovider.AWSClient)(unsafe.Pointer(reflect.ValueOf(p.Meta()).Pointer())) + } + p.SetMeta(nil) + modulePath := "github.com/upbound/provider-aws" + pc := config.NewProvider([]byte(providerSchema), "aws", + modulePath, providerMetadata, + config.WithShortName("aws"), + config.WithRootGroup("aws.upbound.io"), + config.WithIncludeList(CLIReconciledResourceList()), + config.WithTerraformPluginSDKIncludeList(NoForkResourceList()), + config.WithReferenceInjectors([]config.ReferenceInjector{reference.NewInjector(modulePath)}), + config.WithSkipList(skipList), + config.WithFeaturesPackage("internal/features"), + config.WithMainTemplate(hack.MainTemplate), + config.WithTerraformProvider(p), + config.WithDefaultResourceOptions( + GroupKindOverrides(), + KindOverrides(), + RegionAddition(), + TagsAllRemoval(), + IdentifierAssignedByAWS(), + KnownReferencers(), + AddExternalTagsField(), + ResourceConfigurator(), + NamePrefixRemoval(), + DocumentationForTags(), + ), + ) + pc.BasePackages.ControllerMap["internal/controller/eks/clusterauth"] = "eks" + + for _, configure := range ProviderConfiguration { + configure(pc) + } + + pc.ConfigureResources() + return pc, awsClient, nil +} + +// CLIReconciledResourceList returns the list of resources that have external +// name configured in ExternalNameConfigs table and to be reconciled under +// the TF CLI based architecture. +func CLIReconciledResourceList() []string { + l := make([]string, len(CLIReconciledExternalNameConfigs)) + i := 0 + for name := range CLIReconciledExternalNameConfigs { + // Expected format is regex, and we'd like to have exact matches. + l[i] = name + "$" + i++ + } + return l +} + +// NoForkResourceList returns the list of resources that have external +// name configured in ExternalNameConfigs table and to be reconciled under +// the no-fork architecture. +func NoForkResourceList() []string { + l := make([]string, len(NoForkExternalNameConfigs)) + i := 0 + for name := range NoForkExternalNameConfigs { + // Expected format is regex, and we'd like to have exact matches. + l[i] = name + "$" + i++ + } + return l +} + +// Configure configures the specified Provider. +type Configure func(provider *config.Provider) + +// Configurator is a registry for provider Configs. +type Configurator []Configure + +// AddConfig adds a Config to the Configurator registry. +func (c *Configurator) AddConfig(conf Configure) { + *c = append(*c, conf) +} + +// ProviderConfiguration is a global registry to be used by +// the resource providers to register their Config functions. +var ProviderConfiguration = Configurator{} diff --git a/scripts/tag.sh b/scripts/tag.sh new file mode 100755 index 0000000000..fb588ba289 --- /dev/null +++ b/scripts/tag.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +set -aeuo pipefail + +TAGGER="/tmp/buildtagger" +REPO_ROOT="." +EXTRA_BUILDTAGGER_ARGS="${EXTRA_BUILDTAGGER_ARGS:-}" +RESTORE_DEEPCOPY_TAGS="${RESTORE_DEEPCOPY_TAGS:-false}" + +if [[ ! -f "${TAGGER}" ]]; then + curl -fsSL "${BUILDTAGGER_DOWNLOAD_URL}" -o "${TAGGER}" + chmod +x "${TAGGER}" +fi + +"${TAGGER}" --parent-dir "${REPO_ROOT}"/apis --regex "(.+)/.+/.+\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/apis --regex "(.+)/.+/zz_groupversion_info\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/internal/controller --regex "(.+)/.+/zz_controller\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/internal/controller --regex "zz_(.+)_setup\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode file ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/cmd/provider --regex "(.+)/zz_main\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config --regex "(.+)/config\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} + +# constant tags +# apis/zz_register.go -> (apiregister || register || all) && !ignore_autogenerated +"${TAGGER}" --parent-dir "${REPO_ROOT}"/apis/zz_register.go --tag-format "all && !ignore_autogenerated" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# cmd/generator/main.go -> config || generate || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/cmd/generator/main.go --tag-format "all" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/rds/config_test.go -> (rds || all) && !ignore_autogenerated +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/rds/config_test.go --tag-format "(rds || all) && !ignore_autogenerated" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/config_registry.go -> configregistry || register || config || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/registry.go --tag-format "configregistry || all" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/provider.go -> configprovider || register || config || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/provider.go --tag-format "(configprovider || all) && !linter_run" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/overrides.go -> configprovider || register || config || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/overrides.go --tag-format "configprovider || all" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# internal/controller/eks/clusterauth/controller.go -> eks || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/internal/controller/eks/clusterauth/controller.go --tag-format "eks || all" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# internal/controller/eks/clusterauth/eks.go -> eks || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/internal/controller/eks/clusterauth/eks.go --tag-format "eks || all" --mode file ${EXTRA_BUILDTAGGER_ARGS} + +if [[ "${RESTORE_DEEPCOPY_TAGS}" == "true" ]]; then + "${TAGGER}" --parent-dir "${REPO_ROOT}"/apis --regex "zz_generated.deepcopy.go" --tag-format "!ignore_autogenerated" --mode file +fi \ No newline at end of file From cf4d34c59da597aefb7001fa60bea92893488a56 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Fri, 15 Mar 2024 15:12:17 +0300 Subject: [PATCH 2/2] Limit linter runner concurrency to 1 - Set GOGC to 50% for both lint phases - Set linter runner timeout to 90min Signed-off-by: Alper Rifat Ulucinar --- .github/workflows/ci.yml | 1 + .golangci.yml | 12 ++++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a432228f71..9f12d17b65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,4 +99,5 @@ jobs: GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }} SKIP_LINTER_ANALYSIS: false RUN_BUILDTAGGER: true + GOGC: "50" run: make lint diff --git a/.golangci.yml b/.golangci.yml index bf020bf06c..ea12d9929e 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,12 +1,16 @@ -run: - deadline: 20m +# SPDX-FileCopyrightText: 2024 The Crossplane Authors +# +# SPDX-License-Identifier: CC0-1.0 - skip-files: - - "zz_\\..+\\.go$" +run: + timeout: 90m + show-stats: true + concurrency: 1 output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" format: colored-line-number + print-linter-name: true linters-settings: errcheck: