From d8ecc6518a37a44fb7d7c8fa5b37a48bae524485 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Tue, 2 Apr 2024 15:05:08 +0300 Subject: [PATCH] Decrease linter's memory usage - Limit linter runner concurrency to 1 - Set GOGC to 50% for both lint phases - Set linter runner timeout to 90min Signed-off-by: Alper Rifat Ulucinar --- .github/workflows/ci.yml | 84 +++++++++++ .golangci.yml | 12 +- Makefile | 44 +++++- apis/linter_run.go | 17 +++ config/provider.go | 300 +++++++-------------------------------- config/registry.go | 224 +++++++++++++++++++++++++++++ scripts/tag.sh | 40 ++++++ 7 files changed, 463 insertions(+), 258 deletions(-) create mode 100644 apis/linter_run.go create mode 100644 config/registry.go create mode 100755 scripts/tag.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 673f42e1f..1b67ff628 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,91 @@ jobs: uses: upbound/uptest/.github/workflows/provider-ci.yml@standard-runners with: go-version: 1.21 + golangci-skip: true # we will run the linter via "make lint" cleanup-disk: true secrets: UPBOUND_MARKETPLACE_PUSH_ROBOT_USR: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_USR }} UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW: ${{ secrets.UPBOUND_MARKETPLACE_PUSH_ROBOT_PSW }} + + detect-noop: + runs-on: ubuntu-22.04 + outputs: + noop: ${{ steps.noop.outputs.should_skip }} + steps: + - name: Detect No-op Changes + id: noop + uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v5.3.0 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + paths_ignore: '["**.md", "**.png", "**.jpg"]' + do_not_skip: '["workflow_dispatch", "schedule", "push"]' + + lint: + runs-on: ubuntu-22.04 + needs: detect-noop + if: needs.detect-noop.outputs.noop != 'true' + + steps: + - name: Cleanup Disk + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + android: true + dotnet: true + haskell: true + tool-cache: true + large-packages: false + swap-storage: false + + - name: Checkout + uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4 + with: + submodules: true + + - name: Setup Go + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3 + with: + go-version: "1.21" + + - name: Find the Go Build Cache + id: go_cache + run: | + echo "cache=$(make go.cachedir)" >> $GITHUB_OUTPUT && \ + echo "mod_cache=$(make go.mod.cachedir)" >> $GITHUB_OUTPUT && \ + echo "analysis_cache=$HOME/.cache/golangci-lint" >> $GITHUB_OUTPUT && \ + echo "analysis_cache_key=$(make go.lint.analysiskey)" >> $GITHUB_OUTPUT && \ + echo "analysis_cache_key_int=$(make go.lint.analysiskey-interval)" >> $GITHUB_OUTPUT + + + - name: Cache the Go Build Cache + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 + with: + path: ${{ steps.go_cache.outputs.cache }} + key: ${{ runner.os }}-build-lint-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-build-lint- + + - name: Cache Go Dependencies + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 + with: + path: ${{ steps.go_cache.outputs.mod_cache }} + key: ${{ runner.os }}-pkg-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-pkg- + + - name: Cache Linter Analysis + uses: actions/cache@704facf57e6136b1bc63b828d79edcd491f0ee84 # v3 + id: cache-analysis + with: + path: ${{ steps.go_cache.outputs.analysis_cache }} + key: ${{ steps.go_cache.outputs.analysis_cache_key }} + restore-keys: | + ${{ steps.go_cache.outputs.analysis_cache_key_int }} + + - name: Vendor Dependencies + run: make vendor vendor.check + + - name: Lint + env: + GOLANGCI_LINT_CACHE: ${{ steps.go_cache.outputs.analysis_cache }} + SKIP_LINTER_ANALYSIS: false + RUN_BUILDTAGGER: true + GOGC: "50" + run: make lint diff --git a/.golangci.yml b/.golangci.yml index f0fcbf005..00ff91fee 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -3,14 +3,13 @@ # SPDX-License-Identifier: CC0-1.0 run: - deadline: 20m - - skip-files: - - "zz_.*go$" + timeout: 90m + concurrency: 1 output: # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" format: colored-line-number + print-linter-name: true linters-settings: errcheck: @@ -131,6 +130,11 @@ linters: issues: # Excluding configuration per-path and per-linter exclude-rules: + # some group names we have (like "hdinsight") make this linter + # unhappy, so just disable the "misspell" linter on generated files. + - path: zz_.+\.go$ + linters: + - misspell # Exclude some linters from running on tests files. - path: _test(ing)?\.go linters: diff --git a/Makefile b/Makefile index 74bde9d98..7c1012adf 100644 --- a/Makefile +++ b/Makefile @@ -51,7 +51,16 @@ export GOPRIVATE = github.com/upbound/* GO_REQUIRED_VERSION ?= 1.21 # GOLANGCILINT_VERSION is inherited from build submodule by default. # Uncomment below if you need to override the version. -# GOLANGCILINT_VERSION ?= 1.54.0 +GOLANGCILINT_VERSION ?= 1.55.2 + +RUN_BUILDTAGGER ?= true +# if RUN_BUILDTAGGER is set to "true", we will use build constraints +# and use the buildtagger tool to generate the build tags. +ifeq ($(RUN_BUILDTAGGER),true) +GO_LINT_ARGS ?= -v --build-tags all +BUILDTAGGER_VERSION ?= v0.12.0-rc.0.28.gdc5d6f3 +BUILDTAGGER_DOWNLOAD_URL ?= https://s3.us-west-2.amazonaws.com/upbound.official-providers-ci.releases/main/$(BUILDTAGGER_VERSION)/bin/$(SAFEHOST_PLATFORM)/buildtagger +endif # SUBPACKAGES ?= $(shell find cmd/provider -type d -maxdepth 1 -mindepth 1 | cut -d/ -f3) SUBPACKAGES ?= monolith @@ -67,7 +76,7 @@ export SUBPACKAGES := $(SUBPACKAGES) # ==================================================================================== # Setup Kubernetes tools -KIND_VERSION = v0.15.0 +KIND_VERSION = v0.21.0 UP_VERSION = v0.20.0 UP_CHANNEL = stable UPTEST_VERSION = v0.11.1 @@ -304,7 +313,14 @@ go.cachedir: go.mod.cachedir: @go env GOMODCACHE -.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) +go.lint.analysiskey-interval: + @# cache is invalidated at least every 7 days + @echo -n golangci-lint.cache-$$(( $$(date +%s) / (7 * 86400) ))- + +go.lint.analysiskey: + @echo $$(make go.lint.analysiskey-interval)$$(sha1sum go.sum | cut -d' ' -f1) + +.PHONY: cobertura reviewable submodules fallthrough go.mod.cachedir go.cachedir go.lint.analysiskey-interval go.lint.analysiskey run crds.clean $(TERRAFORM_PROVIDER_SCHEMA) build.init: kustomize-crds @@ -318,4 +334,24 @@ kustomize-crds: output.init $(KUSTOMIZE) $(YQ) XDG_CONFIG_HOME=$(PWD)/package $(KUSTOMIZE) build --enable-alpha-plugins $(OUTPUT_DIR)/package/kustomize -o $(OUTPUT_DIR)/package/crds.yaml || $(FAIL) @$(OK) Kustomizing CRDs. -.PHONY: kustomize-crds \ No newline at end of file +.PHONY: kustomize-crds + +ifeq ($(RUN_BUILDTAGGER),true) +lint.init: build-lint-cache +lint.done: delete-build-tags + +build-lint-cache: $(GOLANGCILINT) + @$(INFO) Running golangci-lint with the analysis cache building phase. + @# we run the initial analysis cache build phase using the relatively + @# smaller API group "account", to keep the memory requirements at a + @# minimum. + @(BUILDTAGGER_DOWNLOAD_URL=$(BUILDTAGGER_DOWNLOAD_URL) ./scripts/tag.sh && \ + (([[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] && $(OK) "Skipping analysis cache build phase because it's already been populated") && \ + [[ "${SKIP_LINTER_ANALYSIS}" == "true" ]] || $(GOLANGCILINT) run -v --build-tags analysisservices,configregistry,configprovider,linter_run -v --disable-all --exclude '.*')) || $(FAIL) + @$(OK) Running golangci-lint with the analysis cache building phase. + +delete-build-tags: + @$(INFO) Untagging source files. + @EXTRA_BUILDTAGGER_ARGS="--delete" RESTORE_DEEPCOPY_TAGS="true" ./scripts/tag.sh || $(FAIL) + @$(OK) Untagging source files. +endif \ No newline at end of file diff --git a/apis/linter_run.go b/apis/linter_run.go new file mode 100644 index 000000000..e7ffddaf0 --- /dev/null +++ b/apis/linter_run.go @@ -0,0 +1,17 @@ +//go:build linter_run + +// SPDX-FileCopyrightText: 2024 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package apis + +import "k8s.io/apimachinery/pkg/runtime" + +// AddToSchemes may be used to add all resources defined in the project to a Scheme +var AddToSchemes runtime.SchemeBuilder + +// AddToScheme adds all Resources to the Scheme +func AddToScheme(s *runtime.Scheme) error { + panic(`Must not be called in provider runtime. The provider should not have been built with the "linter_run" build constraint.`) +} diff --git a/config/provider.go b/config/provider.go index 1ebe8f876..2c1edb77a 100644 --- a/config/provider.go +++ b/config/provider.go @@ -5,19 +5,6 @@ package config import ( - // Note(ezgidemirel): we are importing this to embed provider schema document - "context" - _ "embed" - - conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" - tfjson "github.com/hashicorp/terraform-json" - "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" - "github.com/hashicorp/terraform-provider-azurerm/xpprovider" - "github.com/pkg/errors" - - tjconfig "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/registry/reference" - "github.com/upbound/provider-azure/config/alertsmanagement" "github.com/upbound/provider-azure/config/apimanagement" "github.com/upbound/provider-azure/config/appplatform" @@ -28,7 +15,6 @@ import ( "github.com/upbound/provider-azure/config/cache" "github.com/upbound/provider-azure/config/cdn" "github.com/upbound/provider-azure/config/certificateregistration" - "github.com/upbound/provider-azure/config/common" "github.com/upbound/provider-azure/config/compute" "github.com/upbound/provider-azure/config/consumption" "github.com/upbound/provider-azure/config/containerapp" @@ -68,242 +54,56 @@ import ( "github.com/upbound/provider-azure/config/storagesync" "github.com/upbound/provider-azure/config/streamanalytics" "github.com/upbound/provider-azure/config/web" - "github.com/upbound/provider-azure/hack" ) -const ( - resourcePrefix = "azure" - modulePath = "github.com/upbound/provider-azure" -) - -var ( - //go:embed schema.json - providerSchema string - - //go:embed provider-metadata.yaml - providerMetadata []byte -) - -// These resources cannot be generated because of their suffixes colliding with -// kubebuilder-accepted type suffixes. -var skipList = []string{ - "azurerm_mssql_server_extended_auditing_policy", - // group prefix collision - "azurerm_api_management_group", - "azurerm_api_management_product_group", - "azurerm_dedicated_host_group", - "azurerm_storage_disks_pool", - "azurerm_storage_sync_group", - "azurerm_storage_sync_cloud_endpoint", // depends on azurerm_storage_sync_group - "azurerm_virtual_desktop_application_group", - // associated with non-generated - "azurerm_virtual_desktop_workspace_application_group_association", - // generated name too long - "azurerm_network_interface_application_gateway_backend_address_pool_association", - "azurerm_sentinel_data_connector_microsoft_defender_advanced_threat_protection", - // deprecated - "azurerm_virtual_machine_scale_set", - "azurerm_virtual_machine_configuration_policy_assignment", - "azurerm_virtual_machine_scale_set_extension", - "azurerm_sql_server", - "azurerm_video_analyzer", - "azurerm_video_analyzer_edge_module", - "azurerm_sql_managed_database", - "azurerm_sql_managed_instance", - "azurerm_sql_managed_instance_active_directory_administrator", - "azurerm_sql_managed_instance_failover_group", - "azurerm_sql_active_directory_administrator", - "azurerm_sql_database", - "azurerm_sql_elasticpool", - "azurerm_sql_firewall_rule", - "azurerm_site_recovery_replicated_vm", // depends on azurerm_virtual_machine - // irrelevant - "azurerm_virtual_desktop_application", - "azurerm_virtual_desktop_host_pool", - "azurerm_virtual_desktop_workspace", - "azurerm_virtual_desktop_scaling_plan", // depends on azurerm_virtual_desktop_host_pool - "azurerm_virtual_desktop_host_pool_registration_info", // depends on azurerm_virtual_desktop_host_pool - // other upjet issues - "azurerm_container_registry_task", - "azurerm_dashboard", - // doc not found in Terraform Azurerm provider - "azurerm_virtual_network_dns_servers", - // unsupported sensitive field type - "azurerm_security_center_automation", - "azurerm_data_factory_trigger_tumbling_window", - "azurerm_storage_share_file", - "azurerm_sql_virtual_network_rule", - "azurerm_virtual_desktop_workspace", - "azurerm_data_lake_analytics_account", - "azurerm_log_analytics_storage_insights", - "azurerm_virtual_hub_bgp_connection", - "azurerm_automation_dsc_configuration", - "azurerm_automation_dsc_nodeconfiguration", // depends on azurerm_automation_dsc_configuration - "azurerm_monitor_log_profile", - "azurerm_machine_learning_inference_cluster", - "azurerm_sql_failover_group", - "azurerm_logic_app_integration_account_certificate", - "azurerm_container_group", - // Azure are officially halting the preview of Azure Disk Pools, and it will not be made generally available. - "azurerm_disk_pool_iscsi_target", - // DEPRECATED: azurerm_service_plan should be used instead - "azurerm_app_service_source_control_token", - // DEPRECATED: - "azurerm_cdn_frontdoor_route_disable_link_to_default_domain", -} - -// workaround for the TF Azure v3.57.0-based no-fork release: We would like to -// keep the types in the generated CRDs intact -// (prevent number->int type replacements). -func getProviderSchema(s string) (*schema.Provider, error) { - ps := tfjson.ProviderSchemas{} - if err := ps.UnmarshalJSON([]byte(s)); err != nil { - panic(err) - } - if len(ps.Schemas) != 1 { - return nil, errors.Errorf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas)) - } - var rs map[string]*tfjson.Schema - for _, v := range ps.Schemas { - rs = v.ResourceSchemas - break - } - return &schema.Provider{ - ResourcesMap: conversiontfjson.GetV2ResourceMap(rs), - }, nil -} - -// GetProvider returns provider configuration -func GetProvider(ctx context.Context, generationProvider bool) (*tjconfig.Provider, error) { - var p *schema.Provider - var err error - if generationProvider { - p, err = getProviderSchema(providerSchema) - } else { - p, err = xpprovider.GetProviderSchema(ctx) - } - if err != nil { - return nil, errors.Wrapf(err, "cannot get the Terraform provider schema with generation mode set to %t", generationProvider) - } - - pc := tjconfig.NewProvider([]byte(providerSchema), resourcePrefix, modulePath, providerMetadata, - tjconfig.WithShortName("azure"), - tjconfig.WithRootGroup("azure.upbound.io"), - tjconfig.WithIncludeList(CLIReconciledResourceList()), - tjconfig.WithTerraformPluginSDKIncludeList(TerraformPluginSDKResourceList()), - tjconfig.WithSkipList(skipList), - tjconfig.WithDefaultResourceOptions(ResourceConfigurator()), - tjconfig.WithReferenceInjectors([]tjconfig.ReferenceInjector{reference.NewInjector(modulePath)}), - tjconfig.WithFeaturesPackage("internal/features"), - tjconfig.WithMainTemplate(hack.MainTemplate), - tjconfig.WithTerraformProvider(p), - ) - // "azure" group contains resources that actually do not have a specific - // group, e.g. ResourceGroup with APIVersion "azure.upbound.io/v1beta1". - // We need to include the controllers for this group into the base packages - // list to get their controllers packaged together with the config package - // controllers (provider family config package). - for _, c := range []string{"internal/controller/azure/resourcegroup", "internal/controller/azure/resourceproviderregistration", "internal/controller/azure/subscription"} { - pc.BasePackages.ControllerMap[c] = "config" - } - - // API group overrides from Terraform import statements - for _, r := range pc.Resources { - groupKindOverride(r) - } - - for _, configure := range []func(provider *tjconfig.Provider){ - // add custom config functions - authorization.Configure, - alertsmanagement.Configure, - network.Configure, - management.Configure, - media.Configure, - cache.Configure, - resource.Configure, - resources.Configure, - containerapp.Configure, - containerservice.Configure, - postgresql.Configure, - cosmosdb.Configure, - sql.Configure, - storage.Configure, - operationalinsights.Configure, - insights.Configure, - devices.Configure, - datafactory.Configure, - apimanagement.Configure, - healthcareapis.Configure, - logic.Configure, - security.Configure, - base.Configure, - botservice.Configure, - datashare.Configure, - notificationhubs.Configure, - storagesync.Configure, - keyvault.Configure, - eventhub.Configure, - mariadb.Configure, - compute.Configure, - containerregistry.Configure, - dbformysql.Configure, - netapp.Configure, - dataprotection.Configure, - kusto.Configure, - storagecache.Configure, - servicebus.Configure, - consumption.Configure, - streamanalytics.Configure, - costmanagement.Configure, - automation.Configure, - web.Configure, - relay.Configure, - cdn.Configure, - certificateregistration.Configure, - orbital.Configure, - appplatform.Configure, - databricks.Configure, - } { - configure(pc) - } - - pc.ConfigureResources() - - // This function runs after the special configurations were applied. However, if some references were configured in - // the configuration files, the reference generator does not override them. - for _, r := range pc.Resources { - delete(r.References, "resource_group_name") - if err := common.AddCommonReferences(r); err != nil { - panic(err) - } - } - return pc, nil -} - -// CLIReconciledResourceList returns the list of resources that have external -// name configured in ExternalNameConfigs table and to be reconciled under -// the TF CLI based architecture. -func CLIReconciledResourceList() []string { - l := make([]string, len(CLIReconciledExternalNameConfigs)) - i := 0 - for name := range CLIReconciledExternalNameConfigs { - // Expected format is regex and we'd like to have exact matches. - l[i] = name + "$" - i++ - } - return l -} - -// TerraformPluginSDKResourceList returns the list of resources that have external -// name configured in ExternalNameConfigs table and to be reconciled under -// the no-fork architecture. -func TerraformPluginSDKResourceList() []string { - l := make([]string, len(TerraformPluginSDKExternalNameConfigs)) - i := 0 - for name := range TerraformPluginSDKExternalNameConfigs { - // Expected format is regex and we'd like to have exact matches. - l[i] = name + "$" - i++ - } - return l +func init() { + ProviderConfiguration.AddConfig(authorization.Configure) + ProviderConfiguration.AddConfig(alertsmanagement.Configure) + ProviderConfiguration.AddConfig(network.Configure) + ProviderConfiguration.AddConfig(management.Configure) + ProviderConfiguration.AddConfig(media.Configure) + ProviderConfiguration.AddConfig(cache.Configure) + ProviderConfiguration.AddConfig(resource.Configure) + ProviderConfiguration.AddConfig(resources.Configure) + ProviderConfiguration.AddConfig(containerapp.Configure) + ProviderConfiguration.AddConfig(containerservice.Configure) + ProviderConfiguration.AddConfig(postgresql.Configure) + ProviderConfiguration.AddConfig(cosmosdb.Configure) + ProviderConfiguration.AddConfig(sql.Configure) + ProviderConfiguration.AddConfig(storage.Configure) + ProviderConfiguration.AddConfig(operationalinsights.Configure) + ProviderConfiguration.AddConfig(insights.Configure) + ProviderConfiguration.AddConfig(devices.Configure) + ProviderConfiguration.AddConfig(datafactory.Configure) + ProviderConfiguration.AddConfig(apimanagement.Configure) + ProviderConfiguration.AddConfig(healthcareapis.Configure) + ProviderConfiguration.AddConfig(logic.Configure) + ProviderConfiguration.AddConfig(security.Configure) + ProviderConfiguration.AddConfig(base.Configure) + ProviderConfiguration.AddConfig(botservice.Configure) + ProviderConfiguration.AddConfig(datashare.Configure) + ProviderConfiguration.AddConfig(notificationhubs.Configure) + ProviderConfiguration.AddConfig(storagesync.Configure) + ProviderConfiguration.AddConfig(keyvault.Configure) + ProviderConfiguration.AddConfig(eventhub.Configure) + ProviderConfiguration.AddConfig(mariadb.Configure) + ProviderConfiguration.AddConfig(compute.Configure) + ProviderConfiguration.AddConfig(containerregistry.Configure) + ProviderConfiguration.AddConfig(dbformysql.Configure) + ProviderConfiguration.AddConfig(netapp.Configure) + ProviderConfiguration.AddConfig(dataprotection.Configure) + ProviderConfiguration.AddConfig(kusto.Configure) + ProviderConfiguration.AddConfig(storagecache.Configure) + ProviderConfiguration.AddConfig(servicebus.Configure) + ProviderConfiguration.AddConfig(consumption.Configure) + ProviderConfiguration.AddConfig(streamanalytics.Configure) + ProviderConfiguration.AddConfig(costmanagement.Configure) + ProviderConfiguration.AddConfig(automation.Configure) + ProviderConfiguration.AddConfig(web.Configure) + ProviderConfiguration.AddConfig(relay.Configure) + ProviderConfiguration.AddConfig(cdn.Configure) + ProviderConfiguration.AddConfig(certificateregistration.Configure) + ProviderConfiguration.AddConfig(orbital.Configure) + ProviderConfiguration.AddConfig(appplatform.Configure) + ProviderConfiguration.AddConfig(databricks.Configure) } diff --git a/config/registry.go b/config/registry.go new file mode 100644 index 000000000..374518cd6 --- /dev/null +++ b/config/registry.go @@ -0,0 +1,224 @@ +// SPDX-FileCopyrightText: 2024 The Crossplane Authors +// +// SPDX-License-Identifier: CC0-1.0 + +package config + +import ( + "context" + _ "embed" + + "github.com/crossplane/upjet/pkg/config" + tjconfig "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/registry/reference" + conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" + tfjson "github.com/hashicorp/terraform-json" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-provider-azurerm/xpprovider" + "github.com/pkg/errors" + + "github.com/upbound/provider-azure/config/common" + "github.com/upbound/provider-azure/hack" +) + +const ( + resourcePrefix = "azure" + modulePath = "github.com/upbound/provider-azure" +) + +var ( + //go:embed schema.json + providerSchema string + + //go:embed provider-metadata.yaml + providerMetadata []byte +) + +// These resources cannot be generated because of their suffixes colliding with +// kubebuilder-accepted type suffixes. +var skipList = []string{ + "azurerm_mssql_server_extended_auditing_policy", + // group prefix collision + "azurerm_api_management_group", + "azurerm_api_management_product_group", + "azurerm_dedicated_host_group", + "azurerm_storage_disks_pool", + "azurerm_storage_sync_group", + "azurerm_storage_sync_cloud_endpoint", // depends on azurerm_storage_sync_group + "azurerm_virtual_desktop_application_group", + // associated with non-generated + "azurerm_virtual_desktop_workspace_application_group_association", + // generated name too long + "azurerm_network_interface_application_gateway_backend_address_pool_association", + "azurerm_sentinel_data_connector_microsoft_defender_advanced_threat_protection", + // deprecated + "azurerm_virtual_machine_scale_set", + "azurerm_virtual_machine_configuration_policy_assignment", + "azurerm_virtual_machine_scale_set_extension", + "azurerm_sql_server", + "azurerm_video_analyzer", + "azurerm_video_analyzer_edge_module", + "azurerm_sql_managed_database", + "azurerm_sql_managed_instance", + "azurerm_sql_managed_instance_active_directory_administrator", + "azurerm_sql_managed_instance_failover_group", + "azurerm_sql_active_directory_administrator", + "azurerm_sql_database", + "azurerm_sql_elasticpool", + "azurerm_sql_firewall_rule", + "azurerm_site_recovery_replicated_vm", // depends on azurerm_virtual_machine + // irrelevant + "azurerm_virtual_desktop_application", + "azurerm_virtual_desktop_host_pool", + "azurerm_virtual_desktop_workspace", + "azurerm_virtual_desktop_scaling_plan", // depends on azurerm_virtual_desktop_host_pool + "azurerm_virtual_desktop_host_pool_registration_info", // depends on azurerm_virtual_desktop_host_pool + // other upjet issues + "azurerm_container_registry_task", + "azurerm_dashboard", + // doc not found in Terraform Azurerm provider + "azurerm_virtual_network_dns_servers", + // unsupported sensitive field type + "azurerm_security_center_automation", + "azurerm_data_factory_trigger_tumbling_window", + "azurerm_storage_share_file", + "azurerm_sql_virtual_network_rule", + "azurerm_virtual_desktop_workspace", + "azurerm_data_lake_analytics_account", + "azurerm_log_analytics_storage_insights", + "azurerm_virtual_hub_bgp_connection", + "azurerm_automation_dsc_configuration", + "azurerm_automation_dsc_nodeconfiguration", // depends on azurerm_automation_dsc_configuration + "azurerm_monitor_log_profile", + "azurerm_machine_learning_inference_cluster", + "azurerm_sql_failover_group", + "azurerm_logic_app_integration_account_certificate", + "azurerm_container_group", + // Azure are officially halting the preview of Azure Disk Pools, and it will not be made generally available. + "azurerm_disk_pool_iscsi_target", + // DEPRECATED: azurerm_service_plan should be used instead + "azurerm_app_service_source_control_token", + // DEPRECATED: + "azurerm_cdn_frontdoor_route_disable_link_to_default_domain", +} + +// workaround for the TF Azure v3.57.0-based no-fork release: We would like to +// keep the types in the generated CRDs intact +// (prevent number->int type replacements). +func getProviderSchema(s string) (*schema.Provider, error) { + ps := tfjson.ProviderSchemas{} + if err := ps.UnmarshalJSON([]byte(s)); err != nil { + panic(err) + } + if len(ps.Schemas) != 1 { + return nil, errors.Errorf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas)) + } + var rs map[string]*tfjson.Schema + for _, v := range ps.Schemas { + rs = v.ResourceSchemas + break + } + return &schema.Provider{ + ResourcesMap: conversiontfjson.GetV2ResourceMap(rs), + }, nil +} + +// GetProvider returns provider configuration +func GetProvider(ctx context.Context, generationProvider bool) (*tjconfig.Provider, error) { + var p *schema.Provider + var err error + if generationProvider { + p, err = getProviderSchema(providerSchema) + } else { + p, err = xpprovider.GetProviderSchema(ctx) + } + if err != nil { + return nil, errors.Wrapf(err, "cannot get the Terraform provider schema with generation mode set to %t", generationProvider) + } + + pc := tjconfig.NewProvider([]byte(providerSchema), resourcePrefix, modulePath, providerMetadata, + tjconfig.WithShortName("azure"), + tjconfig.WithRootGroup("azure.upbound.io"), + tjconfig.WithIncludeList(CLIReconciledResourceList()), + tjconfig.WithTerraformPluginSDKIncludeList(TerraformPluginSDKResourceList()), + tjconfig.WithSkipList(skipList), + tjconfig.WithDefaultResourceOptions(ResourceConfigurator()), + tjconfig.WithReferenceInjectors([]tjconfig.ReferenceInjector{reference.NewInjector(modulePath)}), + tjconfig.WithFeaturesPackage("internal/features"), + tjconfig.WithMainTemplate(hack.MainTemplate), + tjconfig.WithTerraformProvider(p), + ) + // "azure" group contains resources that actually do not have a specific + // group, e.g. ResourceGroup with APIVersion "azure.upbound.io/v1beta1". + // We need to include the controllers for this group into the base packages + // list to get their controllers packaged together with the config package + // controllers (provider family config package). + for _, c := range []string{"internal/controller/azure/resourcegroup", "internal/controller/azure/resourceproviderregistration", "internal/controller/azure/subscription"} { + pc.BasePackages.ControllerMap[c] = "config" + } + + // API group overrides from Terraform import statements + for _, r := range pc.Resources { + groupKindOverride(r) + } + + // add custom config functions + for _, configure := range ProviderConfiguration { + configure(pc) + } + + pc.ConfigureResources() + + // This function runs after the special configurations were applied. However, if some references were configured in + // the configuration files, the reference generator does not override them. + for _, r := range pc.Resources { + delete(r.References, "resource_group_name") + if err := common.AddCommonReferences(r); err != nil { + panic(err) + } + } + return pc, nil +} + +// CLIReconciledResourceList returns the list of resources that have external +// name configured in ExternalNameConfigs table and to be reconciled under +// the TF CLI based architecture. +func CLIReconciledResourceList() []string { + l := make([]string, len(CLIReconciledExternalNameConfigs)) + i := 0 + for name := range CLIReconciledExternalNameConfigs { + // Expected format is regex and we'd like to have exact matches. + l[i] = name + "$" + i++ + } + return l +} + +// TerraformPluginSDKResourceList returns the list of resources that have external +// name configured in ExternalNameConfigs table and to be reconciled under +// the no-fork architecture. +func TerraformPluginSDKResourceList() []string { + l := make([]string, len(TerraformPluginSDKExternalNameConfigs)) + i := 0 + for name := range TerraformPluginSDKExternalNameConfigs { + // Expected format is regex and we'd like to have exact matches. + l[i] = name + "$" + i++ + } + return l +} + +// Configure configures the specified Provider. +type Configure func(provider *config.Provider) + +// Configurator is a registry for provider Configs. +type Configurator []Configure + +// AddConfig adds a Config to the Configurator registry. +func (c *Configurator) AddConfig(conf Configure) { + *c = append(*c, conf) +} + +// ProviderConfiguration is a global registry to be used by +// the resource providers to register their Config functions. +var ProviderConfiguration = Configurator{} diff --git a/scripts/tag.sh b/scripts/tag.sh new file mode 100755 index 000000000..8357d57e5 --- /dev/null +++ b/scripts/tag.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +set -aeuo pipefail + +# SPDX-FileCopyrightText: 2024 The Crossplane Authors +# +# SPDX-License-Identifier: CC0-1.0 + +TAGGER="/tmp/buildtagger" +REPO_ROOT="." +EXTRA_BUILDTAGGER_ARGS="${EXTRA_BUILDTAGGER_ARGS:-}" +RESTORE_DEEPCOPY_TAGS="${RESTORE_DEEPCOPY_TAGS:-false}" + +if [[ ! -f "${TAGGER}" ]]; then + curl -fsSL "${BUILDTAGGER_DOWNLOAD_URL}" -o "${TAGGER}" + chmod +x "${TAGGER}" +fi + +"${TAGGER}" --parent-dir "${REPO_ROOT}"/apis --regex "(.+)/.+/.+\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/apis --regex "(.+)/.+/zz_groupversion_info\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/internal/controller --regex "(.+)/.+/zz_controller\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/internal/controller --regex "zz_(.+)_setup\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode file ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/cmd/provider --regex "(.+)/zz_main\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config --regex "(.+)/config\.go" --tag-format "(%s || all) && !ignore_autogenerated" --mode dir ${EXTRA_BUILDTAGGER_ARGS} + +# constant tags +# apis/zz_register.go -> (apiregister || register || all) && !ignore_autogenerated +"${TAGGER}" --parent-dir "${REPO_ROOT}"/apis/zz_register.go --tag-format "all && !ignore_autogenerated" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# cmd/generator/main.go -> config || generate || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/cmd/generator/main.go --tag-format "all" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/config_registry.go -> configregistry || register || config || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/registry.go --tag-format "configregistry || all" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/provider.go -> configprovider || register || config || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/provider.go --tag-format "(configprovider || all) && !linter_run" --mode file ${EXTRA_BUILDTAGGER_ARGS} +# config/overrides.go -> configprovider || register || config || all +"${TAGGER}" --parent-dir "${REPO_ROOT}"/config/overrides.go --tag-format "configprovider || all" --mode file ${EXTRA_BUILDTAGGER_ARGS} + +if [[ "${RESTORE_DEEPCOPY_TAGS}" == "true" ]]; then + "${TAGGER}" --parent-dir "${REPO_ROOT}"/apis --regex "zz_generated.deepcopy.go" --tag-format "!ignore_autogenerated" --mode file +fi \ No newline at end of file