From 8278c6eb9616f744b2a6b8789c6c9e8eeb109d97 Mon Sep 17 00:00:00 2001 From: Alper Rifat Ulucinar Date: Mon, 30 Oct 2023 13:46:24 +0300 Subject: [PATCH] Add support for hybrid CLI-based reconciling for configured resources - Add config.Provider.WithNoForkIncludeList to explicitly specify the set of resources to be reconciled under the no-fork architecture. Signed-off-by: Alper Rifat Ulucinar --- pkg/config/provider.go | 65 +++++++++++++++++++++++++++++++++++--- pkg/config/resource.go | 17 +++++++--- pkg/pipeline/controller.go | 7 ++-- 3 files changed, 77 insertions(+), 12 deletions(-) diff --git a/pkg/config/provider.go b/pkg/config/provider.go index 6e406bb5..c72dc4b7 100644 --- a/pkg/config/provider.go +++ b/pkg/config/provider.go @@ -8,10 +8,12 @@ import ( "fmt" "regexp" + tfjson "github.com/hashicorp/terraform-json" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/pkg/errors" "github.com/crossplane/upjet/pkg/registry" + conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson" ) // ResourceConfiguratorFn is a function that implements the ResourceConfigurator @@ -106,12 +108,22 @@ type Provider struct { skippedResourceNames []string // IncludeList is a list of regex for the Terraform resources to be - // included. For example, to include "aws_shield_protection_group" into + // included and reconciled via the Terraform CLI. + // For example, to include "aws_shield_protection_group" into // the generated resources, one can add "aws_shield_protection_group$". // To include whole aws waf group, one can add "aws_waf.*" to the list. // Defaults to []string{".+"} which would include all resources. IncludeList []string + // NoForkIncludeList is a list of regex for the Terraform resources to be + // included and reconciled in the no-fork architecture (without the + // Terraform CLI). + // For example, to include "aws_shield_protection_group" into + // the generated resources, one can add "aws_shield_protection_group$". + // To include whole aws waf group, one can add "aws_waf.*" to the list. + // Defaults to []string{".+"} which would include all resources. + NoForkIncludeList []string + // Resources is a map holding resource configurations where key is Terraform // resource name. Resources map[string]*Resource @@ -158,6 +170,20 @@ func WithIncludeList(l []string) ProviderOption { } } +// WithNoForkIncludeList configures IncludeList for this Provider. +func WithNoForkIncludeList(l []string) ProviderOption { + return func(p *Provider) { + p.NoForkIncludeList = l + } +} + +// WithTerraformProvider configures the TerraformProvider for this Provider. +func WithTerraformProvider(tp *schema.Provider) ProviderOption { + return func(p *Provider) { + p.TerraformProvider = tp + } +} + // WithSkipList configures SkipList for this Provider. func WithSkipList(l []string) ProviderOption { return func(p *Provider) { @@ -202,8 +228,23 @@ func WithMainTemplate(template string) ProviderOption { } } -// NewProvider builds and returns a new Provider from provider native schema. -func NewProvider(resourceMap map[string]*schema.Resource, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo +// NewProvider builds and returns a new Provider from provider +// tfjson schema, that is generated using Terraform CLI with: +// `terraform providers schema --json` +func NewProvider(schema []byte, prefix string, modulePath string, metadata []byte, opts ...ProviderOption) *Provider { // nolint:gocyclo + ps := tfjson.ProviderSchemas{} + if err := ps.UnmarshalJSON(schema); err != nil { + panic(err) + } + if len(ps.Schemas) != 1 { + panic(fmt.Sprintf("there should exactly be 1 provider schema but there are %d", len(ps.Schemas))) + } + var rs map[string]*tfjson.Schema + for _, v := range ps.Schemas { + rs = v.ResourceSchemas + break + } + resourceMap := conversiontfjson.GetV2ResourceMap(rs) providerMetadata, err := registry.NewProviderMetadataFromFile(metadata) if err != nil { panic(errors.Wrap(err, "cannot load provider metadata")) @@ -233,11 +274,27 @@ func NewProvider(resourceMap map[string]*schema.Resource, prefix string, moduleP // There are resources with no schema, that we will address later. fmt.Printf("Skipping resource %s because it has no schema\n", name) } - if len(terraformResource.Schema) == 0 || matches(name, p.SkipList) || !matches(name, p.IncludeList) { + // if in both of the include lists, the new behavior prevails + isNoFork := matches(name, p.NoForkIncludeList) + if len(terraformResource.Schema) == 0 || matches(name, p.SkipList) || (!matches(name, p.IncludeList) && !isNoFork) { p.skippedResourceNames = append(p.skippedResourceNames, name) continue } + if isNoFork { + if p.TerraformProvider == nil || p.TerraformProvider.ResourcesMap[name] == nil { + panic(errors.Errorf("resource %q is configured to be reconciled without the Terraform CLI"+ + "but either config.Provider.TerraformProvider is not configured or the Go schema does not exist for the resource", name)) + } + terraformResource = p.TerraformProvider.ResourcesMap[name] + // TODO: we will need to bump the terraform-plugin-sdk dependency to handle + // schema.Resource.SchemaFunc + if terraformResource.Schema == nil { + p.skippedResourceNames = append(p.skippedResourceNames, name) + continue + } + } p.Resources[name] = DefaultResource(name, terraformResource, providerMetadata.Resources[name], p.DefaultResourceOptions...) + p.Resources[name].useNoForkClient = isNoFork } for i, refInjector := range p.refInjectors { if err := refInjector.InjectReferences(p.Resources); err != nil { diff --git a/pkg/config/resource.go b/pkg/config/resource.go index dd5e5aeb..14b31ac2 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -9,7 +9,6 @@ import ( "fmt" "time" - "github.com/crossplane/upjet/pkg/registry" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" "github.com/pkg/errors" @@ -22,6 +21,8 @@ import ( "github.com/crossplane/crossplane-runtime/pkg/fieldpath" "github.com/crossplane/crossplane-runtime/pkg/reconciler/managed" xpresource "github.com/crossplane/crossplane-runtime/pkg/resource" + + "github.com/crossplane/upjet/pkg/registry" ) // SetIdentifierArgumentsFn sets the name of the resource in Terraform attributes map, @@ -294,10 +295,8 @@ type Resource struct { // databases. UseAsync bool - // UseNoForkClient indicates that a no-fork external client should - // be generated instead of the Terraform CLI-forking client. - UseNoForkClient bool - + // InitializerFns specifies the initializer functions to be used + // for this Resource. InitializerFns []NewInitializerFn // OperationTimeouts allows configuring resource operation timeouts. @@ -339,6 +338,14 @@ type Resource struct { // TerraformCustomDiff allows a resource.Terraformed to customize how its // Terraform InstanceDiff is computed during reconciliation. TerraformCustomDiff CustomDiff + + // useNoForkClient indicates that a no-fork external client should + // be generated instead of the Terraform CLI-forking client. + useNoForkClient bool +} + +func (r *Resource) ShouldUseNoForkClient() bool { + return r.useNoForkClient } // CustomDiff customizes the computed Terraform InstanceDiff. This can be used diff --git a/pkg/pipeline/controller.go b/pkg/pipeline/controller.go index 8305e951..40a7c2d0 100644 --- a/pkg/pipeline/controller.go +++ b/pkg/pipeline/controller.go @@ -9,10 +9,11 @@ import ( "path/filepath" "strings" - "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/pipeline/templates" "github.com/muvaf/typewriter/pkg/wrapper" "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/config" + "github.com/crossplane/upjet/pkg/pipeline/templates" ) // NewControllerGenerator returns a new ControllerGenerator. @@ -49,7 +50,7 @@ func (cg *ControllerGenerator) Generate(cfg *config.Resource, typesPkgPath strin "DisableNameInitializer": cfg.ExternalName.DisableNameInitializer, "TypePackageAlias": ctrlFile.Imports.UsePackage(typesPkgPath), "UseAsync": cfg.UseAsync, - "UseNoForkClient": cfg.UseNoForkClient, + "UseNoForkClient": cfg.ShouldUseNoForkClient(), "ResourceType": cfg.Name, "Initializers": cfg.InitializerFns, }