diff --git a/pkg/config/conversion/conversions.go b/pkg/config/conversion/conversions.go index 6beb91d8..016903ef 100644 --- a/pkg/config/conversion/conversions.go +++ b/pkg/config/conversion/conversions.go @@ -23,7 +23,9 @@ const ( ) const ( - pathForProvider = "spec.forProvider" + pathForProvider = "spec.forProvider" + pathInitProvider = "spec.initProvider" + pathAtProvider = "status.atProvider" ) var ( @@ -32,7 +34,7 @@ var ( _ PavedConversion = &singletonListConverter{} ) -// Conversion is the interface for the API version converters. +// Conversion is the interface for the CRD API version converters. // Conversion implementations registered for a source, target // pair are called in chain so Conversion implementations can be modular, e.g., // a Conversion implementation registered for a specific source and target @@ -176,17 +178,19 @@ func NewCustomConverter(sourceVersion, targetVersion string, converter func(src, type singletonListConverter struct { baseConversion - crdPaths []string - mode Mode + pathPrefixes []string + crdPaths []string + mode ListConversionMode } // NewSingletonListConversion returns a new Conversion from the specified // sourceVersion of an API to the specified targetVersion and uses the // CRD field paths given in crdPaths to convert between the singleton // lists and embedded objects in the given conversion mode. -func NewSingletonListConversion(sourceVersion, targetVersion string, crdPaths []string, mode Mode) Conversion { +func NewSingletonListConversion(sourceVersion, targetVersion string, pathPrefixes []string, crdPaths []string, mode ListConversionMode) Conversion { return &singletonListConverter{ baseConversion: newBaseConversion(sourceVersion, targetVersion), + pathPrefixes: pathPrefixes, crdPaths: crdPaths, mode: mode, } @@ -200,18 +204,24 @@ func (s *singletonListConverter) ConvertPaved(src, target *fieldpath.Paved) (boo if len(s.crdPaths) == 0 { return false, nil } - v, err := src.GetValue(pathForProvider) - if err != nil { - return true, errors.Wrapf(err, "failed to read the %s value for conversion in mode %q", pathForProvider, s.mode) - } - m, ok := v.(map[string]any) - if !ok { - return true, errors.Errorf("value at path %s is not a map[string]any", pathForProvider) - } - if _, err := Convert(m, s.crdPaths, s.mode); err != nil { - return true, errors.Wrapf(err, "failed to convert the source map in mode %q with %s", s.mode, s.baseConversion.String()) + + for _, p := range s.pathPrefixes { + v, err := src.GetValue(p) + if err != nil { + return true, errors.Wrapf(err, "failed to read the %s value for conversion in mode %q", p, s.mode) + } + m, ok := v.(map[string]any) + if !ok { + return true, errors.Errorf("value at path %s is not a map[string]any", p) + } + if _, err := Convert(m, s.crdPaths, s.mode); err != nil { + return true, errors.Wrapf(err, "failed to convert the source map in mode %q with %s", s.mode, s.baseConversion.String()) + } + if err := target.SetValue(p, m); err != nil { + return true, errors.Wrapf(err, "failed to set the %s value for conversion in mode %q", p, s.mode) + } } - return true, errors.Wrapf(target.SetValue(pathForProvider, m), "failed to set the %s value for conversion in mode %q", pathForProvider, s.mode) + return true, nil } type identityConversion struct { @@ -305,5 +315,5 @@ func ExpandParameters(prefixes []string, excludePaths ...string) []string { // excluding paths in the identity conversion. The returned value is // ["spec.forProvider", "spec.initProvider", "status.atProvider"]. func DefaultPathPrefixes() []string { - return []string{"spec.forProvider", "spec.initProvider", "status.atProvider"} + return []string{pathForProvider, pathInitProvider, pathAtProvider} } diff --git a/pkg/config/conversion/conversions_test.go b/pkg/config/conversion/conversions_test.go index e818de86..c8fa4186 100644 --- a/pkg/config/conversion/conversions_test.go +++ b/pkg/config/conversion/conversions_test.go @@ -365,7 +365,7 @@ func TestSingletonListConversion(t *testing.T) { targetVersion string targetMap map[string]any crdPaths []string - mode Mode + mode ListConversionMode } type want struct { converted bool @@ -383,7 +383,7 @@ func TestSingletonListConversion(t *testing.T) { sourceVersion: AllVersions, sourceMap: map[string]any{ "spec": map[string]any{ - "forProvider": map[string]any{ + "initProvider": map[string]any{ "l": []map[string]any{ { "k": "v", @@ -401,7 +401,7 @@ func TestSingletonListConversion(t *testing.T) { converted: true, targetMap: map[string]any{ "spec": map[string]any{ - "forProvider": map[string]any{ + "initProvider": map[string]any{ "l": map[string]any{ "k": "v", }, @@ -416,7 +416,7 @@ func TestSingletonListConversion(t *testing.T) { sourceVersion: AllVersions, sourceMap: map[string]any{ "spec": map[string]any{ - "forProvider": map[string]any{ + "initProvider": map[string]any{ "o": map[string]any{ "k": "v", }, @@ -432,7 +432,7 @@ func TestSingletonListConversion(t *testing.T) { converted: true, targetMap: map[string]any{ "spec": map[string]any{ - "forProvider": map[string]any{ + "initProvider": map[string]any{ "o": []map[string]any{ { "k": "v", @@ -449,7 +449,7 @@ func TestSingletonListConversion(t *testing.T) { sourceVersion: AllVersions, sourceMap: map[string]any{ "spec": map[string]any{ - "forProvider": map[string]any{ + "initProvider": map[string]any{ "o": map[string]any{ "k": "v", }, @@ -468,7 +468,7 @@ func TestSingletonListConversion(t *testing.T) { } for n, tc := range tests { t.Run(n, func(t *testing.T) { - c := NewSingletonListConversion(tc.args.sourceVersion, tc.args.targetVersion, tc.args.crdPaths, tc.args.mode) + c := NewSingletonListConversion(tc.args.sourceVersion, tc.args.targetVersion, []string{pathInitProvider}, tc.args.crdPaths, tc.args.mode) sourceMap, err := roundTrip(tc.args.sourceMap) if err != nil { t.Fatalf("Failed to preprocess tc.args.sourceMap: %v", err) diff --git a/pkg/config/conversion/runtime_conversion.go b/pkg/config/conversion/list_conversion.go similarity index 91% rename from pkg/config/conversion/runtime_conversion.go rename to pkg/config/conversion/list_conversion.go index 1d224167..d7a1fb0f 100644 --- a/pkg/config/conversion/runtime_conversion.go +++ b/pkg/config/conversion/list_conversion.go @@ -14,16 +14,16 @@ import ( "github.com/pkg/errors" ) -// Mode denotes the mode of the runtime API conversion, e.g., +// ListConversionMode denotes the mode of the list-object API conversion, e.g., // conversion of embedded objects into singleton lists. -type Mode int +type ListConversionMode int const ( // ToEmbeddedObject represents a runtime conversion from a singleton list // to an embedded object, i.e., the runtime conversions needed while // reading from the Terraform state and updating the CRD // (for status, late-initialization, etc.) - ToEmbeddedObject Mode = iota + ToEmbeddedObject ListConversionMode = iota // ToSingletonList represents a runtime conversion from an embedded object // to a singleton list, i.e., the runtime conversions needed while passing // the configuration data to the underlying Terraform layer. @@ -36,7 +36,7 @@ const ( ) // String returns a string representation of the conversion mode. -func (m Mode) String() string { +func (m ListConversionMode) String() string { switch m { case ToSingletonList: return "toSingletonList" @@ -79,7 +79,7 @@ func setValue(pv *fieldpath.Paved, v any, fp string) error { // an embedded object will be converted into a singleton list or a singleton // list will be converted into an embedded object) is determined by the mode // parameter. -func Convert(params map[string]any, paths []string, mode Mode) (map[string]any, error) { //nolint:gocyclo // easier to follow as a unit +func Convert(params map[string]any, paths []string, mode ListConversionMode) (map[string]any, error) { //nolint:gocyclo // easier to follow as a unit switch mode { case ToSingletonList: slices.Sort(paths) diff --git a/pkg/config/conversion/runtime_conversion_test.go b/pkg/config/conversion/list_conversion_test.go similarity index 99% rename from pkg/config/conversion/runtime_conversion_test.go rename to pkg/config/conversion/list_conversion_test.go index 1af9e31b..9308a17f 100644 --- a/pkg/config/conversion/runtime_conversion_test.go +++ b/pkg/config/conversion/list_conversion_test.go @@ -18,7 +18,7 @@ func TestConvert(t *testing.T) { type args struct { params map[string]any paths []string - mode Mode + mode ListConversionMode } type want struct { err error @@ -294,7 +294,7 @@ func TestConvert(t *testing.T) { func TestModeString(t *testing.T) { tests := map[string]struct { - m Mode + m ListConversionMode want string }{ "ToSingletonList": { diff --git a/pkg/config/resource.go b/pkg/config/resource.go index 56acc648..6d5184e0 100644 --- a/pkg/config/resource.go +++ b/pkg/config/resource.go @@ -174,6 +174,11 @@ type References map[string]Reference type Reference struct { // Type is the Go type name of the CRD if it is in the same package or // . if it is in a different package. + // Deprecated: Type is deprecated in favor of TerraformName, which provides + // a more stable and less error-prone API compared to Type. TerraformName + // will automatically handle name & version configurations that will affect + // the generated cross-resource reference. This is crucial especially if the + // provider generates multiple versions for its MR APIs. Type string // TerraformName is the name of the Terraform resource // which will be referenced. The supplied resource name is @@ -393,9 +398,19 @@ type Resource struct { // be `ec2.aws.crossplane.io` ShortGroup string - // Version is the version CRD will have. + // Version is the API version being generated for the corresponding CRD. Version string + // ControllerReconcileVersion is the CRD API version the associated + // controller will watch & reconcile. If left unspecified, + // defaults to the value of Version. This configuration parameter + // can be used to have a controller use an older + // API version of the generated CRD instead of the API version being + // generated. Because this configuration parameter's value defaults to + // the value of Version, by default the controllers will reconcile the + // currently generated API versions of their associated CRs. + ControllerReconcileVersion string + // Kind is the kind of the CRD. Kind string @@ -477,8 +492,17 @@ type Resource struct { // index notation (i.e., array/map components do not need indices). ServerSideApplyMergeStrategies ServerSideApplyMergeStrategies + // Conversions is the list of CRD API conversion functions to be invoked + // in-chain by the installed conversion Webhook for the generated CRD. + // This list of conversion.Conversion registered here are responsible for + // doing the conversions between the hub & spoke CRD API versions. Conversions []conversion.Conversion + // TerraformConversions is the list of conversions to be invoked when passing + // data from the Crossplane layer to the Terraform layer and when reading + // data (state) from the Terraform layer to be used in the Crossplane layer. + TerraformConversions []TerraformConversion + // useTerraformPluginSDKClient indicates that a plugin SDK external client should // be generated instead of the Terraform CLI-forking client. useTerraformPluginSDKClient bool diff --git a/pkg/config/tf_conversion.go b/pkg/config/tf_conversion.go new file mode 100644 index 00000000..71ff9514 --- /dev/null +++ b/pkg/config/tf_conversion.go @@ -0,0 +1,72 @@ +// SPDX-FileCopyrightText: 2024 The Crossplane Authors +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "github.com/pkg/errors" + + "github.com/crossplane/upjet/pkg/config/conversion" +) + +// Mode denotes the mode of the runtime Terraform conversion, e.g., +// conversion from Crossplane parameters to Terraform arguments, or +// conversion from Terraform state to Crossplane state. +type Mode int + +const ( + ToTerraform Mode = iota + FromTerraform +) + +// String returns a string representation of the conversion mode. +func (m Mode) String() string { + switch m { + case ToTerraform: + return "toTerraform" + case FromTerraform: + return "fromTerraform" + default: + return "unknown" + } +} + +type TerraformConversion interface { + Convert(params map[string]any, r *Resource, mode Mode) (map[string]any, error) +} + +// ApplyTFConversions applies the configured Terraform conversions on the +// specified params map in the given mode, i.e., from Crossplane layer to the +// Terraform layer or vice versa. +func (r *Resource) ApplyTFConversions(params map[string]any, mode Mode) (map[string]any, error) { + var err error + for _, c := range r.TerraformConversions { + params, err = c.Convert(params, r, mode) + if err != nil { + return nil, err + } + } + return params, nil +} + +type singletonListConversion struct{} + +// NewTFSingletonConversion initializes a new TerraformConversion to convert +// between singleton lists and embedded objects in the exchanged data +// at runtime between the Crossplane & Terraform layers. +func NewTFSingletonConversion() TerraformConversion { + return singletonListConversion{} +} + +func (s singletonListConversion) Convert(params map[string]any, r *Resource, mode Mode) (map[string]any, error) { + var err error + var m map[string]any + switch mode { + case FromTerraform: + m, err = conversion.Convert(params, r.TFListConversionPaths(), conversion.ToEmbeddedObject) + case ToTerraform: + m, err = conversion.Convert(params, r.TFListConversionPaths(), conversion.ToSingletonList) + } + return m, errors.Wrapf(err, "failed to convert between Crossplane and Terraform layers in mode %q", mode) +} diff --git a/pkg/controller/external_tfpluginsdk.go b/pkg/controller/external_tfpluginsdk.go index b04ef2e4..8fed8acd 100644 --- a/pkg/controller/external_tfpluginsdk.go +++ b/pkg/controller/external_tfpluginsdk.go @@ -26,7 +26,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/crossplane/upjet/pkg/config" - "github.com/crossplane/upjet/pkg/config/conversion" "github.com/crossplane/upjet/pkg/metrics" "github.com/crossplane/upjet/pkg/resource" "github.com/crossplane/upjet/pkg/resource/json" @@ -123,7 +122,7 @@ type terraformPluginSDKExternal struct { opTracker *AsyncTracker } -func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externalName string, config *config.Resource, ts terraform.Setup, initParamsMerged bool, kube client.Client) (map[string]any, error) { +func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externalName string, cfg *config.Resource, ts terraform.Setup, initParamsMerged bool, kube client.Client) (map[string]any, error) { params, err := tr.GetMergedParameters(initParamsMerged) if err != nil { return nil, errors.Wrap(err, "cannot get merged parameters") @@ -131,18 +130,18 @@ func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externa if err = resource.GetSensitiveParameters(ctx, &APISecretClient{kube: kube}, tr, params, tr.GetConnectionDetailsMapping()); err != nil { return nil, errors.Wrap(err, "cannot store sensitive parameters into params") } - config.ExternalName.SetIdentifierArgumentFn(params, externalName) - if config.TerraformConfigurationInjector != nil { + cfg.ExternalName.SetIdentifierArgumentFn(params, externalName) + if cfg.TerraformConfigurationInjector != nil { m, err := getJSONMap(tr) if err != nil { return nil, errors.Wrap(err, "cannot get JSON map for the managed resource's spec.forProvider value") } - if err := config.TerraformConfigurationInjector(m, params); err != nil { + if err := cfg.TerraformConfigurationInjector(m, params); err != nil { return nil, errors.Wrap(err, "cannot invoke the configured TerraformConfigurationInjector") } } - tfID, err := config.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) + tfID, err := cfg.ExternalName.GetIDFn(ctx, externalName, params, ts.Map()) if err != nil { return nil, errors.Wrap(err, "cannot get ID") } @@ -151,12 +150,12 @@ func getExtendedParameters(ctx context.Context, tr resource.Terraformed, externa // not all providers may have this attribute // TODO: tags-tags_all implementation is AWS specific. // Consider making this logic independent of provider. - if config.TerraformResource != nil { - if _, ok := config.TerraformResource.CoreConfigSchema().Attributes["tags_all"]; ok { + if cfg.TerraformResource != nil { + if _, ok := cfg.TerraformResource.CoreConfigSchema().Attributes["tags_all"]; ok { params["tags_all"] = params["tags"] } } - return conversion.Convert(params, config.TFListConversionPaths(), conversion.ToSingletonList) + return cfg.ApplyTFConversions(params, config.ToTerraform) } func (c *TerraformPluginSDKConnector) processParamsWithHCLParser(schemaMap map[string]*schema.Schema, params map[string]any) map[string]any { @@ -256,7 +255,7 @@ func (c *TerraformPluginSDKConnector) Connect(ctx context.Context, mg xpresource if err != nil { return nil, errors.Wrap(err, "failed to get the observation") } - tfState, err = conversion.Convert(tfState, c.config.TFListConversionPaths(), conversion.ToSingletonList) + tfState, err = c.config.ApplyTFConversions(tfState, config.ToTerraform) if err != nil { return nil, errors.Wrap(err, "failed to run the API converters on the Terraform state") } @@ -525,7 +524,7 @@ func (n *terraformPluginSDKExternal) Observe(ctx context.Context, mg xpresource. return managed.ExternalObservation{}, errors.Wrap(err, "cannot get connection details") } - stateValueMap, err = conversion.Convert(stateValueMap, n.config.TFListConversionPaths(), conversion.ToEmbeddedObject) + stateValueMap, err = n.config.ApplyTFConversions(stateValueMap, config.FromTerraform) if err != nil { return managed.ExternalObservation{}, errors.Wrap(err, "cannot convert the singleton lists in the observed state value map into embedded objects") } @@ -645,7 +644,7 @@ func (n *terraformPluginSDKExternal) Create(ctx context.Context, mg xpresource.M return managed.ExternalCreation{}, errors.Wrap(err, "cannot get connection details") } - stateValueMap, err = conversion.Convert(stateValueMap, n.config.TFListConversionPaths(), conversion.ToEmbeddedObject) + stateValueMap, err = n.config.ApplyTFConversions(stateValueMap, config.FromTerraform) if err != nil { return managed.ExternalCreation{}, errors.Wrap(err, "cannot convert the singleton lists in the state value map of the newly created resource into embedded objects") } diff --git a/pkg/pipeline/run.go b/pkg/pipeline/run.go index 77665056..317800fe 100644 --- a/pkg/pipeline/run.go +++ b/pkg/pipeline/run.go @@ -112,7 +112,11 @@ func Run(pc *config.Provider, rootDir string) { //nolint:gocyclo if pc.FeaturesPackage != "" { featuresPkgPath = filepath.Join(pc.ModulePath, pc.FeaturesPackage) } - ctrlPkgPath, err := ctrlGen.Generate(resources[name], versionGen.Package().Path(), featuresPkgPath) + watchVersionGen := versionGen + if len(resources[name].ControllerReconcileVersion) != 0 { + watchVersionGen = NewVersionGenerator(rootDir, pc.ModulePath, group, resources[name].ControllerReconcileVersion) + } + ctrlPkgPath, err := ctrlGen.Generate(resources[name], watchVersionGen.Package().Path(), featuresPkgPath) if err != nil { panic(errors.Wrapf(err, "cannot generate controller for resource %s", name)) }