Convert all singleton lists in the MR APIs to embedded objects

- Terraform configuration blocks, even if they have a MaxItems
  constraint of 1, are (almost) always generated as lists. We
  now generate the lists with a MaxItems constraint of 1 as
  embedded objects in our MR APIs.
- This also helps when updating or patching via SSA the
  (previously list) objects. The merging strategy implemented
  by SSA requires configuration for associative lists and
  converting the singleton lists into embedded objects removes
  the configuration need.
- Bump up to v0.28.0
- Temporarily add the $(UPTEST_LOCAL) make target to consume uptest
  from the main channel.

Signed-off-by: Alper Rifat Ulucinar <[email protected]>

Makefile

@@ -77,9 +77,11 @@ export SUBPACKAGES := $(SUBPACKAGES)
 # Setup Kubernetes tools
 
 KIND_VERSION = v0.21.0
-UP_VERSION = v0.20.0
+UP_VERSION = v0.28.0
 UP_CHANNEL = stable
 UPTEST_VERSION = v0.11.1
+UPTEST_LOCAL_VERSION = v0.12.0-2.g635fa3e
+UPTEST_LOCAL_CHANNEL = main
 KUSTOMIZE_VERSION = v5.3.0
 YQ_VERSION = v4.40.5
 UXP_VERSION = 1.14.6-up.1
@@ -89,6 +91,16 @@ export UP_CHANNEL := $(UP_CHANNEL)
 
 -include build/makelib/k8s_tools.mk
 
+# uptest download and install
+UPTEST_LOCAL := $(TOOLS_HOST_DIR)/uptest-$(UPTEST_LOCAL_VERSION)
+
+$(UPTEST_LOCAL):
+	@$(INFO) installing uptest $(UPTEST_LOCAL)
+	@mkdir -p $(TOOLS_HOST_DIR)
+	@curl -fsSLo $(UPTEST_LOCAL) https://s3.us-west-2.amazonaws.com/crossplane.uptest.releases/$(UPTEST_LOCAL_CHANNEL)/$(UPTEST_LOCAL_VERSION)/bin/$(SAFEHOST_PLATFORM)/uptest || $(FAIL)
+	@chmod +x $(UPTEST_LOCAL)
+	@$(OK) installing uptest $(UPTEST_LOCAL)
+
 # ====================================================================================
 # Setup Images
 
@@ -195,9 +207,9 @@ CROSSPLANE_NAMESPACE = upbound-system
 # - UPTEST_EXAMPLE_LIST, a comma-separated list of examples to test
 # - UPTEST_CLOUD_CREDENTIALS (optional), cloud credentials for the provider being tested, e.g. export UPTEST_CLOUD_CREDENTIALS=$(cat ~/azure.json)
 # - UPTEST_DATASOURCE_PATH (optional), see https://github.com/upbound/uptest#injecting-dynamic-values-and-datasource
-uptest: $(UPTEST) $(KUBECTL) $(KUTTL)
+uptest: $(UPTEST_LOCAL) $(KUBECTL) $(KUTTL)
 	@$(INFO) running automated tests
-	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST) e2e "${UPTEST_EXAMPLE_LIST}" --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=cluster/test/setup.sh --default-conditions="Test" || $(FAIL)
+	@KUBECTL=$(KUBECTL) KUTTL=$(KUTTL) CROSSPLANE_NAMESPACE=$(CROSSPLANE_NAMESPACE) $(UPTEST_LOCAL) e2e "${UPTEST_EXAMPLE_LIST}" --data-source="${UPTEST_DATASOURCE_PATH}" --setup-script=cluster/test/setup.sh --default-conditions="Test" || $(FAIL)
 	@$(OK) running automated tests
 
 uptest-local:

apis/generate.go

@@ -11,6 +11,7 @@
 //go:generate rm -rf ../package/crds
 
 // Remove generated files
+//go:generate bash -c "find . \\( -iname 'zz_generated.conversion_hubs.go' -o -iname 'zz_generated.conversion_spokes.go' \\) -delete"
 //go:generate bash -c "find . -type d -empty -delete"
 //go:generate bash -c "find ../internal/controller -iname 'zz_*' -delete"
 //go:generate bash -c "find ../internal/controller -type d -empty -delete"

config/registry.go

@@ -9,7 +9,8 @@ import (
 	_ "embed"
 
 	"github.com/crossplane/upjet/pkg/config"
-	tjconfig "github.com/crossplane/upjet/pkg/config"
+	ujconfig "github.com/crossplane/upjet/pkg/config"
+	"github.com/crossplane/upjet/pkg/config/conversion"
 	"github.com/crossplane/upjet/pkg/registry/reference"
 	conversiontfjson "github.com/crossplane/upjet/pkg/types/conversion/tfjson"
 	tfjson "github.com/hashicorp/terraform-json"
@@ -124,7 +125,7 @@ func getProviderSchema(s string) (*schema.Provider, error) {
 }
 
 // GetProvider returns provider configuration
-func GetProvider(ctx context.Context, generationProvider bool) (*tjconfig.Provider, error) {
+func GetProvider(ctx context.Context, generationProvider bool) (*ujconfig.Provider, error) {
 	var p *schema.Provider
 	var err error
 	if generationProvider {
@@ -136,18 +137,21 @@ func GetProvider(ctx context.Context, generationProvider bool) (*tjconfig.Provid
 		return nil, errors.Wrapf(err, "cannot get the Terraform provider schema with generation mode set to %t", generationProvider)
 	}
 
-	pc := tjconfig.NewProvider([]byte(providerSchema), resourcePrefix, modulePath, providerMetadata,
-		tjconfig.WithShortName("azure"),
-		tjconfig.WithRootGroup("azure.upbound.io"),
-		tjconfig.WithIncludeList(CLIReconciledResourceList()),
-		tjconfig.WithTerraformPluginSDKIncludeList(TerraformPluginSDKResourceList()),
-		tjconfig.WithSkipList(skipList),
-		tjconfig.WithDefaultResourceOptions(ResourceConfigurator()),
-		tjconfig.WithReferenceInjectors([]tjconfig.ReferenceInjector{reference.NewInjector(modulePath)}),
-		tjconfig.WithFeaturesPackage("internal/features"),
-		tjconfig.WithMainTemplate(hack.MainTemplate),
-		tjconfig.WithTerraformProvider(p),
+	pc := ujconfig.NewProvider([]byte(providerSchema), resourcePrefix, modulePath, providerMetadata,
+		ujconfig.WithShortName("azure"),
+		ujconfig.WithRootGroup("azure.upbound.io"),
+		ujconfig.WithIncludeList(CLIReconciledResourceList()),
+		ujconfig.WithTerraformPluginSDKIncludeList(TerraformPluginSDKResourceList()),
+		ujconfig.WithSkipList(skipList),
+		ujconfig.WithDefaultResourceOptions(ResourceConfigurator()),
+		ujconfig.WithReferenceInjectors([]ujconfig.ReferenceInjector{reference.NewInjector(modulePath)}),
+		ujconfig.WithFeaturesPackage("internal/features"),
+		ujconfig.WithMainTemplate(hack.MainTemplate),
+		ujconfig.WithTerraformProvider(p),
+		ujconfig.WithSchemaTraversers(&ujconfig.SingletonListEmbedder{}),
 	)
+
+	bumpVersionsWithEmbeddedLists(pc)
 	// "azure" group contains resources that actually do not have a specific
 	// group, e.g. ResourceGroup with APIVersion "azure.upbound.io/v1beta1".
 	// We need to include the controllers for this group into the base packages
@@ -180,6 +184,26 @@ func GetProvider(ctx context.Context, generationProvider bool) (*tjconfig.Provid
 	return pc, nil
 }
 
+func bumpVersionsWithEmbeddedLists(pc *ujconfig.Provider) {
+	for name, r := range pc.Resources {
+		r := r
+		// nothing to do if no singleton list has been converted to
+		// an embedded object
+		if len(r.CRDListConversionPaths()) == 0 {
+			continue
+		}
+		r.Version = "v1beta2"
+		// we would like to set the storage version to v1beta1 to facilitate
+		// downgrades.
+		r.SetCRDStorageVersion("v1beta1")
+		r.Conversions = []conversion.Conversion{
+			conversion.NewIdentityConversionExpandPaths(conversion.AllVersions, conversion.AllVersions, []string{"spec.forProvider", "spec.initProvider", "status.atProvider"}, r.CRDListConversionPaths()...),
+			conversion.NewSingletonListConversion("v1beta1", "v1beta2", r.CRDListConversionPaths(), conversion.ToEmbeddedObject),
+			conversion.NewSingletonListConversion("v1beta2", "v1beta1", r.CRDListConversionPaths(), conversion.ToSingletonList)}
+		pc.Resources[name] = r
+	}
+}
+
 // CLIReconciledResourceList returns the list of resources that have external
 // name configured in ExternalNameConfigs table and to be reconciled under
 // the TF CLI based architecture.

go.mod

@@ -168,3 +168,5 @@ require (
 replace github.com/hashicorp/terraform-provider-azurerm => github.com/upbound/terraform-provider-azurerm v0.0.0-20240311113236-25a51734d3da
 
 replace github.com/hashicorp/terraform-plugin-sdk/v2 => github.com/hashicorp/terraform-plugin-sdk/v2 v2.29.0
+
+replace github.com/crossplane/upjet => github.com/ulucinar/upbound-upjet v0.0.0-20240422072143-9a463d567f9c

go.sum

@@ -79,8 +79,6 @@ github.com/crossplane/crossplane-runtime v1.16.0-rc.1.0.20240424114634-8641eb2ba
 github.com/crossplane/crossplane-runtime v1.16.0-rc.1.0.20240424114634-8641eb2ba384/go.mod h1:Pz2tdGVMF6KDGzHZOkvKro0nKc8EzK0sb/nSA7pH4Dc=
 github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79 h1:HigXs5tEQxWz0fcj8hzbU2UAZgEM7wPe0XRFOsrtF8Y=
 github.com/crossplane/crossplane-tools v0.0.0-20230925130601-628280f8bf79/go.mod h1:+e4OaFlOcmr0JvINHl/yvEYBrZawzTgj6pQumOH1SS0=
-github.com/crossplane/upjet v1.3.0 h1:qRgcfqLz4M2v7enUku3xEriY5poc5XVbRl98nbvvu+E=
-github.com/crossplane/upjet v1.3.0/go.mod h1:3pDVtCgyBc5f2Zx4K5HEPxxhjndmOc5CHCJNpIivK/g=
 github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg=
 github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
 github.com/dave/jennifer v1.6.0 h1:MQ/6emI2xM7wt0tJzJzyUik2Q3Tcn2eE0vtYgh4GPVI=
@@ -358,6 +356,8 @@ github.com/tombuildsstuff/giovanni v0.20.0 h1:IM/I/iNWMXnPYwcSq8uxV7TKDlv7Nejq0b
 github.com/tombuildsstuff/giovanni v0.20.0/go.mod h1:66KVLYma2whJhEdxPSPL3GQHkulhK+C5CluKfHGfPF4=
 github.com/tombuildsstuff/kermit v0.20240122.1123108 h1:icQaxsv/ANv/KC4Sr0V1trrWA/XIL+3QAVBDpiSTgj8=
 github.com/tombuildsstuff/kermit v0.20240122.1123108/go.mod h1:T3YBVFhRV4qA7SbnRaNE6eapIMpKDA9rG/V7Ocsjlno=
+github.com/ulucinar/upbound-upjet v0.0.0-20240422072143-9a463d567f9c h1:H+IioE227HCCHC9Fz/HRGMv/nSL1pDkOIu9krrTrsng=
+github.com/ulucinar/upbound-upjet v0.0.0-20240422072143-9a463d567f9c/go.mod h1:0bHLtnejZ9bDeyXuBb9MSOQLvKo3+aoTeUBO8N0dGSA=
 github.com/upbound/terraform-provider-azurerm v0.0.0-20240311113236-25a51734d3da h1:Rj/LhOVcK8IOSIBYk7L6LZpuOHsXkkit+JrV4nNmi9o=
 github.com/upbound/terraform-provider-azurerm v0.0.0-20240311113236-25a51734d3da/go.mod h1:GCUoUr4wIRHz+ZYmDI3Y1oE0M64rqJcr3oY60wwJdVc=
 github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=

hack/main.go.tmpl

@@ -23,6 +23,7 @@ import (
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
 	"github.com/crossplane/crossplane-runtime/pkg/statemetrics"
 	tjcontroller "github.com/crossplane/upjet/pkg/controller"
+	"github.com/crossplane/upjet/pkg/controller/conversion"
 	"gopkg.in/alecthomas/kingpin.v2"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -216,6 +217,7 @@ func main() {
 		})), "cannot create default store config")
 	}
 
+	kingpin.FatalIfError(conversion.RegisterConversions(o.Provider), "Cannot initialize the webhook conversion registry")
 	kingpin.FatalIfError(controller.Setup_{{ .Group }}(mgr, o), "Cannot setup Azure controllers")
 	kingpin.FatalIfError(mgr.Start(ctrl.SetupSignalHandler()), "Cannot start controller manager")
 }