[WIP] [DON'T REVIEW] 🌱 Add e2e test to reproduce SSA apiVersion issue

Makefile

@@ -511,11 +511,15 @@ generate-doctoc:
  TRACE=$(TRACE) ./hack/generate-doctoc.sh
 
 .PHONY: generate-e2e-templates
-generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
+generate-e2e-templates: $(KUSTOMIZE) $(addprefix generate-e2e-templates-, v0.4 v1.0 v1.5 v1.6 main) ## Generate cluster templates for all versions
 
 DOCKER_TEMPLATES := test/e2e/data/infrastructure-docker
 INMEMORY_TEMPLATES := test/e2e/data/infrastructure-inmemory
 
+.PHONY: generate-e2e-templates-v0.4
+generate-e2e-templates-v0.4: $(KUSTOMIZE)
+ $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v0.4/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v0.4/cluster-template.yaml
+
 .PHONY: generate-e2e-templates-v1.0
 generate-e2e-templates-v1.0: $(KUSTOMIZE)
  $(KUSTOMIZE) build $(DOCKER_TEMPLATES)/v1.0/cluster-template --load-restrictor LoadRestrictionsNone > $(DOCKER_TEMPLATES)/v1.0/cluster-template.yaml

test/e2e/clusterctl_upgrade_test.go

@@ -34,6 +34,59 @@ var (
  providerDockerPrefix = "docker:v%s"
 )
 
+var _ = Describe("When testing clusterctl upgrades (v0.4=>v1.6=>current) [PR-Blocking]", func() {
+ // Get v0.4 latest stable release
+ version04 := "0.4"
+ stableRelease04, err := GetStableReleaseOfMinor(ctx, version04)
+ Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version04)
+
+ // Get v1.6 latest stable release
+ version16 := "1.6"
+ stableRelease16, err := GetStableReleaseOfMinor(ctx, version16)
+ Expect(err).ToNot(HaveOccurred(), "Failed to get stable version for minor release : %s", version16)
+
+ ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput {
+ return ClusterctlUpgradeSpecInput{
+ E2EConfig: e2eConfig,
+ ClusterctlConfigPath: clusterctlConfigPath,
+ BootstrapClusterProxy: bootstrapClusterProxy,
+ ArtifactFolder: artifactFolder,
+ SkipCleanup: skipCleanup,
+ InfrastructureProvider: ptr.To("docker"),
+ // ### Versions for the initial deployment of providers ###
+ InitWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease04),
+ InitWithCoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease04),
+ InitWithBootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
+ InitWithControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease04)},
+ InitWithInfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease04)},
+ InitWithRuntimeExtensionProviders: []string{},
+ // ### Versions for the first upgrade of providers ### // FIXME: move upgrades to slice & add hook
+ UpgradeWithBinary: fmt.Sprintf(clusterctlDownloadURL, stableRelease16),
+ CoreProvider: fmt.Sprintf(providerCAPIPrefix, stableRelease16),
+ BootstrapProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
+ ControlPlaneProviders: []string{fmt.Sprintf(providerKubeadmPrefix, stableRelease16)},
+ InfrastructureProviders: []string{fmt.Sprintf(providerDockerPrefix, stableRelease16)},
+ RuntimeExtensionProviders: []string{},
+ // Run a final upgrade to latest
+ AdditionalUpgrade: true,
+
+ // Some notes about the version pinning: (FIXME)
+ // We have to pin the providers because with `InitWithProvidersContract` the test would
+ // use the latest version for the contract (which is v1.6.X for v1beta1).
+ // We have to set this to an empty array as clusterctl v1.0 doesn't support
+ // runtime extension providers. If we don't do this the test will automatically
+ // try to deploy the latest version of our test-extension from docker.yaml.
+
+ // NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v1.0/bases.
+ // Note: Both InitWithKubernetesVersion and WorkloadKubernetesVersion should be the highest mgmt cluster version supported by the source Cluster API version.
+ InitWithKubernetesVersion: "v1.23.17",
+ WorkloadKubernetesVersion: "v1.23.17",
+ MgmtFlavor: "topology",
+ WorkloadFlavor: "",
+ }
+ })
+})
+
 var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() {
  // Get v1.0 latest stable release
  version := "1.0"

test/e2e/config/docker.yaml

@@ -35,6 +35,15 @@ providers:
 - name: cluster-api
  type: CoreProvider
  versions:
+ - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+ value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
+ type: "url"
+ contract: v1alpha4
+ replacements:
+ - old: --metrics-addr=127.0.0.1:8080
+ new: --metrics-addr=:8080
+ files:
+ - sourcePath: "../data/shared/v0.4/metadata.yaml"
  - name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
  value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/core-components.yaml"
  type: "url"
@@ -73,6 +82,15 @@ providers:
 - name: kubeadm
  type: BootstrapProvider
  versions:
+ - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+ value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
+ type: "url"
+ contract: v1alpha4
+ replacements:
+ - old: --metrics-addr=127.0.0.1:8080
+ new: --metrics-addr=:8080
+ files:
+ - sourcePath: "../data/shared/v0.4/metadata.yaml"
  - name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
  value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/bootstrap-components.yaml"
  type: "url"
@@ -111,6 +129,15 @@ providers:
 - name: kubeadm
  type: ControlPlaneProvider
  versions:
+ - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+ value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
+ type: "url"
+ contract: v1alpha4
+ replacements:
+ - old: --metrics-addr=127.0.0.1:8080
+ new: --metrics-addr=:8080
+ files:
+ - sourcePath: "../data/shared/v0.4/metadata.yaml"
  - name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
  value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/control-plane-components.yaml"
  type: "url"
@@ -149,6 +176,16 @@ providers:
 - name: docker
  type: InfrastructureProvider
  versions:
+ - name: "{go://sigs.k8s.io/[email protected]}" # latest published release in the v1alpha4 series; this is used for v1alpha4 --> v1beta1 clusterctl upgrades test only.
+ value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
+ type: "url"
+ contract: v1alpha4
+ replacements:
+ - old: --metrics-addr=127.0.0.1:8080
+ new: --metrics-addr=:8080
+ files:
+ - sourcePath: "../data/shared/v0.4/metadata.yaml"
+ - sourcePath: "../data/infrastructure-docker/v0.4/cluster-template.yaml"
  - name: "{go://sigs.k8s.io/[email protected]}" # supported release in the v1beta1 series; this is used for v1beta1 --> main clusterctl upgrades test only.
  value: "https://github.com/kubernetes-sigs/cluster-api/releases/download/{go://sigs.k8s.io/[email protected]}/infrastructure-components-development.yaml"
  type: "url"

test/e2e/data/infrastructure-docker/v0.4/bases/cluster-with-kcp.yaml

@@ -0,0 +1,87 @@
+---
+# DockerCluster object referenced by the Cluster object
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+kind: DockerCluster
+metadata:
+ name: '${CLUSTER_NAME}'
+---
+# Cluster object with
+# - Reference to the KubeadmControlPlane object
+# - the label cni=${CLUSTER_NAME}-crs-0, so the cluster can be selected by the ClusterResourceSet.
+apiVersion: cluster.x-k8s.io/v1alpha4
+kind: Cluster
+metadata:
+ name: '${CLUSTER_NAME}'
+ labels:
+ cni: "${CLUSTER_NAME}-crs-0"
+spec:
+ clusterNetwork:
+ services:
+ cidrBlocks: ['${DOCKER_SERVICE_CIDRS}']
+ pods:
+ cidrBlocks: ['${DOCKER_POD_CIDRS}']
+ serviceDomain: '${DOCKER_SERVICE_DOMAIN}'
+ infrastructureRef:
+ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+ kind: DockerCluster
+ name: '${CLUSTER_NAME}'
+ controlPlaneRef:
+ kind: KubeadmControlPlane
+ apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
+ name: "${CLUSTER_NAME}-control-plane"
+---
+# DockerMachineTemplate object referenced by the KubeadmControlPlane object
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+kind: DockerMachineTemplate
+metadata:
+ name: "${CLUSTER_NAME}-control-plane"
+spec:
+ template:
+ spec:
+ # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+ customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
+ extraMounts:
+ - containerPath: "/var/run/docker.sock"
+ hostPath: "/var/run/docker.sock"
+---
+# KubeadmControlPlane referenced by the Cluster object with
+# - the label kcp-adoption.step2, because it should be created in the second step of the kcp-adoption test.
+kind: KubeadmControlPlane
+apiVersion: controlplane.cluster.x-k8s.io/v1alpha4
+metadata:
+ name: "${CLUSTER_NAME}-control-plane"
+ labels:
+ kcp-adoption.step2: ""
+spec:
+ replicas: ${CONTROL_PLANE_MACHINE_COUNT}
+ machineTemplate:
+ infrastructureRef:
+ kind: DockerMachineTemplate
+ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+ name: "${CLUSTER_NAME}-control-plane"
+ kubeadmConfigSpec:
+ clusterConfiguration:
+ controllerManager:
+ extraArgs: {enable-hostpath-provisioner: 'true'}
+ apiServer:
+ # host.docker.internal is required by kubetest when running on MacOS because of the way ports are proxied.
+ certSANs: [localhost, 127.0.0.1, 0.0.0.0, host.docker.internal]
+ initConfiguration:
+ nodeRegistration:
+ criSocket: unix:///var/run/containerd/containerd.sock
+ kubeletExtraArgs:
+ # We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
+ # This cluster is used in tests where the Kubernetes version is < 1.24
+ cgroup-driver: cgroupfs
+ eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+ fail-swap-on: "false"
+ joinConfiguration:
+ nodeRegistration:
+ criSocket: unix:///var/run/containerd/containerd.sock
+ kubeletExtraArgs:
+ # We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
+ # This cluster is used in tests where the Kubernetes version is < 1.24
+ cgroup-driver: cgroupfs
+ eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+ fail-swap-on: "false"
+ version: "${KUBERNETES_VERSION}"

test/e2e/data/infrastructure-docker/v0.4/bases/crs.yaml

@@ -0,0 +1,24 @@
+---
+# ConfigMap object referenced by the ClusterResourceSet object and with
+# the CNI resource defined in the test config file
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: "cni-${CLUSTER_NAME}-crs-0"
+data: ${CNI_RESOURCES}
+binaryData:
+---
+# ClusterResourceSet object with
+# a selector that targets all the Cluster with label cni=${CLUSTER_NAME}-crs-0
+apiVersion: addons.cluster.x-k8s.io/v1alpha4
+kind: ClusterResourceSet
+metadata:
+ name: "${CLUSTER_NAME}-crs-0"
+spec:
+ strategy: ApplyOnce
+ clusterSelector:
+ matchLabels:
+ cni: "${CLUSTER_NAME}-crs-0"
+ resources:
+ - name: "cni-${CLUSTER_NAME}-crs-0"
+ kind: ConfigMap

test/e2e/data/infrastructure-docker/v0.4/bases/md.yaml

@@ -0,0 +1,57 @@
+---
+# DockerMachineTemplate referenced by the MachineDeployment and with
+# - extraMounts for the docker sock, thus allowing self-hosting test
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+kind: DockerMachineTemplate
+metadata:
+ name: "${CLUSTER_NAME}-md-0"
+spec:
+ template:
+ spec:
+ # NOTE: If the Kubernetes version is changed in `clusterctl_upgrade_test.go` the image and SHA must be updated here.
+ customImage: "kindest/node:v1.23.17@sha256:f77f8cf0b30430ca4128cc7cfafece0c274a118cd0cdb251049664ace0dee4ff"
+ extraMounts:
+ - containerPath: "/var/run/docker.sock"
+ hostPath: "/var/run/docker.sock"
+---
+# KubeadmConfigTemplate referenced by the MachineDeployment
+apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
+kind: KubeadmConfigTemplate
+metadata:
+ name: "${CLUSTER_NAME}-md-0"
+spec:
+ template:
+ spec:
+ joinConfiguration:
+ nodeRegistration:
+ criSocket: unix:///var/run/containerd/containerd.sock
+ kubeletExtraArgs:
+ # We have to pin the cgroupDriver to cgroupfs for Kubernetes < v1.24 because kind does not support systemd for those versions, but kubeadm >= 1.21 defaults to systemd.
+ # This cluster is used in tests where the Kubernetes version is < 1.24
+ cgroup-driver: cgroupfs
+ eviction-hard: 'nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%'
+ fail-swap-on: "false"
+---
+# MachineDeployment object
+apiVersion: cluster.x-k8s.io/v1alpha4
+kind: MachineDeployment
+metadata:
+ name: "${CLUSTER_NAME}-md-0"
+spec:
+ clusterName: "${CLUSTER_NAME}"
+ replicas: ${WORKER_MACHINE_COUNT}
+ selector:
+ matchLabels:
+ template:
+ spec:
+ clusterName: "${CLUSTER_NAME}"
+ version: "${KUBERNETES_VERSION}"
+ bootstrap:
+ configRef:
+ name: "${CLUSTER_NAME}-md-0"
+ apiVersion: bootstrap.cluster.x-k8s.io/v1alpha4
+ kind: KubeadmConfigTemplate
+ infrastructureRef:
+ name: "${CLUSTER_NAME}-md-0"
+ apiVersion: infrastructure.cluster.x-k8s.io/v1alpha4
+ kind: DockerMachineTemplate

test/e2e/data/infrastructure-docker/v0.4/cluster-template/kustomization.yaml

@@ -0,0 +1,4 @@
+bases:
+- ../bases/cluster-with-kcp.yaml
+- ../bases/md.yaml
+- ../bases/crs.yaml

test/e2e/data/shared/v0.4/metadata.yaml

@@ -0,0 +1,12 @@
+apiVersion: clusterctl.cluster.x-k8s.io/v1alpha3
+kind: Metadata
+releaseSeries:
+ - major: 0
+ minor: 4
+ contract: v1alpha4
+ - major: 0
+ minor: 3
+ contract: v1alpha3
+ - major: 0
+ minor: 2
+ contract: v1alpha2

test/e2e/quick_start_test.go

@@ -61,7 +61,7 @@ var _ = Describe("When following the Cluster API quick-start", func() {
  })
 })
 
-var _ = Describe("When following the Cluster API quick-start with ClusterClass [PR-Blocking] [ClusterClass]", func() {
+var _ = Describe("When following the Cluster API quick-start with ClusterClass [ClusterClass]", func() {
  QuickStartSpec(ctx, func() QuickStartSpecInput {
  return QuickStartSpecInput{
  E2EConfig: e2eConfig,