feat: User could customize airgapped install script path and etcd proxy image

bootstrap/api/v1beta1/conversion.go

@@ -43,6 +43,8 @@ func (c *KThreesConfig) ConvertTo(dstRaw ctrlconversion.Hub) error {
  dst.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
  dst.Spec.ServerConfig.DisableCloudController = restored.Spec.ServerConfig.DisableCloudController
  dst.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.ServerConfig.SystemDefaultRegistry
+ dst.Spec.ServerConfig.EtcdProxyImage = restored.Spec.ServerConfig.EtcdProxyImage
+ dst.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.AgentConfig.AirGappedInstallScriptPath
  return nil
 }
 
@@ -96,6 +98,8 @@ func (r *KThreesConfigTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error {
  dst.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
  dst.Spec.Template.Spec.ServerConfig.DisableCloudController = restored.Spec.Template.Spec.ServerConfig.DisableCloudController
  dst.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry
+ dst.Spec.Template.Spec.ServerConfig.EtcdProxyImage = restored.Spec.Template.Spec.ServerConfig.EtcdProxyImage
+ dst.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath
  return nil
 }
 
@@ -159,3 +163,8 @@ func Convert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(in *boot
 
  return nil
 }
+
+// Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig is an autogenerated conversion function.
+func Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *bootstrapv1beta2.KThreesAgentConfig, out *KThreesAgentConfig, s conversion.Scope) error { //nolint: stylecheck
+ return autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in, out, s)
+}

bootstrap/api/v1beta2/kthreesconfig_types.go

@@ -124,6 +124,10 @@ type KThreesServerConfig struct {
  // SystemDefaultRegistry defines private registry to be used for all system images
  // +optional
  SystemDefaultRegistry string `json:"systemDefaultRegistry,omitempty"`
+
+ // Customized etcd proxy image for management cluster to communicate with workload cluster etcd (default: "alpine/socat")
+ // +optional
+ EtcdProxyImage string `json:"etcdProxyImage,omitempty"`
 }
 
 type KThreesAgentConfig struct {
@@ -154,10 +158,16 @@ type KThreesAgentConfig struct {
 
  // AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
  // basically supposing that online container registries and k3s install scripts are not reachable.
- // User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+ // User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
  // on all nodes in the air-gap environment.
  // +optional
  AirGapped bool `json:"airGapped,omitempty"`
+
+ // AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
+ // The install script should be prepared by the user. The value is only
+ // used when AirGapped is set to true (default: "/opt/install.sh").
+ // +optional
+ AirGappedInstallScriptPath string `json:"airGappedInstallScriptPath,omitempty"`
 }
 
 // KThreesConfigStatus defines the observed state of KThreesConfig.

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml

@@ -326,9 +326,15 @@ spec:
  description: |-
  AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
  basically supposing that online container registries and k3s install scripts are not reachable.
- User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+ User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
  on all nodes in the air-gap environment.
  type: boolean
+ airGappedInstallScriptPath:
+ description: |-
+ AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
+ The install script should be prepared by the user. The value is only
+ used when AirGapped is set to true (default: "/opt/install.sh").
+ type: string
  kubeProxyArgs:
  description: KubeProxyArgs Customized flag for kube-proxy process
  items:
@@ -471,6 +477,10 @@ spec:
  the ''cloud-provider=external'' kubelet argument. (default:
  false)'
  type: boolean
+ etcdProxyImage:
+ description: 'Customized etcd proxy image for management cluster
+ to communicate with workload cluster etcd (default: "alpine/socat")'
+ type: string
  httpsListenPort:
  description: 'HTTPSListenPort HTTPS listen port (default: 6443)'
  type: string

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml

@@ -281,9 +281,15 @@ spec:
  description: |-
  AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
  basically supposing that online container registries and k3s install scripts are not reachable.
- User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+ User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
  on all nodes in the air-gap environment.
  type: boolean
+ airGappedInstallScriptPath:
+ description: |-
+ AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
+ The install script should be prepared by the user. The value is only
+ used when AirGapped is set to true (default: "/opt/install.sh").
+ type: string
  kubeProxyArgs:
  description: KubeProxyArgs Customized flag for kube-proxy
  process
@@ -432,6 +438,11 @@ spec:
  the ''cloud-provider=external'' kubelet argument. (default:
  false)'
  type: boolean
+ etcdProxyImage:
+ description: 'Customized etcd proxy image for management
+ cluster to communicate with workload cluster etcd (default:
+ "alpine/socat")'
+ type: string
  httpsListenPort:
  description: 'HTTPSListenPort HTTPS listen port (default:
  6443)'

bootstrap/controllers/kthreesconfig_controller.go

@@ -17,9 +17,11 @@ limitations under the License.
 package controllers
 
 import (
+ "bytes"
  "context"
  "errors"
  "fmt"
+ "html/template"
  "time"
 
  "github.com/go-logr/logr"
@@ -249,23 +251,24 @@ func (r *KThreesConfigReconciler) joinControlplane(ctx context.Context, scope *S
  }
 
  if scope.Config.Spec.IsEtcdEmbedded() {
- etcdProxyFile := bootstrapv1.File{
- Path: etcd.EtcdProxyDaemonsetYamlLocation,
- Content: etcd.EtcdProxyDaemonsetYaml,
- Owner: "root:root",
- Permissions: "0640",
+ etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
+ if err != nil {
+ conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
+ return fmt.Errorf("failed to resolve etcd proxy file: %w", err)
  }
- files = append(files, etcdProxyFile)
+
+ files = append(files, *etcdProxyFile)
  }
 
  cpInput := &cloudinit.ControlPlaneInput{
  BaseUserData: cloudinit.BaseUserData{
- PreK3sCommands: scope.Config.Spec.PreK3sCommands,
- PostK3sCommands: scope.Config.Spec.PostK3sCommands,
- AdditionalFiles: files,
- ConfigFile: workerConfigFile,
- K3sVersion: scope.Config.Spec.Version,
- AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
+ PreK3sCommands: scope.Config.Spec.PreK3sCommands,
+ PostK3sCommands: scope.Config.Spec.PostK3sCommands,
+ AdditionalFiles: files,
+ ConfigFile: workerConfigFile,
+ K3sVersion: scope.Config.Spec.Version,
+ AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
+ AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
  },
  }
 
@@ -320,12 +323,13 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
 
  winput := &cloudinit.WorkerInput{
  BaseUserData: cloudinit.BaseUserData{
- PreK3sCommands: scope.Config.Spec.PreK3sCommands,
- PostK3sCommands: scope.Config.Spec.PostK3sCommands,
- AdditionalFiles: files,
- ConfigFile: workerConfigFile,
- K3sVersion: scope.Config.Spec.Version,
- AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
+ PreK3sCommands: scope.Config.Spec.PreK3sCommands,
+ PostK3sCommands: scope.Config.Spec.PostK3sCommands,
+ AdditionalFiles: files,
+ ConfigFile: workerConfigFile,
+ K3sVersion: scope.Config.Spec.Version,
+ AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
+ AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
  },
  }
 
@@ -380,6 +384,38 @@ func (r *KThreesConfigReconciler) resolveSecretFileContent(ctx context.Context,
  return data, nil
 }
 
+func (r *KThreesConfigReconciler) resolveEtcdProxyFile(cfg *bootstrapv1.KThreesConfig) (*bootstrapv1.File, error) {
+ // Parse the template
+ tpl, err := template.New("etcd-proxy").Parse(etcd.EtcdProxyDaemonsetYamlTemplate)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse etcd-proxy template: %w", err)
+ }
+
+ // If user has set the systemDefaultRegistry, will prefix the image with it.
+ systemDefaultRegistry := cfg.Spec.ServerConfig.SystemDefaultRegistry
+ if systemDefaultRegistry != "" {
+ systemDefaultRegistry = fmt.Sprintf("%s/", systemDefaultRegistry)
+ }
+
+ // Render the template, the image name will be ${EtcdProxyImage} if the user
+ // has set it, otherwise it will be ${SystemDefaultRegistry}alpine/socat
+ var buf bytes.Buffer
+ err = tpl.Execute(&buf, map[string]string{
+ "EtcdProxyImage": cfg.Spec.ServerConfig.EtcdProxyImage,
+ "SystemDefaultRegistry": systemDefaultRegistry,
+ })
+ if err != nil {
+ return nil, fmt.Errorf("failed to render etcd-proxy template: %w", err)
+ }
+
+ return &bootstrapv1.File{
+ Path: etcd.EtcdProxyDaemonsetYamlLocation,
+ Content: buf.String(),
+ Owner: "root:root",
+ Permissions: "0640",
+ }, nil
+}
+
 func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) {
  // initialize the DataSecretAvailableCondition if missing.
  // this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
@@ -465,23 +501,23 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex
  }
 
  if scope.Config.Spec.IsEtcdEmbedded() {
- etcdProxyFile := bootstrapv1.File{
- Path: etcd.EtcdProxyDaemonsetYamlLocation,
- Content: etcd.EtcdProxyDaemonsetYaml,
- Owner: "root:root",
- Permissions: "0640",
+ etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
+ if err != nil {
+ conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
+ return ctrl.Result{}, fmt.Errorf("failed to resolve etcd proxy file: %w", err)
  }
- files = append(files, etcdProxyFile)
+ files = append(files, *etcdProxyFile)
  }
 
  cpinput := &cloudinit.ControlPlaneInput{
  BaseUserData: cloudinit.BaseUserData{
- PreK3sCommands: scope.Config.Spec.PreK3sCommands,
- PostK3sCommands: scope.Config.Spec.PostK3sCommands,
- AdditionalFiles: files,
- ConfigFile: initConfigFile,
- K3sVersion: scope.Config.Spec.Version,
- AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
+ PreK3sCommands: scope.Config.Spec.PreK3sCommands,
+ PostK3sCommands: scope.Config.Spec.PostK3sCommands,
+ AdditionalFiles: files,
+ ConfigFile: initConfigFile,
+ K3sVersion: scope.Config.Spec.Version,
+ AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
+ AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
  },
  Certificates: certificates,
  }

bootstrap/controllers/kthreesconfig_controller_test.go

@@ -0,0 +1,55 @@
+/*
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+ "testing"
+
+ . "github.com/onsi/gomega"
+
+ bootstrapv1 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2"
+)
+
+func TestKThreesConfigReconciler_ResolveEtcdProxyFile(t *testing.T) {
+ g := NewWithT(t)
+ // If EtcdProxyImage is set, it should override the system default registry
+ config := &bootstrapv1.KThreesConfig{
+ Spec: bootstrapv1.KThreesConfigSpec{
+ ServerConfig: bootstrapv1.KThreesServerConfig{
+ EtcdProxyImage: "etcd-proxy-image",
+ SystemDefaultRegistry: "system-default-registry",
+ },
+ },
+ }
+ r := &KThreesConfigReconciler{}
+ etcdProxyFile, err := r.resolveEtcdProxyFile(config)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(etcdProxyFile.Content).To(ContainSubstring("etcd-proxy-image"), "generated etcd proxy image should contain EtcdProxyImage")
+ g.Expect(etcdProxyFile.Content).ToNot(ContainSubstring("system-default-registry"), "system-default-registry should be overwritten by EtcdProxyImage")
+
+ // If EtcdProxyImage is not set, the system default registry should be used
+ config2 := &bootstrapv1.KThreesConfig{
+ Spec: bootstrapv1.KThreesConfigSpec{
+ ServerConfig: bootstrapv1.KThreesServerConfig{
+ SystemDefaultRegistry: "system-default-registry2",
+ },
+ },
+ }
+ etcdProxyFile, err = r.resolveEtcdProxyFile(config2)
+ g.Expect(err).ToNot(HaveOccurred())
+ g.Expect(etcdProxyFile.Content).To(ContainSubstring("system-default-registry2/"), "generated etcd proxy image should be prefixed with SystemDefaultRegistry")
+}

controlplane/api/v1beta1/conversion.go

@@ -94,9 +94,11 @@ func (in *KThreesControlPlane) ConvertTo(dstRaw ctrlconversion.Hub) error {
  dst.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider
  dst.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController = restored.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController
  dst.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry = restored.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry
+ dst.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage = restored.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage
  dst.Spec.MachineTemplate.NodeVolumeDetachTimeout = restored.Spec.MachineTemplate.NodeVolumeDetachTimeout
  dst.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.MachineTemplate.NodeDeletionTimeout
  dst.Status.Version = restored.Status.Version
+ dst.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath
  return nil
 }