Skip to content

Commit

Permalink
feat: user could set airgapped install script path (#140)
Browse files Browse the repository at this point in the history
Signed-off-by: nasusoba <[email protected]>

user could customize etcd proxy image

Signed-off-by: nasusoba <[email protected]>

add test for resolveEtcdProxyFile
  • Loading branch information
nasusoba authored Sep 5, 2024
1 parent f7d2499 commit 429d569
Show file tree
Hide file tree
Showing 20 changed files with 387 additions and 116 deletions.
9 changes: 9 additions & 0 deletions bootstrap/api/v1beta1/conversion.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ func (c *KThreesConfig) ConvertTo(dstRaw ctrlconversion.Hub) error {
dst.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
dst.Spec.ServerConfig.DisableCloudController = restored.Spec.ServerConfig.DisableCloudController
dst.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.ServerConfig.SystemDefaultRegistry
dst.Spec.ServerConfig.EtcdProxyImage = restored.Spec.ServerConfig.EtcdProxyImage
dst.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.AgentConfig.AirGappedInstallScriptPath
return nil
}

Expand Down Expand Up @@ -96,6 +98,8 @@ func (r *KThreesConfigTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error {
dst.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
dst.Spec.Template.Spec.ServerConfig.DisableCloudController = restored.Spec.Template.Spec.ServerConfig.DisableCloudController
dst.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry
dst.Spec.Template.Spec.ServerConfig.EtcdProxyImage = restored.Spec.Template.Spec.ServerConfig.EtcdProxyImage
dst.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath
return nil
}

Expand Down Expand Up @@ -159,3 +163,8 @@ func Convert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(in *boot

return nil
}

// Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig is an autogenerated conversion function.
func Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *bootstrapv1beta2.KThreesAgentConfig, out *KThreesAgentConfig, s conversion.Scope) error { //nolint: stylecheck
return autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in, out, s)
}
17 changes: 7 additions & 10 deletions bootstrap/api/v1beta1/zz_generated.conversion.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

12 changes: 11 additions & 1 deletion bootstrap/api/v1beta2/kthreesconfig_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,10 @@ type KThreesServerConfig struct {
// SystemDefaultRegistry defines private registry to be used for all system images
// +optional
SystemDefaultRegistry string `json:"systemDefaultRegistry,omitempty"`

// Customized etcd proxy image for management cluster to communicate with workload cluster etcd (default: "alpine/socat")
// +optional
EtcdProxyImage string `json:"etcdProxyImage,omitempty"`
}

type KThreesAgentConfig struct {
Expand Down Expand Up @@ -154,10 +158,16 @@ type KThreesAgentConfig struct {

// AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
// basically supposing that online container registries and k3s install scripts are not reachable.
// User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
// User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
// on all nodes in the air-gap environment.
// +optional
AirGapped bool `json:"airGapped,omitempty"`

// AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
// The install script should be prepared by the user. The value is only
// used when AirGapped is set to true (default: "/opt/install.sh").
// +optional
AirGappedInstallScriptPath string `json:"airGappedInstallScriptPath,omitempty"`
}

// KThreesConfigStatus defines the observed state of KThreesConfig.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -326,9 +326,15 @@ spec:
description: |-
AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
basically supposing that online container registries and k3s install scripts are not reachable.
User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
on all nodes in the air-gap environment.
type: boolean
airGappedInstallScriptPath:
description: |-
AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
The install script should be prepared by the user. The value is only
used when AirGapped is set to true (default: "/opt/install.sh").
type: string
kubeProxyArgs:
description: KubeProxyArgs Customized flag for kube-proxy process
items:
Expand Down Expand Up @@ -471,6 +477,10 @@ spec:
the ''cloud-provider=external'' kubelet argument. (default:
false)'
type: boolean
etcdProxyImage:
description: 'Customized etcd proxy image for management cluster
to communicate with workload cluster etcd (default: "alpine/socat")'
type: string
httpsListenPort:
description: 'HTTPSListenPort HTTPS listen port (default: 6443)'
type: string
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -281,9 +281,15 @@ spec:
description: |-
AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
basically supposing that online container registries and k3s install scripts are not reachable.
User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
on all nodes in the air-gap environment.
type: boolean
airGappedInstallScriptPath:
description: |-
AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
The install script should be prepared by the user. The value is only
used when AirGapped is set to true (default: "/opt/install.sh").
type: string
kubeProxyArgs:
description: KubeProxyArgs Customized flag for kube-proxy
process
Expand Down Expand Up @@ -432,6 +438,11 @@ spec:
the ''cloud-provider=external'' kubelet argument. (default:
false)'
type: boolean
etcdProxyImage:
description: 'Customized etcd proxy image for management
cluster to communicate with workload cluster etcd (default:
"alpine/socat")'
type: string
httpsListenPort:
description: 'HTTPSListenPort HTTPS listen port (default:
6443)'
Expand Down
96 changes: 66 additions & 30 deletions bootstrap/controllers/kthreesconfig_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,11 @@ limitations under the License.
package controllers

import (
"bytes"
"context"
"errors"
"fmt"
"html/template"
"time"

"github.com/go-logr/logr"
Expand Down Expand Up @@ -249,23 +251,24 @@ func (r *KThreesConfigReconciler) joinControlplane(ctx context.Context, scope *S
}

if scope.Config.Spec.IsEtcdEmbedded() {
etcdProxyFile := bootstrapv1.File{
Path: etcd.EtcdProxyDaemonsetYamlLocation,
Content: etcd.EtcdProxyDaemonsetYaml,
Owner: "root:root",
Permissions: "0640",
etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
if err != nil {
conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
return fmt.Errorf("failed to resolve etcd proxy file: %w", err)
}
files = append(files, etcdProxyFile)

files = append(files, *etcdProxyFile)
}

cpInput := &cloudinit.ControlPlaneInput{
BaseUserData: cloudinit.BaseUserData{
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
AdditionalFiles: files,
ConfigFile: workerConfigFile,
K3sVersion: scope.Config.Spec.Version,
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
AdditionalFiles: files,
ConfigFile: workerConfigFile,
K3sVersion: scope.Config.Spec.Version,
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
},
}

Expand Down Expand Up @@ -320,12 +323,13 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope)

winput := &cloudinit.WorkerInput{
BaseUserData: cloudinit.BaseUserData{
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
AdditionalFiles: files,
ConfigFile: workerConfigFile,
K3sVersion: scope.Config.Spec.Version,
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
AdditionalFiles: files,
ConfigFile: workerConfigFile,
K3sVersion: scope.Config.Spec.Version,
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
},
}

Expand Down Expand Up @@ -380,6 +384,38 @@ func (r *KThreesConfigReconciler) resolveSecretFileContent(ctx context.Context,
return data, nil
}

func (r *KThreesConfigReconciler) resolveEtcdProxyFile(cfg *bootstrapv1.KThreesConfig) (*bootstrapv1.File, error) {
// Parse the template
tpl, err := template.New("etcd-proxy").Parse(etcd.EtcdProxyDaemonsetYamlTemplate)
if err != nil {
return nil, fmt.Errorf("failed to parse etcd-proxy template: %w", err)
}

// If user has set the systemDefaultRegistry, will prefix the image with it.
systemDefaultRegistry := cfg.Spec.ServerConfig.SystemDefaultRegistry
if systemDefaultRegistry != "" {
systemDefaultRegistry = fmt.Sprintf("%s/", systemDefaultRegistry)
}

// Render the template, the image name will be ${EtcdProxyImage} if the user
// has set it, otherwise it will be ${SystemDefaultRegistry}alpine/socat
var buf bytes.Buffer
err = tpl.Execute(&buf, map[string]string{
"EtcdProxyImage": cfg.Spec.ServerConfig.EtcdProxyImage,
"SystemDefaultRegistry": systemDefaultRegistry,
})
if err != nil {
return nil, fmt.Errorf("failed to render etcd-proxy template: %w", err)
}

return &bootstrapv1.File{
Path: etcd.EtcdProxyDaemonsetYamlLocation,
Content: buf.String(),
Owner: "root:root",
Permissions: "0640",
}, nil
}

func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) {
// initialize the DataSecretAvailableCondition if missing.
// this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
Expand Down Expand Up @@ -465,23 +501,23 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex
}

if scope.Config.Spec.IsEtcdEmbedded() {
etcdProxyFile := bootstrapv1.File{
Path: etcd.EtcdProxyDaemonsetYamlLocation,
Content: etcd.EtcdProxyDaemonsetYaml,
Owner: "root:root",
Permissions: "0640",
etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
if err != nil {
conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
return ctrl.Result{}, fmt.Errorf("failed to resolve etcd proxy file: %w", err)
}
files = append(files, etcdProxyFile)
files = append(files, *etcdProxyFile)
}

cpinput := &cloudinit.ControlPlaneInput{
BaseUserData: cloudinit.BaseUserData{
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
AdditionalFiles: files,
ConfigFile: initConfigFile,
K3sVersion: scope.Config.Spec.Version,
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
PreK3sCommands: scope.Config.Spec.PreK3sCommands,
PostK3sCommands: scope.Config.Spec.PostK3sCommands,
AdditionalFiles: files,
ConfigFile: initConfigFile,
K3sVersion: scope.Config.Spec.Version,
AirGapped: scope.Config.Spec.AgentConfig.AirGapped,
AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
},
Certificates: certificates,
}
Expand Down
55 changes: 55 additions & 0 deletions bootstrap/controllers/kthreesconfig_controller_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controllers

import (
"testing"

. "github.com/onsi/gomega"

bootstrapv1 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2"
)

func TestKThreesConfigReconciler_ResolveEtcdProxyFile(t *testing.T) {
g := NewWithT(t)
// If EtcdProxyImage is set, it should override the system default registry
config := &bootstrapv1.KThreesConfig{
Spec: bootstrapv1.KThreesConfigSpec{
ServerConfig: bootstrapv1.KThreesServerConfig{
EtcdProxyImage: "etcd-proxy-image",
SystemDefaultRegistry: "system-default-registry",
},
},
}
r := &KThreesConfigReconciler{}
etcdProxyFile, err := r.resolveEtcdProxyFile(config)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(etcdProxyFile.Content).To(ContainSubstring("etcd-proxy-image"), "generated etcd proxy image should contain EtcdProxyImage")
g.Expect(etcdProxyFile.Content).ToNot(ContainSubstring("system-default-registry"), "system-default-registry should be overwritten by EtcdProxyImage")

// If EtcdProxyImage is not set, the system default registry should be used
config2 := &bootstrapv1.KThreesConfig{
Spec: bootstrapv1.KThreesConfigSpec{
ServerConfig: bootstrapv1.KThreesServerConfig{
SystemDefaultRegistry: "system-default-registry2",
},
},
}
etcdProxyFile, err = r.resolveEtcdProxyFile(config2)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(etcdProxyFile.Content).To(ContainSubstring("system-default-registry2/"), "generated etcd proxy image should be prefixed with SystemDefaultRegistry")
}
2 changes: 2 additions & 0 deletions controlplane/api/v1beta1/conversion.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,9 +94,11 @@ func (in *KThreesControlPlane) ConvertTo(dstRaw ctrlconversion.Hub) error {
dst.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider
dst.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController = restored.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController
dst.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry = restored.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry
dst.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage = restored.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage
dst.Spec.MachineTemplate.NodeVolumeDetachTimeout = restored.Spec.MachineTemplate.NodeVolumeDetachTimeout
dst.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.MachineTemplate.NodeDeletionTimeout
dst.Status.Version = restored.Status.Version
dst.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath
return nil
}

Expand Down
Loading

0 comments on commit 429d569

Please sign in to comment.