From db0ddadad1fbb6c9e9c09aaaf4ac6acfd712b55f Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Mon, 8 Jan 2024 09:21:45 +0100 Subject: [PATCH] Remove kubelet-config stack after deprecation in 1.26 The kubelet-config component has been deprecated in 1.26. Remove the corresponding resources from the cluster in 1.29. Rename all manifest files in the stack's folder to something that won't be picked up by the stack applier. Leave a note about the stack removal in the stack's folder. Signed-off-by: Tom Wieczorek --- cmd/controller/controller.go | 2 +- go.mod | 2 +- inttest/ap-ha3x3/ha3x3_test.go | 185 ++++++++-- pkg/applier/applier.go | 7 +- pkg/component/controller/kubeletconfig.go | 326 +++--------------- .../controller/kubeletconfig_test.go | 220 +++--------- pkg/config/cli.go | 6 - pkg/config/cli_test.go | 30 +- pkg/constant/constant_shared.go | 1 - 9 files changed, 269 insertions(+), 510 deletions(-) diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 85cbd1850033..d3190ad744b2 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -469,7 +469,7 @@ func (c *command) start(ctx context.Context) error { return err } clusterComponents.Add(ctx, reconciler) - clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory, nodeConfig)) + clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars)) } if !slices.Contains(c.DisableComponents, constant.SystemRbacComponentName) { diff --git a/go.mod b/go.mod index efc9747a1bb6..ccc09ec01616 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,6 @@ require ( github.com/go-playground/validator/v10 v10.16.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/terraform-exec v0.20.0 - github.com/imdario/mergo v0.3.16 github.com/k0sproject/bootloose v0.7.2 github.com/k0sproject/dig v0.2.0 github.com/k0sproject/version v0.4.2 @@ -176,6 +175,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/terraform-json v0.19.0 // indirect github.com/huandu/xstrings v1.4.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/intel/goresctrl v0.3.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect diff --git a/inttest/ap-ha3x3/ha3x3_test.go b/inttest/ap-ha3x3/ha3x3_test.go index 7d07ebb09b3a..d76d656a7729 100644 --- a/inttest/ap-ha3x3/ha3x3_test.go +++ b/inttest/ap-ha3x3/ha3x3_test.go @@ -15,24 +15,30 @@ package ha3x3 import ( + "bytes" + "context" "fmt" "os" + "path/filepath" "strings" "testing" "time" - apv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" + "github.com/avast/retry-go" apconst "github.com/k0sproject/k0s/pkg/autopilot/constant" appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core" + "golang.org/x/exp/slices" "github.com/k0sproject/k0s/inttest/common" aptest "github.com/k0sproject/k0s/inttest/common/autopilot" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" ) type ha3x3Suite struct { common.BootlooseSuite + k0sUpdateVersion string } const haControllerConfig = ` @@ -93,9 +99,6 @@ func (s *ha3x3Suite) SetupTest() { // TestApply applies a well-formed `plan` yaml, and asserts that // all of the correct values across different objects + controllers are correct. func (s *ha3x3Suite) TestApply() { - k0sUpdateVersion := os.Getenv("K0S_UPDATE_TO_VERSION") - s.Require().NotEmpty(k0sUpdateVersion, "env var not set or empty: K0S_UPDATE_TO_VERSION") - planTemplate := ` apiVersion: autopilot.k0sproject.io/v1beta2 kind: Plan @@ -106,7 +109,7 @@ spec: timestamp: now commands: - k0supdate: - version: ` + k0sUpdateVersion + ` + version: ` + s.k0sUpdateVersion + ` platforms: linux-amd64: url: http://localhost/dist/k0s-new @@ -127,58 +130,177 @@ spec: - worker2 ` - manifestFile := "/tmp/happy.yaml" - s.PutFileTemplate(s.ControllerNode(0), manifestFile, planTemplate, nil) + ctx := s.Context() - out, err := s.RunCommandController(0, fmt.Sprintf("/usr/local/bin/k0s kubectl apply -f %s", manifestFile)) - s.T().Logf("kubectl apply output: '%s'", out) + sshController, err := s.SSH(ctx, s.ControllerNode(0)) s.Require().NoError(err) + defer sshController.Disconnect() - ssh, err := s.SSH(s.Context(), s.WorkerNode(0)) + var baseHasOldStack *bool + if version, err := s.GetK0sVersion(s.ControllerNode(0)); s.NoError(err, "Failed to get the base k0s version") { + hasOldStack := version != s.k0sUpdateVersion && strings.HasPrefix(version, "v1.27.") || strings.HasPrefix(version, "v1.28.") + s.T().Logf("Base k0s version: %q, has old stack: %v", version, hasOldStack) + s.checkKubeletConfigStackResources(ctx, sshController, hasOldStack) + baseHasOldStack = &hasOldStack + } + + sshWorker, err := s.SSH(ctx, s.WorkerNode(0)) s.Require().NoError(err) - defer ssh.Disconnect() - out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V") + defer sshWorker.Disconnect() + + iptablesModeBeforeUpdate, err := getIPTablesMode(ctx, sshWorker) + if !s.NoError(err) { + iptablesModeBeforeUpdate = "" + } + + var createPlanOutput bytes.Buffer + err = sshController.Exec(ctx, fmt.Sprintf("k0s kc create -f -"), common.SSHStreams{ + In: strings.NewReader(planTemplate), + Out: &createPlanOutput, + }) s.Require().NoError(err) - iptablesVersionParts := strings.Split(out, " ") - iptablesModeBeforeUpdate := iptablesVersionParts[len(iptablesVersionParts)-1] + s.T().Log(strings.TrimSpace(createPlanOutput.String())) client, err := s.AutopilotClient(s.ControllerNode(0)) s.Require().NoError(err) - s.NotEmpty(client) // The plan has enough information to perform a successful update of k0s, so wait for it. - plan, err := aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted) + s.T().Log("Waiting for autopilot plan to complete") + plan, err := aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted) s.Require().NoError(err) + s.T().Log("Autopilot plan completed") // Ensure all state/status are completed - s.Equal(1, len(plan.Status.Commands)) - cmd := plan.Status.Commands[0] + if s.Len(plan.Status.Commands, 1) { + cmd := plan.Status.Commands[0] + s.Equal(appc.PlanCompleted, cmd.State) + s.Equal(appc.PlanCompleted, cmd.State) + s.NotNil(cmd.K0sUpdate) + s.NotNil(cmd.K0sUpdate.Controllers) + s.NotNil(cmd.K0sUpdate.Workers) + s.Equal(appc.PlanCompleted, cmd.State) + s.NotNil(cmd.K0sUpdate) + s.NotNil(cmd.K0sUpdate.Controllers) + s.NotNil(cmd.K0sUpdate.Workers) - s.Equal(appc.PlanCompleted, cmd.State) - s.NotNil(cmd.K0sUpdate) - s.NotNil(cmd.K0sUpdate.Controllers) - s.NotNil(cmd.K0sUpdate.Workers) + if s.NotNil(cmd.K0sUpdate) { + s.Len(cmd.K0sUpdate.Controllers, s.ControllerCount) + for idx, controller := range cmd.K0sUpdate.Controllers { + s.Equal(appc.SignalCompleted, controller.State, "For controller %d", idx) + } - for _, group := range [][]apv1beta2.PlanCommandTargetStatus{cmd.K0sUpdate.Controllers, cmd.K0sUpdate.Workers} { - for _, node := range group { - s.Equal(appc.SignalCompleted, node.State) + s.Len(cmd.K0sUpdate.Workers, s.WorkerCount) + for idx, worker := range cmd.K0sUpdate.Workers { + s.Equal(appc.SignalCompleted, worker.State, "For worker %d", idx) + } } } if version, err := s.GetK0sVersion(s.ControllerNode(0)); s.NoError(err) { - s.Equal(k0sUpdateVersion, version) + s.Equal(s.k0sUpdateVersion, version) } - out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V") - s.Require().NoError(err) - iptablesVersionParts = strings.Split(out, " ") - iptablesModeAfterUpdate := iptablesVersionParts[len(iptablesVersionParts)-1] - s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate) + if iptablesModeAfterUpdate, err := getIPTablesMode(ctx, sshWorker); s.NoError(err) { + s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate) + } + + if baseHasOldStack != nil { + for idx := 0; idx < s.ControllerCount; idx++ { + func() { + ssh, err := s.SSH(ctx, s.ControllerNode(idx)) + s.Require().NoError(err) + defer ssh.Disconnect() + s.checkKubeletConfigComponentFolders(ctx, ssh, *baseHasOldStack) + }() + } + } + + s.checkKubeletConfigStackResources(ctx, sshController, false) +} + +func (s *ha3x3Suite) checkKubeletConfigComponentFolders(ctx context.Context, ssh *common.SSHConnection, hasOldStack bool) { + if !hasOldStack { + // Expect no kubelet folder at all for recent versions + err := ssh.Exec(ctx, "[ ! -e /var/lib/k0s/manifests/kubelet ]", common.SSHStreams{}) + s.NoError(err, "Failed to check that the kubelet manifest folder doesn't exist") + return + } + + // Expect a kubelet folder with removal files for older k0s versions + + var foundFiles bytes.Buffer + if !s.NoError( + ssh.Exec(ctx, "cd /var/lib/k0s/manifests/kubelet && find . -type f -print0", common.SSHStreams{Out: &foundFiles}), + "Failed to list kubelet manifest folder", + ) { + return + } + + files := strings.Split(strings.TrimSuffix(foundFiles.String(), "\x00"), "\x00") + + // Check that removed.txt is present + if idx := slices.Index(files, "./removed.txt"); idx < 0 { + s.Failf("No removed.txt in kubelet manifests folder", "%v", files) + } else { + files = slices.Delete(files, idx, idx+1) + } + + // Check that all other files are only disabled yaml files. + for _, file := range files { + match, err := filepath.Match("./kubelet-config.yaml.*.removed", file) + s.Require().NoError(err) + if !match { + s.Failf("Unknown file in kubelet manifest folder", "%s in %v", file, files) + } + } +} + +func (s *ha3x3Suite) checkKubeletConfigStackResources(ctx context.Context, ssh *common.SSHConnection, exist bool) { + const cmd = "k0s kc get configmaps,roles,rolebindings -A -l 'k0s.k0sproject.io/stack=kubelet' -oname" + + var out bytes.Buffer + err := retry.Do( + func() error { + out.Reset() + return ssh.Exec(ctx, cmd, common.SSHStreams{Out: &out}) + }, + retry.OnRetry(func(attempt uint, err error) { + s.T().Logf("Failed to check kubelet-config stack resources in attempt #%d, retrying after backoff: %v", attempt+1, err) + }), + retry.Context(ctx), + retry.LastErrorOnly(true), + ) + + if s.NoError(err) { + if exist { + s.NotEmpty(out.String()) + } else { + s.Empty(out.String()) + } + } +} + +func getIPTablesMode(ctx context.Context, ssh *common.SSHConnection) (string, error) { + var out bytes.Buffer + err := ssh.Exec(ctx, "/var/lib/k0s/bin/iptables-save -V", common.SSHStreams{Out: &out}) + if err != nil { + return "", err + } + + version := out.String() + if parts := strings.Split(version, " "); len(parts) == 3 { + return parts[2], nil + } + + return "", fmt.Errorf("expected something like %q, got %q", "iptables-save v1.8.9 (nf_tables)", version) } // TestHA3x3Suite sets up a suite using 3 controllers for quorum, and runs various // autopilot upgrade scenarios against them. func TestHA3x3Suite(t *testing.T) { + k0sUpdateVersion := os.Getenv("K0S_UPDATE_TO_VERSION") + require.NotEmpty(t, k0sUpdateVersion, "env var not set or empty: K0S_UPDATE_TO_VERSION") + suite.Run(t, &ha3x3Suite{ common.BootlooseSuite{ ControllerCount: 3, @@ -186,5 +308,6 @@ func TestHA3x3Suite(t *testing.T) { WithLB: true, LaunchMode: common.LaunchModeOpenRC, }, + k0sUpdateVersion, }) } diff --git a/pkg/applier/applier.go b/pkg/applier/applier.go index 1e38474d866f..a0239070917a 100644 --- a/pkg/applier/applier.go +++ b/pkg/applier/applier.go @@ -19,7 +19,6 @@ package applier import ( "context" "fmt" - "path" "path/filepath" "github.com/k0sproject/k0s/pkg/kubernetes" @@ -35,6 +34,10 @@ import ( // manifestFilePattern is the glob pattern that all applicable manifest files need to match. const manifestFilePattern = "*.yaml" +func FindManifestFilesInDir(dir string) ([]string, error) { + return filepath.Glob(filepath.Join(dir, manifestFilePattern)) +} + // Applier manages all the "static" manifests and applies them on the k8s API type Applier struct { Name string @@ -96,7 +99,7 @@ func (a *Applier) Apply(ctx context.Context) error { return err } - files, err := filepath.Glob(path.Join(a.Dir, manifestFilePattern)) + files, err := FindManifestFilesInDir(a.Dir) if err != nil { return err } diff --git a/pkg/component/controller/kubeletconfig.go b/pkg/component/controller/kubeletconfig.go index 449093613d2c..44b2532c58ee 100644 --- a/pkg/component/controller/kubeletconfig.go +++ b/pkg/component/controller/kubeletconfig.go @@ -17,315 +17,77 @@ limitations under the License. package controller import ( - "bytes" "context" - "crypto/tls" - "encoding/json" - "fmt" - "io" - "path" + "errors" + "os" "path/filepath" - "reflect" - - "github.com/imdario/mergo" - "github.com/k0sproject/k0s/pkg/component/manager" - "github.com/k0sproject/k0s/pkg/config" - k8sutil "github.com/k0sproject/k0s/pkg/kubernetes" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/yaml" "github.com/k0sproject/k0s/internal/pkg/dir" "github.com/k0sproject/k0s/internal/pkg/file" - "github.com/k0sproject/k0s/internal/pkg/templatewriter" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/applier" + "github.com/k0sproject/k0s/pkg/component/manager" + "github.com/k0sproject/k0s/pkg/config" "github.com/k0sproject/k0s/pkg/constant" + + "github.com/sirupsen/logrus" ) // Dummy checks so we catch easily if we miss some interface implementation -var _ manager.Reconciler = (*KubeletConfig)(nil) var _ manager.Component = (*KubeletConfig)(nil) -// KubeletConfig is the reconciler for generic kubelet configs +// KubeletConfig is the old, replaced reconciler for generic kubelet configs. type KubeletConfig struct { - log logrus.FieldLogger - - kubeClientFactory k8sutil.ClientFactoryInterface - k0sVars *config.CfgVars - previousProfiles v1beta1.WorkerProfiles - nodeConfig *v1beta1.ClusterConfig -} - -// NewKubeletConfig creates new KubeletConfig reconciler -func NewKubeletConfig(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInterface, nodeConfig *v1beta1.ClusterConfig) *KubeletConfig { - return &KubeletConfig{ - log: logrus.WithFields(logrus.Fields{"component": "kubeletconfig"}), - - kubeClientFactory: clientFactory, - k0sVars: k0sVars, - nodeConfig: nodeConfig, - } -} - -// Init does nothing -func (k *KubeletConfig) Init(_ context.Context) error { - return nil -} - -// Stop does nothign, nothing actually running -func (k *KubeletConfig) Stop() error { - return nil -} - -// Run dumps the needed manifest objects -func (k *KubeletConfig) Start(_ context.Context) error { - - return nil + k0sVars *config.CfgVars } -// Reconcile detects changes in configuration and applies them to the component -func (k *KubeletConfig) Reconcile(ctx context.Context, clusterSpec *v1beta1.ClusterConfig) error { - k.log.Debug("reconcile method called for: KubeletConfig") - // Check if we actually need to reconcile anything - defaultProfilesExist, err := k.defaultProfilesExist(ctx) - if err != nil { - return err - } - if defaultProfilesExist && reflect.DeepEqual(k.previousProfiles, clusterSpec.Spec.WorkerProfiles) { - k.log.Debugf("default profiles exist and no change in user specified profiles, nothing to reconcile") - return nil - } - - manifest, err := k.createProfiles(clusterSpec) - if err != nil { - return fmt.Errorf("failed to build final manifest: %v", err) - } - - if err := k.save(manifest.Bytes()); err != nil { - return fmt.Errorf("can't write manifest with config maps: %v", err) - } - k.previousProfiles = clusterSpec.Spec.WorkerProfiles - - return nil +// NewKubeletConfig creates a new KubeletConfig reconciler that merely +// uninstalls itself, if it still exists. +func NewKubeletConfig(k0sVars *config.CfgVars) *KubeletConfig { + return &KubeletConfig{k0sVars} } -func (k *KubeletConfig) defaultProfilesExist(ctx context.Context) (bool, error) { - c, err := k.kubeClientFactory.GetClient() - if err != nil { - return false, err - } - defaultProfileName := formatProfileName("default") - _, err = c.CoreV1().ConfigMaps("kube-system").Get(ctx, defaultProfileName, v1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } else if err != nil { - return false, err - } - return true, nil -} - -func (k *KubeletConfig) createProfiles(clusterSpec *v1beta1.ClusterConfig) (*bytes.Buffer, error) { - dnsAddress, err := k.nodeConfig.Spec.Network.DNSAddress() - if err != nil { - return nil, fmt.Errorf("failed to get DNS address for kubelet config: %v", err) - } - manifest := bytes.NewBuffer([]byte{}) - defaultProfile := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - defaultProfile["cgroupsPerQOS"] = true - - winDefaultProfile := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - winDefaultProfile["cgroupsPerQOS"] = false +func (k *KubeletConfig) Init(context.Context) error { + kubeletDir := filepath.Join(k.k0sVars.ManifestsDir, "kubelet") - if err := k.writeConfigMapWithProfile(manifest, "default", defaultProfile); err != nil { - return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err) - } - if err := k.writeConfigMapWithProfile(manifest, "default-windows", winDefaultProfile); err != nil { - return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err) - } - configMapNames := []string{ - formatProfileName("default"), - formatProfileName("default-windows"), - } - for _, profile := range clusterSpec.Spec.WorkerProfiles { - profileConfig := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - - var workerValues unstructuredYamlObject - err := json.Unmarshal(profile.Config.Raw, &workerValues) - if err != nil { - return nil, fmt.Errorf("failed to decode worker profile values: %v", err) - } - merged, err := mergeProfiles(&profileConfig, workerValues) - if err != nil { - return nil, fmt.Errorf("can't merge profile `%s` with default profile: %v", profile.Name, err) - } - - if err := k.writeConfigMapWithProfile(manifest, - profile.Name, - merged); err != nil { - return nil, fmt.Errorf("can't write manifest for profile config map: %v", err) - } - configMapNames = append(configMapNames, formatProfileName(profile.Name)) - } - if err := k.writeRbacRoleBindings(manifest, configMapNames); err != nil { - return nil, fmt.Errorf("can't write manifest for rbac bindings: %v", err) - } - return manifest, nil -} - -func (k *KubeletConfig) save(data []byte) error { - kubeletDir := path.Join(k.k0sVars.ManifestsDir, "kubelet") err := dir.Init(kubeletDir, constant.ManifestsDirMode) if err != nil { return err } - filePath := filepath.Join(kubeletDir, "kubelet-config.yaml") - if err := file.WriteContentAtomically(filePath, data, constant.CertMode); err != nil { - return fmt.Errorf("can't write kubelet configuration config map: %v", err) - } - - deprecationNotice := []byte(`The kubelet-config component has been replaced by the worker-config component in k0s 1.26. -It is scheduled for removal in k0s 1.27. -`) - - if err := file.WriteContentAtomically(filepath.Join(kubeletDir, "deprecated.txt"), deprecationNotice, constant.CertMode); err != nil { - k.log.WithError(err).Warn("Failed to write deprecation notice") - } - - return nil -} - -type unstructuredYamlObject map[string]interface{} - -func (k *KubeletConfig) writeConfigMapWithProfile(w io.Writer, name string, profile unstructuredYamlObject) error { - profileYaml, err := yaml.Marshal(profile) - if err != nil { - return err - } - tw := templatewriter.TemplateWriter{ - Name: "kubelet-config", - Template: kubeletConfigsManifestTemplate, - Data: struct { - Name string - KubeletConfigYAML string - }{ - Name: formatProfileName(name), - KubeletConfigYAML: string(profileYaml), - }, - } - return tw.WriteToBuffer(w) -} - -func formatProfileName(name string) string { - return fmt.Sprintf("kubelet-config-%s-%s", name, constant.KubernetesMajorMinorVersion) -} - -func (k *KubeletConfig) writeRbacRoleBindings(w io.Writer, configMapNames []string) error { - tw := templatewriter.TemplateWriter{ - Name: "kubelet-config-rbac", - Template: rbacRoleAndBindingsManifestTemplate, - Data: struct { - ConfigMapNames []string - }{ - ConfigMapNames: configMapNames, - }, + var errs []error + + // Iterate over all files that would be read by the manifest applier and + // rename them, so they won't be applied anymore. + if manifests, err := applier.FindManifestFilesInDir(kubeletDir); err != nil { + errs = append(errs, err) + } else { + for _, manifest := range manifests { + // Reserve a new unique file name to preserve the file's contents. + f, err := os.CreateTemp(filepath.Dir(manifest), filepath.Base(manifest)+".*.removed") + if err != nil { + errs = append(errs, err) + continue + } + errs = append(errs, f.Close()) + + // Rename the file, overwriting the target. + errs = append(errs, os.Rename(manifest, f.Name())) + } } - return tw.WriteToBuffer(w) -} + const removalNotice = `The kubelet-config component has been replaced by the worker-config component in k0s 1.26. +It has been removed in k0s 1.29. +` -func getDefaultProfile(dnsAddress string, clusterDomain string) unstructuredYamlObject { - // the motivation to keep it like this instead of the yaml template: - // - it's easier to merge programatically defined structure - // - apart from map[string]interface there is no good way to define free-form mapping + errs = append(errs, file.WriteContentAtomically(filepath.Join(kubeletDir, "removed.txt"), []byte(removalNotice), constant.CertMode)) - cipherSuites := make([]string, len(constant.AllowedTLS12CipherSuiteIDs)) - for i, cipherSuite := range constant.AllowedTLS12CipherSuiteIDs { - cipherSuites[i] = tls.CipherSuiteName(cipherSuite) + // Remove a potential deprecation notice + if err := os.Remove(filepath.Join(kubeletDir, "deprecated.txt")); err != nil && !errors.Is(err, os.ErrNotExist) { + logrus.WithField("component", "kubelet-config").WithError(err).Warn("Failed to delete deprecation notice") } - // for the authentication.x509.clientCAFile and volumePluginDir we want to use later binding so we put template placeholder instead of actual value there - profile := unstructuredYamlObject{ - "apiVersion": "kubelet.config.k8s.io/v1beta1", - "kind": "KubeletConfiguration", - "clusterDNS": []string{dnsAddress}, - "clusterDomain": clusterDomain, - "tlsCipherSuites": cipherSuites, - "failSwapOn": false, - "rotateCertificates": true, - "serverTLSBootstrap": true, - "eventRecordQPS": 0, - } - return profile + return errors.Join(errs...) } -const kubeletConfigsManifestTemplate = `--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{.Name}} - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -data: - kubelet: | -{{ .KubeletConfigYAML | nindent 4 }} -` - -const rbacRoleAndBindingsManifestTemplate = `--- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: system:bootstrappers:kubelet-configmaps - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -rules: -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: -{{- range .ConfigMapNames }} - - "{{ . -}}" -{{ end }} - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system:bootstrappers:kubelet-configmaps - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: system:bootstrappers:kubelet-configmaps -subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes -` - -// mergeInto merges b to the a, a is modified inplace -func mergeProfiles(a *unstructuredYamlObject, b unstructuredYamlObject) (unstructuredYamlObject, error) { - if err := mergo.Merge(a, b, mergo.WithOverride); err != nil { - return nil, err - } - return *a, nil -} +func (k *KubeletConfig) Start(context.Context) error { return nil } +func (k *KubeletConfig) Stop() error { return nil } diff --git a/pkg/component/controller/kubeletconfig_test.go b/pkg/component/controller/kubeletconfig_test.go index fa337b01a04d..7a9d80c3e7e5 100644 --- a/pkg/component/controller/kubeletconfig_test.go +++ b/pkg/component/controller/kubeletconfig_test.go @@ -14,186 +14,80 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package controller_test import ( - "encoding/json" - "strings" + "context" + "os" + "path/filepath" + "syscall" "testing" - "github.com/k0sproject/k0s/internal/testutil" - "k8s.io/apimachinery/pkg/runtime" - - helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/applier" + "github.com/k0sproject/k0s/pkg/component/controller" "github.com/k0sproject/k0s/pkg/config" - + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "sigs.k8s.io/yaml" ) -func Test_KubeletConfig(t *testing.T) { - cfg := v1beta1.DefaultClusterConfig() - k0sVars, err := config.NewCfgVars(nil, t.TempDir()) +func Test_KubeletConfig_Nonexistent(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests", "kubelet") + + k0sVars, err := config.NewCfgVars(nil, tmp) require.NoError(t, err) - dnsAddr, _ := cfg.Spec.Network.DNSAddress() - t.Run("default_profile_only", func(t *testing.T) { - k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory(), cfg) - - t.Log("starting to run...") - buf, err := k.createProfiles(cfg) - require.NoError(t, err) - if err != nil { - t.FailNow() - } - manifestYamls := strings.Split(strings.TrimSuffix(buf.String(), "---"), "---")[1:] - t.Run("output_must_have_3_manifests", func(t *testing.T) { - require.Len(t, manifestYamls, 4, "Must have exactly 4 generated manifests per profile") - requireConfigMap(t, manifestYamls[0], "kubelet-config-default-1.29") - requireConfigMap(t, manifestYamls[1], "kubelet-config-default-windows-1.29") - requireRole(t, manifestYamls[2], []string{ - formatProfileName("default"), - formatProfileName("default-windows"), - }) - requireRoleBinding(t, manifestYamls[3]) - }) - }) - t.Run("default_profile_must_pass_down_cluster_domain", func(t *testing.T) { - profile := getDefaultProfile(dnsAddr, "cluster.local.custom") - require.Equal(t, string( - "cluster.local.custom", - ), profile["clusterDomain"]) - }) - t.Run("with_user_provided_profiles", func(t *testing.T) { - k, cfgWithUserProvidedProfiles := defaultConfigWithUserProvidedProfiles(t) - buf, err := k.createProfiles(cfgWithUserProvidedProfiles) - require.NoError(t, err) - manifestYamls := strings.Split(strings.TrimSuffix(buf.String(), "---"), "---")[1:] - expectedManifestsCount := 6 - require.Len(t, manifestYamls, expectedManifestsCount, "Must have exactly 6 generated manifests per profile") - - t.Run("final_output_must_have_manifests_for_profiles", func(t *testing.T) { - // check that each profile has config map, role and role binding - var resourceNamesForRole []string - for idx, profileName := range []string{"default", "default-windows", "profile_XXX", "profile_YYY"} { - fullName := "kubelet-config-" + profileName + "-1.29" - resourceNamesForRole = append(resourceNamesForRole, formatProfileName(profileName)) - requireConfigMap(t, manifestYamls[idx], fullName) - } - requireRole(t, manifestYamls[len(resourceNamesForRole)], resourceNamesForRole) - }) - t.Run("user_profile_X_must_be_merged_with_default_profile", func(t *testing.T) { - profileXXX := struct { - Data map[string]string `yaml:"data"` - }{} - - profileYYY := struct { - Data map[string]string `yaml:"data"` - }{} - - require.NoError(t, yaml.Unmarshal([]byte(manifestYamls[2]), &profileXXX)) - require.NoError(t, yaml.Unmarshal([]byte(manifestYamls[3]), &profileYYY)) - - // manually apple the same changes to default config and check that there is no diff - defaultProfileKubeletConfig := getDefaultProfile(dnsAddr, "cluster.local") - defaultProfileKubeletConfig["authentication"] = map[string]interface{}{ - "anonymous": map[string]interface{}{ - "enabled": false, - }, - } - defaultWithChangesXXX, err := yaml.Marshal(defaultProfileKubeletConfig) - require.NoError(t, err) - - defaultProfileKubeletConfig = getDefaultProfile(dnsAddr, "cluster.local") - defaultProfileKubeletConfig["authentication"] = map[string]interface{}{ - "webhook": map[string]interface{}{ - "cacheTTL": "15s", - }, - } - defaultWithChangesYYY, err := yaml.Marshal(defaultProfileKubeletConfig) - - require.NoError(t, err) - - require.YAMLEq(t, string(defaultWithChangesXXX), profileXXX.Data["kubelet"]) - require.YAMLEq(t, string(defaultWithChangesYYY), profileYYY.Data["kubelet"]) - }) - }) + + underTest := controller.NewKubeletConfig(k0sVars) + assert.NoError(t, underTest.Init(context.TODO())) + assert.DirExists(t, dir, "Kubelet manifest directory wasn't created") } -func defaultConfigWithUserProvidedProfiles(t *testing.T) (*KubeletConfig, *v1beta1.ClusterConfig) { - cfg := v1beta1.DefaultClusterConfig() - k0sVars, err := config.NewCfgVars(nil, t.TempDir()) +func Test_KubeletConfig_ManifestDirObstructed(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests") + require.NoError(t, os.WriteFile(dir, []byte("obstructed"), 0644)) + + k0sVars, err := config.NewCfgVars(nil, tmp) require.NoError(t, err) - k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory(), cfg) - - cfgProfileX := map[string]interface{}{ - "authentication": map[string]interface{}{ - "anonymous": map[string]interface{}{ - "enabled": false, - }, - }, - } - wcx, err := json.Marshal(cfgProfileX) - if err != nil { - t.Fatal(err) - } - cfg.Spec.WorkerProfiles = append(cfg.Spec.WorkerProfiles, - v1beta1.WorkerProfile{ - Name: "profile_XXX", - Config: &runtime.RawExtension{Raw: wcx}, - }, - ) - - cfgProfileY := map[string]interface{}{ - "authentication": map[string]interface{}{ - "webhook": map[string]interface{}{ - "cacheTTL": "15s", - }, - }, - } - wcy, err := json.Marshal(cfgProfileY) - if err != nil { - t.Fatal(err) - } + underTest := controller.NewKubeletConfig(k0sVars) + err = underTest.Init(context.TODO()) - cfg.Spec.WorkerProfiles = append(cfg.Spec.WorkerProfiles, - v1beta1.WorkerProfile{ - Name: "profile_YYY", - Config: &runtime.RawExtension{Raw: wcy}, - }, - ) - return k, cfg + if pathErr := (*os.PathError)(nil); assert.ErrorAs(t, err, &pathErr) { + assert.Equal(t, pathErr.Path, dir) + assert.Equal(t, pathErr.Op, "mkdir") + assert.Equal(t, pathErr.Err, syscall.ENOTDIR) + } } -func requireConfigMap(t *testing.T, spec string, name string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "ConfigMap", dst["kind"]) - require.Equal(t, name, dst["metadata"].(map[string]interface{})["name"]) - spec, foundSpec := dst["data"].(map[string]interface{})["kubelet"].(string) - require.True(t, foundSpec, "kubelet config map must have embedded kubelet config") - require.True(t, strings.TrimSpace(spec) != "", "kubelet config map must have non-empty embedded kubelet config") -} +func Test_KubeletConfig_RenamesManifestFiles(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests", "kubelet") + require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "deprecated.txt"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.yaml"), nil, 0644)) + + foundFiles, err := applier.FindManifestFilesInDir(dir) + require.NoError(t, err, "Failed to find manifests in kubelet dir") + require.NotEmpty(t, foundFiles, "No manifests in kubelet dir; did the applier change its file patterns?") + + k0sVars, err := config.NewCfgVars(nil, tmp) + require.NoError(t, err) -func requireRole(t *testing.T, spec string, expectedResourceNames []string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "Role", dst["kind"]) - require.Equal(t, "system:bootstrappers:kubelet-configmaps", dst["metadata"].(map[string]interface{})["name"]) - var currentResourceNames []string - for _, el := range dst["rules"].([]interface{})[0].(map[string]interface{})["resourceNames"].([]interface{}) { - currentResourceNames = append(currentResourceNames, el.(string)) + underTest := controller.NewKubeletConfig(k0sVars) + assert.NoError(t, underTest.Init(context.TODO())) + + assert.NoFileExists(t, filepath.Join(dir, "deprecated.txt")) + assert.FileExists(t, filepath.Join(dir, "removed.txt")) + if matches, err := filepath.Glob(filepath.Join(dir, "manifest.yaml.*.removed")); assert.NoError(t, err) { + assert.Len(t, matches, 1, "Expected a single removed manifest file") + } + if entries, err := os.ReadDir(dir); assert.NoError(t, err) { + assert.Len(t, entries, 2, "Expected exactly two files in kubelet folder") } - require.Equal(t, expectedResourceNames, currentResourceNames) -} -func requireRoleBinding(t *testing.T, spec string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "RoleBinding", dst["kind"]) - require.Equal(t, "system:bootstrappers:kubelet-configmaps", dst["metadata"].(map[string]interface{})["name"]) + foundFiles, err = applier.FindManifestFilesInDir(dir) + if assert.NoError(t, err, "Failed to find manifests in kubelet dir") { + assert.Empty(t, foundFiles, "Some manifests were still found in kubelet dir") + } } diff --git a/pkg/config/cli.go b/pkg/config/cli.go index c91b4210879e..90a4ff47997d 100644 --- a/pkg/config/cli.go +++ b/pkg/config/cli.go @@ -105,12 +105,6 @@ func (o *ControllerOptions) Normalize() error { constant.APIConfigComponentName, "--enable-dynamic-config", ) } - - case constant.KubeletConfigComponentName: - logrus.Warnf("Usage of deprecated component name %q, please switch to %q", - constant.KubeletConfigComponentName, constant.WorkerConfigComponentName, - ) - disabledComponent = constant.WorkerConfigComponentName } if !slices.Contains(availableComponents, disabledComponent) { diff --git a/pkg/config/cli_test.go b/pkg/config/cli_test.go index b4e724404f54..4d9e2c4eba3a 100644 --- a/pkg/config/cli_test.go +++ b/pkg/config/cli_test.go @@ -44,30 +44,14 @@ func TestControllerOptions_Normalize(t *testing.T) { assert.ErrorContains(t, err, "unknown component i-dont-exist") }) - for _, test := range []struct { - name string - disabled, expected []string - }{ - { - "removesDuplicateComponents", - []string{"helm", "kube-proxy", "coredns", "kube-proxy", "autopilot"}, - []string{"helm", "kube-proxy", "coredns", "autopilot"}, - }, - { - "replacesDeprecation", - []string{"helm", "kubelet-config", "coredns", "kubelet-config", "autopilot"}, - []string{"helm", "worker-config", "coredns", "autopilot"}, - }, - { - "replacesDeprecationAvoidingDuplicates", - []string{"helm", "kubelet-config", "coredns", "kubelet-config", "worker-config", "autopilot"}, - []string{"helm", "worker-config", "coredns", "autopilot"}, - }, - } { - underTest := ControllerOptions{DisableComponents: test.disabled} + t.Run("removesDuplicateComponents", func(t *testing.T) { + disabled := []string{"helm", "kube-proxy", "coredns", "kube-proxy", "autopilot"} + expected := []string{"helm", "kube-proxy", "coredns", "autopilot"} + + underTest := ControllerOptions{DisableComponents: disabled} err := underTest.Normalize() require.NoError(t, err) - assert.Equal(t, test.expected, underTest.DisableComponents) - } + assert.Equal(t, expected, underTest.DisableComponents) + }) } diff --git a/pkg/constant/constant_shared.go b/pkg/constant/constant_shared.go index 1c91a4f968da..9fcff851301a 100644 --- a/pkg/constant/constant_shared.go +++ b/pkg/constant/constant_shared.go @@ -106,7 +106,6 @@ const ( KubeControllerManagerComponentName = "kube-controller-manager" KubeProxyComponentName = "kube-proxy" KubeSchedulerComponentName = "kube-scheduler" - KubeletConfigComponentName = "kubelet-config" // Deprecated: replaced by worker-config WorkerConfigComponentName = "worker-config" MetricsServerComponentName = "metrics-server" NetworkProviderComponentName = "network-provider"