From a6a6fa122b5c09703c2f836351dc62c5bc2ac522 Mon Sep 17 00:00:00 2001 From: Tom Wieczorek Date: Fri, 3 Mar 2023 16:09:58 +0100 Subject: [PATCH] Remove kubelet-config stack after deprecation in 1.26 The kubelet-config component has been deprecated in 1.26. Remove the corresponding resources from the cluster in 1.29. Rename all manifest files in the stack's folder to something that won't be picked up by the stack applier. Leave a note about the stack removal in the stack's folder. Signed-off-by: Tom Wieczorek --- cmd/controller/controller.go | 2 +- go.mod | 2 +- inttest/ap-ha3x3/ha3x3_test.go | 78 ++++- pkg/applier/applier.go | 7 +- pkg/component/controller/kubeletconfig.go | 319 +++--------------- .../controller/kubeletconfig_test.go | 216 +++--------- pkg/config/cli.go | 6 - pkg/config/cli_test.go | 30 +- pkg/constant/constant_shared.go | 1 - 9 files changed, 181 insertions(+), 480 deletions(-) diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 85cbd1850033..d3190ad744b2 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -469,7 +469,7 @@ func (c *command) start(ctx context.Context) error { return err } clusterComponents.Add(ctx, reconciler) - clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory, nodeConfig)) + clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars)) } if !slices.Contains(c.DisableComponents, constant.SystemRbacComponentName) { diff --git a/go.mod b/go.mod index 95c2ffefec65..abf00fdd1240 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/go-playground/validator/v10 v10.16.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/terraform-exec v0.19.0 - github.com/imdario/mergo v0.3.16 github.com/k0sproject/bootloose v0.7.2 github.com/k0sproject/dig v0.2.0 github.com/k0sproject/version v0.4.2 @@ -174,6 +173,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/terraform-json v0.17.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/intel/goresctrl v0.3.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect diff --git a/inttest/ap-ha3x3/ha3x3_test.go b/inttest/ap-ha3x3/ha3x3_test.go index 3b052cee1a4b..133615bbc25d 100644 --- a/inttest/ap-ha3x3/ha3x3_test.go +++ b/inttest/ap-ha3x3/ha3x3_test.go @@ -15,14 +15,18 @@ package ha3x3 import ( + "context" "fmt" + "path/filepath" "strings" "testing" "time" + "github.com/avast/retry-go" apv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" apconst "github.com/k0sproject/k0s/pkg/autopilot/constant" appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core" + "golang.org/x/exp/slices" "github.com/k0sproject/k0s/inttest/common" aptest "github.com/k0sproject/k0s/inttest/common/autopilot" @@ -92,6 +96,14 @@ func (s *ha3x3Suite) SetupTest() { // TestApply applies a well-formed `plan` yaml, and asserts that // all of the correct values across different objects + controllers are correct. func (s *ha3x3Suite) TestApply() { + ctx := s.Context() + baseK0sVersion, err := s.GetK0sVersion(s.ControllerNode(0)) + if s.NoError(err, "Failed to get the base k0s version") { + s.T().Logf("Base k0s version: %s", baseK0sVersion) + } else { + baseK0sVersion = "Error: " + err.Error() + } + planTemplate := ` apiVersion: autopilot.k0sproject.io/v1beta2 kind: Plan @@ -130,10 +142,10 @@ spec: s.T().Logf("kubectl apply output: '%s'", out) s.Require().NoError(err) - ssh, err := s.SSH(s.Context(), s.WorkerNode(0)) + ssh, err := s.SSH(ctx, s.WorkerNode(0)) s.Require().NoError(err) defer ssh.Disconnect() - out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V") + out, err = ssh.ExecWithOutput(ctx, "/var/lib/k0s/bin/iptables-save -V") s.Require().NoError(err) iptablesVersionParts := strings.Split(out, " ") iptablesModeBeforeUpdate := iptablesVersionParts[len(iptablesVersionParts)-1] @@ -143,7 +155,7 @@ spec: s.NotEmpty(client) // The plan has enough information to perform a successful update of k0s, so wait for it. - plan, err := aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted) + plan, err := aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted) s.Require().NoError(err) // Ensure all state/status are completed @@ -165,11 +177,69 @@ spec: s.Equal(s.K0sUpdateVersion, version) } - out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V") + out, err = ssh.ExecWithOutput(ctx, "/var/lib/k0s/bin/iptables-save -V") s.Require().NoError(err) iptablesVersionParts = strings.Split(out, " ") iptablesModeAfterUpdate := iptablesVersionParts[len(iptablesVersionParts)-1] s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate) + + s.checkKubeletConfigComponentFolders(ctx, ssh, baseK0sVersion) + s.checkKubeletConfigStackResources(ctx, ssh) +} + +func (s *ha3x3Suite) checkKubeletConfigComponentFolders(ctx context.Context, ssh *common.SSHConnection, baseK0sVersion string) { + var expected []string + switch { + case strings.HasPrefix(baseK0sVersion, "v1.24") || strings.HasPrefix(baseK0sVersion, "v1.25"): + expected = []string{"./removed.txt", "./kubelet-config.yaml.*.removed"} + case strings.HasPrefix(baseK0sVersion, "v1.26"): + expected = []string{"./deprecated.txt", "./removed.txt", "./kubelet-config.yaml.*.removed"} + case strings.HasPrefix(baseK0sVersion, "v1.27"): + expected = []string{"./removed.txt"} + default: // Expect no kubelet folder at all subsequent versions + s.NoError( + ssh.Exec(ctx, "[ ! -e /var/lib/k0s/manifests/kubelet ]", common.SSHStreams{}), + "The kubelet manifests folder exists", + ) + return + } + + findFiles, err := ssh.ExecWithOutput(ctx, "cd /var/lib/k0s/manifests/kubelet && find -print0 . -type f") + if !s.NoError(err, "Failed to list kubelet manifests folder") { + return + } + + files := strings.Split(findFiles, "\x00") + for _, expected := range expected { + idx := slices.IndexFunc(files, func(candidate string) bool { + matched, err := filepath.Match(expected, candidate) + s.Require().NoError(err) + return matched + }) + if s.GreaterOrEqual(idx, 0, "No %s in kubelet manifests folder", expected) { + files = append(files[:idx], files[idx+1:]...) + } + } + + s.Empty(files, "Some unexpected files detected in kubelet manifests folder") +} + +func (s *ha3x3Suite) checkKubeletConfigStackResources(ctx context.Context, ssh *common.SSHConnection) { + var kubeletStackResources string + err := retry.Do( + func() (err error) { + kubeletStackResources, err = ssh.ExecWithOutput(ctx, "kubectl get configmaps,roles,rolebindings -A -l 'k0s.k0sproject.io/stack=kubelet' -oname") + return + }, + retry.OnRetry(func(attempt uint, err error) { + s.T().Logf("Failed to execute kubectl in attempt #%d, retrying after backoff: %v", attempt+1, err) + }), + retry.Context(ctx), + retry.LastErrorOnly(true), + ) + if s.NoError(err) { + s.Empty(kubeletStackResources) + } } // TestHA3x3Suite sets up a suite using 3 controllers for quorum, and runs various diff --git a/pkg/applier/applier.go b/pkg/applier/applier.go index c42653f2664b..86c0f56c9f67 100644 --- a/pkg/applier/applier.go +++ b/pkg/applier/applier.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "github.com/k0sproject/k0s/pkg/kubernetes" @@ -40,6 +39,10 @@ import ( // manifestFilePattern is the glob pattern that all applicable manifest files need to match. const manifestFilePattern = "*.yaml" +func FindManifestFilesInDir(dir string) ([]string, error) { + return filepath.Glob(filepath.Join(dir, manifestFilePattern)) +} + // Applier manages all the "static" manifests and applies them on the k8s API type Applier struct { Name string @@ -107,7 +110,7 @@ func (a *Applier) Apply(ctx context.Context) error { return err } - files, err := filepath.Glob(path.Join(a.Dir, manifestFilePattern)) + files, err := FindManifestFilesInDir(a.Dir) if err != nil { return err } diff --git a/pkg/component/controller/kubeletconfig.go b/pkg/component/controller/kubeletconfig.go index 449093613d2c..6a01a227aed5 100644 --- a/pkg/component/controller/kubeletconfig.go +++ b/pkg/component/controller/kubeletconfig.go @@ -17,315 +17,76 @@ limitations under the License. package controller import ( - "bytes" "context" - "crypto/tls" - "encoding/json" - "fmt" - "io" - "path" + "errors" + "os" "path/filepath" - "reflect" - - "github.com/imdario/mergo" - "github.com/k0sproject/k0s/pkg/component/manager" - "github.com/k0sproject/k0s/pkg/config" - k8sutil "github.com/k0sproject/k0s/pkg/kubernetes" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/yaml" "github.com/k0sproject/k0s/internal/pkg/dir" "github.com/k0sproject/k0s/internal/pkg/file" - "github.com/k0sproject/k0s/internal/pkg/templatewriter" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/applier" + "github.com/k0sproject/k0s/pkg/component/manager" + "github.com/k0sproject/k0s/pkg/config" "github.com/k0sproject/k0s/pkg/constant" ) // Dummy checks so we catch easily if we miss some interface implementation -var _ manager.Reconciler = (*KubeletConfig)(nil) var _ manager.Component = (*KubeletConfig)(nil) // KubeletConfig is the reconciler for generic kubelet configs type KubeletConfig struct { - log logrus.FieldLogger - - kubeClientFactory k8sutil.ClientFactoryInterface - k0sVars *config.CfgVars - previousProfiles v1beta1.WorkerProfiles - nodeConfig *v1beta1.ClusterConfig + k0sVars *config.CfgVars } // NewKubeletConfig creates new KubeletConfig reconciler -func NewKubeletConfig(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInterface, nodeConfig *v1beta1.ClusterConfig) *KubeletConfig { - return &KubeletConfig{ - log: logrus.WithFields(logrus.Fields{"component": "kubeletconfig"}), - - kubeClientFactory: clientFactory, - k0sVars: k0sVars, - nodeConfig: nodeConfig, - } -} - -// Init does nothing -func (k *KubeletConfig) Init(_ context.Context) error { - return nil -} - -// Stop does nothign, nothing actually running -func (k *KubeletConfig) Stop() error { - return nil +func NewKubeletConfig(k0sVars *config.CfgVars) *KubeletConfig { + return &KubeletConfig{k0sVars} } -// Run dumps the needed manifest objects -func (k *KubeletConfig) Start(_ context.Context) error { - - return nil -} - -// Reconcile detects changes in configuration and applies them to the component -func (k *KubeletConfig) Reconcile(ctx context.Context, clusterSpec *v1beta1.ClusterConfig) error { - k.log.Debug("reconcile method called for: KubeletConfig") - // Check if we actually need to reconcile anything - defaultProfilesExist, err := k.defaultProfilesExist(ctx) +func (k *KubeletConfig) Init(context.Context) error { + kubeletDir := filepath.Join(k.k0sVars.ManifestsDir, "kubelet") + err := dir.Init(kubeletDir, constant.ManifestsDirMode) if err != nil { return err } - if defaultProfilesExist && reflect.DeepEqual(k.previousProfiles, clusterSpec.Spec.WorkerProfiles) { - k.log.Debugf("default profiles exist and no change in user specified profiles, nothing to reconcile") - return nil - } - - manifest, err := k.createProfiles(clusterSpec) - if err != nil { - return fmt.Errorf("failed to build final manifest: %v", err) - } - if err := k.save(manifest.Bytes()); err != nil { - return fmt.Errorf("can't write manifest with config maps: %v", err) - } - k.previousProfiles = clusterSpec.Spec.WorkerProfiles - - return nil -} - -func (k *KubeletConfig) defaultProfilesExist(ctx context.Context) (bool, error) { - c, err := k.kubeClientFactory.GetClient() - if err != nil { - return false, err - } - defaultProfileName := formatProfileName("default") - _, err = c.CoreV1().ConfigMaps("kube-system").Get(ctx, defaultProfileName, v1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } else if err != nil { - return false, err - } - return true, nil -} + var errs []error -func (k *KubeletConfig) createProfiles(clusterSpec *v1beta1.ClusterConfig) (*bytes.Buffer, error) { - dnsAddress, err := k.nodeConfig.Spec.Network.DNSAddress() + // Iterate over all files that would be read by the manifest applier and + // rename them, so they won't be applied anymore. + manifests, err := applier.FindManifestFilesInDir(kubeletDir) if err != nil { - return nil, fmt.Errorf("failed to get DNS address for kubelet config: %v", err) - } - manifest := bytes.NewBuffer([]byte{}) - defaultProfile := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - defaultProfile["cgroupsPerQOS"] = true - - winDefaultProfile := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - winDefaultProfile["cgroupsPerQOS"] = false - - if err := k.writeConfigMapWithProfile(manifest, "default", defaultProfile); err != nil { - return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err) - } - if err := k.writeConfigMapWithProfile(manifest, "default-windows", winDefaultProfile); err != nil { - return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err) - } - configMapNames := []string{ - formatProfileName("default"), - formatProfileName("default-windows"), - } - for _, profile := range clusterSpec.Spec.WorkerProfiles { - profileConfig := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - - var workerValues unstructuredYamlObject - err := json.Unmarshal(profile.Config.Raw, &workerValues) - if err != nil { - return nil, fmt.Errorf("failed to decode worker profile values: %v", err) - } - merged, err := mergeProfiles(&profileConfig, workerValues) - if err != nil { - return nil, fmt.Errorf("can't merge profile `%s` with default profile: %v", profile.Name, err) + errs = append(errs, err) + } else { + for _, manifest := range manifests { + // Reserve a new unique file name to preserve the file's contents. + f, err := os.CreateTemp(filepath.Dir(manifest), filepath.Base(manifest)+".*.removed") + if err != nil { + errs = append(errs, err) + continue + } + if err := f.Close(); err != nil { + errs = append(errs, err) + } + + // Rename the file, overwriting the target. + if err = os.Rename(manifest, f.Name()); err != nil { + errs = append(errs, err) + } } - - if err := k.writeConfigMapWithProfile(manifest, - profile.Name, - merged); err != nil { - return nil, fmt.Errorf("can't write manifest for profile config map: %v", err) - } - configMapNames = append(configMapNames, formatProfileName(profile.Name)) - } - if err := k.writeRbacRoleBindings(manifest, configMapNames); err != nil { - return nil, fmt.Errorf("can't write manifest for rbac bindings: %v", err) - } - return manifest, nil -} - -func (k *KubeletConfig) save(data []byte) error { - kubeletDir := path.Join(k.k0sVars.ManifestsDir, "kubelet") - err := dir.Init(kubeletDir, constant.ManifestsDirMode) - if err != nil { - return err } - filePath := filepath.Join(kubeletDir, "kubelet-config.yaml") - if err := file.WriteContentAtomically(filePath, data, constant.CertMode); err != nil { - return fmt.Errorf("can't write kubelet configuration config map: %v", err) - } - - deprecationNotice := []byte(`The kubelet-config component has been replaced by the worker-config component in k0s 1.26. -It is scheduled for removal in k0s 1.27. -`) - - if err := file.WriteContentAtomically(filepath.Join(kubeletDir, "deprecated.txt"), deprecationNotice, constant.CertMode); err != nil { - k.log.WithError(err).Warn("Failed to write deprecation notice") - } - - return nil -} - -type unstructuredYamlObject map[string]interface{} + const removalNotice = `The kubelet-config component has been replaced by the worker-config component in k0s 1.26. +It has been removed in k0s 1.27. +` -func (k *KubeletConfig) writeConfigMapWithProfile(w io.Writer, name string, profile unstructuredYamlObject) error { - profileYaml, err := yaml.Marshal(profile) + err = file.WriteContentAtomically(filepath.Join(kubeletDir, "removed.txt"), []byte(removalNotice), constant.CertMode) if err != nil { - return err + errs = append(errs, err) } - tw := templatewriter.TemplateWriter{ - Name: "kubelet-config", - Template: kubeletConfigsManifestTemplate, - Data: struct { - Name string - KubeletConfigYAML string - }{ - Name: formatProfileName(name), - KubeletConfigYAML: string(profileYaml), - }, - } - return tw.WriteToBuffer(w) -} - -func formatProfileName(name string) string { - return fmt.Sprintf("kubelet-config-%s-%s", name, constant.KubernetesMajorMinorVersion) -} - -func (k *KubeletConfig) writeRbacRoleBindings(w io.Writer, configMapNames []string) error { - tw := templatewriter.TemplateWriter{ - Name: "kubelet-config-rbac", - Template: rbacRoleAndBindingsManifestTemplate, - Data: struct { - ConfigMapNames []string - }{ - ConfigMapNames: configMapNames, - }, - } - - return tw.WriteToBuffer(w) -} - -func getDefaultProfile(dnsAddress string, clusterDomain string) unstructuredYamlObject { - // the motivation to keep it like this instead of the yaml template: - // - it's easier to merge programatically defined structure - // - apart from map[string]interface there is no good way to define free-form mapping - cipherSuites := make([]string, len(constant.AllowedTLS12CipherSuiteIDs)) - for i, cipherSuite := range constant.AllowedTLS12CipherSuiteIDs { - cipherSuites[i] = tls.CipherSuiteName(cipherSuite) - } - - // for the authentication.x509.clientCAFile and volumePluginDir we want to use later binding so we put template placeholder instead of actual value there - profile := unstructuredYamlObject{ - "apiVersion": "kubelet.config.k8s.io/v1beta1", - "kind": "KubeletConfiguration", - "clusterDNS": []string{dnsAddress}, - "clusterDomain": clusterDomain, - "tlsCipherSuites": cipherSuites, - "failSwapOn": false, - "rotateCertificates": true, - "serverTLSBootstrap": true, - "eventRecordQPS": 0, - } - return profile + return errors.Join(errs...) } -const kubeletConfigsManifestTemplate = `--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{.Name}} - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -data: - kubelet: | -{{ .KubeletConfigYAML | nindent 4 }} -` - -const rbacRoleAndBindingsManifestTemplate = `--- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: system:bootstrappers:kubelet-configmaps - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -rules: -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: -{{- range .ConfigMapNames }} - - "{{ . -}}" -{{ end }} - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system:bootstrappers:kubelet-configmaps - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: system:bootstrappers:kubelet-configmaps -subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes -` - -// mergeInto merges b to the a, a is modified inplace -func mergeProfiles(a *unstructuredYamlObject, b unstructuredYamlObject) (unstructuredYamlObject, error) { - if err := mergo.Merge(a, b, mergo.WithOverride); err != nil { - return nil, err - } - return *a, nil -} +func (k *KubeletConfig) Start(context.Context) error { return nil } +func (k *KubeletConfig) Stop() error { return nil } diff --git a/pkg/component/controller/kubeletconfig_test.go b/pkg/component/controller/kubeletconfig_test.go index 1aadebdee6a8..2db1ff945201 100644 --- a/pkg/component/controller/kubeletconfig_test.go +++ b/pkg/component/controller/kubeletconfig_test.go @@ -14,186 +14,76 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package controller_test import ( - "encoding/json" - "strings" + "context" + "errors" + "os" + "path/filepath" + "syscall" "testing" - "github.com/k0sproject/k0s/internal/testutil" - "k8s.io/apimachinery/pkg/runtime" - - helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/component/controller" "github.com/k0sproject/k0s/pkg/config" - + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "sigs.k8s.io/yaml" ) -func Test_KubeletConfig(t *testing.T) { - cfg := v1beta1.DefaultClusterConfig() - k0sVars, err := config.NewCfgVars(nil, t.TempDir()) +func Test_KubeletConfig_Nonexistent(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests", "kubelet") + + k0sVars, err := config.NewCfgVars(nil, tmp) require.NoError(t, err) - dnsAddr, _ := cfg.Spec.Network.DNSAddress() - t.Run("default_profile_only", func(t *testing.T) { - k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory(), cfg) - - t.Log("starting to run...") - buf, err := k.createProfiles(cfg) - require.NoError(t, err) - if err != nil { - t.FailNow() - } - manifestYamls := strings.Split(strings.TrimSuffix(buf.String(), "---"), "---")[1:] - t.Run("output_must_have_3_manifests", func(t *testing.T) { - require.Len(t, manifestYamls, 4, "Must have exactly 4 generated manifests per profile") - requireConfigMap(t, manifestYamls[0], "kubelet-config-default-1.28") - requireConfigMap(t, manifestYamls[1], "kubelet-config-default-windows-1.28") - requireRole(t, manifestYamls[2], []string{ - formatProfileName("default"), - formatProfileName("default-windows"), - }) - requireRoleBinding(t, manifestYamls[3]) - }) - }) - t.Run("default_profile_must_pass_down_cluster_domain", func(t *testing.T) { - profile := getDefaultProfile(dnsAddr, "cluster.local.custom") - require.Equal(t, string( - "cluster.local.custom", - ), profile["clusterDomain"]) - }) - t.Run("with_user_provided_profiles", func(t *testing.T) { - k, cfgWithUserProvidedProfiles := defaultConfigWithUserProvidedProfiles(t) - buf, err := k.createProfiles(cfgWithUserProvidedProfiles) - require.NoError(t, err) - manifestYamls := strings.Split(strings.TrimSuffix(buf.String(), "---"), "---")[1:] - expectedManifestsCount := 6 - require.Len(t, manifestYamls, expectedManifestsCount, "Must have exactly 6 generated manifests per profile") - - t.Run("final_output_must_have_manifests_for_profiles", func(t *testing.T) { - // check that each profile has config map, role and role binding - var resourceNamesForRole []string - for idx, profileName := range []string{"default", "default-windows", "profile_XXX", "profile_YYY"} { - fullName := "kubelet-config-" + profileName + "-1.28" - resourceNamesForRole = append(resourceNamesForRole, formatProfileName(profileName)) - requireConfigMap(t, manifestYamls[idx], fullName) - } - requireRole(t, manifestYamls[len(resourceNamesForRole)], resourceNamesForRole) - }) - t.Run("user_profile_X_must_be_merged_with_default_profile", func(t *testing.T) { - profileXXX := struct { - Data map[string]string `yaml:"data"` - }{} - - profileYYY := struct { - Data map[string]string `yaml:"data"` - }{} - - require.NoError(t, yaml.Unmarshal([]byte(manifestYamls[2]), &profileXXX)) - require.NoError(t, yaml.Unmarshal([]byte(manifestYamls[3]), &profileYYY)) - - // manually apple the same changes to default config and check that there is no diff - defaultProfileKubeletConfig := getDefaultProfile(dnsAddr, "cluster.local") - defaultProfileKubeletConfig["authentication"] = map[string]interface{}{ - "anonymous": map[string]interface{}{ - "enabled": false, - }, - } - defaultWithChangesXXX, err := yaml.Marshal(defaultProfileKubeletConfig) - require.NoError(t, err) - - defaultProfileKubeletConfig = getDefaultProfile(dnsAddr, "cluster.local") - defaultProfileKubeletConfig["authentication"] = map[string]interface{}{ - "webhook": map[string]interface{}{ - "cacheTTL": "15s", - }, - } - defaultWithChangesYYY, err := yaml.Marshal(defaultProfileKubeletConfig) - - require.NoError(t, err) - - require.YAMLEq(t, string(defaultWithChangesXXX), profileXXX.Data["kubelet"]) - require.YAMLEq(t, string(defaultWithChangesYYY), profileYYY.Data["kubelet"]) - }) - }) + + underTest := controller.NewKubeletConfig(k0sVars) + assert.NoError(t, underTest.Init(context.TODO())) + + assert.FileExists(t, filepath.Join(dir, "removed.txt")) + if entries, err := os.ReadDir(dir); assert.NoError(t, err) { + assert.Len(t, entries, 1, "Expected a single file in kubelet folder") + } } -func defaultConfigWithUserProvidedProfiles(t *testing.T) (*KubeletConfig, *v1beta1.ClusterConfig) { - cfg := v1beta1.DefaultClusterConfig() - k0sVars, err := config.NewCfgVars(nil, t.TempDir()) +func Test_KubeletConfig_RenamesManifestFiles(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests", "kubelet") + require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "deprecation.txt"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.yaml"), nil, 0644)) + + k0sVars, err := config.NewCfgVars(nil, tmp) require.NoError(t, err) - k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory(), cfg) - - cfgProfileX := map[string]interface{}{ - "authentication": map[string]interface{}{ - "anonymous": map[string]interface{}{ - "enabled": false, - }, - }, - } - wcx, err := json.Marshal(cfgProfileX) - if err != nil { - t.Fatal(err) + + underTest := controller.NewKubeletConfig(k0sVars) + assert.NoError(t, underTest.Init(context.TODO())) + + assert.FileExists(t, filepath.Join(dir, "deprecation.txt")) + assert.FileExists(t, filepath.Join(dir, "removed.txt")) + if matches, err := filepath.Glob(filepath.Join(dir, "manifest.yaml.*.removed")); assert.NoError(t, err) { + assert.Len(t, matches, 1, "Expected a single removed manifest file") } - cfg.Spec.WorkerProfiles = append(cfg.Spec.WorkerProfiles, - v1beta1.WorkerProfile{ - Name: "profile_XXX", - Config: &runtime.RawExtension{Raw: wcx}, - }, - ) - - cfgProfileY := map[string]interface{}{ - "authentication": map[string]interface{}{ - "webhook": map[string]interface{}{ - "cacheTTL": "15s", - }, - }, + if entries, err := os.ReadDir(dir); assert.NoError(t, err) { + assert.Len(t, entries, 3, "Expected exactly three files in kubelet folder") } +} - wcy, err := json.Marshal(cfgProfileY) - if err != nil { - t.Fatal(err) - } +func Test_KubeletConfig_ManifestDirObstructed(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests") + require.NoError(t, os.WriteFile(dir, []byte("obstructed"), 0644)) - cfg.Spec.WorkerProfiles = append(cfg.Spec.WorkerProfiles, - v1beta1.WorkerProfile{ - Name: "profile_YYY", - Config: &runtime.RawExtension{Raw: wcy}, - }, - ) - return k, cfg -} + k0sVars, err := config.NewCfgVars(nil, tmp) + require.NoError(t, err) -func requireConfigMap(t *testing.T, spec string, name string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "ConfigMap", dst["kind"]) - require.Equal(t, name, dst["metadata"].(map[string]interface{})["name"]) - spec, foundSpec := dst["data"].(map[string]interface{})["kubelet"].(string) - require.True(t, foundSpec, "kubelet config map must have embedded kubelet config") - require.True(t, strings.TrimSpace(spec) != "", "kubelet config map must have non-empty embedded kubelet config") -} + underTest := controller.NewKubeletConfig(k0sVars) + err = underTest.Init(context.TODO()) -func requireRole(t *testing.T, spec string, expectedResourceNames []string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "Role", dst["kind"]) - require.Equal(t, "system:bootstrappers:kubelet-configmaps", dst["metadata"].(map[string]interface{})["name"]) - var currentResourceNames []string - for _, el := range dst["rules"].([]interface{})[0].(map[string]interface{})["resourceNames"].([]interface{}) { - currentResourceNames = append(currentResourceNames, el.(string)) + assert.NotNil(t, errors.Unwrap(err), "Not wrapping a single error: %v", err) + var pathErr *os.PathError + if assert.ErrorAs(t, err, &pathErr) { + assert.Equal(t, dir, pathErr.Path) + assert.ErrorIs(t, err, syscall.ENOTDIR) } - require.Equal(t, expectedResourceNames, currentResourceNames) -} - -func requireRoleBinding(t *testing.T, spec string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "RoleBinding", dst["kind"]) - require.Equal(t, "system:bootstrappers:kubelet-configmaps", dst["metadata"].(map[string]interface{})["name"]) } diff --git a/pkg/config/cli.go b/pkg/config/cli.go index c91b4210879e..90a4ff47997d 100644 --- a/pkg/config/cli.go +++ b/pkg/config/cli.go @@ -105,12 +105,6 @@ func (o *ControllerOptions) Normalize() error { constant.APIConfigComponentName, "--enable-dynamic-config", ) } - - case constant.KubeletConfigComponentName: - logrus.Warnf("Usage of deprecated component name %q, please switch to %q", - constant.KubeletConfigComponentName, constant.WorkerConfigComponentName, - ) - disabledComponent = constant.WorkerConfigComponentName } if !slices.Contains(availableComponents, disabledComponent) { diff --git a/pkg/config/cli_test.go b/pkg/config/cli_test.go index b4e724404f54..4d9e2c4eba3a 100644 --- a/pkg/config/cli_test.go +++ b/pkg/config/cli_test.go @@ -44,30 +44,14 @@ func TestControllerOptions_Normalize(t *testing.T) { assert.ErrorContains(t, err, "unknown component i-dont-exist") }) - for _, test := range []struct { - name string - disabled, expected []string - }{ - { - "removesDuplicateComponents", - []string{"helm", "kube-proxy", "coredns", "kube-proxy", "autopilot"}, - []string{"helm", "kube-proxy", "coredns", "autopilot"}, - }, - { - "replacesDeprecation", - []string{"helm", "kubelet-config", "coredns", "kubelet-config", "autopilot"}, - []string{"helm", "worker-config", "coredns", "autopilot"}, - }, - { - "replacesDeprecationAvoidingDuplicates", - []string{"helm", "kubelet-config", "coredns", "kubelet-config", "worker-config", "autopilot"}, - []string{"helm", "worker-config", "coredns", "autopilot"}, - }, - } { - underTest := ControllerOptions{DisableComponents: test.disabled} + t.Run("removesDuplicateComponents", func(t *testing.T) { + disabled := []string{"helm", "kube-proxy", "coredns", "kube-proxy", "autopilot"} + expected := []string{"helm", "kube-proxy", "coredns", "autopilot"} + + underTest := ControllerOptions{DisableComponents: disabled} err := underTest.Normalize() require.NoError(t, err) - assert.Equal(t, test.expected, underTest.DisableComponents) - } + assert.Equal(t, expected, underTest.DisableComponents) + }) } diff --git a/pkg/constant/constant_shared.go b/pkg/constant/constant_shared.go index 9b6dda7215ce..fe817b00fadc 100644 --- a/pkg/constant/constant_shared.go +++ b/pkg/constant/constant_shared.go @@ -106,7 +106,6 @@ const ( KubeControllerManagerComponentName = "kube-controller-manager" KubeProxyComponentName = "kube-proxy" KubeSchedulerComponentName = "kube-scheduler" - KubeletConfigComponentName = "kubelet-config" // Deprecated: replaced by worker-config WorkerConfigComponentName = "worker-config" MetricsServerComponentName = "metrics-server" NetworkProviderComponentName = "network-provider"