diff --git a/cmd/controller/controller.go b/cmd/controller/controller.go index 85cbd1850033..d3190ad744b2 100644 --- a/cmd/controller/controller.go +++ b/cmd/controller/controller.go @@ -469,7 +469,7 @@ func (c *command) start(ctx context.Context) error { return err } clusterComponents.Add(ctx, reconciler) - clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory, nodeConfig)) + clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars)) } if !slices.Contains(c.DisableComponents, constant.SystemRbacComponentName) { diff --git a/go.mod b/go.mod index 95c2ffefec65..abf00fdd1240 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,6 @@ require ( github.com/go-playground/validator/v10 v10.16.0 github.com/google/go-cmp v0.6.0 github.com/hashicorp/terraform-exec v0.19.0 - github.com/imdario/mergo v0.3.16 github.com/k0sproject/bootloose v0.7.2 github.com/k0sproject/dig v0.2.0 github.com/k0sproject/version v0.4.2 @@ -174,6 +173,7 @@ require ( github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/terraform-json v0.17.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/intel/goresctrl v0.3.0 // indirect github.com/jmoiron/sqlx v1.3.5 // indirect diff --git a/inttest/ap-ha3x3/ha3x3_test.go b/inttest/ap-ha3x3/ha3x3_test.go index 3b052cee1a4b..133615bbc25d 100644 --- a/inttest/ap-ha3x3/ha3x3_test.go +++ b/inttest/ap-ha3x3/ha3x3_test.go @@ -15,14 +15,18 @@ package ha3x3 import ( + "context" "fmt" + "path/filepath" "strings" "testing" "time" + "github.com/avast/retry-go" apv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2" apconst "github.com/k0sproject/k0s/pkg/autopilot/constant" appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core" + "golang.org/x/exp/slices" "github.com/k0sproject/k0s/inttest/common" aptest "github.com/k0sproject/k0s/inttest/common/autopilot" @@ -92,6 +96,14 @@ func (s *ha3x3Suite) SetupTest() { // TestApply applies a well-formed `plan` yaml, and asserts that // all of the correct values across different objects + controllers are correct. func (s *ha3x3Suite) TestApply() { + ctx := s.Context() + baseK0sVersion, err := s.GetK0sVersion(s.ControllerNode(0)) + if s.NoError(err, "Failed to get the base k0s version") { + s.T().Logf("Base k0s version: %s", baseK0sVersion) + } else { + baseK0sVersion = "Error: " + err.Error() + } + planTemplate := ` apiVersion: autopilot.k0sproject.io/v1beta2 kind: Plan @@ -130,10 +142,10 @@ spec: s.T().Logf("kubectl apply output: '%s'", out) s.Require().NoError(err) - ssh, err := s.SSH(s.Context(), s.WorkerNode(0)) + ssh, err := s.SSH(ctx, s.WorkerNode(0)) s.Require().NoError(err) defer ssh.Disconnect() - out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V") + out, err = ssh.ExecWithOutput(ctx, "/var/lib/k0s/bin/iptables-save -V") s.Require().NoError(err) iptablesVersionParts := strings.Split(out, " ") iptablesModeBeforeUpdate := iptablesVersionParts[len(iptablesVersionParts)-1] @@ -143,7 +155,7 @@ spec: s.NotEmpty(client) // The plan has enough information to perform a successful update of k0s, so wait for it. - plan, err := aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted) + plan, err := aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted) s.Require().NoError(err) // Ensure all state/status are completed @@ -165,11 +177,69 @@ spec: s.Equal(s.K0sUpdateVersion, version) } - out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V") + out, err = ssh.ExecWithOutput(ctx, "/var/lib/k0s/bin/iptables-save -V") s.Require().NoError(err) iptablesVersionParts = strings.Split(out, " ") iptablesModeAfterUpdate := iptablesVersionParts[len(iptablesVersionParts)-1] s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate) + + s.checkKubeletConfigComponentFolders(ctx, ssh, baseK0sVersion) + s.checkKubeletConfigStackResources(ctx, ssh) +} + +func (s *ha3x3Suite) checkKubeletConfigComponentFolders(ctx context.Context, ssh *common.SSHConnection, baseK0sVersion string) { + var expected []string + switch { + case strings.HasPrefix(baseK0sVersion, "v1.24") || strings.HasPrefix(baseK0sVersion, "v1.25"): + expected = []string{"./removed.txt", "./kubelet-config.yaml.*.removed"} + case strings.HasPrefix(baseK0sVersion, "v1.26"): + expected = []string{"./deprecated.txt", "./removed.txt", "./kubelet-config.yaml.*.removed"} + case strings.HasPrefix(baseK0sVersion, "v1.27"): + expected = []string{"./removed.txt"} + default: // Expect no kubelet folder at all subsequent versions + s.NoError( + ssh.Exec(ctx, "[ ! -e /var/lib/k0s/manifests/kubelet ]", common.SSHStreams{}), + "The kubelet manifests folder exists", + ) + return + } + + findFiles, err := ssh.ExecWithOutput(ctx, "cd /var/lib/k0s/manifests/kubelet && find -print0 . -type f") + if !s.NoError(err, "Failed to list kubelet manifests folder") { + return + } + + files := strings.Split(findFiles, "\x00") + for _, expected := range expected { + idx := slices.IndexFunc(files, func(candidate string) bool { + matched, err := filepath.Match(expected, candidate) + s.Require().NoError(err) + return matched + }) + if s.GreaterOrEqual(idx, 0, "No %s in kubelet manifests folder", expected) { + files = append(files[:idx], files[idx+1:]...) + } + } + + s.Empty(files, "Some unexpected files detected in kubelet manifests folder") +} + +func (s *ha3x3Suite) checkKubeletConfigStackResources(ctx context.Context, ssh *common.SSHConnection) { + var kubeletStackResources string + err := retry.Do( + func() (err error) { + kubeletStackResources, err = ssh.ExecWithOutput(ctx, "kubectl get configmaps,roles,rolebindings -A -l 'k0s.k0sproject.io/stack=kubelet' -oname") + return + }, + retry.OnRetry(func(attempt uint, err error) { + s.T().Logf("Failed to execute kubectl in attempt #%d, retrying after backoff: %v", attempt+1, err) + }), + retry.Context(ctx), + retry.LastErrorOnly(true), + ) + if s.NoError(err) { + s.Empty(kubeletStackResources) + } } // TestHA3x3Suite sets up a suite using 3 controllers for quorum, and runs various diff --git a/pkg/applier/applier.go b/pkg/applier/applier.go index c42653f2664b..86c0f56c9f67 100644 --- a/pkg/applier/applier.go +++ b/pkg/applier/applier.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "os" - "path" "path/filepath" "github.com/k0sproject/k0s/pkg/kubernetes" @@ -40,6 +39,10 @@ import ( // manifestFilePattern is the glob pattern that all applicable manifest files need to match. const manifestFilePattern = "*.yaml" +func FindManifestFilesInDir(dir string) ([]string, error) { + return filepath.Glob(filepath.Join(dir, manifestFilePattern)) +} + // Applier manages all the "static" manifests and applies them on the k8s API type Applier struct { Name string @@ -107,7 +110,7 @@ func (a *Applier) Apply(ctx context.Context) error { return err } - files, err := filepath.Glob(path.Join(a.Dir, manifestFilePattern)) + files, err := FindManifestFilesInDir(a.Dir) if err != nil { return err } diff --git a/pkg/component/controller/kubeletconfig.go b/pkg/component/controller/kubeletconfig.go index 449093613d2c..6a01a227aed5 100644 --- a/pkg/component/controller/kubeletconfig.go +++ b/pkg/component/controller/kubeletconfig.go @@ -17,315 +17,76 @@ limitations under the License. package controller import ( - "bytes" "context" - "crypto/tls" - "encoding/json" - "fmt" - "io" - "path" + "errors" + "os" "path/filepath" - "reflect" - - "github.com/imdario/mergo" - "github.com/k0sproject/k0s/pkg/component/manager" - "github.com/k0sproject/k0s/pkg/config" - k8sutil "github.com/k0sproject/k0s/pkg/kubernetes" - "github.com/sirupsen/logrus" - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/yaml" "github.com/k0sproject/k0s/internal/pkg/dir" "github.com/k0sproject/k0s/internal/pkg/file" - "github.com/k0sproject/k0s/internal/pkg/templatewriter" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/applier" + "github.com/k0sproject/k0s/pkg/component/manager" + "github.com/k0sproject/k0s/pkg/config" "github.com/k0sproject/k0s/pkg/constant" ) // Dummy checks so we catch easily if we miss some interface implementation -var _ manager.Reconciler = (*KubeletConfig)(nil) var _ manager.Component = (*KubeletConfig)(nil) // KubeletConfig is the reconciler for generic kubelet configs type KubeletConfig struct { - log logrus.FieldLogger - - kubeClientFactory k8sutil.ClientFactoryInterface - k0sVars *config.CfgVars - previousProfiles v1beta1.WorkerProfiles - nodeConfig *v1beta1.ClusterConfig + k0sVars *config.CfgVars } // NewKubeletConfig creates new KubeletConfig reconciler -func NewKubeletConfig(k0sVars *config.CfgVars, clientFactory k8sutil.ClientFactoryInterface, nodeConfig *v1beta1.ClusterConfig) *KubeletConfig { - return &KubeletConfig{ - log: logrus.WithFields(logrus.Fields{"component": "kubeletconfig"}), - - kubeClientFactory: clientFactory, - k0sVars: k0sVars, - nodeConfig: nodeConfig, - } -} - -// Init does nothing -func (k *KubeletConfig) Init(_ context.Context) error { - return nil -} - -// Stop does nothign, nothing actually running -func (k *KubeletConfig) Stop() error { - return nil +func NewKubeletConfig(k0sVars *config.CfgVars) *KubeletConfig { + return &KubeletConfig{k0sVars} } -// Run dumps the needed manifest objects -func (k *KubeletConfig) Start(_ context.Context) error { - - return nil -} - -// Reconcile detects changes in configuration and applies them to the component -func (k *KubeletConfig) Reconcile(ctx context.Context, clusterSpec *v1beta1.ClusterConfig) error { - k.log.Debug("reconcile method called for: KubeletConfig") - // Check if we actually need to reconcile anything - defaultProfilesExist, err := k.defaultProfilesExist(ctx) +func (k *KubeletConfig) Init(context.Context) error { + kubeletDir := filepath.Join(k.k0sVars.ManifestsDir, "kubelet") + err := dir.Init(kubeletDir, constant.ManifestsDirMode) if err != nil { return err } - if defaultProfilesExist && reflect.DeepEqual(k.previousProfiles, clusterSpec.Spec.WorkerProfiles) { - k.log.Debugf("default profiles exist and no change in user specified profiles, nothing to reconcile") - return nil - } - - manifest, err := k.createProfiles(clusterSpec) - if err != nil { - return fmt.Errorf("failed to build final manifest: %v", err) - } - if err := k.save(manifest.Bytes()); err != nil { - return fmt.Errorf("can't write manifest with config maps: %v", err) - } - k.previousProfiles = clusterSpec.Spec.WorkerProfiles - - return nil -} - -func (k *KubeletConfig) defaultProfilesExist(ctx context.Context) (bool, error) { - c, err := k.kubeClientFactory.GetClient() - if err != nil { - return false, err - } - defaultProfileName := formatProfileName("default") - _, err = c.CoreV1().ConfigMaps("kube-system").Get(ctx, defaultProfileName, v1.GetOptions{}) - if err != nil && errors.IsNotFound(err) { - return false, nil - } else if err != nil { - return false, err - } - return true, nil -} + var errs []error -func (k *KubeletConfig) createProfiles(clusterSpec *v1beta1.ClusterConfig) (*bytes.Buffer, error) { - dnsAddress, err := k.nodeConfig.Spec.Network.DNSAddress() + // Iterate over all files that would be read by the manifest applier and + // rename them, so they won't be applied anymore. + manifests, err := applier.FindManifestFilesInDir(kubeletDir) if err != nil { - return nil, fmt.Errorf("failed to get DNS address for kubelet config: %v", err) - } - manifest := bytes.NewBuffer([]byte{}) - defaultProfile := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - defaultProfile["cgroupsPerQOS"] = true - - winDefaultProfile := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - winDefaultProfile["cgroupsPerQOS"] = false - - if err := k.writeConfigMapWithProfile(manifest, "default", defaultProfile); err != nil { - return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err) - } - if err := k.writeConfigMapWithProfile(manifest, "default-windows", winDefaultProfile); err != nil { - return nil, fmt.Errorf("can't write manifest for default profile config map: %v", err) - } - configMapNames := []string{ - formatProfileName("default"), - formatProfileName("default-windows"), - } - for _, profile := range clusterSpec.Spec.WorkerProfiles { - profileConfig := getDefaultProfile(dnsAddress, clusterSpec.Spec.Network.ClusterDomain) - - var workerValues unstructuredYamlObject - err := json.Unmarshal(profile.Config.Raw, &workerValues) - if err != nil { - return nil, fmt.Errorf("failed to decode worker profile values: %v", err) - } - merged, err := mergeProfiles(&profileConfig, workerValues) - if err != nil { - return nil, fmt.Errorf("can't merge profile `%s` with default profile: %v", profile.Name, err) + errs = append(errs, err) + } else { + for _, manifest := range manifests { + // Reserve a new unique file name to preserve the file's contents. + f, err := os.CreateTemp(filepath.Dir(manifest), filepath.Base(manifest)+".*.removed") + if err != nil { + errs = append(errs, err) + continue + } + if err := f.Close(); err != nil { + errs = append(errs, err) + } + + // Rename the file, overwriting the target. + if err = os.Rename(manifest, f.Name()); err != nil { + errs = append(errs, err) + } } - - if err := k.writeConfigMapWithProfile(manifest, - profile.Name, - merged); err != nil { - return nil, fmt.Errorf("can't write manifest for profile config map: %v", err) - } - configMapNames = append(configMapNames, formatProfileName(profile.Name)) - } - if err := k.writeRbacRoleBindings(manifest, configMapNames); err != nil { - return nil, fmt.Errorf("can't write manifest for rbac bindings: %v", err) - } - return manifest, nil -} - -func (k *KubeletConfig) save(data []byte) error { - kubeletDir := path.Join(k.k0sVars.ManifestsDir, "kubelet") - err := dir.Init(kubeletDir, constant.ManifestsDirMode) - if err != nil { - return err } - filePath := filepath.Join(kubeletDir, "kubelet-config.yaml") - if err := file.WriteContentAtomically(filePath, data, constant.CertMode); err != nil { - return fmt.Errorf("can't write kubelet configuration config map: %v", err) - } - - deprecationNotice := []byte(`The kubelet-config component has been replaced by the worker-config component in k0s 1.26. -It is scheduled for removal in k0s 1.27. -`) - - if err := file.WriteContentAtomically(filepath.Join(kubeletDir, "deprecated.txt"), deprecationNotice, constant.CertMode); err != nil { - k.log.WithError(err).Warn("Failed to write deprecation notice") - } - - return nil -} - -type unstructuredYamlObject map[string]interface{} + const removalNotice = `The kubelet-config component has been replaced by the worker-config component in k0s 1.26. +It has been removed in k0s 1.27. +` -func (k *KubeletConfig) writeConfigMapWithProfile(w io.Writer, name string, profile unstructuredYamlObject) error { - profileYaml, err := yaml.Marshal(profile) + err = file.WriteContentAtomically(filepath.Join(kubeletDir, "removed.txt"), []byte(removalNotice), constant.CertMode) if err != nil { - return err + errs = append(errs, err) } - tw := templatewriter.TemplateWriter{ - Name: "kubelet-config", - Template: kubeletConfigsManifestTemplate, - Data: struct { - Name string - KubeletConfigYAML string - }{ - Name: formatProfileName(name), - KubeletConfigYAML: string(profileYaml), - }, - } - return tw.WriteToBuffer(w) -} - -func formatProfileName(name string) string { - return fmt.Sprintf("kubelet-config-%s-%s", name, constant.KubernetesMajorMinorVersion) -} - -func (k *KubeletConfig) writeRbacRoleBindings(w io.Writer, configMapNames []string) error { - tw := templatewriter.TemplateWriter{ - Name: "kubelet-config-rbac", - Template: rbacRoleAndBindingsManifestTemplate, - Data: struct { - ConfigMapNames []string - }{ - ConfigMapNames: configMapNames, - }, - } - - return tw.WriteToBuffer(w) -} - -func getDefaultProfile(dnsAddress string, clusterDomain string) unstructuredYamlObject { - // the motivation to keep it like this instead of the yaml template: - // - it's easier to merge programatically defined structure - // - apart from map[string]interface there is no good way to define free-form mapping - cipherSuites := make([]string, len(constant.AllowedTLS12CipherSuiteIDs)) - for i, cipherSuite := range constant.AllowedTLS12CipherSuiteIDs { - cipherSuites[i] = tls.CipherSuiteName(cipherSuite) - } - - // for the authentication.x509.clientCAFile and volumePluginDir we want to use later binding so we put template placeholder instead of actual value there - profile := unstructuredYamlObject{ - "apiVersion": "kubelet.config.k8s.io/v1beta1", - "kind": "KubeletConfiguration", - "clusterDNS": []string{dnsAddress}, - "clusterDomain": clusterDomain, - "tlsCipherSuites": cipherSuites, - "failSwapOn": false, - "rotateCertificates": true, - "serverTLSBootstrap": true, - "eventRecordQPS": 0, - } - return profile + return errors.Join(errs...) } -const kubeletConfigsManifestTemplate = `--- -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{.Name}} - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -data: - kubelet: | -{{ .KubeletConfigYAML | nindent 4 }} -` - -const rbacRoleAndBindingsManifestTemplate = `--- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: system:bootstrappers:kubelet-configmaps - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -rules: -- apiGroups: [""] - resources: ["configmaps"] - resourceNames: -{{- range .ConfigMapNames }} - - "{{ . -}}" -{{ end }} - verbs: ["get"] ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: system:bootstrappers:kubelet-configmaps - namespace: kube-system - labels: - k0s.k0sproject.io/deprecated-since: "1.26" - annotations: - k0s.k0sproject.io/deprecated: | - The kubelet-config component has been replaced by the worker-config component in k0s 1.26. - It is scheduled for removal in k0s 1.27. -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: system:bootstrappers:kubelet-configmaps -subjects: - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:bootstrappers - - apiGroup: rbac.authorization.k8s.io - kind: Group - name: system:nodes -` - -// mergeInto merges b to the a, a is modified inplace -func mergeProfiles(a *unstructuredYamlObject, b unstructuredYamlObject) (unstructuredYamlObject, error) { - if err := mergo.Merge(a, b, mergo.WithOverride); err != nil { - return nil, err - } - return *a, nil -} +func (k *KubeletConfig) Start(context.Context) error { return nil } +func (k *KubeletConfig) Stop() error { return nil } diff --git a/pkg/component/controller/kubeletconfig_test.go b/pkg/component/controller/kubeletconfig_test.go index 1aadebdee6a8..2db1ff945201 100644 --- a/pkg/component/controller/kubeletconfig_test.go +++ b/pkg/component/controller/kubeletconfig_test.go @@ -14,186 +14,76 @@ See the License for the specific language governing permissions and limitations under the License. */ -package controller +package controller_test import ( - "encoding/json" - "strings" + "context" + "errors" + "os" + "path/filepath" + "syscall" "testing" - "github.com/k0sproject/k0s/internal/testutil" - "k8s.io/apimachinery/pkg/runtime" - - helmv1beta1 "github.com/k0sproject/k0s/pkg/apis/helm/v1beta1" - "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" + "github.com/k0sproject/k0s/pkg/component/controller" "github.com/k0sproject/k0s/pkg/config" - + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "sigs.k8s.io/yaml" ) -func Test_KubeletConfig(t *testing.T) { - cfg := v1beta1.DefaultClusterConfig() - k0sVars, err := config.NewCfgVars(nil, t.TempDir()) +func Test_KubeletConfig_Nonexistent(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests", "kubelet") + + k0sVars, err := config.NewCfgVars(nil, tmp) require.NoError(t, err) - dnsAddr, _ := cfg.Spec.Network.DNSAddress() - t.Run("default_profile_only", func(t *testing.T) { - k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory(), cfg) - - t.Log("starting to run...") - buf, err := k.createProfiles(cfg) - require.NoError(t, err) - if err != nil { - t.FailNow() - } - manifestYamls := strings.Split(strings.TrimSuffix(buf.String(), "---"), "---")[1:] - t.Run("output_must_have_3_manifests", func(t *testing.T) { - require.Len(t, manifestYamls, 4, "Must have exactly 4 generated manifests per profile") - requireConfigMap(t, manifestYamls[0], "kubelet-config-default-1.28") - requireConfigMap(t, manifestYamls[1], "kubelet-config-default-windows-1.28") - requireRole(t, manifestYamls[2], []string{ - formatProfileName("default"), - formatProfileName("default-windows"), - }) - requireRoleBinding(t, manifestYamls[3]) - }) - }) - t.Run("default_profile_must_pass_down_cluster_domain", func(t *testing.T) { - profile := getDefaultProfile(dnsAddr, "cluster.local.custom") - require.Equal(t, string( - "cluster.local.custom", - ), profile["clusterDomain"]) - }) - t.Run("with_user_provided_profiles", func(t *testing.T) { - k, cfgWithUserProvidedProfiles := defaultConfigWithUserProvidedProfiles(t) - buf, err := k.createProfiles(cfgWithUserProvidedProfiles) - require.NoError(t, err) - manifestYamls := strings.Split(strings.TrimSuffix(buf.String(), "---"), "---")[1:] - expectedManifestsCount := 6 - require.Len(t, manifestYamls, expectedManifestsCount, "Must have exactly 6 generated manifests per profile") - - t.Run("final_output_must_have_manifests_for_profiles", func(t *testing.T) { - // check that each profile has config map, role and role binding - var resourceNamesForRole []string - for idx, profileName := range []string{"default", "default-windows", "profile_XXX", "profile_YYY"} { - fullName := "kubelet-config-" + profileName + "-1.28" - resourceNamesForRole = append(resourceNamesForRole, formatProfileName(profileName)) - requireConfigMap(t, manifestYamls[idx], fullName) - } - requireRole(t, manifestYamls[len(resourceNamesForRole)], resourceNamesForRole) - }) - t.Run("user_profile_X_must_be_merged_with_default_profile", func(t *testing.T) { - profileXXX := struct { - Data map[string]string `yaml:"data"` - }{} - - profileYYY := struct { - Data map[string]string `yaml:"data"` - }{} - - require.NoError(t, yaml.Unmarshal([]byte(manifestYamls[2]), &profileXXX)) - require.NoError(t, yaml.Unmarshal([]byte(manifestYamls[3]), &profileYYY)) - - // manually apple the same changes to default config and check that there is no diff - defaultProfileKubeletConfig := getDefaultProfile(dnsAddr, "cluster.local") - defaultProfileKubeletConfig["authentication"] = map[string]interface{}{ - "anonymous": map[string]interface{}{ - "enabled": false, - }, - } - defaultWithChangesXXX, err := yaml.Marshal(defaultProfileKubeletConfig) - require.NoError(t, err) - - defaultProfileKubeletConfig = getDefaultProfile(dnsAddr, "cluster.local") - defaultProfileKubeletConfig["authentication"] = map[string]interface{}{ - "webhook": map[string]interface{}{ - "cacheTTL": "15s", - }, - } - defaultWithChangesYYY, err := yaml.Marshal(defaultProfileKubeletConfig) - - require.NoError(t, err) - - require.YAMLEq(t, string(defaultWithChangesXXX), profileXXX.Data["kubelet"]) - require.YAMLEq(t, string(defaultWithChangesYYY), profileYYY.Data["kubelet"]) - }) - }) + + underTest := controller.NewKubeletConfig(k0sVars) + assert.NoError(t, underTest.Init(context.TODO())) + + assert.FileExists(t, filepath.Join(dir, "removed.txt")) + if entries, err := os.ReadDir(dir); assert.NoError(t, err) { + assert.Len(t, entries, 1, "Expected a single file in kubelet folder") + } } -func defaultConfigWithUserProvidedProfiles(t *testing.T) (*KubeletConfig, *v1beta1.ClusterConfig) { - cfg := v1beta1.DefaultClusterConfig() - k0sVars, err := config.NewCfgVars(nil, t.TempDir()) +func Test_KubeletConfig_RenamesManifestFiles(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests", "kubelet") + require.NoError(t, os.MkdirAll(dir, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "deprecation.txt"), nil, 0644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "manifest.yaml"), nil, 0644)) + + k0sVars, err := config.NewCfgVars(nil, tmp) require.NoError(t, err) - k := NewKubeletConfig(k0sVars, testutil.NewFakeClientFactory(), cfg) - - cfgProfileX := map[string]interface{}{ - "authentication": map[string]interface{}{ - "anonymous": map[string]interface{}{ - "enabled": false, - }, - }, - } - wcx, err := json.Marshal(cfgProfileX) - if err != nil { - t.Fatal(err) + + underTest := controller.NewKubeletConfig(k0sVars) + assert.NoError(t, underTest.Init(context.TODO())) + + assert.FileExists(t, filepath.Join(dir, "deprecation.txt")) + assert.FileExists(t, filepath.Join(dir, "removed.txt")) + if matches, err := filepath.Glob(filepath.Join(dir, "manifest.yaml.*.removed")); assert.NoError(t, err) { + assert.Len(t, matches, 1, "Expected a single removed manifest file") } - cfg.Spec.WorkerProfiles = append(cfg.Spec.WorkerProfiles, - v1beta1.WorkerProfile{ - Name: "profile_XXX", - Config: &runtime.RawExtension{Raw: wcx}, - }, - ) - - cfgProfileY := map[string]interface{}{ - "authentication": map[string]interface{}{ - "webhook": map[string]interface{}{ - "cacheTTL": "15s", - }, - }, + if entries, err := os.ReadDir(dir); assert.NoError(t, err) { + assert.Len(t, entries, 3, "Expected exactly three files in kubelet folder") } +} - wcy, err := json.Marshal(cfgProfileY) - if err != nil { - t.Fatal(err) - } +func Test_KubeletConfig_ManifestDirObstructed(t *testing.T) { + tmp := t.TempDir() + dir := filepath.Join(tmp, "manifests") + require.NoError(t, os.WriteFile(dir, []byte("obstructed"), 0644)) - cfg.Spec.WorkerProfiles = append(cfg.Spec.WorkerProfiles, - v1beta1.WorkerProfile{ - Name: "profile_YYY", - Config: &runtime.RawExtension{Raw: wcy}, - }, - ) - return k, cfg -} + k0sVars, err := config.NewCfgVars(nil, tmp) + require.NoError(t, err) -func requireConfigMap(t *testing.T, spec string, name string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "ConfigMap", dst["kind"]) - require.Equal(t, name, dst["metadata"].(map[string]interface{})["name"]) - spec, foundSpec := dst["data"].(map[string]interface{})["kubelet"].(string) - require.True(t, foundSpec, "kubelet config map must have embedded kubelet config") - require.True(t, strings.TrimSpace(spec) != "", "kubelet config map must have non-empty embedded kubelet config") -} + underTest := controller.NewKubeletConfig(k0sVars) + err = underTest.Init(context.TODO()) -func requireRole(t *testing.T, spec string, expectedResourceNames []string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "Role", dst["kind"]) - require.Equal(t, "system:bootstrappers:kubelet-configmaps", dst["metadata"].(map[string]interface{})["name"]) - var currentResourceNames []string - for _, el := range dst["rules"].([]interface{})[0].(map[string]interface{})["resourceNames"].([]interface{}) { - currentResourceNames = append(currentResourceNames, el.(string)) + assert.NotNil(t, errors.Unwrap(err), "Not wrapping a single error: %v", err) + var pathErr *os.PathError + if assert.ErrorAs(t, err, &pathErr) { + assert.Equal(t, dir, pathErr.Path) + assert.ErrorIs(t, err, syscall.ENOTDIR) } - require.Equal(t, expectedResourceNames, currentResourceNames) -} - -func requireRoleBinding(t *testing.T, spec string) { - dst := map[string]interface{}{} - require.NoError(t, yaml.Unmarshal([]byte(spec), &dst)) - dst = helmv1beta1.CleanUpGenericMap(dst) - require.Equal(t, "RoleBinding", dst["kind"]) - require.Equal(t, "system:bootstrappers:kubelet-configmaps", dst["metadata"].(map[string]interface{})["name"]) } diff --git a/pkg/config/cli.go b/pkg/config/cli.go index c91b4210879e..90a4ff47997d 100644 --- a/pkg/config/cli.go +++ b/pkg/config/cli.go @@ -105,12 +105,6 @@ func (o *ControllerOptions) Normalize() error { constant.APIConfigComponentName, "--enable-dynamic-config", ) } - - case constant.KubeletConfigComponentName: - logrus.Warnf("Usage of deprecated component name %q, please switch to %q", - constant.KubeletConfigComponentName, constant.WorkerConfigComponentName, - ) - disabledComponent = constant.WorkerConfigComponentName } if !slices.Contains(availableComponents, disabledComponent) { diff --git a/pkg/config/cli_test.go b/pkg/config/cli_test.go index b4e724404f54..4d9e2c4eba3a 100644 --- a/pkg/config/cli_test.go +++ b/pkg/config/cli_test.go @@ -44,30 +44,14 @@ func TestControllerOptions_Normalize(t *testing.T) { assert.ErrorContains(t, err, "unknown component i-dont-exist") }) - for _, test := range []struct { - name string - disabled, expected []string - }{ - { - "removesDuplicateComponents", - []string{"helm", "kube-proxy", "coredns", "kube-proxy", "autopilot"}, - []string{"helm", "kube-proxy", "coredns", "autopilot"}, - }, - { - "replacesDeprecation", - []string{"helm", "kubelet-config", "coredns", "kubelet-config", "autopilot"}, - []string{"helm", "worker-config", "coredns", "autopilot"}, - }, - { - "replacesDeprecationAvoidingDuplicates", - []string{"helm", "kubelet-config", "coredns", "kubelet-config", "worker-config", "autopilot"}, - []string{"helm", "worker-config", "coredns", "autopilot"}, - }, - } { - underTest := ControllerOptions{DisableComponents: test.disabled} + t.Run("removesDuplicateComponents", func(t *testing.T) { + disabled := []string{"helm", "kube-proxy", "coredns", "kube-proxy", "autopilot"} + expected := []string{"helm", "kube-proxy", "coredns", "autopilot"} + + underTest := ControllerOptions{DisableComponents: disabled} err := underTest.Normalize() require.NoError(t, err) - assert.Equal(t, test.expected, underTest.DisableComponents) - } + assert.Equal(t, expected, underTest.DisableComponents) + }) } diff --git a/pkg/constant/constant_shared.go b/pkg/constant/constant_shared.go index 9b6dda7215ce..fe817b00fadc 100644 --- a/pkg/constant/constant_shared.go +++ b/pkg/constant/constant_shared.go @@ -106,7 +106,6 @@ const ( KubeControllerManagerComponentName = "kube-controller-manager" KubeProxyComponentName = "kube-proxy" KubeSchedulerComponentName = "kube-scheduler" - KubeletConfigComponentName = "kubelet-config" // Deprecated: replaced by worker-config WorkerConfigComponentName = "worker-config" MetricsServerComponentName = "metrics-server" NetworkProviderComponentName = "network-provider"