Remove kubelet-config stack after deprecation in 1.26

The kubelet-config component has been deprecated in 1.26. Remove the
corresponding resources from the cluster in 1.29. Rename all manifest
files in the stack's folder to something that won't be picked up by the
stack applier. Leave a note about the stack removal in the stack's
folder.

Signed-off-by: Tom Wieczorek <[email protected]>

cmd/controller/controller.go

@@ -469,7 +469,7 @@ func (c *command) start(ctx context.Context) error {
 			return err
 		}
 		clusterComponents.Add(ctx, reconciler)
-		clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory, nodeConfig))
+		clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars))
 	}
 
 	if !slices.Contains(c.DisableComponents, constant.SystemRbacComponentName) {

go.mod

@@ -26,7 +26,6 @@ require (
 	github.com/go-playground/validator/v10 v10.16.0
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/terraform-exec v0.19.0
-	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/bootloose v0.7.2
 	github.com/k0sproject/dig v0.2.0
 	github.com/k0sproject/version v0.4.2
@@ -174,6 +173,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/terraform-json v0.17.1 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/intel/goresctrl v0.3.0 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect

inttest/ap-ha3x3/ha3x3_test.go

@@ -15,14 +15,18 @@
 package ha3x3
 
 import (
+	"context"
 	"fmt"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/avast/retry-go"
 	apv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
 	apconst "github.com/k0sproject/k0s/pkg/autopilot/constant"
 	appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core"
+	"golang.org/x/exp/slices"
 
 	"github.com/k0sproject/k0s/inttest/common"
 	aptest "github.com/k0sproject/k0s/inttest/common/autopilot"
@@ -92,6 +96,14 @@ func (s *ha3x3Suite) SetupTest() {
 // TestApply applies a well-formed `plan` yaml, and asserts that
 // all of the correct values across different objects + controllers are correct.
 func (s *ha3x3Suite) TestApply() {
+	ctx := s.Context()
+	baseK0sVersion, err := s.GetK0sVersion(s.ControllerNode(0))
+	if s.NoError(err, "Failed to get the base k0s version") {
+		s.T().Logf("Base k0s version: %s", baseK0sVersion)
+	} else {
+		baseK0sVersion = "Error: " + err.Error()
+	}
+
 	planTemplate := `
 apiVersion: autopilot.k0sproject.io/v1beta2
 kind: Plan
@@ -130,10 +142,10 @@ spec:
 	s.T().Logf("kubectl apply output: '%s'", out)
 	s.Require().NoError(err)
 
-	ssh, err := s.SSH(s.Context(), s.WorkerNode(0))
+	ssh, err := s.SSH(ctx, s.WorkerNode(0))
 	s.Require().NoError(err)
 	defer ssh.Disconnect()
-	out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V")
+	out, err = ssh.ExecWithOutput(ctx, "/var/lib/k0s/bin/iptables-save -V")
 	s.Require().NoError(err)
 	iptablesVersionParts := strings.Split(out, " ")
 	iptablesModeBeforeUpdate := iptablesVersionParts[len(iptablesVersionParts)-1]
@@ -143,7 +155,7 @@ spec:
 	s.NotEmpty(client)
 
 	// The plan has enough information to perform a successful update of k0s, so wait for it.
-	plan, err := aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted)
+	plan, err := aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted)
 	s.Require().NoError(err)
 
 	// Ensure all state/status are completed
@@ -165,11 +177,69 @@ spec:
 		s.Equal(s.K0sUpdateVersion, version)
 	}
 
-	out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V")
+	out, err = ssh.ExecWithOutput(ctx, "/var/lib/k0s/bin/iptables-save -V")
 	s.Require().NoError(err)
 	iptablesVersionParts = strings.Split(out, " ")
 	iptablesModeAfterUpdate := iptablesVersionParts[len(iptablesVersionParts)-1]
 	s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate)
+
+	s.checkKubeletConfigComponentFolders(ctx, ssh, baseK0sVersion)
+	s.checkKubeletConfigStackResources(ctx, ssh)
+}
+
+func (s *ha3x3Suite) checkKubeletConfigComponentFolders(ctx context.Context, ssh *common.SSHConnection, baseK0sVersion string) {
+	var expected []string
+	switch {
+	case strings.HasPrefix(baseK0sVersion, "v1.24") || strings.HasPrefix(baseK0sVersion, "v1.25"):
+		expected = []string{"./removed.txt", "./kubelet-config.yaml.*.removed"}
+	case strings.HasPrefix(baseK0sVersion, "v1.26"):
+		expected = []string{"./deprecated.txt", "./removed.txt", "./kubelet-config.yaml.*.removed"}
+	case strings.HasPrefix(baseK0sVersion, "v1.27"):
+		expected = []string{"./removed.txt"}
+	default: // Expect no kubelet folder at all subsequent versions
+		s.NoError(
+			ssh.Exec(ctx, "[ ! -e /var/lib/k0s/manifests/kubelet ]", common.SSHStreams{}),
+			"The kubelet manifests folder exists",
+		)
+		return
+	}
+
+	findFiles, err := ssh.ExecWithOutput(ctx, "cd /var/lib/k0s/manifests/kubelet && find -print0 . -type f")
+	if !s.NoError(err, "Failed to list kubelet manifests folder") {
+		return
+	}
+
+	files := strings.Split(findFiles, "\x00")
+	for _, expected := range expected {
+		idx := slices.IndexFunc(files, func(candidate string) bool {
+			matched, err := filepath.Match(expected, candidate)
+			s.Require().NoError(err)
+			return matched
+		})
+		if s.GreaterOrEqual(idx, 0, "No %s in kubelet manifests folder", expected) {
+			files = append(files[:idx], files[idx+1:]...)
+		}
+	}
+
+	s.Empty(files, "Some unexpected files detected in kubelet manifests folder")
+}
+
+func (s *ha3x3Suite) checkKubeletConfigStackResources(ctx context.Context, ssh *common.SSHConnection) {
+	var kubeletStackResources string
+	err := retry.Do(
+		func() (err error) {
+			kubeletStackResources, err = ssh.ExecWithOutput(ctx, "kubectl get configmaps,roles,rolebindings -A -l 'k0s.k0sproject.io/stack=kubelet' -oname")
+			return
+		},
+		retry.OnRetry(func(attempt uint, err error) {
+			s.T().Logf("Failed to execute kubectl in attempt #%d, retrying after backoff: %v", attempt+1, err)
+		}),
+		retry.Context(ctx),
+		retry.LastErrorOnly(true),
+	)
+	if s.NoError(err) {
+		s.Empty(kubeletStackResources)
+	}
 }
 
 // TestHA3x3Suite sets up a suite using 3 controllers for quorum, and runs various

pkg/applier/applier.go

@@ -21,7 +21,6 @@ import (
 	"errors"
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
 
 	"github.com/k0sproject/k0s/pkg/kubernetes"
@@ -40,6 +39,10 @@ import (
 // manifestFilePattern is the glob pattern that all applicable manifest files need to match.
 const manifestFilePattern = "*.yaml"
 
+func FindManifestFilesInDir(dir string) ([]string, error) {
+	return filepath.Glob(filepath.Join(dir, manifestFilePattern))
+}
+
 // Applier manages all the "static" manifests and applies them on the k8s API
 type Applier struct {
 	Name string
@@ -107,7 +110,7 @@ func (a *Applier) Apply(ctx context.Context) error {
 		return err
 	}
 
-	files, err := filepath.Glob(path.Join(a.Dir, manifestFilePattern))
+	files, err := FindManifestFilesInDir(a.Dir)
 	if err != nil {
 		return err
 	}