Remove kubelet-config stack after deprecation in 1.26

The kubelet-config component has been deprecated in 1.26. Remove the
corresponding resources from the cluster in 1.29. Rename all manifest
files in the stack's folder to something that won't be picked up by the
stack applier. Leave a note about the stack removal in the stack's
folder.

Signed-off-by: Tom Wieczorek <[email protected]>

cmd/controller/controller.go

@@ -469,7 +469,7 @@ func (c *command) start(ctx context.Context) error {
 			return err
 		}
 		clusterComponents.Add(ctx, reconciler)
-		clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars, adminClientFactory, nodeConfig))
+		clusterComponents.Add(ctx, controller.NewKubeletConfig(c.K0sVars))
 	}
 
 	if !slices.Contains(c.DisableComponents, constant.SystemRbacComponentName) {

go.mod

@@ -29,7 +29,6 @@ require (
 	github.com/go-playground/validator/v10 v10.16.0
 	github.com/google/go-cmp v0.6.0
 	github.com/hashicorp/terraform-exec v0.20.0
-	github.com/imdario/mergo v0.3.16
 	github.com/k0sproject/bootloose v0.7.2
 	github.com/k0sproject/dig v0.2.0
 	github.com/k0sproject/version v0.4.2
@@ -176,6 +175,7 @@ require (
 	github.com/hashicorp/go-version v1.6.0 // indirect
 	github.com/hashicorp/terraform-json v0.19.0 // indirect
 	github.com/huandu/xstrings v1.4.0 // indirect
+	github.com/imdario/mergo v0.3.16 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/intel/goresctrl v0.3.0 // indirect
 	github.com/jmoiron/sqlx v1.3.5 // indirect

inttest/ap-ha3x3/ha3x3_test.go

@@ -15,24 +15,30 @@
 package ha3x3
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
-	apv1beta2 "github.com/k0sproject/k0s/pkg/apis/autopilot/v1beta2"
+	"github.com/avast/retry-go"
 	apconst "github.com/k0sproject/k0s/pkg/autopilot/constant"
 	appc "github.com/k0sproject/k0s/pkg/autopilot/controller/plans/core"
+	"golang.org/x/exp/slices"
 
 	"github.com/k0sproject/k0s/inttest/common"
 	aptest "github.com/k0sproject/k0s/inttest/common/autopilot"
 
+	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/suite"
 )
 
 type ha3x3Suite struct {
 	common.BootlooseSuite
+	k0sUpdateVersion string
 }
 
 const haControllerConfig = `
@@ -93,9 +99,6 @@ func (s *ha3x3Suite) SetupTest() {
 // TestApply applies a well-formed `plan` yaml, and asserts that
 // all of the correct values across different objects + controllers are correct.
 func (s *ha3x3Suite) TestApply() {
-	k0sUpdateVersion := os.Getenv("K0S_UPDATE_TO_VERSION")
-	s.Require().NotEmpty(k0sUpdateVersion, "env var not set or empty: K0S_UPDATE_TO_VERSION")
-
 	planTemplate := `
 apiVersion: autopilot.k0sproject.io/v1beta2
 kind: Plan
@@ -106,7 +109,7 @@ spec:
   timestamp: now
   commands:
     - k0supdate:
-        version: ` + k0sUpdateVersion + `
+        version: ` + s.k0sUpdateVersion + `
         platforms:
           linux-amd64:
             url: http://localhost/dist/k0s-new
@@ -127,64 +130,184 @@ spec:
                   - worker2
 `
 
-	manifestFile := "/tmp/happy.yaml"
-	s.PutFileTemplate(s.ControllerNode(0), manifestFile, planTemplate, nil)
+	ctx := s.Context()
 
-	out, err := s.RunCommandController(0, fmt.Sprintf("/usr/local/bin/k0s kubectl apply -f %s", manifestFile))
-	s.T().Logf("kubectl apply output: '%s'", out)
+	sshController, err := s.SSH(ctx, s.ControllerNode(0))
 	s.Require().NoError(err)
+	defer sshController.Disconnect()
 
-	ssh, err := s.SSH(s.Context(), s.WorkerNode(0))
+	var baseHasOldStack *bool
+	if version, err := s.GetK0sVersion(s.ControllerNode(0)); s.NoError(err, "Failed to get the base k0s version") {
+		hasOldStack := version != s.k0sUpdateVersion && strings.HasPrefix(version, "v1.27.") || strings.HasPrefix(version, "v1.28.")
+		s.T().Logf("Base k0s version: %q, has old stack: %v", version, hasOldStack)
+		s.checkKubeletConfigStackResources(ctx, sshController, hasOldStack)
+		baseHasOldStack = &hasOldStack
+	}
+
+	sshWorker, err := s.SSH(ctx, s.WorkerNode(0))
 	s.Require().NoError(err)
-	defer ssh.Disconnect()
-	out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V")
+	defer sshWorker.Disconnect()
+
+	iptablesModeBeforeUpdate, err := getIPTablesMode(ctx, sshWorker)
+	if !s.NoError(err) {
+		iptablesModeBeforeUpdate = ""
+	}
+
+	var createPlanOutput bytes.Buffer
+	err = sshController.Exec(ctx, fmt.Sprintf("k0s kc create -f -"), common.SSHStreams{
+		In:  strings.NewReader(planTemplate),
+		Out: &createPlanOutput,
+	})
 	s.Require().NoError(err)
-	iptablesVersionParts := strings.Split(out, " ")
-	iptablesModeBeforeUpdate := iptablesVersionParts[len(iptablesVersionParts)-1]
+	s.T().Log(strings.TrimSpace(createPlanOutput.String()))
 
 	client, err := s.AutopilotClient(s.ControllerNode(0))
 	s.Require().NoError(err)
-	s.NotEmpty(client)
 
 	// The plan has enough information to perform a successful update of k0s, so wait for it.
-	plan, err := aptest.WaitForPlanState(s.Context(), client, apconst.AutopilotName, appc.PlanCompleted)
+	s.T().Log("Waiting for autopilot plan to complete")
+	plan, err := aptest.WaitForPlanState(ctx, client, apconst.AutopilotName, appc.PlanCompleted)
 	s.Require().NoError(err)
+	s.T().Log("Autopilot plan completed")
 
 	// Ensure all state/status are completed
-	s.Equal(1, len(plan.Status.Commands))
-	cmd := plan.Status.Commands[0]
+	if s.Len(plan.Status.Commands, 1) {
+		cmd := plan.Status.Commands[0]
+		s.Equal(appc.PlanCompleted, cmd.State)
+		s.Equal(appc.PlanCompleted, cmd.State)
+		s.NotNil(cmd.K0sUpdate)
+		s.NotNil(cmd.K0sUpdate.Controllers)
+		s.NotNil(cmd.K0sUpdate.Workers)
+		s.Equal(appc.PlanCompleted, cmd.State)
+		s.NotNil(cmd.K0sUpdate)
+		s.NotNil(cmd.K0sUpdate.Controllers)
+		s.NotNil(cmd.K0sUpdate.Workers)
 
-	s.Equal(appc.PlanCompleted, cmd.State)
-	s.NotNil(cmd.K0sUpdate)
-	s.NotNil(cmd.K0sUpdate.Controllers)
-	s.NotNil(cmd.K0sUpdate.Workers)
+		if s.NotNil(cmd.K0sUpdate) {
+			s.Len(cmd.K0sUpdate.Controllers, s.ControllerCount)
+			for idx, controller := range cmd.K0sUpdate.Controllers {
+				s.Equal(appc.SignalCompleted, controller.State, "For controller %d", idx)
+			}
 
-	for _, group := range [][]apv1beta2.PlanCommandTargetStatus{cmd.K0sUpdate.Controllers, cmd.K0sUpdate.Workers} {
-		for _, node := range group {
-			s.Equal(appc.SignalCompleted, node.State)
+			s.Len(cmd.K0sUpdate.Workers, s.WorkerCount)
+			for idx, worker := range cmd.K0sUpdate.Workers {
+				s.Equal(appc.SignalCompleted, worker.State, "For worker %d", idx)
+			}
 		}
 	}
 
 	if version, err := s.GetK0sVersion(s.ControllerNode(0)); s.NoError(err) {
-		s.Equal(k0sUpdateVersion, version)
+		s.Equal(s.k0sUpdateVersion, version)
 	}
 
-	out, err = ssh.ExecWithOutput(s.Context(), "/var/lib/k0s/bin/iptables-save -V")
-	s.Require().NoError(err)
-	iptablesVersionParts = strings.Split(out, " ")
-	iptablesModeAfterUpdate := iptablesVersionParts[len(iptablesVersionParts)-1]
-	s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate)
+	if iptablesModeAfterUpdate, err := getIPTablesMode(ctx, sshWorker); s.NoError(err) {
+		s.Equal(iptablesModeBeforeUpdate, iptablesModeAfterUpdate)
+	}
+
+	if baseHasOldStack != nil {
+		for idx := 0; idx < s.ControllerCount; idx++ {
+			func() {
+				ssh, err := s.SSH(ctx, s.ControllerNode(idx))
+				s.Require().NoError(err)
+				defer ssh.Disconnect()
+				s.checkKubeletConfigComponentFolders(ctx, ssh, *baseHasOldStack)
+			}()
+		}
+	}
+
+	s.checkKubeletConfigStackResources(ctx, sshController, false)
+}
+
+func (s *ha3x3Suite) checkKubeletConfigComponentFolders(ctx context.Context, ssh *common.SSHConnection, hasOldStack bool) {
+	if !hasOldStack {
+		// Expect no kubelet folder at all for recent versions
+		err := ssh.Exec(ctx, "[ ! -e /var/lib/k0s/manifests/kubelet ]", common.SSHStreams{})
+		s.NoError(err, "Failed to check that the kubelet manifest folder doesn't exist")
+		return
+	}
+
+	// Expect a kubelet folder with removal files for older k0s versions
+
+	var foundFiles bytes.Buffer
+	if !s.NoError(
+		ssh.Exec(ctx, "cd /var/lib/k0s/manifests/kubelet && find . -type f -print0", common.SSHStreams{Out: &foundFiles}),
+		"Failed to list kubelet manifest folder",
+	) {
+		return
+	}
+
+	files := strings.Split(strings.TrimSuffix(foundFiles.String(), "\x00"), "\x00")
+
+	// Check that removed.txt is present
+	if idx := slices.Index(files, "./removed.txt"); idx < 0 {
+		s.Failf("No removed.txt in kubelet manifests folder", "%v", files)
+	} else {
+		files = slices.Delete(files, idx, idx+1)
+	}
+
+	// Check that all other files are only disabled yaml files.
+	for _, file := range files {
+		match, err := filepath.Match("./kubelet-config.yaml.*.removed", file)
+		s.Require().NoError(err)
+		if !match {
+			s.Failf("Unknown file in kubelet manifest folder", "%s in %v", file, files)
+		}
+	}
+}
+
+func (s *ha3x3Suite) checkKubeletConfigStackResources(ctx context.Context, ssh *common.SSHConnection, exist bool) {
+	const cmd = "k0s kc get configmaps,roles,rolebindings -A -l 'k0s.k0sproject.io/stack=kubelet' -oname"
+
+	var out bytes.Buffer
+	err := retry.Do(
+		func() error {
+			out.Reset()
+			return ssh.Exec(ctx, cmd, common.SSHStreams{Out: &out})
+		},
+		retry.OnRetry(func(attempt uint, err error) {
+			s.T().Logf("Failed to check kubelet-config stack resources in attempt #%d, retrying after backoff: %v", attempt+1, err)
+		}),
+		retry.Context(ctx),
+		retry.LastErrorOnly(true),
+	)
+
+	if s.NoError(err) {
+		if exist {
+			s.NotEmpty(out.String())
+		} else {
+			s.Empty(out.String())
+		}
+	}
+}
+
+func getIPTablesMode(ctx context.Context, ssh *common.SSHConnection) (string, error) {
+	var out bytes.Buffer
+	err := ssh.Exec(ctx, "/var/lib/k0s/bin/iptables-save -V", common.SSHStreams{Out: &out})
+	if err != nil {
+		return "", err
+	}
+
+	version := out.String()
+	if parts := strings.Split(version, " "); len(parts) == 3 {
+		return parts[2], nil
+	}
+
+	return "", fmt.Errorf("expected something like %q, got %q", "iptables-save v1.8.9 (nf_tables)", version)
 }
 
 // TestHA3x3Suite sets up a suite using 3 controllers for quorum, and runs various
 // autopilot upgrade scenarios against them.
 func TestHA3x3Suite(t *testing.T) {
+	k0sUpdateVersion := os.Getenv("K0S_UPDATE_TO_VERSION")
+	require.NotEmpty(t, k0sUpdateVersion, "env var not set or empty: K0S_UPDATE_TO_VERSION")
+
 	suite.Run(t, &ha3x3Suite{
 		common.BootlooseSuite{
 			ControllerCount: 3,
 			WorkerCount:     3,
 			WithLB:          true,
 			LaunchMode:      common.LaunchModeOpenRC,
 		},
+		k0sUpdateVersion,
 	})
 }

pkg/applier/applier.go

@@ -19,7 +19,6 @@ package applier
 import (
 	"context"
 	"fmt"
-	"path"
 	"path/filepath"
 
 	"github.com/k0sproject/k0s/pkg/kubernetes"
@@ -35,6 +34,10 @@ import (
 // manifestFilePattern is the glob pattern that all applicable manifest files need to match.
 const manifestFilePattern = "*.yaml"
 
+func FindManifestFilesInDir(dir string) ([]string, error) {
+	return filepath.Glob(filepath.Join(dir, manifestFilePattern))
+}
+
 // Applier manages all the "static" manifests and applies them on the k8s API
 type Applier struct {
 	Name string
@@ -96,7 +99,7 @@ func (a *Applier) Apply(ctx context.Context) error {
 		return err
 	}
 
-	files, err := filepath.Glob(path.Join(a.Dir, manifestFilePattern))
+	files, err := FindManifestFilesInDir(a.Dir)
 	if err != nil {
 		return err
 	}