diff --git a/cmd/reset/reset.go b/cmd/reset/reset.go index 9e40c220e392..13253e803ec7 100644 --- a/cmd/reset/reset.go +++ b/cmd/reset/reset.go @@ -68,7 +68,7 @@ func (c *command) reset() error { } // Get Cleanup Config - cfg, err := cleanup.NewConfig(c.K0sVars, c.CfgFile, c.WorkerOptions.CriSocket) + cfg, err := cleanup.NewConfig(c.Debug, c.K0sVars, c.WorkerOptions.CriSocket) if err != nil { return fmt.Errorf("failed to configure cleanup: %w", err) } diff --git a/pkg/cleanup/cleanup.go b/pkg/cleanup/cleanup.go index 30fea04d5f41..c07cbba24321 100644 --- a/pkg/cleanup/cleanup.go +++ b/pkg/cleanup/cleanup.go @@ -19,63 +19,43 @@ package cleanup import ( "errors" "fmt" - "os/exec" + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" "github.com/k0sproject/k0s/pkg/component/worker" + workerconfig "github.com/k0sproject/k0s/pkg/component/worker/config" + "github.com/k0sproject/k0s/pkg/component/worker/containerd" "github.com/k0sproject/k0s/pkg/config" - + "github.com/k0sproject/k0s/pkg/constant" "github.com/k0sproject/k0s/pkg/container/runtime" + "github.com/sirupsen/logrus" ) type Config struct { - cfgFile string - containerd *containerdConfig - containerRuntime runtime.ContainerRuntime - dataDir string - k0sVars *config.CfgVars - runDir string + cleanupSteps []Step } -type containerdConfig struct { - binPath string - cmd *exec.Cmd - socketPath string -} - -func NewConfig(k0sVars *config.CfgVars, cfgFile string, criSocketFlag string) (*Config, error) { - runDir := "/run/k0s" // https://github.com/k0sproject/k0s/pull/591/commits/c3f932de85a0b209908ad39b817750efc4987395 - - var containerdCfg *containerdConfig +func NewConfig(debug bool, k0sVars *config.CfgVars, criSocketFlag string) (*Config, error) { + cfg, err := k0sVars.NodeConfig() + if err != nil { + logrus.Errorf("failed to get cluster setup: %v", err) + } - runtimeEndpoint, err := worker.GetContainerRuntimeEndpoint(criSocketFlag, runDir) + containers, err := newContainersStep(debug, k0sVars, criSocketFlag) if err != nil { return nil, err } - if criSocketFlag == "" { - containerdCfg = &containerdConfig{ - binPath: fmt.Sprintf("%s/%s", k0sVars.DataDir, "bin/containerd"), - socketPath: runtimeEndpoint.Path, - } - } - - return &Config{ - cfgFile: cfgFile, - containerd: containerdCfg, - containerRuntime: runtime.NewContainerRuntime(runtimeEndpoint), - dataDir: k0sVars.DataDir, - runDir: runDir, - k0sVars: k0sVars, - }, nil -} -func (c *Config) Cleanup() error { - var errs []error cleanupSteps := []Step{ - &containers{Config: c}, - &users{Config: c}, - &services{Config: c}, - &directories{Config: c}, + containers, + &users{ + systemUsers: cfg.Spec.Install.SystemUsers, + }, + &services{}, + &directories{ + dataDir: k0sVars.DataDir, + runDir: k0sVars.RunDir, + }, &cni{}, } @@ -83,7 +63,13 @@ func (c *Config) Cleanup() error { cleanupSteps = append(cleanupSteps, bridge) } - for _, step := range cleanupSteps { + return &Config{cleanupSteps}, nil +} + +func (c *Config) Cleanup() error { + var errs []error + + for _, step := range c.cleanupSteps { logrus.Info("* ", step.Name()) err := step.Run() if err != nil { @@ -97,6 +83,32 @@ func (c *Config) Cleanup() error { return nil } +func newContainersStep(debug bool, k0sVars *config.CfgVars, criSocketFlag string) (*containers, error) { + runtimeEndpoint, err := worker.GetContainerRuntimeEndpoint(criSocketFlag, k0sVars.RunDir) + if err != nil { + return nil, err + } + + containers := containers{ + containerRuntime: runtime.NewContainerRuntime(runtimeEndpoint), + } + + if criSocketFlag == "" { + logLevel := "error" + if debug { + logLevel = "debug" + } + containers.managedContainerd = containerd.NewComponent(logLevel, k0sVars, &workerconfig.Profile{ + PauseImage: &k0sv1beta1.ImageSpec{ + Image: constant.KubePauseContainerImage, + Version: constant.KubePauseContainerImageVersion, + }, + }) + } + + return &containers, nil +} + // Step interface is used to implement cleanup steps type Step interface { // Run impelements specific cleanup operations diff --git a/pkg/cleanup/containers.go b/pkg/cleanup/containers.go index b3d849dc60c4..53f881ebfa95 100644 --- a/pkg/cleanup/containers.go +++ b/pkg/cleanup/containers.go @@ -20,14 +20,11 @@ import ( "context" "errors" "fmt" - "io/fs" "os" - "os/exec" - "path/filepath" "strings" - "time" - "github.com/k0sproject/k0s/internal/pkg/file" + "github.com/k0sproject/k0s/pkg/component/worker/containerd" + "github.com/k0sproject/k0s/pkg/container/runtime" "github.com/avast/retry-go" "github.com/sirupsen/logrus" @@ -35,7 +32,8 @@ import ( ) type containers struct { - Config *Config + managedContainerd *containerd.Component + containerRuntime runtime.ContainerRuntime } // Name returns the name of the step @@ -46,23 +44,27 @@ func (c *containers) Name() string { // Run removes all the pods and mounts and stops containers afterwards // Run starts containerd if custom CRI is not configured func (c *containers) Run() error { - if !c.isCustomCriUsed() { - if err := c.startContainerd(); err != nil { - if errors.Is(err, fs.ErrNotExist) || errors.Is(err, exec.ErrNotFound) { - logrus.Debugf("containerd binary not found. Skipping container cleanup") - return nil - } - return fmt.Errorf("failed to start containerd: %w", err) + if c.managedContainerd != nil { + ctx := context.TODO() + if err := c.managedContainerd.Init(ctx); err != nil { + logrus.WithError(err).Warn("Failed to initialize containerd, skipping container cleanup") + return nil + } + if err := c.managedContainerd.Start(ctx); err != nil { + logrus.WithError(err).Warn("Failed to start containerd, skipping container cleanup") + return nil } + defer func() { + if err := c.managedContainerd.Stop(); err != nil { + logrus.WithError(err).Warn("Failed to stop containerd") + } + }() } if err := c.stopAllContainers(); err != nil { logrus.Debugf("error stopping containers: %v", err) } - if !c.isCustomCriUsed() { - c.stopContainerd() - } return nil } @@ -91,48 +93,6 @@ func removeMount(path string) error { return errors.Join(errs...) } -func (c *containers) isCustomCriUsed() bool { - return c.Config.containerd == nil -} - -func (c *containers) startContainerd() error { - logrus.Debugf("starting containerd") - args := []string{ - "--root=" + filepath.Join(c.Config.dataDir, "containerd"), - "--state=" + filepath.Join(c.Config.runDir, "containerd"), - "--address=" + c.Config.containerd.socketPath, - } - if file.Exists("/etc/k0s/containerd.toml") { - args = append(args, "--config=/etc/k0s/containerd.toml") - } - cmd := exec.Command(c.Config.containerd.binPath, args...) - if err := cmd.Start(); err != nil { - return err - } - - c.Config.containerd.cmd = cmd - logrus.Debugf("started containerd successfully") - - return nil -} - -func (c *containers) stopContainerd() { - logrus.Debug("attempting to stop containerd") - logrus.Debugf("found containerd pid: %v", c.Config.containerd.cmd.Process.Pid) - if err := c.Config.containerd.cmd.Process.Signal(os.Interrupt); err != nil { - logrus.Errorf("failed to kill containerd: %v", err) - } - // if process, didn't exit, wait a few seconds and send SIGKILL - if c.Config.containerd.cmd.ProcessState.ExitCode() != -1 { - time.Sleep(5 * time.Second) - - if err := c.Config.containerd.cmd.Process.Kill(); err != nil { - logrus.Errorf("failed to send SIGKILL to containerd: %v", err) - } - } - logrus.Debug("successfully stopped containerd") -} - func (c *containers) stopAllContainers() error { var errs []error @@ -141,7 +101,7 @@ func (c *containers) stopAllContainers() error { err := retry.Do(func() error { logrus.Debugf("trying to list all pods") var err error - pods, err = c.Config.containerRuntime.ListContainers(ctx) + pods, err = c.containerRuntime.ListContainers(ctx) if err != nil { return err } @@ -158,7 +118,7 @@ func (c *containers) stopAllContainers() error { for _, pod := range pods { logrus.Debugf("stopping container: %v", pod) - err := c.Config.containerRuntime.StopContainer(ctx, pod) + err := c.containerRuntime.StopContainer(ctx, pod) if err != nil { if strings.Contains(err.Error(), "443: connect: connection refused") { // on a single node instance, we will see "connection refused" error. this is to be expected @@ -168,13 +128,13 @@ func (c *containers) stopAllContainers() error { errs = append(errs, fmt.Errorf("failed to stop running pod %s: %w", pod, err)) } } - err = c.Config.containerRuntime.RemoveContainer(ctx, pod) + err = c.containerRuntime.RemoveContainer(ctx, pod) if err != nil { errs = append(errs, fmt.Errorf("failed to remove pod %s: %w", pod, err)) } } - pods, err = c.Config.containerRuntime.ListContainers(ctx) + pods, err = c.containerRuntime.ListContainers(ctx) if err == nil && len(pods) == 0 { logrus.Info("successfully removed k0s containers!") } diff --git a/pkg/cleanup/directories.go b/pkg/cleanup/directories.go index 4a155d6e559f..b1c11f849405 100644 --- a/pkg/cleanup/directories.go +++ b/pkg/cleanup/directories.go @@ -28,7 +28,8 @@ import ( ) type directories struct { - Config *Config + dataDir string + runDir string } // Name returns the name of the step @@ -65,11 +66,11 @@ func (d *directories) Run() error { v := procMounts[i] // avoid unmount datadir if its mounted on separate partition // k0s didn't mount it so leave it alone - if v.Path == d.Config.k0sVars.DataDir { + if v.Path == d.dataDir { dataDirMounted = true continue } - if isUnderPath(v.Path, filepath.Join(d.Config.dataDir, "kubelet")) || isUnderPath(v.Path, d.Config.k0sVars.DataDir) { + if isUnderPath(v.Path, filepath.Join(d.dataDir, "kubelet")) || isUnderPath(v.Path, d.dataDir) { logrus.Debugf("%v is mounted! attempting to unmount...", v.Path) if err = mounter.Unmount(v.Path); err != nil { // if we fail to unmount, try lazy unmount so @@ -84,23 +85,23 @@ func (d *directories) Run() error { } if dataDirMounted { - logrus.Debugf("removing the contents of mounted data-dir (%s)", d.Config.dataDir) + logrus.Debugf("removing the contents of mounted data-dir (%s)", d.dataDir) } else { - logrus.Debugf("removing k0s generated data-dir (%s)", d.Config.dataDir) + logrus.Debugf("removing k0s generated data-dir (%s)", d.dataDir) } - if err := os.RemoveAll(d.Config.dataDir); err != nil { + if err := os.RemoveAll(d.dataDir); err != nil { if !dataDirMounted { return fmt.Errorf("failed to delete k0s generated data-dir: %w", err) } - if !errorIsUnlinkat(err, d.Config.dataDir) { + if !errorIsUnlinkat(err, d.dataDir) { return fmt.Errorf("failed to delete contents of mounted data-dir: %w", err) } } - logrus.Debugf("deleting k0s generated run-dir (%s)", d.Config.runDir) - if err := os.RemoveAll(d.Config.runDir); err != nil { - return fmt.Errorf("failed to delete %s: %w", d.Config.runDir, err) + logrus.Debugf("deleting k0s generated run-dir (%s)", d.runDir) + if err := os.RemoveAll(d.runDir); err != nil { + return fmt.Errorf("failed to delete %s: %w", d.runDir, err) } return nil diff --git a/pkg/cleanup/services.go b/pkg/cleanup/services.go index 9d6665b42d5c..9f91ddc377ae 100644 --- a/pkg/cleanup/services.go +++ b/pkg/cleanup/services.go @@ -24,9 +24,7 @@ import ( "github.com/k0sproject/k0s/pkg/install" ) -type services struct { - Config *Config -} +type services struct{} // Name returns the name of the step func (s *services) Name() string { diff --git a/pkg/cleanup/users.go b/pkg/cleanup/users.go index 8a0f3bb060a7..9bd2b4cf699e 100644 --- a/pkg/cleanup/users.go +++ b/pkg/cleanup/users.go @@ -17,12 +17,14 @@ limitations under the License. package cleanup import ( + k0sv1beta1 "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1" "github.com/k0sproject/k0s/pkg/install" + "github.com/sirupsen/logrus" ) type users struct { - Config *Config + systemUsers *k0sv1beta1.SystemUser } // Name returns the name of the step @@ -32,11 +34,7 @@ func (u *users) Name() string { // Run removes all controller users that are present on the host func (u *users) Run() error { - cfg, err := u.Config.k0sVars.NodeConfig() - if err != nil { - logrus.Errorf("failed to get cluster setup: %v", err) - } - if err := install.DeleteControllerUsers(cfg.Spec.Install.SystemUsers); err != nil { + if err := install.DeleteControllerUsers(u.systemUsers); err != nil { // don't fail, just notify on delete error logrus.Warnf("failed to delete controller users: %v", err) }