Skip to content

Commit

Permalink
Pass along CRI config as return value
Browse files Browse the repository at this point in the history 
Return the CRI config as a string instead of having the CRIConfigurer
write it to disk. This reduces the number of config options required by
the Configurer, and makes testing less dependent on the file system.

Let the containerd component write the file instead.

Remove the constants for the paths to containerd-cri.toml and use k0s'
RunDir instead. This already takes into account the differences between
Windows and UNIX-like operating systems, and also takes into account the
different defaults when running as non-root user.

Remove the escapedPath function and inline it directly before the
containerd component generates the template.

Signed-off-by: Tom Wieczorek <[email protected]>
  • Loading branch information
@twz123
twz123 committed Feb 21, 2024
1 parent 0b40888 commit 8d47f5f
Show file tree
Hide file tree
Showing 3 changed files with 62 additions and 118 deletions.
27 changes: 21 additions & 6 deletions pkg/component/worker/containerd.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,19 +197,34 @@ func (c *Containerd) setupConfig() error {
}
containerdConfigurer := containerd.NewConfigurer(c.Profile.PauseImage, filepath.Join(c.importsPath, "*.toml"))

imports, err := containerdConfigurer.HandleImports()
criConfig, err := containerdConfigurer.HandleImports()
if err != nil {
return fmt.Errorf("can't handle imports: %w", err)
}

criConfigPath := filepath.Join(c.K0sVars.RunDir, "containerd-cri.toml")
err = file.WriteContentAtomically(criConfigPath, []byte(criConfig.Config), 0644)
if err != nil {
return fmt.Errorf("can't create containerd CRI config: %w", err)
}

var data struct{ Imports []string }
data.Imports = append(criConfig.ImportPaths, criConfigPath)

// double escape for windows because containerd expects
// double backslash in the configuration but golang templates
// unescape double slash to a single slash
if runtime.GOOS == "windows" {
for i := range data.Imports {
data.Imports[i] = strings.ReplaceAll(data.Imports[i], "\\", "\\\\")
}
}

output := bytes.NewBuffer([]byte{})
tw := templatewriter.TemplateWriter{
Name: "containerdconfig",
Template: confTmpl,
Data: struct {
Imports []string
}{
Imports: imports,
},
Data: data,
}
if err := tw.WriteToBuffer(output); err != nil {
return fmt.Errorf("can't create containerd config: %w", err)
Expand Down
64 changes: 20 additions & 44 deletions pkg/component/worker/containerd/criconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@ import (
"os"
"path/filepath"
"runtime"
"strings"

"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
"github.com/mesosphere/toml-merge/pkg/patch"
Expand All @@ -33,39 +32,32 @@ import (
criconfig "github.com/containerd/containerd/pkg/cri/config"
)

// TODO: move to K0sVars
const containerdCRIConfigPathPosix = "/run/k0s/containerd-cri.toml"
const containerdCRIConfigPathWindows = "C:\\var\\lib\\k0s\\run\\containerd-cri.toml"
type CRIConfig struct {
Config string
ImportPaths []string
}

type CRIConfigurer struct {
loadPath string
pauseImage string
criRuntimePath string
loadPath string
pauseImage string

log *logrus.Entry
}

func NewConfigurer(pauseImage *v1beta1.ImageSpec, importsPath string) *CRIConfigurer {
c := &CRIConfigurer{
return &CRIConfigurer{
loadPath: importsPath,
pauseImage: pauseImage.URI(),
log: logrus.WithField("component", "containerd"),
}
if runtime.GOOS == "windows" {
c.criRuntimePath = containerdCRIConfigPathWindows

} else {
c.criRuntimePath = containerdCRIConfigPathPosix
}
return c
}

// HandleImports Resolves containerd imports from the import glob path.
// If the partial config has CRI plugin enabled, it will add to the runc CRI config (single file).
// if no CRI plugin is found, it will add the file as-is to imports list returned.
// Once all files are processed the concatenated CRI config file is written and added to the imports list.
func (c *CRIConfigurer) HandleImports() ([]string, error) {
var imports []string
func (c *CRIConfigurer) HandleImports() (*CRIConfig, error) {
var importPaths []string
var criConfigBuffer bytes.Buffer

// Add default runc based CRI config
Expand All @@ -74,54 +66,38 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) {
return nil, err
}

files, err := filepath.Glob(c.loadPath)
c.log.Debugf("found containerd config files: %v", files)
filePaths, err := filepath.Glob(c.loadPath)
c.log.Debugf("found containerd config files: %v", filePaths)
if err != nil {
return nil, err
}

finalConfig := criConfigBuffer.String()
for _, file := range files {
data, err := os.ReadFile(file)
for _, filePath := range filePaths {
data, err := os.ReadFile(filePath)
if err != nil {
return nil, err
}
c.log.Debugf("parsing containerd config file: %s", file)
c.log.Debugf("parsing containerd config file: %s", filePath)
hasCRI, err := hasCRIPluginConfig(data)
if err != nil {
return nil, err
}
if hasCRI {
c.log.Infof("found CRI plugin config in %s, merging to CRI config", file)
c.log.Infof("found CRI plugin config in %s, merging to CRI config", filePath)
// Merge to the existing CRI config
finalConfig, err = patch.TOMLString(finalConfig, patch.FilePatches(file))
finalConfig, err = patch.TOMLString(finalConfig, patch.FilePatches(filePath))
if err != nil {
return nil, fmt.Errorf("failed to merge CRI config from %s: %w", file, err)
return nil, fmt.Errorf("failed to merge CRI config from %s: %w", filePath, err)
}
} else {
c.log.Debugf("adding %s as-is to imports", file)
c.log.Debugf("adding %s as-is to imports", filePath)
// Add file to imports
imports = append(imports, escapedPath(file))
importPaths = append(importPaths, filePath)
}
}
// Write the CRI config to a file and add it to imports
err = os.WriteFile(c.criRuntimePath, []byte(finalConfig), 0644)
if err != nil {
return nil, err
}
imports = append(imports, escapedPath(c.criRuntimePath))

return imports, nil
}

func escapedPath(s string) string {
// double escape for windows because containerd expects
// double backslash in the configuration but golang templates
// unescape double slash to a single slash
if runtime.GOOS == "windows" {
return strings.ReplaceAll(s, "\\", "\\\\")
}
return s
return &CRIConfig{Config: finalConfig, ImportPaths: importPaths}, nil
}

// We need to use custom struct so we can unmarshal the CRI plugin config only
Expand Down
89 changes: 21 additions & 68 deletions pkg/component/worker/containerd/criconfig_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ import (
"path/filepath"
"testing"

"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"

serverconfig "github.com/containerd/containerd/services/server/config"

"github.com/sirupsen/logrus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
Expand Down Expand Up @@ -54,100 +55,52 @@ version = 2
func TestCRIConfigurer_HandleImports(t *testing.T) {
t.Run("should merge CRI config snippets", func(t *testing.T) {
importsPath := t.TempDir()
criRuntimePath := filepath.Join(t.TempDir(), "cri.toml")
criRuntimeConfig := `
[plugins]
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "zfs"
`
err := os.WriteFile(filepath.Join(importsPath, "foo.toml"), []byte(criRuntimeConfig), 0644)
require.NoError(t, err)
c := CRIConfigurer{
loadPath: filepath.Join(importsPath, "*.toml"),
criRuntimePath: criRuntimePath,
log: logrus.New().WithField("test", t.Name()),
}
_, err = c.HandleImports()
c := NewConfigurer(&v1beta1.ImageSpec{}, filepath.Join(importsPath, "*.toml"))
criConfig, err := c.HandleImports()
require.NoError(t, err)
assert.Empty(t, criConfig.ImportPaths, "CRI config snippets should be merged, not imported")

// Dump the config for inspection
b, _ := os.ReadFile(criRuntimePath)
t.Logf("CRI config:\n%s", string(b))
t.Logf("CRI config:\n%s", criConfig.Config)

criConfigPath := filepath.Join(t.TempDir(), "cri.toml")
require.NoError(t, os.WriteFile(criConfigPath, []byte(criConfig.Config), 0644))

// Load the criRuntimeConfig and verify the settings are correct
containerdConfig := &serverconfig.Config{}
err = serverconfig.LoadConfig(criRuntimePath, containerdConfig)
require.NoError(t, err)
var containerdConfig serverconfig.Config
require.NoError(t, serverconfig.LoadConfig(criConfigPath, &containerdConfig))

criConfig := containerdConfig.Plugins["io.containerd.grpc.v1.cri"]
snapshotter := criConfig.GetPath([]string{"containerd", "snapshotter"})
criPlugins := containerdConfig.Plugins["io.containerd.grpc.v1.cri"]
snapshotter := criPlugins.GetPath([]string{"containerd", "snapshotter"})
require.Equal(t, "zfs", snapshotter)
})

t.Run("should have single import for CRI if there's nothing in imports dir", func(t *testing.T) {
criRuntimePath := filepath.Join(t.TempDir(), "cri.toml")
c := CRIConfigurer{
loadPath: filepath.Join(t.TempDir(), "*.toml"),
criRuntimePath: criRuntimePath,
log: logrus.New().WithField("test", t.Name()),
}
imports, err := c.HandleImports()
t.Run("should have no imports if imports dir is empty", func(t *testing.T) {
c := NewConfigurer(&v1beta1.ImageSpec{}, filepath.Join(t.TempDir(), "*.toml"))
criConfig, err := c.HandleImports()
assert.NoError(t, err)
if assert.Len(t, imports, 1) {
assert.Equal(t, escapedPath(criRuntimePath), imports[0])
}
})

t.Run("should have single import for all CRI configs", func(t *testing.T) {
importsPath := t.TempDir()
criRuntimePath := filepath.Join(t.TempDir(), "cri.toml")
criRuntimeConfig := `
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
endpoint = ["https://registry-1.docker.io"]
`
err := os.WriteFile(filepath.Join(importsPath, "foo.toml"), []byte(criRuntimeConfig), 0644)
require.NoError(t, err)
c := CRIConfigurer{
loadPath: filepath.Join(importsPath, "*.toml"),
criRuntimePath: criRuntimePath,
log: logrus.New().WithField("test", t.Name()),
}
imports, err := c.HandleImports()
require.NoError(t, err)
require.Len(t, imports, 1)
require.Contains(t, imports, escapedPath(criRuntimePath))

// Load the criRuntimeConfig and verify the settings are correct
containerdConfig := &serverconfig.Config{}
err = serverconfig.LoadConfig(criRuntimePath, containerdConfig)
require.NoError(t, err)

criConfig := containerdConfig.Plugins["io.containerd.grpc.v1.cri"]
ep := criConfig.GetPath([]string{"registry", "mirrors", "docker.io", "endpoint"})
assert.Equal(t, []any{"https://registry-1.docker.io"}, ep)
assert.Empty(t, criConfig.ImportPaths)
})

t.Run("should have two imports when one non CRI snippet", func(t *testing.T) {
t.Run("should import non-CRI config snippets", func(t *testing.T) {
importsPath := t.TempDir()
criRuntimePath := filepath.Join(t.TempDir(), "cri.toml")
criRuntimeConfig := `
foo = "bar"
version = 2
`
nonCriConfigPath := filepath.Join(importsPath, "foo.toml")
err := os.WriteFile(nonCriConfigPath, []byte(criRuntimeConfig), 0644)
require.NoError(t, err)
c := CRIConfigurer{
loadPath: filepath.Join(importsPath, "*.toml"),
criRuntimePath: criRuntimePath,
log: logrus.New().WithField("test", t.Name()),
}
imports, err := c.HandleImports()
c := NewConfigurer(&v1beta1.ImageSpec{}, filepath.Join(importsPath, "*.toml"))
criConfig, err := c.HandleImports()
assert.NoError(t, err)
if assert.Len(t, imports, 2) {
assert.Contains(t, imports, escapedPath(criRuntimePath))
assert.Contains(t, imports, escapedPath(nonCriConfigPath))
}
assert.Equal(t, []string{nonCriConfigPath}, criConfig.ImportPaths)
})
}

0 comments on commit 8d47f5f

Please sign in to comment.