Actually merge CRI configs from drop-ins instead of concatenating them

Uses https://github.com/mesosphere/toml-merge for merging the CRI config snippets. If a config drop-in is not a CRI config, it behaves as previously and just adds it to imports.

Fixes #3283

Signed-off-by: Jussi Nummelin <[email protected]>
(cherry picked from commit 3fa95ec)

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/k0sproject/dig v0.2.0
 	github.com/kardianos/service v1.2.2
 	github.com/logrusorgru/aurora/v3 v3.0.0
+	github.com/mesosphere/toml-merge v0.2.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b

go.sum

@@ -606,6 +606,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
 github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/mesosphere/toml-merge v0.2.0 h1:stCUgrwbictiebeHRqEJ1NfQl/h5noyFKR0LBWjWXxQ=
+github.com/mesosphere/toml-merge v0.2.0/go.mod h1:WYpgeqeG5puUtv2NREGyOIqTnYuWswyo7CBgx6QK80s=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4=

pkg/component/worker/containerd/criconfig.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1"
 	"github.com/k0sproject/k0s/pkg/constant"
+	"github.com/mesosphere/toml-merge/pkg/patch"
 	"github.com/pelletier/go-toml"
 	"github.com/sirupsen/logrus"
 
@@ -75,6 +76,8 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	finalConfig := criConfigBuffer.String()
 	for _, file := range files {
 		data, err := os.ReadFile(file)
 		if err != nil {
@@ -86,13 +89,11 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) {
 			return nil, err
 		}
 		if hasCRI {
-			c.log.Debugf("found CRI plugin config in %s, appending to CRI config", file)
-			// Append all CRI plugin configs into single file
-			// and add it to imports
-			criConfigBuffer.WriteString(fmt.Sprintf("# appended from %s\n", file))
-			_, err := criConfigBuffer.Write(data)
+			c.log.Infof("found CRI plugin config in %s, merging to CRI config", file)
+			// Merge to the existing CRI config
+			finalConfig, err = patch.TOMLString(finalConfig, patch.FilePatches(file))
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("failed to merge CRI config from %s: %w", file, err)
 			}
 		} else {
 			c.log.Debugf("adding %s as-is to imports", file)
@@ -102,7 +103,7 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) {
 	}
 
 	// Write the CRI config to a file and add it to imports
-	err = os.WriteFile(c.criRuntimePath, criConfigBuffer.Bytes(), 0644)
+	err = os.WriteFile(c.criRuntimePath, []byte(finalConfig), 0644)
 	if err != nil {
 		return nil, err
 	}

pkg/component/worker/containerd/criconfig_test.go

@@ -21,6 +21,7 @@ import (
 	"path/filepath"
 	"testing"
 
+	srvconfig "github.com/containerd/containerd/services/server/config"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
@@ -52,6 +53,39 @@ version = 2
 }
 
 func TestCRIConfigurer_HandleImports(t *testing.T) {
+	t.Run("should merge CRI configs", func(t *testing.T) {
+		tmp := t.TempDir()
+		testLoadPath := filepath.Join(tmp, "*.toml")
+		criRuntimePath := filepath.Join(t.TempDir(), "cri.toml")
+		criRuntimeConfig := `
+[plugins]
+	[plugins."io.containerd.grpc.v1.cri".containerd]
+    	snapshotter = "zfs"
+`
+		err := os.WriteFile(filepath.Join(tmp, "foo.toml"), []byte(criRuntimeConfig), 0644)
+		require.NoError(t, err)
+		c := CRIConfigurer{
+			loadPath:       testLoadPath,
+			criRuntimePath: criRuntimePath,
+			log:            logrus.New().WithField("test", t.Name()),
+		}
+		_, err = c.HandleImports()
+		require.NoError(t, err)
+
+		// Dump the config for inspection
+		b, _ := os.ReadFile(criRuntimePath)
+		t.Logf("cri config:\n%s", string(b))
+
+		// Load the criRuntimeConfig and verify the settings are correct
+		containerdConfig := &srvconfig.Config{}
+		err = srvconfig.LoadConfig(criRuntimePath, containerdConfig)
+		require.NoError(t, err)
+
+		criConfig := containerdConfig.Plugins["io.containerd.grpc.v1.cri"]
+		snapshotter := criConfig.GetPath([]string{"containerd", "snapshotter"})
+		require.Equal(t, "zfs", snapshotter)
+	})
+
 	t.Run("should have single import for CRI if there's nothing in imports dir", func(t *testing.T) {
 		testLoadPath := filepath.Join(t.TempDir(), "*.toml")
 		criRuntimePath := filepath.Join(t.TempDir(), "cri.toml")
@@ -73,6 +107,7 @@ func TestCRIConfigurer_HandleImports(t *testing.T) {
 		criRuntimeConfig := `
 [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
   [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+    endpoint = ["https://registry-1.docker.io"]
 `
 		err := os.WriteFile(filepath.Join(tmp, "foo.toml"), []byte(criRuntimeConfig), 0644)
 		require.NoError(t, err)
@@ -85,8 +120,15 @@ func TestCRIConfigurer_HandleImports(t *testing.T) {
 		require.NoError(t, err)
 		require.Len(t, imports, 1)
 		require.Contains(t, imports, criRuntimePath)
-		criCfg := loadFile(t, criRuntimePath)
-		require.Contains(t, criCfg, criRuntimeConfig)
+
+		// Load the criRuntimeConfig and verify the settings are correct
+		containerdConfig := &srvconfig.Config{}
+		err = srvconfig.LoadConfig(criRuntimePath, containerdConfig)
+		require.NoError(t, err)
+
+		criConfig := containerdConfig.Plugins["io.containerd.grpc.v1.cri"]
+		ep := criConfig.GetPath([]string{"registry", "mirrors", "docker.io", "endpoint"})
+		require.Equal(t, []interface{}{"https://registry-1.docker.io"}, ep)
 	})
 
 	t.Run("should have two imports when one non CRI snippet", func(t *testing.T) {
@@ -112,11 +154,3 @@ version = 2
 		require.Contains(t, imports, nonCriConfigPath)
 	})
 }
-
-func loadFile(t *testing.T, path string) string {
-	data, err := os.ReadFile(path)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return string(data)
-}