From 2f878a2ec8bbf49c089a80ae2fc41c08793b5292 Mon Sep 17 00:00:00 2001 From: Jussi Nummelin Date: Thu, 24 Aug 2023 13:05:39 +0300 Subject: [PATCH] Actually merge CRI configs from drop-ins instead of concatenating them Uses https://github.com/mesosphere/toml-merge for merging the CRI config snippets. If a config drop-in is not a CRI config, it behaves as previously and just adds it to imports. Fixes #3283 Signed-off-by: Jussi Nummelin (cherry picked from commit 3fa95ecbb031836cdc5dbb1ad2fa524598f18bae) --- go.mod | 1 + go.sum | 2 + pkg/component/worker/containerd/criconfig.go | 15 +++--- .../worker/containerd/criconfig_test.go | 54 +++++++++++++++---- 4 files changed, 55 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index fa97ca3c34a2..4b5e56ea6879 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/k0sproject/dig v0.2.0 github.com/kardianos/service v1.2.2 github.com/logrusorgru/aurora/v3 v3.0.0 + github.com/mesosphere/toml-merge v0.2.0 github.com/mitchellh/go-homedir v1.1.0 github.com/olekukonko/tablewriter v0.0.5 github.com/opencontainers/image-spec v1.1.0-rc2.0.20221005185240-3a7f492d3f1b diff --git a/go.sum b/go.sum index 4f0efde1917a..ed15b0a1c0bf 100644 --- a/go.sum +++ b/go.sum @@ -606,6 +606,8 @@ github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mesosphere/toml-merge v0.2.0 h1:stCUgrwbictiebeHRqEJ1NfQl/h5noyFKR0LBWjWXxQ= +github.com/mesosphere/toml-merge v0.2.0/go.mod h1:WYpgeqeG5puUtv2NREGyOIqTnYuWswyo7CBgx6QK80s= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.5/go.mod h1:v8+iFts2sPIKUV1ltktPXMCC8fumSKFItNcD2cLtRR4= diff --git a/pkg/component/worker/containerd/criconfig.go b/pkg/component/worker/containerd/criconfig.go index d1af5579fe76..614b588e5aa4 100644 --- a/pkg/component/worker/containerd/criconfig.go +++ b/pkg/component/worker/containerd/criconfig.go @@ -25,6 +25,7 @@ import ( "github.com/k0sproject/k0s/pkg/apis/k0s.k0sproject.io/v1beta1" "github.com/k0sproject/k0s/pkg/constant" + "github.com/mesosphere/toml-merge/pkg/patch" "github.com/pelletier/go-toml" "github.com/sirupsen/logrus" @@ -75,6 +76,8 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) { if err != nil { return nil, err } + + finalConfig := criConfigBuffer.String() for _, file := range files { data, err := os.ReadFile(file) if err != nil { @@ -86,13 +89,11 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) { return nil, err } if hasCRI { - c.log.Debugf("found CRI plugin config in %s, appending to CRI config", file) - // Append all CRI plugin configs into single file - // and add it to imports - criConfigBuffer.WriteString(fmt.Sprintf("# appended from %s\n", file)) - _, err := criConfigBuffer.Write(data) + c.log.Infof("found CRI plugin config in %s, merging to CRI config", file) + // Merge to the existing CRI config + finalConfig, err = patch.TOMLString(finalConfig, patch.FilePatches(file)) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to merge CRI config from %s: %w", file, err) } } else { c.log.Debugf("adding %s as-is to imports", file) @@ -102,7 +103,7 @@ func (c *CRIConfigurer) HandleImports() ([]string, error) { } // Write the CRI config to a file and add it to imports - err = os.WriteFile(c.criRuntimePath, criConfigBuffer.Bytes(), 0644) + err = os.WriteFile(c.criRuntimePath, []byte(finalConfig), 0644) if err != nil { return nil, err } diff --git a/pkg/component/worker/containerd/criconfig_test.go b/pkg/component/worker/containerd/criconfig_test.go index 43b01ed0a8da..c19424406718 100644 --- a/pkg/component/worker/containerd/criconfig_test.go +++ b/pkg/component/worker/containerd/criconfig_test.go @@ -21,6 +21,7 @@ import ( "path/filepath" "testing" + srvconfig "github.com/containerd/containerd/services/server/config" "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" ) @@ -52,6 +53,39 @@ version = 2 } func TestCRIConfigurer_HandleImports(t *testing.T) { + t.Run("should merge CRI configs", func(t *testing.T) { + tmp := t.TempDir() + testLoadPath := filepath.Join(tmp, "*.toml") + criRuntimePath := filepath.Join(t.TempDir(), "cri.toml") + criRuntimeConfig := ` +[plugins] + [plugins."io.containerd.grpc.v1.cri".containerd] + snapshotter = "zfs" +` + err := os.WriteFile(filepath.Join(tmp, "foo.toml"), []byte(criRuntimeConfig), 0644) + require.NoError(t, err) + c := CRIConfigurer{ + loadPath: testLoadPath, + criRuntimePath: criRuntimePath, + log: logrus.New().WithField("test", t.Name()), + } + _, err = c.HandleImports() + require.NoError(t, err) + + // Dump the config for inspection + b, _ := os.ReadFile(criRuntimePath) + t.Logf("cri config:\n%s", string(b)) + + // Load the criRuntimeConfig and verify the settings are correct + containerdConfig := &srvconfig.Config{} + err = srvconfig.LoadConfig(criRuntimePath, containerdConfig) + require.NoError(t, err) + + criConfig := containerdConfig.Plugins["io.containerd.grpc.v1.cri"] + snapshotter := criConfig.GetPath([]string{"containerd", "snapshotter"}) + require.Equal(t, "zfs", snapshotter) + }) + t.Run("should have single import for CRI if there's nothing in imports dir", func(t *testing.T) { testLoadPath := filepath.Join(t.TempDir(), "*.toml") criRuntimePath := filepath.Join(t.TempDir(), "cri.toml") @@ -73,6 +107,7 @@ func TestCRIConfigurer_HandleImports(t *testing.T) { criRuntimeConfig := ` [plugins."io.containerd.grpc.v1.cri".registry.mirrors] [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"] + endpoint = ["https://registry-1.docker.io"] ` err := os.WriteFile(filepath.Join(tmp, "foo.toml"), []byte(criRuntimeConfig), 0644) require.NoError(t, err) @@ -85,8 +120,15 @@ func TestCRIConfigurer_HandleImports(t *testing.T) { require.NoError(t, err) require.Len(t, imports, 1) require.Contains(t, imports, criRuntimePath) - criCfg := loadFile(t, criRuntimePath) - require.Contains(t, criCfg, criRuntimeConfig) + + // Load the criRuntimeConfig and verify the settings are correct + containerdConfig := &srvconfig.Config{} + err = srvconfig.LoadConfig(criRuntimePath, containerdConfig) + require.NoError(t, err) + + criConfig := containerdConfig.Plugins["io.containerd.grpc.v1.cri"] + ep := criConfig.GetPath([]string{"registry", "mirrors", "docker.io", "endpoint"}) + require.Equal(t, []interface{}{"https://registry-1.docker.io"}, ep) }) t.Run("should have two imports when one non CRI snippet", func(t *testing.T) { @@ -112,11 +154,3 @@ version = 2 require.Contains(t, imports, nonCriConfigPath) }) } - -func loadFile(t *testing.T, path string) string { - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - return string(data) -}