Merge pull request k0sproject#3902 from jnummelin/feat/kube-router-ex…

…tra-args

Add `extraArgs` config for kube-router

docs/configuration.md

@@ -91,6 +91,7 @@ spec:
       mtu: 0
       peerRouterASNs: ""
       peerRouterIPs: ""
+      extraArgs:
     nodeLocalLoadBalancing:
       enabled: false
       envoyProxy:
@@ -213,15 +214,16 @@ CALICO_IPV6POOL_CIDR: "{{ spec.network.dualStack.IPv6podCIDR }}"
 #### `spec.network.kuberouter`
 
 | Element          | Description                                                                                                                                                                                                                                                               |
-| ---------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+|------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | `autoMTU`        | Autodetection of used MTU (default: `true`).                                                                                                                                                                                                                              |
 | `mtu`            | Override MTU setting, if `autoMTU` must be set to `false`).                                                                                                                                                                                                               |
 | `metricsPort`    | Kube-router metrics server port. Set to 0 to disable metrics  (default: `8080`).                                                                                                                                                                                          |
-| `peerRouterIPs`  | Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                                                                        |
-| `peerRouterASNs` | Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                                                                             |
+| `peerRouterIPs`  | DEPRECATED: Use extraArgs with peerRouterIPs instead. Comma-separated list of [global peer addresses](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                  |
+| `peerRouterASNs` | DEPRECATED: Use extraArgs with peerRouterASNs instead. Comma-separated list of [global peer ASNs](https://github.com/cloudnativelabs/kube-router/blob/master/docs/bgp.md#global-external-bgp-peers).                                                                      |
 | `hairpin`        | Hairpin mode, supported modes `Enabled`: enabled cluster wide, `Allowed`: must be allowed per service [using annotations](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode), `Disabled`: doesn't work at all (default: Enabled) |
 | `hairpinMode`    | **Deprecated** Use `hairpin` instead. If both `hairpin` and `hairpinMode` are defined, this is ignored. If only hairpinMode is configured explicitly activates hairpinMode (https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#hairpin-mode).  |
-| `ipMasq`         | IP masquerade for traffic originating from the pod network, and destined outside of it (default: false) |
+| `ipMasq`         | IP masquerade for traffic originating from the pod network, and destined outside of it (default: false)                                                                                                                                                                   |
+| `extraArgs`      | Extra arguments to pass to kube-router. Can be also used to override any k0s managed args. For reference, see kube-router [documentation](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md#command-line-options). (default: empty)           |
 
 **Note**: Kube-router allows many networking aspects to be configured per node, service, and pod (for more information, refer to the [Kube-router user guide](https://github.com/cloudnativelabs/kube-router/blob/master/docs/user-guide.md)).
 

pkg/apis/k0s/v1beta1/kuberouter.go

@@ -33,9 +33,14 @@ type KubeRouter struct {
 	// IP masquerade for traffic originating from the pod network, and destined outside of it (default: false)
 	IPMasq bool `json:"ipMasq"`
 	// Comma-separated list of global peer addresses
+	// DEPRECATED: Use extraArgs with peerRouterASNs instead
 	PeerRouterASNs string `json:"peerRouterASNs"`
 	// Comma-separated list of global peer ASNs
+	// DEPRECATED: Use extraArgs with peerRouterIPs instead
 	PeerRouterIPs string `json:"peerRouterIPs"`
+	// ExtraArgs are extra arguments to pass to kube-router
+	// Can be also used to override the default k0s managed kube-router arguments
+	ExtraArgs map[string]string `json:"extraArgs,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Enabled;Allowed;Disabled

pkg/component/controller/kuberouter.go

@@ -20,7 +20,9 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"reflect"
 
+	"github.com/k0sproject/k0s/internal/pkg/stringmap"
 	"github.com/k0sproject/k0s/internal/pkg/templatewriter"
 	"github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/k0sproject/k0s/pkg/component/manager"
@@ -49,12 +51,12 @@ type kubeRouterConfig struct {
 	MetricsPort       int
 	CNIInstallerImage string
 	CNIImage          string
-	GlobalHairpin     bool
 	CNIHairpin        bool
 	IPMasq            bool
 	PeerRouterIPs     string
 	PeerRouterASNs    string
 	PullPolicy        string
+	Args              []string
 }
 
 // NewKubeRouter creates new KubeRouter reconciler component
@@ -73,25 +75,26 @@ func (k *KubeRouter) Init(_ context.Context) error { return nil }
 // Stop no-op as nothing running
 func (k *KubeRouter) Stop() error { return nil }
 
-func getHairpinConfig(cfg *kubeRouterConfig, krc *v1beta1.KubeRouter) {
+func getHairpinConfig(krc *v1beta1.KubeRouter) (cniHairpin bool, globalHairpin bool) {
 	// Configure hairpin
 	switch krc.Hairpin {
 	case v1beta1.HairpinUndefined:
 		// If Hairpin is undefined, then we honor HairpinMode
 		if krc.HairpinMode {
-			cfg.CNIHairpin = true
-			cfg.GlobalHairpin = true
+			cniHairpin = true
+			globalHairpin = true
 		}
 	case v1beta1.HairpinDisabled:
-		cfg.CNIHairpin = false
-		cfg.GlobalHairpin = false
+		cniHairpin = false
+		globalHairpin = false
 	case v1beta1.HairpinAllowed:
-		cfg.CNIHairpin = true
-		cfg.GlobalHairpin = false
+		cniHairpin = true
+		globalHairpin = false
 	case v1beta1.HairpinEnabled:
-		cfg.CNIHairpin = true
-		cfg.GlobalHairpin = true
+		cniHairpin = true
+		globalHairpin = true
 	}
+	return
 }
 
 // Reconcile detects changes in configuration and applies them to the component
@@ -106,20 +109,44 @@ func (k *KubeRouter) Reconcile(_ context.Context, clusterConfig *v1beta1.Cluster
 		return fmt.Errorf("cannot change CNI provider from %s to %s", existingCNI, constant.CNIProviderKubeRouter)
 	}
 
+	cniHairpin, globalHairpin := getHairpinConfig(clusterConfig.Spec.Network.KubeRouter)
+
+	args := stringmap.StringMap{
+		// k0s set default args
+		"run-router":           "true",
+		"run-firewall":         "true",
+		"run-service-proxy":    "false",
+		"bgp-graceful-restart": "true",
+		// Args from config values
+		"auto-mtu":     fmt.Sprintf("%t", clusterConfig.Spec.Network.KubeRouter.AutoMTU),
+		"metrics-port": fmt.Sprintf("%d", clusterConfig.Spec.Network.KubeRouter.MetricsPort),
+		"hairpin-mode": fmt.Sprintf("%t", globalHairpin),
+	}
+
+	// We should not add peering flags if the values are empty
+	if clusterConfig.Spec.Network.KubeRouter.PeerRouterASNs != "" {
+		args["peer-router-asns"] = clusterConfig.Spec.Network.KubeRouter.PeerRouterASNs
+	}
+	if clusterConfig.Spec.Network.KubeRouter.PeerRouterIPs != "" {
+		args["peer-router-ips"] = clusterConfig.Spec.Network.KubeRouter.PeerRouterIPs
+	}
+
+	// Override or add args from config
+	args.Merge(clusterConfig.Spec.Network.KubeRouter.ExtraArgs)
+
 	cfg := kubeRouterConfig{
 		AutoMTU:           clusterConfig.Spec.Network.KubeRouter.AutoMTU,
 		MTU:               clusterConfig.Spec.Network.KubeRouter.MTU,
 		MetricsPort:       clusterConfig.Spec.Network.KubeRouter.MetricsPort,
-		PeerRouterIPs:     clusterConfig.Spec.Network.KubeRouter.PeerRouterIPs,
-		PeerRouterASNs:    clusterConfig.Spec.Network.KubeRouter.PeerRouterASNs,
 		IPMasq:            clusterConfig.Spec.Network.KubeRouter.IPMasq,
+		CNIHairpin:        cniHairpin,
 		CNIImage:          clusterConfig.Spec.Images.KubeRouter.CNI.URI(),
 		CNIInstallerImage: clusterConfig.Spec.Images.KubeRouter.CNIInstaller.URI(),
 		PullPolicy:        clusterConfig.Spec.Images.DefaultPullPolicy,
+		Args:              args.ToDashedArgs(),
 	}
-	getHairpinConfig(&cfg, clusterConfig.Spec.Network.KubeRouter)
 
-	if cfg == k.previousConfig {
+	if reflect.DeepEqual(k.previousConfig, cfg) {
 		k.log.Info("config matches with previous, not reconciling anything")
 		return nil
 	}
@@ -278,20 +305,8 @@ spec:
         image: {{ .CNIImage }}
         imagePullPolicy: {{ .PullPolicy }}
         args:
-        - "--run-router=true"
-        - "--run-firewall=true"
-        - "--run-service-proxy=false"
-        - "--bgp-graceful-restart=true"
-        - "--metrics-port={{ .MetricsPort }}"
-        - "--hairpin-mode={{ .GlobalHairpin }}"
-        {{- if not .AutoMTU }}
-        - "--auto-mtu=false"
-        {{- end }}
-        {{- if .PeerRouterIPs }}
-        - "--peer-router-ips={{ .PeerRouterIPs }}"
-        {{- end }}
-        {{- if .PeerRouterASNs }}
-        - "--peer-router-asns={{ .PeerRouterASNs }}"
+        {{- range .Args }}
+        - {{ . | printf "%q" }}
         {{- end }}
         env:
         - name: NODE_NAME

pkg/component/controller/kuberouter_test.go

@@ -77,40 +77,50 @@ func TestKubeRouterConfig(t *testing.T) {
 }
 
 type hairpinTest struct {
-	krc    *v1beta1.KubeRouter
-	result kubeRouterConfig
+	krc                 *v1beta1.KubeRouter
+	resultCNIHairpin    bool
+	resultGlobalHairpin bool
 }
 
 func TestGetHairpinConfig(t *testing.T) {
 	hairpinTests := []hairpinTest{
 		{
-			krc:    &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinUndefined, HairpinMode: true},
-			result: kubeRouterConfig{CNIHairpin: true, GlobalHairpin: true},
+			krc:                 &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinUndefined, HairpinMode: true},
+			resultCNIHairpin:    true,
+			resultGlobalHairpin: true,
 		},
 		{
-			krc:    &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinUndefined, HairpinMode: false},
-			result: kubeRouterConfig{CNIHairpin: false, GlobalHairpin: false},
+			krc:                 &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinUndefined, HairpinMode: false},
+			resultCNIHairpin:    false,
+			resultGlobalHairpin: false,
 		},
 		{
-			krc:    &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinAllowed, HairpinMode: true},
-			result: kubeRouterConfig{CNIHairpin: true, GlobalHairpin: false},
+			krc:                 &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinAllowed, HairpinMode: true},
+			resultCNIHairpin:    true,
+			resultGlobalHairpin: false,
 		},
 		{
-			krc:    &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinDisabled, HairpinMode: true},
-			result: kubeRouterConfig{CNIHairpin: false, GlobalHairpin: false},
+			krc:                 &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinDisabled, HairpinMode: true},
+			resultCNIHairpin:    false,
+			resultGlobalHairpin: false,
 		},
 		{
-			krc:    &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinEnabled, HairpinMode: false},
-			result: kubeRouterConfig{CNIHairpin: true, GlobalHairpin: true},
+			krc:                 &v1beta1.KubeRouter{Hairpin: v1beta1.HairpinEnabled, HairpinMode: false},
+			resultCNIHairpin:    true,
+			resultGlobalHairpin: true,
 		},
 	}
 
 	for _, test := range hairpinTests {
 		cfg := &kubeRouterConfig{}
-		getHairpinConfig(cfg, test.krc)
-		if cfg.CNIHairpin != test.result.CNIHairpin || cfg.GlobalHairpin != test.result.GlobalHairpin {
-			t.Fatalf("Hairpin configuration (%#v) does not match exepected output (%#v) ", cfg, test.result)
+		cniHairpin, globalHairpin := getHairpinConfig(test.krc)
+		if cniHairpin != test.resultCNIHairpin {
+			t.Fatalf("CNI hairpin configuration (%#v) does not match exepected output (%#v) ", cfg, test.resultCNIHairpin)
 		}
+		if globalHairpin != test.resultGlobalHairpin {
+			t.Fatalf("Global hairpin configuration (%#v) does not match exepected output (%#v) ", cfg, test.resultGlobalHairpin)
+		}
+
 	}
 }
 
@@ -182,6 +192,38 @@ func TestKubeRouterManualMTUManifests(t *testing.T) {
 	require.Equal(t, float64(1234), p.Dig("mtu"))
 }
 
+func TestExtraArgs(t *testing.T) {
+	k0sVars, err := config.NewCfgVars(nil, t.TempDir())
+	require.NoError(t, err)
+	cfg := v1beta1.DefaultClusterConfig()
+	cfg.Spec.Network.Calico = nil
+	cfg.Spec.Network.Provider = "kuberouter"
+	cfg.Spec.Network.KubeRouter = v1beta1.DefaultKubeRouter()
+	cfg.Spec.Network.KubeRouter.ExtraArgs = map[string]string{
+		// Add some random arg
+		"foo": "bar",
+		// Override the default arg
+		"run-firewall": "false",
+	}
+
+	saver := inMemorySaver{}
+	kr := NewKubeRouter(k0sVars, saver)
+	require.NoError(t, kr.Reconcile(context.Background(), cfg))
+	require.NoError(t, kr.Stop())
+
+	manifestData, foundRaw := saver["kube-router.yaml"]
+	require.True(t, foundRaw, "must have manifests for kube-router")
+
+	resources, err := testutil.ParseManifests(manifestData)
+	require.NoError(t, err)
+	ds, err := findDaemonset(resources)
+	require.NoError(t, err)
+	require.NotNil(t, ds)
+
+	assert.Contains(t, ds.Spec.Template.Spec.Containers[0].Args, "--run-firewall=false")
+	assert.Contains(t, ds.Spec.Template.Spec.Containers[0].Args, "--foo=bar")
+}
+
 func findConfig(resources []*unstructured.Unstructured) (corev1.ConfigMap, error) {
 	var cm corev1.ConfigMap
 	for _, r := range resources {

static/manifests/v1beta1/CustomResourceDefinition/k0s.k0sproject.io_clusterconfigs.yaml

@@ -428,6 +428,13 @@ spec:
                       autoMTU:
                         description: 'Auto-detection of used MTU (default: true)'
                         type: boolean
+                      extraArgs:
+                        additionalProperties:
+                          type: string
+                        description: ExtraArgs are extra arguments to pass to kube-router
+                          Can be also used to override the default k0s managed kube-router
+                          arguments
+                        type: object
                       hairpin:
                         default: Enabled
                         description: 'Admits three values: "Enabled" enables it globally,
@@ -456,10 +463,12 @@ spec:
                           false)
                         type: integer
                       peerRouterASNs:
-                        description: Comma-separated list of global peer addresses
+                        description: 'Comma-separated list of global peer addresses
+                          DEPRECATED: Use extraArgs with peerRouterASNs instead'
                         type: string
                       peerRouterIPs:
-                        description: Comma-separated list of global peer ASNs
+                        description: 'Comma-separated list of global peer ASNs DEPRECATED:
+                          Use extraArgs with peerRouterIPs instead'
                         type: string
                     type: object
                   nodeLocalLoadBalancing: