feat(kuma-cp): allow skipping certain label propagation on multizone (#…

…11918)

## Motivation

Users might not want certain labels to be propagated between control
planes.

## Implementation information

Two config values, one for zone and one for global.

## Supporting documentation

Closes #11416

<!--
> Changelog: skip
-->
<!--
Uncomment the above section to explicitly set a [`> Changelog:` entry
here](https://github.com/kumahq/kuma/blob/master/CONTRIBUTING.md#submitting-a-patch)?
-->

---------

Signed-off-by: Mike Beaumont <[email protected]>

pkg/config/loader_test.go

@@ -267,6 +267,7 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Multizone.Global.KDS.ZoneHealthCheck.PollInterval.Duration).To(Equal(11 * time.Second))
 			Expect(cfg.Multizone.Global.KDS.ZoneHealthCheck.Timeout.Duration).To(Equal(110 * time.Second))
 			Expect(cfg.Multizone.Global.KDS.Tracing.Enabled).To(BeFalse())
+			Expect(cfg.Multizone.Global.KDS.Labels.SkipPrefixes).To(Equal([]string{"argocd.argoproj.io"}))
 			Expect(cfg.Multizone.Zone.GlobalAddress).To(Equal("grpc://1.1.1.1:5685"))
 			Expect(cfg.Multizone.Zone.Name).To(Equal("zone-1"))
 			Expect(cfg.Multizone.Zone.KDS.RootCAFile).To(Equal("/rootCa"))
@@ -276,6 +277,7 @@ var _ = Describe("Config loader", func() {
 			Expect(cfg.Multizone.Zone.KDS.NackBackoff.Duration).To(Equal(21 * time.Second))
 			Expect(cfg.Multizone.Zone.KDS.ResponseBackoff.Duration).To(Equal(2 * time.Second))
 			Expect(cfg.Multizone.Zone.KDS.TlsSkipVerify).To(BeTrue())
+			Expect(cfg.Multizone.Zone.KDS.Labels.SkipPrefixes).To(Equal([]string{"argocd.argoproj.io"}))
 
 			Expect(cfg.Defaults.SkipMeshCreation).To(BeTrue())
 			Expect(cfg.Defaults.SkipTenantResources).To(BeTrue())
@@ -622,6 +624,8 @@ multizone:
         timeout: 110s
       tracing:
         enabled: false
+      labels:
+        skipPrefixes: ["argocd.argoproj.io"]
   zone:
     globalAddress: "grpc://1.1.1.1:5685"
     name: "zone-1"
@@ -633,6 +637,8 @@ multizone:
       nackBackoff: 21s
       responseBackoff: 2s
       tlsSkipVerify: true
+      labels:
+        skipPrefixes: ["argocd.argoproj.io"]
     disableOriginLabelValidation: true
     ingressUpdateInterval: 2s
 dnsServer:
@@ -966,6 +972,7 @@ meshService:
 				"KUMA_MULTIZONE_GLOBAL_KDS_ZONE_HEALTH_CHECK_POLL_INTERVAL":                                "11s",
 				"KUMA_MULTIZONE_GLOBAL_KDS_ZONE_HEALTH_CHECK_TIMEOUT":                                      "110s",
 				"KUMA_MULTIZONE_GLOBAL_KDS_TRACING_ENABLED":                                                "false",
+				"KUMA_MULTIZONE_GLOBAL_KDS_LABELS_SKIP_PREFIXES":                                           "argocd.argoproj.io",
 				"KUMA_MULTIZONE_ZONE_GLOBAL_ADDRESS":                                                       "grpc://1.1.1.1:5685",
 				"KUMA_MULTIZONE_ZONE_NAME":                                                                 "zone-1",
 				"KUMA_MULTIZONE_ZONE_KDS_ROOT_CA_FILE":                                                     "/rootCa",
@@ -977,6 +984,7 @@ meshService:
 				"KUMA_MULTIZONE_ZONE_KDS_TLS_SKIP_VERIFY":                                                  "true",
 				"KUMA_MULTIZONE_ZONE_DISABLE_ORIGIN_LABEL_VALIDATION":                                      "true",
 				"KUMA_MULTIZONE_ZONE_INGRESS_UPDATE_INTERVAL":                                              "2s",
+				"KUMA_MULTIZONE_ZONE_KDS_LABELS_SKIP_PREFIXES":                                             "argocd.argoproj.io",
 				"KUMA_MULTIZONE_GLOBAL_KDS_ZONE_INSIGHT_FLUSH_INTERVAL":                                    "5s",
 				"KUMA_DEFAULTS_SKIP_MESH_CREATION":                                                         "true",
 				"KUMA_DEFAULTS_SKIP_HOSTNAME_GENERATORS":                                                   "true",

pkg/config/multizone/kds.go

@@ -43,6 +43,8 @@ type KdsServerConfig struct {
 	// ZoneHealthCheck holds config for ensuring zones are online
 	ZoneHealthCheck ZoneHealthCheckConfig `json:"zoneHealthCheck"`
 	Tracing         KDSServerTracing      `json:"tracing"`
+	// Labels allows for customizing label handling
+	Labels GlobalLabels `json:"labels"`
 }
 
 var _ config.Config = &KdsServerConfig{}
@@ -103,6 +105,8 @@ type KdsClientConfig struct {
 	// ResponseBackoff is a time Zone CP waits before sending ACK/NACK.
 	// This is a way to slow down Global CP from sending resources too often.
 	ResponseBackoff config_types.Duration `json:"responseBackoff" envconfig:"kuma_multizone_zone_kds_response_backoff"`
+	// Labels allows for customizing label handling
+	Labels ZoneLabels `json:"labels"`
 }
 
 var _ config.Config = &KdsClientConfig{}

pkg/config/multizone/sync.go

@@ -0,0 +1,10 @@
+package multizone
+
+type GlobalLabels struct {
+	// Labels with any of these prefixes won't be synced between control planes
+	SkipPrefixes []string `json:"skipPrefixes,omitempty" envconfig:"kuma_multizone_global_kds_labels_skip_prefixes"`
+}
+type ZoneLabels struct {
+	// Labels with any of these prefixes won't be synced between control planes
+	SkipPrefixes []string `json:"skipPrefixes,omitempty" envconfig:"kuma_multizone_zone_kds_labels_skip_prefixes"`
+}

pkg/kds/context/context.go

@@ -69,7 +69,10 @@ func DefaultContext(
 	}
 
 	globalMappers := []reconcile_v2.ResourceMapper{
-		UpdateResourceMeta(util.WithLabel(mesh_proto.ResourceOriginLabel, string(mesh_proto.GlobalResourceOrigin))),
+		UpdateResourceMeta(
+			util.WithLabel(mesh_proto.ResourceOriginLabel, string(mesh_proto.GlobalResourceOrigin)),
+			util.WithoutLabelPrefixes(cfg.Multizone.Global.KDS.Labels.SkipPrefixes...),
+		),
 		reconcile_v2.If(
 			reconcile_v2.And(
 				reconcile_v2.TypeIs(system.GlobalSecretType),
@@ -104,6 +107,7 @@ func DefaultContext(
 			util.WithLabel(mesh_proto.ZoneTag, cfg.Multizone.Zone.Name),
 			util.WithoutLabel(mesh_proto.DeletionGracePeriodStartedLabel),
 			util.If(util.IsKubernetes(cfg.Store.Type), util.PopulateNamespaceLabelFromNameExtension()),
+			util.WithoutLabelPrefixes(cfg.Multizone.Zone.KDS.Labels.SkipPrefixes...),
 		),
 		MapInsightResourcesZeroGeneration,
 		reconcile_v2.If(

pkg/kds/util/meta.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"strings"
 	"time"
 
 	"golang.org/x/exp/maps"
@@ -59,6 +60,18 @@ func WithoutLabel(key string) CloneResourceMetaOpt {
 	}
 }
 
+func WithoutLabelPrefixes(prefixes ...string) CloneResourceMetaOpt {
+	return func(m *resourceMeta) {
+		for label := range m.labels {
+			for _, prefix := range prefixes {
+				if strings.HasPrefix(label, prefix) {
+					delete(m.labels, label)
+				}
+			}
+		}
+	}
+}
+
 func If(condition func(resource model.ResourceMeta) bool, fn CloneResourceMetaOpt) CloneResourceMetaOpt {
 	return func(meta *resourceMeta) {
 		if condition(meta) {

test/e2e_env/multizone/sync/sync.go

@@ -22,7 +22,27 @@ func Sync() {
 	meshName := "sync"
 
 	BeforeAll(func() {
-		Expect(multizone.Global.Install(MTLSMeshUniversal(meshName))).To(Succeed())
+		Expect(
+			multizone.Global.Install(MTLSMeshUniversal(meshName)),
+		).To(Succeed())
+		Expect(
+			multizone.Global.Install(YamlUniversal(fmt.Sprintf(`
+type: MeshTrafficPermission
+name: allow-to-client
+mesh: %s
+labels:
+  argocd.argoproj.io/instance: something
+spec:
+  targetRef:
+    kind: Mesh
+  from:
+    - targetRef:
+        kind: MeshService
+        name: client-server_kuma-test_svc_80 # this is just something to sync
+      default:
+        action: Allow
+`, meshName)),
+			)).To(Succeed())
 		Expect(WaitForMesh(meshName, multizone.Zones())).To(Succeed())
 
 		group := errgroup.Group{}
@@ -105,6 +125,17 @@ func Sync() {
 				g.Expect(strings.Count(out, "Online")).To(Equal(2))
 			}, "30s", "1s").Should(Succeed())
 		})
+
+		It("should drop unwanted labels", func() {
+			Eventually(func(g Gomega) {
+				out, err := multizone.Global.GetKumactlOptions().RunKumactlAndGetOutput("get", "meshtrafficpermissions", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, "argocd.argoproj.io")).To(Equal(1))
+				out, err = multizone.KubeZone1.GetKumactlOptions().RunKumactlAndGetOutput("get", "meshtrafficpermissions", "--mesh", meshName, "-o", "yaml")
+				g.Expect(err).ToNot(HaveOccurred())
+				g.Expect(strings.Count(out, "argocd.argoproj.io")).To(Equal(0))
+			}, "30s", "1s").Should(Succeed())
+		})
 	})
 
 	Context("from Global to Zone", func() {

test/framework/envs/multizone/env.go

@@ -67,6 +67,7 @@ func setupKubeZone(wg *sync.WaitGroup, clusterName string, extraOptions ...frame
 		// 100s and 80s are values that we also use in mesh-perf when we put a lot of pressure on the CP.
 		framework.WithEnv("KUMA_RUNTIME_KUBERNETES_LEADER_ELECTION_LEASE_DURATION", "100s"),
 		framework.WithEnv("KUMA_RUNTIME_KUBERNETES_LEADER_ELECTION_RENEW_DEADLINE", "80s"),
+		framework.WithEnv("KUMA_MULTIZONE_ZONE_KDS_LABELS_SKIP_PREFIXES", "argocd.argoproj.io"),
 	}
 	options = append(options, extraOptions...)
 	zone := NewK8sCluster(NewTestingT(), clusterName, Verbose)
@@ -87,6 +88,7 @@ func setupUniZone(wg *sync.WaitGroup, clusterName string, extraOptions ...framew
 			WithIngressEnvoyAdminTunnel(),
 			WithEnv("KUMA_XDS_DATAPLANE_DEREGISTRATION_DELAY", "0s"), // we have only 1 Kuma CP instance so there is no risk setting this to 0
 			WithEnv("KUMA_MULTIZONE_ZONE_KDS_NACK_BACKOFF", "1s"),
+			WithEnv("KUMA_MULTIZONE_ZONE_KDS_LABELS_SKIP_PREFIXES", "argocd.argoproj.io"),
 		},
 		extraOptions...,
 	)
@@ -110,6 +112,7 @@ func SetupAndGetState() []byte {
 	globalOptions := append(
 		[]framework.KumaDeploymentOption{
 			WithEnv("KUMA_MULTIZONE_GLOBAL_KDS_NACK_BACKOFF", "1s"),
+			WithEnv("KUMA_MULTIZONE_GLOBAL_KDS_LABELS_SKIP_PREFIXES", "argocd.argoproj.io"),
 		},
 		framework.KumaDeploymentOptionsFromConfig(framework.Config.KumaCpConfig.Multizone.Global)...)
 	Expect(Global.Install(Kuma(core.Global, globalOptions...))).To(Succeed())