From 05da3c9358c0420476e87ee3550d9fe8727098d5 Mon Sep 17 00:00:00 2001 From: Ivan Mikheykin Date: Fri, 14 Jun 2024 14:55:28 +0300 Subject: [PATCH] refactor(kube-api-rewriter): rewrite labels, annotations and finalizers - Add KubeVirt and CDI rules for labels, annotations and finalizers. - Add rewriter for labelSelector in queries. - Add rewriters for affinity, labelSelectors in different resources. - Add metadata patch rewriters: nodes, services, deployments, ds, ... - Add webhooks patch rewriters: validatingwebhookconfigurations, mutatingwebhookconfigurations - Add more resources for rewrite: jobs, services, deployments, sts, ds, pvc, pod, ... - Enable rewrite for 3rd party resources: servicemonitors, prometheusrules, ... - Rename labels in our templates. Add kubectl.kubernetes.io/default-container annotation. - Add virt-operator patch to rename install-strategy labels: there is no way to rewrite these labels with kube-api-rewriter. - Simplify some Rewrite* methods using transformers - Create PrefixedNameRewriter to hold original-renamed indexes. Signed-off-by: Ivan Mikheykin --- .../kube-api-proxy/cmd/kube-api-proxy/main.go | 2 +- .../pkg/kubevirt/kubevirt_rules.go | 56 +++- .../kube-api-proxy/pkg/rewriter/3rdparty.go | 32 +++ .../pkg/rewriter/admission_configuration.go | 27 ++ .../kube-api-proxy/pkg/rewriter/affinity.go | 151 +++++++++++ images/kube-api-proxy/pkg/rewriter/app.go | 74 +++--- .../kube-api-proxy/pkg/rewriter/app_test.go | 246 ++++++++++++++++++ images/kube-api-proxy/pkg/rewriter/core.go | 96 ++++++- .../kube-api-proxy/pkg/rewriter/core_test.go | 209 +++++++++++++++ images/kube-api-proxy/pkg/rewriter/list.go | 11 +- images/kube-api-proxy/pkg/rewriter/map.go | 18 +- .../kube-api-proxy/pkg/rewriter/metadata.go | 77 ++++++ images/kube-api-proxy/pkg/rewriter/policy.go | 7 +- .../pkg/rewriter/prefixed_name_rewriter.go | 141 ++++++++++ .../kube-api-proxy/pkg/rewriter/resource.go | 66 +---- .../pkg/rewriter/rule_rewriter.go | 117 ++++++--- .../pkg/rewriter/rule_rewriter_test.go | 47 +++- images/kube-api-proxy/pkg/rewriter/rules.go | 186 +++---------- .../pkg/rewriter/target_request.go | 35 ++- .../pkg/rewriter/transformers.go | 111 ++++++++ .../016-rename-install-strategy-labels.patch | 31 +++ templates/_helpers.tpl | 54 ---- templates/_kube_api_rewriter.tpl | 57 ++++ templates/cdi/_helpers.tpl | 1 + templates/cdi/cdi-operator/deployment.yaml | 4 + templates/cdi/service-monitor.yaml | 2 +- templates/kubevirt/_helpers.tpl | 1 + templates/kubevirt/service-monitor.yaml | 2 +- templates/kubevirt/virt-api/vpa.yaml | 2 +- templates/kubevirt/virt-controller/vpa.yaml | 2 +- templates/kubevirt/virt-handler/vpa.yaml | 2 +- .../kubevirt/virt-operator/deployment.yaml | 11 +- 32 files changed, 1462 insertions(+), 416 deletions(-) create mode 100644 images/kube-api-proxy/pkg/rewriter/3rdparty.go create mode 100644 images/kube-api-proxy/pkg/rewriter/affinity.go create mode 100644 images/kube-api-proxy/pkg/rewriter/app_test.go create mode 100644 images/kube-api-proxy/pkg/rewriter/core_test.go create mode 100644 images/kube-api-proxy/pkg/rewriter/metadata.go create mode 100644 images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go create mode 100644 images/kube-api-proxy/pkg/rewriter/transformers.go create mode 100644 images/virt-artifact/patches/016-rename-install-strategy-labels.patch create mode 100644 templates/_kube_api_rewriter.tpl diff --git a/images/kube-api-proxy/cmd/kube-api-proxy/main.go b/images/kube-api-proxy/cmd/kube-api-proxy/main.go index 52143233b..ef111b1f1 100644 --- a/images/kube-api-proxy/cmd/kube-api-proxy/main.go +++ b/images/kube-api-proxy/cmd/kube-api-proxy/main.go @@ -79,7 +79,7 @@ func main() { } rewriteRules = rulesFromFile } - rewriteRules.Complete() + rewriteRules.Init() proxies := make([]*server.HTTPServer, 0) diff --git a/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go b/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go index f80bac018..f653e8281 100644 --- a/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go +++ b/images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go @@ -20,6 +20,12 @@ import ( . "kube-api-proxy/pkg/rewriter" ) +const ( + internalPrefix = "internal.virtualization.deckhouse.io" + nodePrefix = "node.virtualization.deckhouse.io" + dvpPrefix = "virtualization.deckhouse.io" +) + var KubevirtRewriteRules = &RewriteRules{ KindPrefix: "DVPInternal", // KV ResourceTypePrefix: "dvpinternal", // kv @@ -30,16 +36,54 @@ var KubevirtRewriteRules = &RewriteRules{ Webhooks: KubevirtWebhooks, Labels: MetadataReplace{ Names: []MetadataReplaceRule{ - {Old: "cdi.kubevirt.io", New: "cdi.internal.virtualization.deckhouse.io"}, - {Old: "kubevirt.io", New: "kubevirt.internal.virtualization.deckhouse.io"}, + {Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix}, + {Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix}, + {Original: "prometheus.kubevirt.io", Renamed: "prometheus.kubevirt." + internalPrefix}, + {Original: "prometheus.cdi.kubevirt.io", Renamed: "prometheus.cdi." + internalPrefix}, + // Special cases. + {Original: "node-labeller.kubevirt.io/skip-node", Renamed: "node-labeller." + dvpPrefix + "/skip-node"}, + {Original: "node-labeller.kubevirt.io/obsolete-host-model", Renamed: "node-labeller." + internalPrefix + "/obsolete-host-model"}, + }, + Prefixes: []MetadataReplaceRule{ + // CDI related labels. + {Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix}, + {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, + {Original: "prometheus.cdi.kubevirt.io", Renamed: "prometheus.cdi." + internalPrefix}, + {Original: "upload.cdi.kubevirt.io", Renamed: "upload.cdi." + internalPrefix}, + // KubeVirt related labels. + {Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix}, + {Original: "prometheus.kubevirt.io", Renamed: "prometheus.kubevirt." + internalPrefix}, + {Original: "operator.kubevirt.io", Renamed: "operator.kubevirt." + internalPrefix}, + {Original: "vm.kubevirt.io", Renamed: "vm.kubevirt." + internalPrefix}, + // Node features related labels. + // Note: these labels are not "internal". + {Original: "cpu-feature.node.kubevirt.io", Renamed: "cpu-feature." + nodePrefix}, + {Original: "cpu-model-migration.node.kubevirt.io", Renamed: "cpu-model-migration." + nodePrefix}, + {Original: "cpu-model.node.kubevirt.io", Renamed: "cpu-model." + nodePrefix}, + {Original: "cpu-timer.node.kubevirt.io", Renamed: "cpu-timer." + nodePrefix}, + {Original: "cpu-vendor.node.kubevirt.io", Renamed: "cpu-vendor." + nodePrefix}, + {Original: "scheduling.node.kubevirt.io", Renamed: "scheduling." + nodePrefix}, + {Original: "host-model-cpu.node.kubevirt.io", Renamed: "host-model-cpu." + nodePrefix}, + {Original: "host-model-required-features.node.kubevirt.io", Renamed: "host-model-required-features." + nodePrefix}, + {Original: "hyperv.node.kubevirt.io", Renamed: "hyperv." + nodePrefix}, }, + }, + Annotations: MetadataReplace{ + Prefixes: []MetadataReplaceRule{ + // CDI related annotations. + {Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix}, + {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, + // KubeVirt related annotations. + {Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix}, + {Original: "certificates.kubevirt.io", Renamed: "certificates.kubevirt." + internalPrefix}, + }, + }, + Finalizers: MetadataReplace{ Prefixes: []MetadataReplaceRule{ - {Old: "cdi.kubevirt.io", New: "cdi.internal.virtualization.deckhouse.io"}, - {Old: "upload.cdi.kubevirt.io", New: "upload.cdi.internal.virtualization.deckhouse.io"}, - {Old: "kubevirt.io", New: "kubevirt.internal.virtualization.deckhouse.io"}, + {Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix}, + {Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix}, }, }, - Annotations: MetadataReplace{}, } // TODO create generator in golang to produce below rules from Kubevirt and CDI sources so proxy can work with future versions. diff --git a/images/kube-api-proxy/pkg/rewriter/3rdparty.go b/images/kube-api-proxy/pkg/rewriter/3rdparty.go new file mode 100644 index 000000000..915d73ef0 --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/3rdparty.go @@ -0,0 +1,32 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +// Rewrite routines for 3rd party resources, i.e. ServiceMonitor. + +const ( + PrometheusRuleKind = "PrometheusRule" + PrometheusRuleListKind = "PrometheusRuleList" + ServiceMonitorKind = "ServiceMonitor" + ServiceMonitorListKind = "ServiceMonitorList" +) + +func RewriteServiceMonitorOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + return TransformObject(obj, "spec.selector", func(obj []byte) ([]byte, error) { + return rewriteLabelSelector(rules, obj, action) + }) +} diff --git a/images/kube-api-proxy/pkg/rewriter/admission_configuration.go b/images/kube-api-proxy/pkg/rewriter/admission_configuration.go index 562102a81..81b0eb1b0 100644 --- a/images/kube-api-proxy/pkg/rewriter/admission_configuration.go +++ b/images/kube-api-proxy/pkg/rewriter/admission_configuration.go @@ -16,6 +16,8 @@ limitations under the License. package rewriter +import "github.com/tidwall/gjson" + const ( ValidatingWebhookConfigurationKind = "ValidatingWebhookConfiguration" ValidatingWebhookConfigurationListKind = "ValidatingWebhookConfigurationList" @@ -60,3 +62,28 @@ func RewriteMutatingOrList(rules *RewriteRules, obj []byte, action Action) ([]by }) }) } + +func RenameWebhookConfigurationPatch(rules *RewriteRules, obj []byte) ([]byte, error) { + obj, err := RenameMetadataPatch(rules, obj) + if err != nil { + return nil, err + } + + return TransformPatch(obj, func(mergePatch []byte) ([]byte, error) { + return RewriteArray(mergePatch, "webhooks", func(webhook []byte) ([]byte, error) { + return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) { + return restoreRoleRule(rules, item) + }) + }) + }, func(jsonPatch []byte) ([]byte, error) { + path := gjson.GetBytes(jsonPatch, "path").String() + if path == "/webhooks" { + return RewriteArray(jsonPatch, "value", func(webhook []byte) ([]byte, error) { + return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) { + return renameRoleRule(rules, item) + }) + }) + } + return jsonPatch, nil + }) +} diff --git a/images/kube-api-proxy/pkg/rewriter/affinity.go b/images/kube-api-proxy/pkg/rewriter/affinity.go new file mode 100644 index 000000000..34cbc912d --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/affinity.go @@ -0,0 +1,151 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +// RewriteAffinity renames or restores labels in labelSelector of affinity structure. +// See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity +func RewriteAffinity(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { + return TransformObject(obj, path, func(affinity []byte) ([]byte, error) { + rwrAffinity, err := TransformObject(affinity, "nodeAffinity", func(item []byte) ([]byte, error) { + return rewriteNodeAffinity(rules, item, action) + }) + if err != nil { + return nil, err + } + + rwrAffinity, err = TransformObject(rwrAffinity, "podAffinity", func(item []byte) ([]byte, error) { + return rewritePodAffinity(rules, item, action) + }) + if err != nil { + return nil, err + } + + return TransformObject(rwrAffinity, "podAntiAffinity", func(item []byte) ([]byte, error) { + return rewritePodAffinity(rules, item, action) + }) + + }) +} + +// rewriteNodeAffinity rewrites labels in nodeAffinity structure. +// nodeAffinity: +// +// requiredDuringSchedulingIgnoredDuringExecution: +// nodeSelectorTerms []NodeSelector -> rewrite each item: key in each matchExpressions and matchFields +// preferredDuringSchedulingIgnoredDuringExecution: -> array of PreferredSchedulingTerm: +// preference NodeSelector -> rewrite key in each matchExpressions and matchFields +// weight: +func rewriteNodeAffinity(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + // Rewrite an array of nodeSelectorTerms in requiredDuringSchedulingIgnoredDuringExecution field. + var err error + obj, err = TransformObject(obj, "requiredDuringSchedulingIgnoredDuringExecution", func(affinityTerm []byte) ([]byte, error) { + return RewriteArray(affinityTerm, "nodeSelectorTerms", func(item []byte) ([]byte, error) { + return rewriteNodeSelectorTerm(rules, item, action) + }) + }) + if err != nil { + return nil, err + } + + // Rewrite an array of weightedNodeSelectorTerms in preferredDuringSchedulingIgnoredDuringExecution field. + return RewriteArray(obj, "preferredDuringSchedulingIgnoredDuringExecution", func(item []byte) ([]byte, error) { + return TransformObject(item, "preference", func(preference []byte) ([]byte, error) { + return rewriteNodeSelectorTerm(rules, preference, action) + }) + }) +} + +// rewriteNodeSelectorTerm renames or restores key fields in matchLabels or matchExpressions of NodeSelectorTerm. +func rewriteNodeSelectorTerm(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + obj, err := RewriteArray(obj, "matchLabels", func(item []byte) ([]byte, error) { + return rewriteSelectorRequirement(rules, item, action) + }) + if err != nil { + return nil, err + } + return RewriteArray(obj, "matchExpressions", func(item []byte) ([]byte, error) { + return rewriteSelectorRequirement(rules, item, action) + }) +} + +func rewriteSelectorRequirement(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + return TransformString(obj, "key", func(field string) string { + return rules.LabelsRewriter().Rewrite(field, action) + }) +} + +// rewritePodAffinity rewrites PodAffinity and PodAntiAffinity structures. +// PodAffinity and PodAntiAffinity structures are the same: +// +// requiredDuringSchedulingIgnoredDuringExecution -> array of PodAffinityTerm structures: +// labelSelector: +// matchLabels -> rewrite map +// matchExpressions -> rewrite key in each item +// topologyKey -> rewrite as label name +// namespaceSelector -> rewrite as labelSelector +// preferredDuringSchedulingIgnoredDuringExecution -> array of WeightedPodAffinityTerm: +// weight +// podAffinityTerm PodAffinityTerm -> rewrite as described above +func rewritePodAffinity(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + // Rewrite an array of PodAffinityTerms in requiredDuringSchedulingIgnoredDuringExecution field. + obj, err := RewriteArray(obj, "requiredDuringSchedulingIgnoredDuringExecution", func(affinityTerm []byte) ([]byte, error) { + return rewritePodAffinityTerm(rules, affinityTerm, action) + }) + if err != nil { + return nil, err + } + + // Rewrite an array of WeightedPodAffinityTerms in requiredDuringSchedulingIgnoredDuringExecution field. + return RewriteArray(obj, "preferredDuringSchedulingIgnoredDuringExecution", func(affinityTerm []byte) ([]byte, error) { + return TransformObject(affinityTerm, "podAffinityTerm", func(podAffinityTerm []byte) ([]byte, error) { + return rewritePodAffinityTerm(rules, podAffinityTerm, action) + }) + }) +} + +func rewritePodAffinityTerm(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + obj, err := TransformObject(obj, "labelSelector", func(labelSelector []byte) ([]byte, error) { + return rewriteLabelSelector(rules, labelSelector, action) + }) + if err != nil { + return nil, err + } + + obj, err = TransformString(obj, "topologyKey", func(field string) string { + return rules.LabelsRewriter().Rewrite(field, action) + }) + if err != nil { + return nil, err + } + + return TransformObject(obj, "namespaceSelector", func(selector []byte) ([]byte, error) { + return rewriteLabelSelector(rules, selector, action) + }) +} + +// rewriteLabelSelector rewrites matchLabels and matchExpressions. It is similar to rewriteNodeSelectorTerm +// but matchLabels is a map here, not an array of requirements. +func rewriteLabelSelector(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + obj, err := RewriteLabelsMap(rules, obj, "matchLabels", action) + if err != nil { + return nil, err + } + + return RewriteArray(obj, "matchExpressions", func(item []byte) ([]byte, error) { + return rewriteSelectorRequirement(rules, item, action) + }) +} diff --git a/images/kube-api-proxy/pkg/rewriter/app.go b/images/kube-api-proxy/pkg/rewriter/app.go index ead7de303..23a1ae20c 100644 --- a/images/kube-api-proxy/pkg/rewriter/app.go +++ b/images/kube-api-proxy/pkg/rewriter/app.go @@ -16,6 +16,8 @@ limitations under the License. package rewriter +import "github.com/tidwall/gjson" + const ( DeploymentKind = "Deployment" DeploymentListKind = "DeploymentList" @@ -26,58 +28,64 @@ const ( ) func RewriteDeploymentOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - if action == Rename { - return RewriteResourceOrList(obj, DeploymentListKind, renameSpecLabelsAnno(rules)) - } - return RewriteResourceOrList(obj, DeploymentListKind, restoreSpecLabelsAnno(rules)) + return RewriteResourceOrList(obj, DeploymentListKind, func(singleObj []byte) ([]byte, error) { + return RewriteSpecTemplateLabelsAnno(rules, singleObj, "spec", action) + }) } func RewriteDaemonSetOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - if action == Rename { - return RewriteResourceOrList(obj, DaemonSetListKind, renameSpecLabelsAnno(rules)) - } - return RewriteResourceOrList(obj, DaemonSetListKind, restoreSpecLabelsAnno(rules)) + return RewriteResourceOrList(obj, DaemonSetListKind, func(singleObj []byte) ([]byte, error) { + return RewriteSpecTemplateLabelsAnno(rules, singleObj, "spec", action) + }) } func RewriteStatefulSetOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - if action == Rename { - return RewriteResourceOrList(obj, StatefulSetListKind, renameSpecLabelsAnno(rules)) - } - return RewriteResourceOrList(obj, StatefulSetListKind, restoreSpecLabelsAnno(rules)) + return RewriteResourceOrList(obj, StatefulSetListKind, func(singleObj []byte) ([]byte, error) { + return RewriteSpecTemplateLabelsAnno(rules, singleObj, "spec", action) + }) } -func renameSpecLabelsAnno(rules *RewriteRules) func(singleObj []byte) ([]byte, error) { - return func(singleObj []byte) ([]byte, error) { - singleObj, err := RewriteMapOfStrings(singleObj, "spec.template.metadata.labels", rules.RenameLabels) - if err != nil { - return nil, err - } - singleObj, err = RewriteMapOfStrings(singleObj, "spec.selector.matchLabels", rules.RenameLabels) - if err != nil { - return nil, err +func RenameSpecTemplatePatch(rules *RewriteRules, obj []byte) ([]byte, error) { + obj, err := RenameMetadataPatch(rules, obj) + if err != nil { + return nil, err + } + + return TransformPatch(obj, func(mergePatch []byte) ([]byte, error) { + return RewriteSpecTemplateLabelsAnno(rules, mergePatch, "spec", Rename) + }, func(jsonPatch []byte) ([]byte, error) { + path := gjson.GetBytes(jsonPatch, "path").String() + if path == "/spec" { + return RewriteSpecTemplateLabelsAnno(rules, jsonPatch, "value", Rename) } - singleObj, err = RewriteMapOfStrings(singleObj, "spec.template.spec.nodeSelector", rules.RenameLabels) + return jsonPatch, nil + }) +} + +// RewriteSpecTemplateLabelsAnno transforms labels and annotations in spec fields: +// - selector as LabelSelector +// - template.metadata.labels as labels map +// - template.metadata.annotations as annotations map +// - template.affinity as Affinity +// - template.nodeSelector as labels map. +func RewriteSpecTemplateLabelsAnno(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { + return TransformObject(obj, path, func(obj []byte) ([]byte, error) { + obj, err := RewriteLabelsMap(rules, obj, "template.metadata.labels", action) if err != nil { return nil, err } - return RewriteMapOfStrings(singleObj, "spec.template.metadata.annotations", rules.RenameAnnotations) - } -} - -func restoreSpecLabelsAnno(rules *RewriteRules) func(singleObj []byte) ([]byte, error) { - return func(singleObj []byte) ([]byte, error) { - singleObj, err := RewriteMapOfStrings(singleObj, "spec.template.metadata.labels", rules.RestoreLabels) + obj, err = RewriteLabelsMap(rules, obj, "selector.matchLabels", action) if err != nil { return nil, err } - singleObj, err = RewriteMapOfStrings(singleObj, "spec.selector.matchLabels", rules.RestoreLabels) + obj, err = RewriteLabelsMap(rules, obj, "template.spec.nodeSelector", action) if err != nil { return nil, err } - singleObj, err = RewriteMapOfStrings(singleObj, "spec.template.spec.nodeSelector", rules.RestoreLabels) + obj, err = RewriteAffinity(rules, obj, "template.spec.affinity", action) if err != nil { return nil, err } - return RewriteMapOfStrings(singleObj, "spec.template.metadata.annotations", rules.RestoreAnnotations) - } + return RewriteAnnotationsMap(rules, obj, "template.metadata.annotations", action) + }) } diff --git a/images/kube-api-proxy/pkg/rewriter/app_test.go b/images/kube-api-proxy/pkg/rewriter/app_test.go new file mode 100644 index 000000000..9a75ab84d --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/app_test.go @@ -0,0 +1,246 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +import ( + "bufio" + "bytes" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func createTestRewriterForApp() *RuleBasedRewriter { + apiGroupRules := map[string]APIGroupRule{ + "original.group.io": { + GroupRule: GroupRule{ + Group: "original.group.io", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + }, + ResourceRules: map[string]ResourceRule{ + "someresources": { + Kind: "SomeResource", + ListKind: "SomeResourceList", + Plural: "someresources", + Singular: "someresource", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + Categories: []string{"all"}, + ShortNames: []string{"sr", "srs"}, + }, + "anotherresources": { + Kind: "AnotherResource", + ListKind: "AnotherResourceList", + Plural: "anotherresources", + Singular: "anotherresource", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + ShortNames: []string{"ar"}, + }, + }, + }, + "other.group.io": { + GroupRule: GroupRule{ + Group: "original.group.io", + Versions: []string{"v2alpha3"}, + PreferredVersion: "v2alpha3", + }, + ResourceRules: map[string]ResourceRule{ + "otherresources": { + Kind: "OtherResource", + ListKind: "OtherResourceList", + Plural: "otherresources", + Singular: "otherresource", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + ShortNames: []string{"or"}, + }, + }, + }, + } + + rules := &RewriteRules{ + KindPrefix: "Prefixed", // KV + ResourceTypePrefix: "prefixed", // kv + ShortNamePrefix: "p", + Categories: []string{"prefixed"}, + RenamedGroup: "prefixed.resources.group.io", + Rules: apiGroupRules, + Labels: MetadataReplace{ + Prefixes: []MetadataReplaceRule{ + {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + {Original: "component.labelgroup.io", Renamed: "component.replacedlabelgroup.io"}, + }, + Names: []MetadataReplaceRule{ + {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + }, + }, + Annotations: MetadataReplace{ + Prefixes: []MetadataReplaceRule{ + {Original: "annogroup.io", Renamed: "replacedannogroup.io"}, + {Original: "component.annogroup.io", Renamed: "component.replacedannogroup.io"}, + }, + Names: []MetadataReplaceRule{ + {Original: "annogroup.io", Renamed: "replacedannogroup.io"}, + }, + }, + } + rules.Init() + return &RuleBasedRewriter{ + Rules: rules, + } +} + +func TestRenameDeploymentLabels(t *testing.T) { + deploymentReq := `POST /apis/apps/v1/deployments/testdeployment HTTP/1.1 +Host: 127.0.0.1 + +` + deploymentBody := `{ +"apiVersion": "apiextensions.k8s.io/v1", +"kind": "Deployment", +"metadata": { + "name":"testdeployment", + "labels":{ + "labelgroup.io": "labelValue", + "labelgroup.io/labelName": "labelValue", + "component.labelgroup.io/labelName": "labelValue" + }, + "annotations": { + "annogroup.io": "annoValue", + "annogroup.io/annoName": "annoValue", + "component.annogroup.io/annoName": "annoValue" + } +}, +"spec": { + "replicas": 1, + "selector": { + "matchLabels": { + "labelgroup.io": "labelValue", + "labelgroup.io/labelName": "labelValue", + "component.labelgroup.io/labelName": "labelValue" + } + }, + "template": { + "metadata": { + "name":"testdeployment", + "labels":{ + "labelgroup.io": "labelValue", + "labelgroup.io/labelName": "labelValue", + "component.labelgroup.io/labelName": "labelValue" + }, + "annotations": { + "annogroup.io": "annoValue", + "annogroup.io/annoName": "annoValue", + "component.annogroup.io/annoName": "annoValue" + } + }, + "spec": { + "nodeSelector": { + "labelgroup.io": "labelValue", + "labelgroup.io/labelName": "labelValue", + "component.labelgroup.io/labelName": "labelValue" + }, + "affinity": { + "podAntiAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "labelSelector": { + "matchExpressions":[{ + "key": "labelgroup.io", + "operator":"In", + "values": ["some-value"] + }] + }, + "topologyKey": "kubernetes.io/hostname" + }, + "weight": 1 + } + ] + }, + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "preference": { + "matchExpressions":[{ + "key": "labelgroup.io", + "operator":"In", + "values": ["some-value"] + }] + }, + "weight": 1 + } + ] + } + }, + "containers": [] + } + } +} +}` + req, err := http.ReadRequest(bufio.NewReader(bytes.NewBufferString(deploymentReq + deploymentBody))) + require.NoError(t, err, "should parse hardcoded http request") + require.NotNil(t, req.URL, "should parse url in hardcoded http request") + + rwr := createTestRewriterForApp() + targetReq := NewTargetRequest(rwr, req) + require.NotNil(t, targetReq, "should get TargetRequest") + require.True(t, targetReq.ShouldRewriteRequest(), "should rewrite request") + require.True(t, targetReq.ShouldRewriteResponse(), "should rewrite response") + // require.Equal(t, origGroup, targetReq.OrigGroup(), "should set proper orig group") + + resultBytes, err := rwr.RewriteJSONPayload(targetReq, []byte(deploymentBody), Rename) + if err != nil { + t.Fatalf("should rename Deployment without error: %v", err) + } + if resultBytes == nil { + t.Fatalf("should rename Deployment: %v", err) + } + + tests := []struct { + path string + expected string + }{ + {`metadata.labels.replacedlabelgroup\.io`, "labelValue"}, + {`metadata.labels.labelgroup\.io`, ""}, + {`metadata.labels.replacedlabelgroup\.io/labelName`, "labelValue"}, + {`metadata.labels.labelgroup\.io/labelName`, ""}, + {`metadata.labels.component\.replacedlabelgroup\.io/labelName`, "labelValue"}, + {`metadata.labels.component\.labelgroup\.io/labelName`, ""}, + {`metadata.annotations.replacedannogroup\.io`, "annoValue"}, + {`metadata.annotations.annogroup\.io`, ""}, + {`metadata.annotations.replacedannogroup\.io/annoName`, "annoValue"}, + {`metadata.annotations.annogroup\.io/annoName`, ""}, + {`metadata.annotations.component\.replacedannogroup\.io/annoName`, "annoValue"}, + {`metadata.annotations.component\.annogroup\.io/annoName`, ""}, + {`spec.template.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.podAffinityTerm.labelSelector.matchExpressions.0.key`, "replacedlabelgroup.io"}, + {`spec.template.spec.affinity.nodeAffinity.preferredDuringSchedulingIgnoredDuringExecution.0.preference.matchExpressions.0.key`, "replacedlabelgroup.io"}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + actual := gjson.GetBytes(resultBytes, tt.path).String() + if actual != tt.expected { + t.Fatalf("%s value should be %s, got %s", tt.path, tt.expected, actual) + } + }) + } +} diff --git a/images/kube-api-proxy/pkg/rewriter/core.go b/images/kube-api-proxy/pkg/rewriter/core.go index 7965c7ca9..782dcf8de 100644 --- a/images/kube-api-proxy/pkg/rewriter/core.go +++ b/images/kube-api-proxy/pkg/rewriter/core.go @@ -16,18 +16,100 @@ limitations under the License. package rewriter +import ( + "github.com/tidwall/gjson" +) + const ( - PodKind = "Pod" - PodListKind = "PodList" + PodKind = "Pod" + PodListKind = "PodList" + ServiceKind = "Service" + ServiceListKind = "ServiceList" + JobKind = "Job" + JobListKind = "JobList" + PersistentVolumeClaimKind = "PersistentVolumeClaim" + PersistentVolumeClaimListKind = "PersistentVolumeClaimList" ) func RewritePodOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - if action == Rename { - return RewriteResourceOrList(obj, PodListKind, func(singleObj []byte) ([]byte, error) { - return RewriteMapOfStrings(singleObj, "spec.nodeSelector", rules.RenameLabels) + return RewriteResourceOrList(obj, PodListKind, func(singleObj []byte) ([]byte, error) { + singleObj, err := RewriteLabelsMap(rules, singleObj, "spec.nodeSelector", action) + if err != nil { + return nil, err + } + return RewriteAffinity(rules, singleObj, "spec.affinity", action) + }) +} + +func RewriteServiceOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + return RewriteResourceOrList(obj, ServiceListKind, func(singleObj []byte) ([]byte, error) { + return RewriteLabelsMap(rules, singleObj, "spec.selector", action) + }) +} + +// RewriteJobOrList transforms known fields in the Job manifest. +func RewriteJobOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + return RewriteResourceOrList(obj, JobListKind, func(singleObj []byte) ([]byte, error) { + return RewriteSpecTemplateLabelsAnno(rules, singleObj, "spec", action) + }) +} + +// RewritePVCOrList transforms known fields in the PersistentVolumeClaim manifest. +func RewritePVCOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + return RewriteResourceOrList(obj, PersistentVolumeClaimListKind, func(singleObj []byte) ([]byte, error) { + singleObj, err := TransformObject(singleObj, "spec.dataSource", func(specDataSource []byte) ([]byte, error) { + return RewriteAPIGroupAndKind(rules, specDataSource, action) + }) + if err != nil { + return nil, err + } + return TransformObject(singleObj, "spec.dataSourceRef", func(specDataSourceRef []byte) ([]byte, error) { + return RewriteAPIGroupAndKind(rules, specDataSourceRef, action) }) + }) +} + +func RenameServicePatch(rules *RewriteRules, obj []byte) ([]byte, error) { + obj, err := RenameMetadataPatch(rules, obj) + if err != nil { + return nil, err } - return RewriteResourceOrList(obj, PodListKind, func(singleObj []byte) ([]byte, error) { - return RewriteMapOfStrings(singleObj, "spec.nodeSelector", rules.RestoreLabels) + + // Also rename patch on spec field. + return TransformPatch(obj, nil, func(jsonPatch []byte) ([]byte, error) { + path := gjson.GetBytes(jsonPatch, "path").String() + switch path { + case "/spec": + return RewriteLabelsMap(rules, jsonPatch, "value.selector", Rename) + } + return jsonPatch, nil + }) +} + +func RewriteAPIGroupAndKind(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + var err error + kind := gjson.GetBytes(obj, "kind").String() + + obj, err = TransformString(obj, "kind", func(field string) string { + if action == Rename { + return rules.RenameKind(field) + } + return rules.RestoreKind(field) + }) + if err != nil { + return nil, err + } + + return TransformString(obj, "apiGroup", func(apiGroup string) string { + if action == Rename { + return rules.RenamedGroup + } + // Renamed to original is a one-to-many relation, so we + // need an original kind to get proper group for Restore action. + groupRule, _ := rules.KindRules(apiGroup, rules.RestoreKind(kind)) + if groupRule == nil { + return apiGroup + } + return groupRule.Group }) } diff --git a/images/kube-api-proxy/pkg/rewriter/core_test.go b/images/kube-api-proxy/pkg/rewriter/core_test.go new file mode 100644 index 000000000..ca4cdf97c --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/core_test.go @@ -0,0 +1,209 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +import ( + "bufio" + "bytes" + "net/http" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func createTestRewriterForCore() *RuleBasedRewriter { + apiGroupRules := map[string]APIGroupRule{ + "original.group.io": { + GroupRule: GroupRule{ + Group: "original.group.io", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + }, + ResourceRules: map[string]ResourceRule{ + "someresources": { + Kind: "SomeResource", + ListKind: "SomeResourceList", + Plural: "someresources", + Singular: "someresource", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + Categories: []string{"all"}, + ShortNames: []string{"sr", "srs"}, + }, + "anotherresources": { + Kind: "AnotherResource", + ListKind: "AnotherResourceList", + Plural: "anotherresources", + Singular: "anotherresource", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + ShortNames: []string{"ar"}, + }, + }, + }, + "other.group.io": { + GroupRule: GroupRule{ + Group: "original.group.io", + Versions: []string{"v2alpha3"}, + PreferredVersion: "v2alpha3", + }, + ResourceRules: map[string]ResourceRule{ + "otherresources": { + Kind: "OtherResource", + ListKind: "OtherResourceList", + Plural: "otherresources", + Singular: "otherresource", + Versions: []string{"v1", "v1alpha1"}, + PreferredVersion: "v1", + ShortNames: []string{"or"}, + }, + }, + }, + } + + rules := &RewriteRules{ + KindPrefix: "Prefixed", // KV + ResourceTypePrefix: "prefixed", // kv + ShortNamePrefix: "p", + Categories: []string{"prefixed"}, + RenamedGroup: "prefixed.resources.group.io", + Rules: apiGroupRules, + Labels: MetadataReplace{ + Prefixes: []MetadataReplaceRule{ + {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + {Original: "component.labelgroup.io", Renamed: "component.replacedlabelgroup.io"}, + }, + Names: []MetadataReplaceRule{ + {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + }, + }, + Annotations: MetadataReplace{ + Prefixes: []MetadataReplaceRule{ + {Original: "annogroup.io", Renamed: "replacedannogroup.io"}, + {Original: "component.annogroup.io", Renamed: "component.replacedannogroup.io"}, + }, + Names: []MetadataReplaceRule{ + {Original: "annogroup.io", Renamed: "replacedannogroup.io"}, + }, + }, + } + rules.Init() + return &RuleBasedRewriter{ + Rules: rules, + } +} + +func TestRewriteServicePatch(t *testing.T) { + serviceReq := `PATCH /api/v1/namespaces/default/services/testservice HTTP/1.1 +Host: 127.0.0.1 + +` + servicePatch := `[{ + "op":"replace", + "path":"/spec", + "value": { + "selector":{ "labelgroup.io":"true" } + } +}]` + + req, err := http.ReadRequest(bufio.NewReader(bytes.NewBufferString(serviceReq + servicePatch))) + require.NoError(t, err, "should parse hardcoded http request") + require.NotNil(t, req.URL, "should parse url in hardcoded http request") + + rwr := createTestRewriterForCore() + targetReq := NewTargetRequest(rwr, req) + require.NotNil(t, targetReq, "should get TargetRequest") + require.True(t, targetReq.ShouldRewriteRequest(), "should rewrite request") + require.True(t, targetReq.ShouldRewriteResponse(), "should rewrite response") + // require.Equal(t, origGroup, targetReq.OrigGroup(), "should set proper orig group") + + resultBytes, err := rwr.RewritePatch(targetReq, []byte(servicePatch)) + if err != nil { + t.Fatalf("should rename Service patch without error: %v", err) + } + if resultBytes == nil { + t.Fatalf("should rename Service patch: %v", err) + } + + tests := []struct { + path string + expected string + }{ + {`0.value.selector.labelgroup\.io`, ""}, + {`0.value.selector.replacedlabelgroup\.io`, "true"}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + actual := gjson.GetBytes(resultBytes, tt.path).String() + if actual != tt.expected { + t.Fatalf("%s value should be %s, got %s", tt.path, tt.expected, actual) + } + }) + } + +} + +func TestRewriteMetadataPatch(t *testing.T) { + serviceReq := `PATCH /apis/admissionregistration.k8s.io/v1/validatingwebhookconfigurations/test-validator HTTP/1.1 +Host: 127.0.0.1 + +` + servicePatch := `[{ + "op":"replace", + "path":"/metadata/labels", + "value": {"labelgroup.io":"true" } +}]` + + req, err := http.ReadRequest(bufio.NewReader(bytes.NewBufferString(serviceReq + servicePatch))) + require.NoError(t, err, "should parse hardcoded http request") + require.NotNil(t, req.URL, "should parse url in hardcoded http request") + + rwr := createTestRewriterForCore() + targetReq := NewTargetRequest(rwr, req) + require.NotNil(t, targetReq, "should get TargetRequest") + require.True(t, targetReq.ShouldRewriteRequest(), "should rewrite request") + require.True(t, targetReq.ShouldRewriteResponse(), "should rewrite response") + // require.Equal(t, origGroup, targetReq.OrigGroup(), "should set proper orig group") + + resultBytes, err := rwr.RewritePatch(targetReq, []byte(servicePatch)) + if err != nil { + t.Fatalf("should rename Service patch without error: %v", err) + } + if resultBytes == nil { + t.Fatalf("should rename Service patch: %v", err) + } + + tests := []struct { + path string + expected string + }{ + {`0.value.labelgroup\.io`, ""}, + {`0.value.replacedlabelgroup\.io`, "true"}, + } + + for _, tt := range tests { + t.Run(tt.path, func(t *testing.T) { + actual := gjson.GetBytes(resultBytes, tt.path).String() + if actual != tt.expected { + t.Fatalf("%s value should be %s, got %s", tt.path, tt.expected, actual) + } + }) + } + +} diff --git a/images/kube-api-proxy/pkg/rewriter/list.go b/images/kube-api-proxy/pkg/rewriter/list.go index 7740ca637..1bdc8b695 100644 --- a/images/kube-api-proxy/pkg/rewriter/list.go +++ b/images/kube-api-proxy/pkg/rewriter/list.go @@ -17,11 +17,14 @@ limitations under the License. package rewriter import ( + "strings" + "github.com/tidwall/gjson" "github.com/tidwall/sjson" - "strings" ) +// TODO merge this file into transformers.go + // RewriteResourceOrList is a helper to transform a single resource or a list of resources. func RewriteResourceOrList(payload []byte, listKind string, transformFn func(singleObj []byte) ([]byte, error)) ([]byte, error) { kind := gjson.GetBytes(payload, "kind").String() @@ -43,9 +46,11 @@ func RewriteResourceOrList2(payload []byte, transformFn func(singleObj []byte) ( return RewriteArray(payload, "items", transformFn) } +// RewriteArray gets array by path and transforms each item using transformFn. +// Use Root path to transform object itself. func RewriteArray(obj []byte, arrayPath string, transformFn func(item []byte) ([]byte, error)) ([]byte, error) { // Transform each item in list. Put back original items if transformFn returns nil bytes. - items := gjson.GetBytes(obj, arrayPath).Array() + items := GetBytes(obj, arrayPath).Array() if len(items) == 0 { return obj, nil } @@ -65,5 +70,5 @@ func RewriteArray(obj []byte, arrayPath string, transformFn func(item []byte) ([ } } - return sjson.SetRawBytes(obj, arrayPath, rwrItems) + return SetRawBytes(obj, arrayPath, rwrItems) } diff --git a/images/kube-api-proxy/pkg/rewriter/map.go b/images/kube-api-proxy/pkg/rewriter/map.go index 482860835..83d51db77 100644 --- a/images/kube-api-proxy/pkg/rewriter/map.go +++ b/images/kube-api-proxy/pkg/rewriter/map.go @@ -21,25 +21,19 @@ import ( "github.com/tidwall/sjson" ) -func RewriteMapOfStrings(obj []byte, mapPath string, transformFn func(map[string]string) map[string]string) ([]byte, error) { +// TODO merge this file into transformers.go + +// RewriteMapStringString transforms map[string]string value addressed by path. +func RewriteMapStringString(obj []byte, mapPath string, transformFn func(k, v string) (string, string)) ([]byte, error) { m := gjson.GetBytes(obj, mapPath).Map() if len(m) == 0 { return obj, nil } newMap := make(map[string]string, len(m)) for k, v := range m { - newMap[k] = v.String() + newK, newV := transformFn(k, v.String()) + newMap[newK] = newV } - newMap = transformFn(newMap) - - return sjson.SetBytes(obj, mapPath, newMap) -} -func RewriteMap(obj []byte, mapPath string, transformFn func(map[string]gjson.Result) interface{}) ([]byte, error) { - m := gjson.GetBytes(obj, mapPath).Map() - if len(m) == 0 { - return obj, nil - } - newMap := transformFn(m) return sjson.SetBytes(obj, mapPath, newMap) } diff --git a/images/kube-api-proxy/pkg/rewriter/metadata.go b/images/kube-api-proxy/pkg/rewriter/metadata.go new file mode 100644 index 000000000..476bf548b --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/metadata.go @@ -0,0 +1,77 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +import "github.com/tidwall/gjson" + +func RewriteMetadata(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { + //var err error + obj, err := RewriteLabelsMap(rules, obj, "metadata.labels", action) + if err != nil { + return nil, err + } + obj, err = RewriteAnnotationsMap(rules, obj, "metadata.annotations", action) + if err != nil { + return nil, err + } + return RewriteFinalizers(rules, obj, "metadata.finalizers", action) +} + +// RenameMetadataPatch transforms known metadata fields in patches. +// Example: +// - merge patch on metadata: +// {"metadata": { "labels": {"kubevirt.io/schedulable": "false", "cpumanager": "false"}, "annotations": {"kubevirt.io/heartbeat": "2024-06-07T23:27:53Z"}}} +// - JSON patch on metadata: +// [{"op":"test", "path":"/metadata/labels", "value":{"label":"value"}}, +// +// {"op":"replace", "path":"/metadata/labels", "value":{"label":"newValue"}}] +func RenameMetadataPatch(rules *RewriteRules, patch []byte) ([]byte, error) { + return TransformPatch(patch, + func(mergePatch []byte) ([]byte, error) { + return RewriteMetadata(rules, mergePatch, Rename) + }, + func(jsonPatch []byte) ([]byte, error) { + path := gjson.GetBytes(jsonPatch, "path").String() + switch path { + case "/metadata/labels": + return RewriteLabelsMap(rules, jsonPatch, "value", Rename) + case "/metadata/annotations": + return RewriteAnnotationsMap(rules, jsonPatch, "value", Rename) + case "/metadata/finalizers": + return RewriteFinalizers(rules, jsonPatch, "value", Rename) + } + return jsonPatch, nil + }) +} + +func RewriteLabelsMap(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { + return RewriteMapStringString(obj, path, func(k, v string) (string, string) { + return rules.LabelsRewriter().Rewrite(k, action), v + }) +} + +func RewriteAnnotationsMap(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { + return RewriteMapStringString(obj, path, func(k, v string) (string, string) { + return rules.AnnotationsRewriter().Rewrite(k, action), v + }) +} + +func RewriteFinalizers(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) { + return TransformArrayOfStrings(obj, path, func(finalizer string) string { + return rules.FinalizersRewriter().Rewrite(finalizer, action) + }) +} diff --git a/images/kube-api-proxy/pkg/rewriter/policy.go b/images/kube-api-proxy/pkg/rewriter/policy.go index 5d7755553..68b3b178f 100644 --- a/images/kube-api-proxy/pkg/rewriter/policy.go +++ b/images/kube-api-proxy/pkg/rewriter/policy.go @@ -22,12 +22,7 @@ const ( ) func RewritePDBOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - if action == Rename { - return RewriteResourceOrList(obj, PodDisruptionBudgetListKind, func(singleObj []byte) ([]byte, error) { - return RewriteMapOfStrings(singleObj, "spec.selector", rules.RenameLabels) - }) - } return RewriteResourceOrList(obj, PodDisruptionBudgetListKind, func(singleObj []byte) ([]byte, error) { - return RewriteMapOfStrings(singleObj, "spec.selector", rules.RestoreLabels) + return RewriteLabelsMap(rules, singleObj, "spec.selector", action) }) } diff --git a/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go b/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go new file mode 100644 index 000000000..185b0c6f6 --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/prefixed_name_rewriter.go @@ -0,0 +1,141 @@ +package rewriter + +import "strings" + +type PrefixedNameRewriter struct { + namesRenameIdx map[string]string + namesRestoreIdx map[string]string + prefixRenameIdx map[string]string + prefixRestoreIdx map[string]string +} + +func NewPrefixedNameRewriter(replaceRules MetadataReplace) *PrefixedNameRewriter { + return &PrefixedNameRewriter{ + namesRenameIdx: indexRules(replaceRules.Names), + namesRestoreIdx: indexRulesReverse(replaceRules.Names), + prefixRenameIdx: indexRules(replaceRules.Prefixes), + prefixRestoreIdx: indexRulesReverse(replaceRules.Prefixes), + } +} + +func (p *PrefixedNameRewriter) Rewrite(name string, action Action) string { + switch action { + case Rename: + return p.rename(name) + case Restore: + return p.restore(name) + } + return name +} + +func (p *PrefixedNameRewriter) RewriteSlice(names []string, action Action) []string { + switch action { + case Rename: + return p.rewriteSlice(names, p.rename) + case Restore: + return p.rewriteSlice(names, p.restore) + } + return names +} + +func (p *PrefixedNameRewriter) RewriteMap(names map[string]string, action Action) map[string]string { + switch action { + case Rename: + return p.rewriteMap(names, p.rename) + case Restore: + return p.rewriteMap(names, p.restore) + } + return names +} + +func (p *PrefixedNameRewriter) Rename(name string) string { + return p.rename(name) +} + +func (p *PrefixedNameRewriter) Restore(name string) string { + return p.restore(name) +} + +func (p *PrefixedNameRewriter) RenameSlice(names []string) []string { + return p.rewriteSlice(names, p.rename) +} + +func (p *PrefixedNameRewriter) RestoreSlice(names []string) []string { + return p.rewriteSlice(names, p.restore) +} + +func (p *PrefixedNameRewriter) RenameMap(names map[string]string) map[string]string { + return p.rewriteMap(names, p.rename) +} + +func (p *PrefixedNameRewriter) RestoreMap(names map[string]string) map[string]string { + return p.rewriteMap(names, p.restore) +} + +func (p *PrefixedNameRewriter) rewriteMap(names map[string]string, fn func(string) string) map[string]string { + if names == nil { + return nil + } + result := make(map[string]string) + for name, value := range names { + result[fn(name)] = value + } + return result +} + +func (p *PrefixedNameRewriter) rewriteSlice(names []string, fn func(string) string) []string { + if names == nil { + return nil + } + result := make([]string, 0, len(names)) + for _, name := range names { + result = append(result, fn(name)) + } + return result +} + +func (p *PrefixedNameRewriter) rename(name string) string { + if renamed, ok := p.namesRenameIdx[name]; ok { + return renamed + } + // No exact name, find prefix. + prefix, remainder, found := strings.Cut(name, "/") + if !found { + return name + } + if renamedPrefix, ok := p.prefixRenameIdx[prefix]; ok { + return renamedPrefix + "/" + remainder + } + return name +} + +func (p *PrefixedNameRewriter) restore(name string) string { + if restored, ok := p.namesRestoreIdx[name]; ok { + return restored + } + // No exact name, find prefix. + prefix, remainder, found := strings.Cut(name, "/") + if !found { + return name + } + if restoredPrefix, ok := p.prefixRestoreIdx[prefix]; ok { + return restoredPrefix + "/" + remainder + } + return name +} + +func indexRules(rules []MetadataReplaceRule) map[string]string { + idx := make(map[string]string, len(rules)) + for _, rule := range rules { + idx[rule.Original] = rule.Renamed + } + return idx +} + +func indexRulesReverse(rules []MetadataReplaceRule) map[string]string { + idx := make(map[string]string, len(rules)) + for _, rule := range rules { + idx[rule.Renamed] = rule.Original + } + return idx +} diff --git a/images/kube-api-proxy/pkg/rewriter/resource.go b/images/kube-api-proxy/pkg/rewriter/resource.go index 67a2b51b2..8eef7ade8 100644 --- a/images/kube-api-proxy/pkg/rewriter/resource.go +++ b/images/kube-api-proxy/pkg/rewriter/resource.go @@ -358,68 +358,6 @@ func RenameManagedFields(rules *RewriteRules, obj []byte) ([]byte, error) { return sjson.SetRawBytes(obj, "metadata.managedFields", newFields) } -func RewriteMetadata(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - newObj, err := RewriteMetadataLabels(rules, obj, action) - if err != nil { - return nil, err - } - return RewriteMetadataAnnotations(rules, newObj, action) -} - -func RewriteMetadataLabels(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - labels := gjson.GetBytes(obj, "metadata.labels").Map() - if len(labels) == 0 { - return obj, nil - } - - newLabels := make(map[string]string, len(labels)) - for k, v := range labels { - newLabels[k] = v.String() - } - switch action { - case Rename: - newLabels = rules.RenameLabels(newLabels) - case Restore: - newLabels = rules.RestoreLabels(newLabels) - } - - return sjson.SetBytes(obj, "metadata.labels", newLabels) -} - -func RewriteMetadataAnnotations(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - annos := gjson.GetBytes(obj, "metadata.annotations").Map() - if len(annos) == 0 { - return obj, nil - } - newAnnos := make(map[string]string, len(annos)) - for k, v := range annos { - newAnnos[k] = v.String() - } - switch action { - case Rename: - newAnnos = rules.RenameAnnotations(newAnnos) - case Restore: - newAnnos = rules.RestoreAnnotations(newAnnos) - } - - return sjson.SetBytes(obj, "metadata.annotations", newAnnos) -} - -func RewriteFinalizers(rules *RewriteRules, obj []byte, action Action) ([]byte, error) { - fins := gjson.GetBytes(obj, "spec.finalizers").Array() - if len(fins) == 0 { - return obj, nil - } - newFins := make([]string, len(fins)) - for i, f := range fins { - newFins[i] = f.String() - } - switch action { - case Rename: - newFins = rules.RenameFinalizers(newFins) - case Restore: - newFins = rules.RestoreFinalizers(newFins) - } - - return sjson.SetBytes(obj, "spec.finalizers", newFins) +func RenameResourcePatch(rules *RewriteRules, patch []byte) ([]byte, error) { + return RenameMetadataPatch(rules, patch) } diff --git a/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go b/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go index 3fdbaec26..fe13357e1 100644 --- a/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go +++ b/images/kube-api-proxy/pkg/rewriter/rule_rewriter.go @@ -22,6 +22,7 @@ import ( "strings" "github.com/tidwall/gjson" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) type RuleBasedRewriter struct { @@ -157,23 +158,39 @@ func (rw *RuleBasedRewriter) rewriteLabelSelector(rawQuery string) string { if lsq == "" { return rawQuery } - listLabels := strings.Split(lsq, ",") - labels := make(map[string]string, len(listLabels)) - for _, l := range listLabels { - ll := strings.Split(l, "=") - if (len(ll)>1) { - labels[ll[0]] = ll[1] - } else { - labels[ll[0]] = "" - } + + labelSelector, err := metav1.ParseToLabelSelector(lsq) + if err != nil { + // The labelSelector is not well-formed. We pass it through, so + // API Server will return an error. + return rawQuery } - labels = rw.Rules.RenameLabels(labels) - count := 0 - for k, v := range labels { - listLabels[count] = k + "=" + v - count++ + + // Return early if labelSelector is empty, e.g. ?labelSelector=&limit=500 + if labelSelector == nil { + return rawQuery } - q.Set("labelSelector", strings.Join(listLabels, ",")) + + rwrMatchLabels := rw.Rules.LabelsRewriter().RenameMap(labelSelector.MatchLabels) + + rwrMatchExpressions := make([]metav1.LabelSelectorRequirement, 0) + for _, expr := range labelSelector.MatchExpressions { + rwrExpr := expr + rwrExpr.Key = rw.Rules.LabelsRewriter().Rename(rwrExpr.Key) + rwrMatchExpressions = append(rwrMatchExpressions, rwrExpr) + } + + rwrLabelSelector := &metav1.LabelSelector{ + MatchLabels: rwrMatchLabels, + MatchExpressions: rwrMatchExpressions, + } + + res, err := metav1.LabelSelectorAsSelector(rwrLabelSelector) + if err != nil { + return rawQuery + } + + q.Set("labelSelector", res.String()) return q.Encode() } @@ -238,10 +255,20 @@ func (rw *RuleBasedRewriter) RewriteJSONPayload(targetReq *TargetRequest, obj [] rwrBytes, err = RewriteStatefulSetOrList(rw.Rules, obj, action) case DaemonSetKind, DaemonSetListKind: rwrBytes, err = RewriteDaemonSetOrList(rw.Rules, obj, action) - case PodKind: + case PodKind, PodListKind: rwrBytes, err = RewritePodOrList(rw.Rules, obj, action) - case PodDisruptionBudgetKind: + case PodDisruptionBudgetKind, PodDisruptionBudgetListKind: rwrBytes, err = RewritePDBOrList(rw.Rules, obj, action) + case JobKind, JobListKind: + rwrBytes, err = RewriteJobOrList(rw.Rules, obj, action) + case ServiceKind, ServiceListKind: + rwrBytes, err = RewriteServiceOrList(rw.Rules, obj, action) + case PersistentVolumeClaimKind, PersistentVolumeClaimListKind: + rwrBytes, err = RewritePVCOrList(rw.Rules, obj, action) + + case ServiceMonitorKind, ServiceMonitorListKind: + rwrBytes, err = RewriteServiceMonitorOrList(rw.Rules, obj, action) + default: if targetReq.IsCore() { rwrBytes, err = RewriteOwnerReferences(rw.Rules, obj, action) @@ -261,20 +288,11 @@ func (rw *RuleBasedRewriter) RewriteJSONPayload(targetReq *TargetRequest, obj [] return obj, err } - rwrBytes, err = RewriteResourceOrList2(rwrBytes, func(singleObj []byte) ([]byte, error) { - return RewriteFinalizers(rw.Rules, singleObj, action) - }) - if err != nil { - return obj, err - } - if shouldRewriteOwnerReferences(kind) { rwrBytes, err = RewriteOwnerReferences(rw.Rules, rwrBytes, action) - } - - // Return obj bytes as-is in case of the error. - if err != nil { - return obj, err + if err != nil { + return obj, err + } } return rwrBytes, nil @@ -282,18 +300,33 @@ func (rw *RuleBasedRewriter) RewriteJSONPayload(targetReq *TargetRequest, obj [] // RewritePatch rewrites patches for some known objects. // Only rename action is required for patches. -func (rw *RuleBasedRewriter) RewritePatch(targetReq *TargetRequest, obj []byte) ([]byte, error) { - if targetReq.IsCRD() { - // Check if CRD is known. - _, resRule := rw.Rules.ResourceRules(targetReq.OrigGroup(), targetReq.OrigResourceType()) - if resRule == nil { - return obj, nil +func (rw *RuleBasedRewriter) RewritePatch(targetReq *TargetRequest, patchBytes []byte) ([]byte, error) { + _, resRule := rw.Rules.ResourceRules(targetReq.OrigGroup(), targetReq.OrigResourceType()) + if resRule != nil { + if targetReq.IsCRD() { + return RenameCRDPatch(rw.Rules, resRule, patchBytes) } + return RenameResourcePatch(rw.Rules, patchBytes) + } - return RenameCRDPatch(rw.Rules, resRule, obj) + switch targetReq.OrigResourceType() { + case "services": + return RenameServicePatch(rw.Rules, patchBytes) + case "deployments", + "daemonsets", + "statefulsets": + return RenameSpecTemplatePatch(rw.Rules, patchBytes) + case "validatingwebhookconfigurations", + "mutatingwebhookconfigurations": + return RenameWebhookConfigurationPatch(rw.Rules, patchBytes) + case "nodes", + "apiservices", + "secrets", + "configmaps": + return RenameMetadataPatch(rw.Rules, patchBytes) } - return obj, nil + return patchBytes, nil } func shouldRewriteOwnerReferences(resourceType string) bool { @@ -307,10 +340,18 @@ func shouldRewriteOwnerReferences(resourceType string) bool { ClusterRoleBindingKind, ClusterRoleBindingListKind, APIServiceKind, APIServiceListKind, DeploymentKind, DeploymentListKind, + DaemonSetKind, DaemonSetListKind, + StatefulSetKind, StatefulSetListKind, + PodKind, PodListKind, + JobKind, JobListKind, ValidatingWebhookConfigurationKind, ValidatingWebhookConfigurationListKind, MutatingWebhookConfigurationKind, - MutatingWebhookConfigurationListKind: + MutatingWebhookConfigurationListKind, + ServiceKind, ServiceListKind, + PersistentVolumeClaimKind, PersistentVolumeClaimListKind, + PrometheusRuleKind, PrometheusRuleListKind, + ServiceMonitorKind, ServiceMonitorListKind: return true } diff --git a/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go b/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go index bf1dd448a..60a595b90 100644 --- a/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go +++ b/images/kube-api-proxy/pkg/rewriter/rule_rewriter_test.go @@ -82,19 +82,20 @@ func createTestRewriter() *RuleBasedRewriter { Rules: apiGroupRules, Labels: MetadataReplace{ Prefixes: []MetadataReplaceRule{ - {Old: "original.prefix", New: "rewrite.prefix"}, + {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, + {Original: "component.labelgroup.io", Renamed: "component.replacedlabelgroup.io"}, }, Names: []MetadataReplaceRule{ - {Old: "original.label.io", New: "rewrite.label.io"}, + {Original: "labelgroup.io", Renamed: "replacedlabelgroup.io"}, }, }, Annotations: MetadataReplace{ Names: []MetadataReplaceRule{ - {Old: "original.annotation.io", New: "rewrite.annotation.io"}, + {Original: "annogroup.io", Renamed: "replacedanno.io"}, }, }, } - rules.Complete() + rules.Init() return &RuleBasedRewriter{ Rules: rules, } @@ -105,7 +106,7 @@ func TestRewriteAPIEndpoint(t *testing.T) { name string path string expectPath string - exepctQuery string + expectQuery string }{ { "rewritable group", @@ -144,10 +145,34 @@ func TestRewriteAPIEndpoint(t *testing.T) { "", }, { - "rewritable labelSelector", - "/api/v1/namespaces/d8-virtualization/pods?labelSelector=original.label.io%3Dlabelvalue&limit=500", + "labelSelector one label name", + "/api/v1/namespaces/nsname/pods?labelSelector=labelgroup.io&limit=0", + "/api/v1/namespaces/nsname/pods", + "labelSelector=replacedlabelgroup.io&limit=0", + }, + { + "labelSelector one prefixed label", + "/api/v1/pods?labelSelector=labelgroup.io%2Fsome-attr&limit=500", + "/api/v1/pods", + "labelSelector=replacedlabelgroup.io%2Fsome-attr&limit=500", + }, + { + "labelSelector label name and value", + "/api/v1/namespaces/d8-virtualization/pods?labelSelector=labelgroup.io%3Dlabelvalue&limit=500", + "/api/v1/namespaces/d8-virtualization/pods", + "labelSelector=replacedlabelgroup.io%3Dlabelvalue&limit=500", + }, + { + "labelSelector prefixed label and value", + "/api/v1/namespaces/d8-virtualization/pods?labelSelector=component.labelgroup.io%2Fsome-attr%3Dlabelvalue&limit=500", + "/api/v1/namespaces/d8-virtualization/pods", + "labelSelector=component.replacedlabelgroup.io%2Fsome-attr%3Dlabelvalue&limit=500", + }, + { + "labelSelector label name not in values", + "/api/v1/namespaces/d8-virtualization/pods?labelSelector=labelgroup.io+notin+%28value-one%2Cvalue-two%29&limit=500", "/api/v1/namespaces/d8-virtualization/pods", - "labelSelector=rewrite.label.io%3Dlabelvalue&limit=500", + "labelSelector=replacedlabelgroup.io+notin+%28value-one%2Cvalue-two%29&limit=500", }, } @@ -164,10 +189,10 @@ func TestRewriteAPIEndpoint(t *testing.T) { if tt.expectPath == "" { require.Nil(t, newEp, "should not rewrite path '%s', got %+v", tt.path, newEp) } - require.NotNil(t, newEp, "should rewrite path '%s', got nil originEndpoint") + require.NotNil(t, newEp, "should rewrite path '%s', got nil endpoint. Original ep: %#v", ep) - require.Equal(t, tt.expectPath, newEp.Path(), "expect rewrite for path '%s' to be '%s', got '%s'", tt.path, tt.expectPath, ep.Path()) - require.Equal(t, tt.exepctQuery, newEp.RawQuery, "expect rewrite query for path %q to be '%s', got '%s'", tt.path, tt.exepctQuery, ep.RawQuery) + require.Equal(t, tt.expectPath, newEp.Path(), "expect rewrite for path '%s' to be '%s', got '%s', newEp: %#v", tt.path, tt.expectPath, newEp.Path(), newEp) + require.Equal(t, tt.expectQuery, newEp.RawQuery, "expect rewrite query for path %q to be '%s', got '%s', newEp: %#v", tt.path, tt.expectQuery, newEp.RawQuery, newEp) }) } diff --git a/images/kube-api-proxy/pkg/rewriter/rules.go b/images/kube-api-proxy/pkg/rewriter/rules.go index aa4232dde..0bb1b1f84 100644 --- a/images/kube-api-proxy/pkg/rewriter/rules.go +++ b/images/kube-api-proxy/pkg/rewriter/rules.go @@ -31,15 +31,18 @@ type RewriteRules struct { Labels MetadataReplace `json:"labels"` Annotations MetadataReplace `json:"annotations"` Finalizers MetadataReplace `json:"finalizers"` - labelsIndexer *MetadataIndexer - annoIndexer *MetadataIndexer - finalizerIndexer *MetadataIndexer + + // TODO move these indexed rewriters into the RuleBasedRewriter. + labelsRewriter *PrefixedNameRewriter + annotationsRewriter *PrefixedNameRewriter + finalizersRewriter *PrefixedNameRewriter } -func (rr *RewriteRules) Complete() { - rr.labelsIndexer = rr.Labels.Complete() - rr.annoIndexer = rr.Annotations.Complete() - rr.finalizerIndexer = rr.Finalizers.Complete() +// Init should be called before using rules in the RuleBasedRewriter. +func (rr *RewriteRules) Init() { + rr.labelsRewriter = NewPrefixedNameRewriter(rr.Labels) + rr.annotationsRewriter = NewPrefixedNameRewriter(rr.Annotations) + rr.finalizersRewriter = NewPrefixedNameRewriter(rr.Finalizers) } type APIGroupRule struct { @@ -75,57 +78,9 @@ type MetadataReplace struct { Names []MetadataReplaceRule } -func (mr *MetadataReplace) Complete() *MetadataIndexer { - namesOldToNew := make(map[string]string, len(mr.Names)) - namesNewToOld := make(map[string]string, len(mr.Names)) - for _, l := range mr.Names { - namesOldToNew[l.Old] = l.New - namesNewToOld[l.New] = l.Old - } - prefixOldToNew := make(map[string]string, len(mr.Prefixes)) - prefixNewToOld := make(map[string]string, len(mr.Prefixes)) - for _, l := range mr.Prefixes { - prefixOldToNew[l.Old] = l.New - prefixNewToOld[l.New] = l.Old - } - return &MetadataIndexer{ - namesOldToNew: namesOldToNew, - namesNewToOld: namesNewToOld, - prefixOldToNew: prefixOldToNew, - prefixNewToOld: prefixNewToOld, - } -} - -type MetadataIndexer struct { - namesOldToNew map[string]string - namesNewToOld map[string]string - prefixOldToNew map[string]string - prefixNewToOld map[string]string -} - -func (mi *MetadataIndexer) GetOld(s string) (string, bool) { - v, found := mi.namesNewToOld[s] - return v, found -} - -func (mi *MetadataIndexer) GetNew(s string) (string, bool) { - v, found := mi.namesOldToNew[s] - return v, found -} - -func (mi *MetadataIndexer) GetOldPrefix(s string) (string, bool) { - v, found := mi.prefixNewToOld[s] - return v, found -} - -func (mi *MetadataIndexer) GetNewPrefix(s string) (string, bool) { - v, found := mi.prefixOldToNew[s] - return v, found -} - type MetadataReplaceRule struct { - Old string `json:"old"` - New string `json:"new"` + Original string `json:"original"` + Renamed string `json:"renamed"` } // GetAPIGroupList returns an array of groups in format applicable to use in APIGroupList: @@ -236,6 +191,17 @@ func (rr *RewriteRules) GroupResourceRules(resourceType string) (*GroupRule, *Re return nil, nil } +func (rr *RewriteRules) GroupResourceRulesByKind(kind string) (*GroupRule, *ResourceRule) { + for _, group := range rr.Rules { + for _, res := range group.ResourceRules { + if res.Kind == kind { + return &group.GroupRule, &res + } + } + } + return nil, nil +} + func (rr *RewriteRules) RenameResource(resource string) string { return rr.ResourceTypePrefix + resource } @@ -259,6 +225,11 @@ func (rr *RewriteRules) RestoreApiVersion(apiVersion string, group string) strin } func (rr *RewriteRules) RenameApiVersion(apiVersion string) string { + // Check if apiVersion is just a group name. + if !strings.Contains(apiVersion, "/") && rr.HasGroup(apiVersion) { + return rr.RenamedGroup + } + // Replace group, keep version. apiVerParts := strings.Split(apiVersion, "/") if len(apiVerParts) != 2 { @@ -297,103 +268,14 @@ func (rr *RewriteRules) RestoreShortNames(shortNames []string) []string { return newNames } -func (rr *RewriteRules) RenameLabel(label string) string { - return rr.rename(label, rr.labelsIndexer) - -} - -func (rr *RewriteRules) RestoreLabel(label string) string { - return rr.restore(label, rr.labelsIndexer) -} - -func (rr *RewriteRules) RenameLabels(labels map[string]string) map[string]string { - return rr.rewriteMaps(labels, rr.RenameLabel) +func (rr *RewriteRules) LabelsRewriter() *PrefixedNameRewriter { + return rr.labelsRewriter } -func (rr *RewriteRules) RestoreLabels(labels map[string]string) map[string]string { - return rr.rewriteMaps(labels, rr.RestoreLabel) +func (rr *RewriteRules) AnnotationsRewriter() *PrefixedNameRewriter { + return rr.annotationsRewriter } -func (rr *RewriteRules) RenameAnnotation(anno string) string { - return rr.rename(anno, rr.annoIndexer) -} - -func (rr *RewriteRules) RestoreAnnotation(anno string) string { - return rr.restore(anno, rr.annoIndexer) -} - -func (rr *RewriteRules) RenameAnnotations(annotations map[string]string) map[string]string { - return rr.rewriteMaps(annotations, rr.RenameAnnotation) - -} - -func (rr *RewriteRules) RestoreAnnotations(annotations map[string]string) map[string]string { - return rr.rewriteMaps(annotations, rr.RestoreAnnotation) -} - -func (rr *RewriteRules) RenameFinalizer(fin string) string { - return rr.rename(fin, rr.annoIndexer) -} - -func (rr *RewriteRules) RestoreFinalizer(fin string) string { - return rr.restore(fin, rr.annoIndexer) -} - -func (rr *RewriteRules) RenameFinalizers(fins []string) []string { - return rr.rewriteSlices(fins, rr.RenameFinalizer) - -} - -func (rr *RewriteRules) RestoreFinalizers(fins []string) []string { - return rr.rewriteSlices(fins, rr.RestoreFinalizer) -} - -func (rr *RewriteRules) rewriteMaps(m map[string]string, fn func(s string) string) map[string]string { - result := make(map[string]string, len(m)) - for k, v := range m { - result[fn(k)] = v - } - return result -} - -func (rr *RewriteRules) rewriteSlices(s []string, fn func(s string) string) []string { - result := make([]string, len(s)) - for i, ss := range s { - result[i] = fn(ss) - } - return result -} - -func (rr *RewriteRules) rename(s string, indexer *MetadataIndexer) string { - if indexer == nil { - return s - } - if v, ok := indexer.GetNew(s); ok { - return v - } - prefix, _, found := strings.Cut(s, "/") - if !found { - return s - } - if v, ok := indexer.GetNewPrefix(prefix); ok { - return v + strings.TrimPrefix(s, prefix) - } - return s -} - -func (rr *RewriteRules) restore(s string, indexer *MetadataIndexer) string { - if indexer == nil { - return s - } - if v, ok := indexer.GetOld(s); ok { - return v - } - prefix, _, found := strings.Cut(s, "/") - if !found { - return s - } - if v, ok := indexer.GetOldPrefix(prefix); ok { - return v + strings.TrimPrefix(s, prefix) - } - return s +func (rr *RewriteRules) FinalizersRewriter() *PrefixedNameRewriter { + return rr.finalizersRewriter } diff --git a/images/kube-api-proxy/pkg/rewriter/target_request.go b/images/kube-api-proxy/pkg/rewriter/target_request.go index 2afaa3f1a..f4467bc30 100644 --- a/images/kube-api-proxy/pkg/rewriter/target_request.go +++ b/images/kube-api-proxy/pkg/rewriter/target_request.go @@ -157,7 +157,7 @@ func (tr *TargetRequest) ShouldRewriteRequest() bool { return true } - return shouldRewriteResource(tr.originEndpoint.ResourceType, tr.originEndpoint.IsCore) + return shouldRewriteResource(tr.originEndpoint.ResourceType) } } @@ -203,7 +203,7 @@ func (tr *TargetRequest) ShouldRewriteResponse() bool { return true } - return shouldRewriteResource(tr.originEndpoint.ResourceType, tr.originEndpoint.IsCore) + return shouldRewriteResource(tr.originEndpoint.ResourceType) } func (tr *TargetRequest) ResourceForLog() string { @@ -268,24 +268,15 @@ func (tr *TargetRequest) ResourceForLog() string { return "UNKNOWN" } -func shouldRewriteResource(kind string, isCore bool) bool { - // Some core resources should be rewritten. - if isCore { - switch kind { - case "pods", - "configmaps", - "secrets", - "services", - "serviceaccounts": - - return true - } - return false - } - - // Rewrite special resources. - switch kind { - case "mutatingwebhookconfigurations", +func shouldRewriteResource(resourceType string) bool { + switch resourceType { + case "nodes", + "pods", + "configmaps", + "secrets", + "services", + "serviceaccounts", + "mutatingwebhookconfigurations", "validatingwebhookconfigurations", "clusterroles", "roles", @@ -294,6 +285,10 @@ func shouldRewriteResource(kind string, isCore bool) bool { "deployments", "statefulsets", "daemonsets", + "jobs", + "persistentvolumeclaims", + "prometheusrules", + "servicemonitors", "poddisruptionbudgets", "controllerrevisions", "apiservices": diff --git a/images/kube-api-proxy/pkg/rewriter/transformers.go b/images/kube-api-proxy/pkg/rewriter/transformers.go new file mode 100644 index 000000000..ef68ec873 --- /dev/null +++ b/images/kube-api-proxy/pkg/rewriter/transformers.go @@ -0,0 +1,111 @@ +/* +Copyright 2024 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package rewriter + +import ( + "encoding/json" + + "github.com/tidwall/gjson" + "github.com/tidwall/sjson" +) + +// TransformString transforms string value addressed by path. +func TransformString(obj []byte, path string, transformFn func(field string) string) ([]byte, error) { + pathStr := gjson.GetBytes(obj, path) + if !pathStr.Exists() { + return obj, nil + } + rwrString := transformFn(pathStr.String()) + return sjson.SetBytes(obj, path, rwrString) +} + +// TransformObject transforms object value addressed by path. +func TransformObject(obj []byte, path string, transformFn func(item []byte) ([]byte, error)) ([]byte, error) { + pathObj := gjson.GetBytes(obj, path) + if !pathObj.IsObject() { + return obj, nil + } + rwrObj, err := transformFn([]byte(pathObj.Raw)) + if err != nil { + return nil, err + } + return sjson.SetRawBytes(obj, path, rwrObj) +} + +// TransformArrayOfStrings transforms array value addressed by path. +func TransformArrayOfStrings(obj []byte, arrayPath string, transformFn func(item string) string) ([]byte, error) { + // Transform each item in list. Put back original items if transformFn returns nil bytes. + items := gjson.GetBytes(obj, arrayPath).Array() + if len(items) == 0 { + return obj, nil + } + rwrItems := make([]string, len(items)) + for i, item := range items { + rwrItems[i] = transformFn(item.String()) + } + + return sjson.SetBytes(obj, arrayPath, rwrItems) +} + +// TransformPatch treats obj as a JSON patch or Merge patch and calls +// a corresponding transformFn. +func TransformPatch( + obj []byte, + transformMerge func(mergePatch []byte) ([]byte, error), + transformJSON func(jsonPatch []byte) ([]byte, error)) ([]byte, error) { + if len(obj) == 0 { + return obj, nil + } + // Merge patch for Kubernetes resource is always starts with the curly bracket. + if string(obj[0]) == "{" && transformMerge != nil { + return transformMerge(obj) + } + + // JSON patch should start with the square bracket. + if string(obj[0]) == "[" && transformJSON != nil { + return RewriteArray(obj, Root, transformJSON) + } + + // Return patch as-is in other cases. + return obj, nil +} + +// Helpers for traversing JSON objects with support for root path. +// gjson supports @this, but sjson don't, so unique alias is used. + +const Root = "@ROOT" + +func GetBytes(obj []byte, path string) gjson.Result { + if path == Root { + return gjson.ParseBytes(obj) + } + return gjson.GetBytes(obj, path) +} + +func SetBytes(obj []byte, path string, value interface{}) ([]byte, error) { + if path == Root { + return json.Marshal(value) + } + return sjson.SetBytes(obj, path, value) +} + +func SetRawBytes(obj []byte, path string, value []byte) ([]byte, error) { + if path == Root { + return value, nil + } + return sjson.SetRawBytes(obj, path, value) +} diff --git a/images/virt-artifact/patches/016-rename-install-strategy-labels.patch b/images/virt-artifact/patches/016-rename-install-strategy-labels.patch new file mode 100644 index 000000000..f035259e5 --- /dev/null +++ b/images/virt-artifact/patches/016-rename-install-strategy-labels.patch @@ -0,0 +1,31 @@ +diff --git a/staging/src/kubevirt.io/api/core/v1/types.go b/staging/src/kubevirt.io/api/core/v1/types.go +index afea9c85d..e633c3919 100644 +--- a/staging/src/kubevirt.io/api/core/v1/types.go ++++ b/staging/src/kubevirt.io/api/core/v1/types.go +@@ -828,13 +828,13 @@ const ( + ManagedByLabelOperatorValue = "virt-operator" + ManagedByLabelOperatorOldValue = "kubevirt-operator" + // This annotation represents the kubevirt version for an install strategy configmap. +- InstallStrategyVersionAnnotation = "kubevirt.io/install-strategy-version" ++ InstallStrategyVersionAnnotation = "install.internal.virtualization.deckhouse.io/install-strategy-version" + // This annotation represents the kubevirt registry used for an install strategy configmap. +- InstallStrategyRegistryAnnotation = "kubevirt.io/install-strategy-registry" ++ InstallStrategyRegistryAnnotation = "install.internal.virtualization.deckhouse.io/install-strategy-registry" + // This annotation represents the kubevirt deployment identifier used for an install strategy configmap. +- InstallStrategyIdentifierAnnotation = "kubevirt.io/install-strategy-identifier" ++ InstallStrategyIdentifierAnnotation = "install.internal.virtualization.deckhouse.io/install-strategy-identifier" + // This annotation shows the enconding used for the manifests in the Install Strategy ConfigMap. +- InstallStrategyConfigMapEncoding = "kubevirt.io/install-strategy-cm-encoding" ++ InstallStrategyConfigMapEncoding = "install.internal.virtualization.deckhouse.io/install-strategy-cm-encoding" + // This annotation is a hash of all customizations that live under spec.CustomizeComponents + KubeVirtCustomizeComponentAnnotationHash = "kubevirt.io/customizer-identifier" + // This annotation represents the kubevirt generation that was used to create a resource +@@ -845,7 +845,7 @@ const ( + EphemeralProvisioningObject string = "kubevirt.io/ephemeral-provisioning" + + // This label indicates the object is a part of the install strategy retrieval process. +- InstallStrategyLabel = "kubevirt.io/install-strategy" ++ InstallStrategyLabel = "install.internal.virtualization.deckhouse.io/install-strategy" + + // Set by virt-operator to coordinate component deletion + VirtOperatorComponentFinalizer string = "kubevirt.io/virtOperatorFinalizer" diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl index dc38399ab..f2dc14b3d 100644 --- a/templates/_helpers.tpl +++ b/templates/_helpers.tpl @@ -21,57 +21,3 @@ spec: - {{ $labelValue }} topologyKey: kubernetes.io/hostname {{- end -}} - -{{- define "kubeproxy_resources" -}} -cpu: 100m -memory: 150Mi -{{- end -}} - -{{- define "nowebhook_kubeproxy_patch" -}} - '{{ include "nowebhook_kubeproxy_patch_tmpl" . | fromYaml | toJson }}' -{{- end }} - -{{- define "nowebhook_kubeproxy_patch_tmpl" -}} - {{- $ctx := index . 0 -}} - {{- $containerName := index . 1 -}} - {{- $proxyImage := include "helm_lib_module_image" (list $ctx "kubeApiProxy") }} -metadata: - annotations: - kubectl.kubernetes.io/default-container: {{ $containerName }} -spec: - template: - spec: - volumes: - - name: kube-api-proxy-kubeconfig - configMap: - name: kube-api-proxy-kubeconfig - containers: - - name: proxy - image: {{ $proxyImage }} - imagePullPolicy: IfNotPresent - env: - - name: LOG_LEVEL - value: Debug - resources: - requests: - {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }} - {{- if not ( $ctx.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "kubeproxy_resources" . | nindent 12 }} - {{- end }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault - terminationMessagePath: /dev/termination-log - terminationMessagePolicy: File - - name: {{ $containerName }} - env: - - name: KUBECONFIG - value: /kubeconfig.local/proxy.kubeconfig - volumeMounts: - - name: kube-api-proxy-kubeconfig - mountPath: /kubeconfig.local -{{- end -}} diff --git a/templates/_kube_api_rewriter.tpl b/templates/_kube_api_rewriter.tpl new file mode 100644 index 000000000..680ba665a --- /dev/null +++ b/templates/_kube_api_rewriter.tpl @@ -0,0 +1,57 @@ +{{- define "kube_api_rewriter.env" -}} +- name: LOG_LEVEL + value: Debug +{{- end -}} + +{{- define "kubeproxy_resources" -}} +cpu: 100m +memory: 150Mi +{{- end -}} + +{{- define "nowebhook_kubeproxy_patch" -}} + '{{ include "nowebhook_kubeproxy_patch_tmpl" . | fromYaml | toJson }}' +{{- end }} + +{{- define "nowebhook_kubeproxy_patch_tmpl" -}} + {{- $ctx := index . 0 -}} + {{- $containerName := index . 1 -}} + {{- $proxyImage := include "helm_lib_module_image" (list $ctx "kubeApiProxy") }} +metadata: + annotations: + kubectl.kubernetes.io/default-container: {{ $containerName }} +spec: + template: + spec: + volumes: + - name: kube-api-proxy-kubeconfig + configMap: + name: kube-api-proxy-kubeconfig + containers: + - name: proxy + image: {{ $proxyImage }} + imagePullPolicy: IfNotPresent + env: + {{- include "kube_api_rewriter.env" . | nindent 8 }} + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 12 }} + {{- if not ( $ctx.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "kubeproxy_resources" . | nindent 12 }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + - name: {{ $containerName }} + env: + - name: KUBECONFIG + value: /kubeconfig.local/proxy.kubeconfig + volumeMounts: + - name: kube-api-proxy-kubeconfig + mountPath: /kubeconfig.local +{{- end -}} diff --git a/templates/cdi/_helpers.tpl b/templates/cdi/_helpers.tpl index 221db64ba..c33d6181b 100644 --- a/templates/cdi/_helpers.tpl +++ b/templates/cdi/_helpers.tpl @@ -38,6 +38,7 @@ spec: name: webhook-proxy protocol: TCP env: + {{- include "kube_api_rewriter.env" . | nindent 8 }} - name: WEBHOOK_ADDRESS value: "https://127.0.0.1:8443" - name: WEBHOOK_CERT_FILE diff --git a/templates/cdi/cdi-operator/deployment.yaml b/templates/cdi/cdi-operator/deployment.yaml index 791f06a5e..ddd84ca81 100644 --- a/templates/cdi/cdi-operator/deployment.yaml +++ b/templates/cdi/cdi-operator/deployment.yaml @@ -71,10 +71,14 @@ spec: metadata: labels: app: cdi-operator + annotations: + kubectl.kubernetes.io/default-container: cdi-operator spec: {{- include "helm_lib_pod_anti_affinity_for_ha" (list . (dict "app" "cdi-operator")) | nindent 6 }} containers: - name: proxy + env: + {{- include "kube_api_rewriter.env" . | nindent 8 }} image: {{ include "helm_lib_module_image" (list . "kubeApiProxy") }} imagePullPolicy: IfNotPresent securityContext: diff --git a/templates/cdi/service-monitor.yaml b/templates/cdi/service-monitor.yaml index 6404000a4..bbc93e0ff 100644 --- a/templates/cdi/service-monitor.yaml +++ b/templates/cdi/service-monitor.yaml @@ -34,5 +34,5 @@ spec: - d8-{{ .Chart.Name }} selector: matchLabels: - prometheus.cdi.kubevirt.io: "true" + prometheus.cdi.internal.virtualization.deckhouse.io: "true" {{- end }} diff --git a/templates/kubevirt/_helpers.tpl b/templates/kubevirt/_helpers.tpl index 4ac1abff3..8950fadeb 100644 --- a/templates/kubevirt/_helpers.tpl +++ b/templates/kubevirt/_helpers.tpl @@ -46,6 +46,7 @@ spec: value: /etc/virt-api/certificates/tls.crt - name: WEBHOOK_KEY_FILE value: /etc/virt-api/certificates/tls.key + {{- include "kube_api_rewriter.env" . | nindent 12 }} volumeMounts: - name: kubevirt-virt-api-certs mountPath: /etc/virt-api/certificates diff --git a/templates/kubevirt/service-monitor.yaml b/templates/kubevirt/service-monitor.yaml index 3dce7cd70..2ac3d1413 100644 --- a/templates/kubevirt/service-monitor.yaml +++ b/templates/kubevirt/service-monitor.yaml @@ -34,5 +34,5 @@ spec: - d8-{{ .Chart.Name }} selector: matchLabels: - prometheus.kubevirt.io: "true" + prometheus.kubevirt.internal.virtualization.deckhouse.io: "true" {{- end }} diff --git a/templates/kubevirt/virt-api/vpa.yaml b/templates/kubevirt/virt-api/vpa.yaml index 4da64c1ec..0dd8db613 100644 --- a/templates/kubevirt/virt-api/vpa.yaml +++ b/templates/kubevirt/virt-api/vpa.yaml @@ -5,7 +5,7 @@ kind: VerticalPodAutoscaler metadata: name: virt-api namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "kubevirt.io" "virt-api" "workload-resource-policy.deckhouse.io" "master")) | nindent 2 }} + {{- include "helm_lib_module_labels" (list . (dict "kubevirt.internal.virtualization.deckhouse.io" "virt-api" "workload-resource-policy.deckhouse.io" "master")) | nindent 2 }} spec: targetRef: apiVersion: "apps/v1" diff --git a/templates/kubevirt/virt-controller/vpa.yaml b/templates/kubevirt/virt-controller/vpa.yaml index 873595d04..ab6b178c7 100644 --- a/templates/kubevirt/virt-controller/vpa.yaml +++ b/templates/kubevirt/virt-controller/vpa.yaml @@ -5,7 +5,7 @@ kind: VerticalPodAutoscaler metadata: name: virt-controller namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "kubevirt.io" "virt-controller")) | nindent 2 }} + {{- include "helm_lib_module_labels" (list . (dict "kubevirt.internal.virtualization.deckhouse.io" "virt-controller")) | nindent 2 }} spec: targetRef: apiVersion: "apps/v1" diff --git a/templates/kubevirt/virt-handler/vpa.yaml b/templates/kubevirt/virt-handler/vpa.yaml index 6fd3db501..b8d3c48c3 100644 --- a/templates/kubevirt/virt-handler/vpa.yaml +++ b/templates/kubevirt/virt-handler/vpa.yaml @@ -5,7 +5,7 @@ kind: VerticalPodAutoscaler metadata: name: virt-handler namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "kubevirt.io" "virt-handler" "workload-resource-policy.deckhouse.io" "every-node")) | nindent 2 }} + {{- include "helm_lib_module_labels" (list . (dict "kubevirt.internal.virtualization.deckhouse.io" "virt-handler" "workload-resource-policy.deckhouse.io" "every-node")) | nindent 2 }} spec: targetRef: apiVersion: "apps/v1" diff --git a/templates/kubevirt/virt-operator/deployment.yaml b/templates/kubevirt/virt-operator/deployment.yaml index 36166118f..8d4a1e4a2 100644 --- a/templates/kubevirt/virt-operator/deployment.yaml +++ b/templates/kubevirt/virt-operator/deployment.yaml @@ -25,7 +25,7 @@ kind: VerticalPodAutoscaler metadata: name: virt-operator namespace: d8-{{ .Chart.Name }} - {{- include "helm_lib_module_labels" (list . (dict "kubevirt.io" "virt-operator" "workload-resource-policy.deckhouse.io" "master")) | nindent 2 }} + {{- include "helm_lib_module_labels" (list . (dict "kubevirt.internal.virtualization.deckhouse.io" "virt-operator" "workload-resource-policy.deckhouse.io" "master")) | nindent 2 }} spec: targetRef: apiVersion: "apps/v1" @@ -69,12 +69,14 @@ spec: selector: matchLabels: app: virt-operator - kubevirt.io: virt-operator + kubevirt.internal.virtualization.deckhouse.io: virt-operator template: metadata: + annotations: + kubectl.kubernetes.io/default-container: virt-operator labels: app: virt-operator - kubevirt.io: virt-operator + kubevirt.internal.virtualization.deckhouse.io: virt-operator name: virt-operator spec: affinity: @@ -83,7 +85,7 @@ spec: - podAffinityTerm: labelSelector: matchExpressions: - - key: kubevirt.io + - key: kubevirt.internal.virtualization.deckhouse.io operator: In values: - virt-operator @@ -119,6 +121,7 @@ spec: value: "/etc/virt-operator/certificates/tls.crt" - name: WEBHOOK_KEY_FILE value: "/etc/virt-operator/certificates/tls.key" + {{- include "kube_api_rewriter.env" . | nindent 10 }} volumeMounts: - mountPath: /etc/virt-operator/certificates name: kubevirt-operator-certs