Skip to content

Commit

Permalink
refactor(kube-api-rewriter): rewrite labels, annotations and finalizers
Browse files Browse the repository at this point in the history
- Add KubeVirt and CDI rules for labels, annotations and finalizers.
- Add rewriter for labelSelector in queries.
- Add rewriters for affinity, labelSelectors in different resources.
- Add metadata patch rewriters: nodes, services, deployments, ds, ...
- Add webhooks patch rewriters: validatingwebhookconfigurations, mutatingwebhookconfigurations
- Add more resources for rewrite: jobs, services, deployments, sts, ds, pvc, pod, ...
- Enable rewrite for 3rd party resources: servicemonitors, prometheusrules, ...
- Rename labels in our templates. Add kubectl.kubernetes.io/default-container annotation.
- Add virt-operator patch to rename install-strategy labels: there is no way to rewrite these labels with kube-api-rewriter.
- Simplify some Rewrite* methods using transformers
- Create PrefixedNameRewriter to hold original-renamed indexes.

Signed-off-by: Ivan Mikheykin <[email protected]>
  • Loading branch information
diafour committed Jun 17, 2024
1 parent 6bd3bae commit 05da3c9
Show file tree
Hide file tree
Showing 32 changed files with 1,462 additions and 416 deletions.
2 changes: 1 addition & 1 deletion images/kube-api-proxy/cmd/kube-api-proxy/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ func main() {
}
rewriteRules = rulesFromFile
}
rewriteRules.Complete()
rewriteRules.Init()

proxies := make([]*server.HTTPServer, 0)

Expand Down
56 changes: 50 additions & 6 deletions images/kube-api-proxy/pkg/kubevirt/kubevirt_rules.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ import (
. "kube-api-proxy/pkg/rewriter"
)

const (
internalPrefix = "internal.virtualization.deckhouse.io"
nodePrefix = "node.virtualization.deckhouse.io"
dvpPrefix = "virtualization.deckhouse.io"
)

var KubevirtRewriteRules = &RewriteRules{
KindPrefix: "DVPInternal", // KV
ResourceTypePrefix: "dvpinternal", // kv
Expand All @@ -30,16 +36,54 @@ var KubevirtRewriteRules = &RewriteRules{
Webhooks: KubevirtWebhooks,
Labels: MetadataReplace{
Names: []MetadataReplaceRule{
{Old: "cdi.kubevirt.io", New: "cdi.internal.virtualization.deckhouse.io"},
{Old: "kubevirt.io", New: "kubevirt.internal.virtualization.deckhouse.io"},
{Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix},
{Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix},
{Original: "prometheus.kubevirt.io", Renamed: "prometheus.kubevirt." + internalPrefix},
{Original: "prometheus.cdi.kubevirt.io", Renamed: "prometheus.cdi." + internalPrefix},
// Special cases.
{Original: "node-labeller.kubevirt.io/skip-node", Renamed: "node-labeller." + dvpPrefix + "/skip-node"},
{Original: "node-labeller.kubevirt.io/obsolete-host-model", Renamed: "node-labeller." + internalPrefix + "/obsolete-host-model"},
},
Prefixes: []MetadataReplaceRule{
// CDI related labels.
{Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix},
{Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix},
{Original: "prometheus.cdi.kubevirt.io", Renamed: "prometheus.cdi." + internalPrefix},
{Original: "upload.cdi.kubevirt.io", Renamed: "upload.cdi." + internalPrefix},
// KubeVirt related labels.
{Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix},
{Original: "prometheus.kubevirt.io", Renamed: "prometheus.kubevirt." + internalPrefix},
{Original: "operator.kubevirt.io", Renamed: "operator.kubevirt." + internalPrefix},
{Original: "vm.kubevirt.io", Renamed: "vm.kubevirt." + internalPrefix},
// Node features related labels.
// Note: these labels are not "internal".
{Original: "cpu-feature.node.kubevirt.io", Renamed: "cpu-feature." + nodePrefix},
{Original: "cpu-model-migration.node.kubevirt.io", Renamed: "cpu-model-migration." + nodePrefix},
{Original: "cpu-model.node.kubevirt.io", Renamed: "cpu-model." + nodePrefix},
{Original: "cpu-timer.node.kubevirt.io", Renamed: "cpu-timer." + nodePrefix},
{Original: "cpu-vendor.node.kubevirt.io", Renamed: "cpu-vendor." + nodePrefix},
{Original: "scheduling.node.kubevirt.io", Renamed: "scheduling." + nodePrefix},
{Original: "host-model-cpu.node.kubevirt.io", Renamed: "host-model-cpu." + nodePrefix},
{Original: "host-model-required-features.node.kubevirt.io", Renamed: "host-model-required-features." + nodePrefix},
{Original: "hyperv.node.kubevirt.io", Renamed: "hyperv." + nodePrefix},
},
},
Annotations: MetadataReplace{
Prefixes: []MetadataReplaceRule{
// CDI related annotations.
{Original: "cdi.kubevirt.io", Renamed: "cdi." + internalPrefix},
{Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix},
// KubeVirt related annotations.
{Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix},
{Original: "certificates.kubevirt.io", Renamed: "certificates.kubevirt." + internalPrefix},
},
},
Finalizers: MetadataReplace{
Prefixes: []MetadataReplaceRule{
{Old: "cdi.kubevirt.io", New: "cdi.internal.virtualization.deckhouse.io"},
{Old: "upload.cdi.kubevirt.io", New: "upload.cdi.internal.virtualization.deckhouse.io"},
{Old: "kubevirt.io", New: "kubevirt.internal.virtualization.deckhouse.io"},
{Original: "kubevirt.io", Renamed: "kubevirt." + internalPrefix},
{Original: "operator.cdi.kubevirt.io", Renamed: "operator.cdi." + internalPrefix},
},
},
Annotations: MetadataReplace{},
}

// TODO create generator in golang to produce below rules from Kubevirt and CDI sources so proxy can work with future versions.
Expand Down
32 changes: 32 additions & 0 deletions images/kube-api-proxy/pkg/rewriter/3rdparty.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
Copyright 2024 Flant JSC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rewriter

// Rewrite routines for 3rd party resources, i.e. ServiceMonitor.

const (
PrometheusRuleKind = "PrometheusRule"
PrometheusRuleListKind = "PrometheusRuleList"
ServiceMonitorKind = "ServiceMonitor"
ServiceMonitorListKind = "ServiceMonitorList"
)

func RewriteServiceMonitorOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
return TransformObject(obj, "spec.selector", func(obj []byte) ([]byte, error) {
return rewriteLabelSelector(rules, obj, action)
})
}
27 changes: 27 additions & 0 deletions images/kube-api-proxy/pkg/rewriter/admission_configuration.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ limitations under the License.

package rewriter

import "github.com/tidwall/gjson"

const (
ValidatingWebhookConfigurationKind = "ValidatingWebhookConfiguration"
ValidatingWebhookConfigurationListKind = "ValidatingWebhookConfigurationList"
Expand Down Expand Up @@ -60,3 +62,28 @@ func RewriteMutatingOrList(rules *RewriteRules, obj []byte, action Action) ([]by
})
})
}

func RenameWebhookConfigurationPatch(rules *RewriteRules, obj []byte) ([]byte, error) {
obj, err := RenameMetadataPatch(rules, obj)
if err != nil {
return nil, err
}

return TransformPatch(obj, func(mergePatch []byte) ([]byte, error) {
return RewriteArray(mergePatch, "webhooks", func(webhook []byte) ([]byte, error) {
return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
return restoreRoleRule(rules, item)
})
})
}, func(jsonPatch []byte) ([]byte, error) {
path := gjson.GetBytes(jsonPatch, "path").String()
if path == "/webhooks" {
return RewriteArray(jsonPatch, "value", func(webhook []byte) ([]byte, error) {
return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
return renameRoleRule(rules, item)
})
})
}
return jsonPatch, nil
})
}
151 changes: 151 additions & 0 deletions images/kube-api-proxy/pkg/rewriter/affinity.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
/*
Copyright 2024 Flant JSC
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rewriter

// RewriteAffinity renames or restores labels in labelSelector of affinity structure.
// See https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity
func RewriteAffinity(rules *RewriteRules, obj []byte, path string, action Action) ([]byte, error) {
return TransformObject(obj, path, func(affinity []byte) ([]byte, error) {
rwrAffinity, err := TransformObject(affinity, "nodeAffinity", func(item []byte) ([]byte, error) {
return rewriteNodeAffinity(rules, item, action)
})
if err != nil {
return nil, err
}

rwrAffinity, err = TransformObject(rwrAffinity, "podAffinity", func(item []byte) ([]byte, error) {
return rewritePodAffinity(rules, item, action)
})
if err != nil {
return nil, err
}

return TransformObject(rwrAffinity, "podAntiAffinity", func(item []byte) ([]byte, error) {
return rewritePodAffinity(rules, item, action)
})

})
}

// rewriteNodeAffinity rewrites labels in nodeAffinity structure.
// nodeAffinity:
//
// requiredDuringSchedulingIgnoredDuringExecution:
// nodeSelectorTerms []NodeSelector -> rewrite each item: key in each matchExpressions and matchFields
// preferredDuringSchedulingIgnoredDuringExecution: -> array of PreferredSchedulingTerm:
// preference NodeSelector -> rewrite key in each matchExpressions and matchFields
// weight:
func rewriteNodeAffinity(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
// Rewrite an array of nodeSelectorTerms in requiredDuringSchedulingIgnoredDuringExecution field.
var err error
obj, err = TransformObject(obj, "requiredDuringSchedulingIgnoredDuringExecution", func(affinityTerm []byte) ([]byte, error) {
return RewriteArray(affinityTerm, "nodeSelectorTerms", func(item []byte) ([]byte, error) {
return rewriteNodeSelectorTerm(rules, item, action)
})
})
if err != nil {
return nil, err
}

// Rewrite an array of weightedNodeSelectorTerms in preferredDuringSchedulingIgnoredDuringExecution field.
return RewriteArray(obj, "preferredDuringSchedulingIgnoredDuringExecution", func(item []byte) ([]byte, error) {
return TransformObject(item, "preference", func(preference []byte) ([]byte, error) {
return rewriteNodeSelectorTerm(rules, preference, action)
})
})
}

// rewriteNodeSelectorTerm renames or restores key fields in matchLabels or matchExpressions of NodeSelectorTerm.
func rewriteNodeSelectorTerm(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
obj, err := RewriteArray(obj, "matchLabels", func(item []byte) ([]byte, error) {
return rewriteSelectorRequirement(rules, item, action)
})
if err != nil {
return nil, err
}
return RewriteArray(obj, "matchExpressions", func(item []byte) ([]byte, error) {
return rewriteSelectorRequirement(rules, item, action)
})
}

func rewriteSelectorRequirement(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
return TransformString(obj, "key", func(field string) string {
return rules.LabelsRewriter().Rewrite(field, action)
})
}

// rewritePodAffinity rewrites PodAffinity and PodAntiAffinity structures.
// PodAffinity and PodAntiAffinity structures are the same:
//
// requiredDuringSchedulingIgnoredDuringExecution -> array of PodAffinityTerm structures:
// labelSelector:
// matchLabels -> rewrite map
// matchExpressions -> rewrite key in each item
// topologyKey -> rewrite as label name
// namespaceSelector -> rewrite as labelSelector
// preferredDuringSchedulingIgnoredDuringExecution -> array of WeightedPodAffinityTerm:
// weight
// podAffinityTerm PodAffinityTerm -> rewrite as described above
func rewritePodAffinity(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
// Rewrite an array of PodAffinityTerms in requiredDuringSchedulingIgnoredDuringExecution field.
obj, err := RewriteArray(obj, "requiredDuringSchedulingIgnoredDuringExecution", func(affinityTerm []byte) ([]byte, error) {
return rewritePodAffinityTerm(rules, affinityTerm, action)
})
if err != nil {
return nil, err
}

// Rewrite an array of WeightedPodAffinityTerms in requiredDuringSchedulingIgnoredDuringExecution field.
return RewriteArray(obj, "preferredDuringSchedulingIgnoredDuringExecution", func(affinityTerm []byte) ([]byte, error) {
return TransformObject(affinityTerm, "podAffinityTerm", func(podAffinityTerm []byte) ([]byte, error) {
return rewritePodAffinityTerm(rules, podAffinityTerm, action)
})
})
}

func rewritePodAffinityTerm(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
obj, err := TransformObject(obj, "labelSelector", func(labelSelector []byte) ([]byte, error) {
return rewriteLabelSelector(rules, labelSelector, action)
})
if err != nil {
return nil, err
}

obj, err = TransformString(obj, "topologyKey", func(field string) string {
return rules.LabelsRewriter().Rewrite(field, action)
})
if err != nil {
return nil, err
}

return TransformObject(obj, "namespaceSelector", func(selector []byte) ([]byte, error) {
return rewriteLabelSelector(rules, selector, action)
})
}

// rewriteLabelSelector rewrites matchLabels and matchExpressions. It is similar to rewriteNodeSelectorTerm
// but matchLabels is a map here, not an array of requirements.
func rewriteLabelSelector(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
obj, err := RewriteLabelsMap(rules, obj, "matchLabels", action)
if err != nil {
return nil, err
}

return RewriteArray(obj, "matchExpressions", func(item []byte) ([]byte, error) {
return rewriteSelectorRequirement(rules, item, action)
})
}
Loading

0 comments on commit 05da3c9

Please sign in to comment.