fix(kube-api-rewriter): fix discovery and ValidatingAdmissionPolicy r…

…ewrite (#475) Fix array construction for discovery rewrite in v1.30. Add rewriting for ValidatingAdmissionPolicy and ValidatingAdmissionPolicyBinding. --------- Signed-off-by: yaroslavborbat <[email protected]>
deckhouse · Oct 28, 2024 · b429542 · b429542
1 parent 56a26c6
commit b429542
Show file tree

Hide file tree

Showing 10 changed files with 183 additions and 69 deletions.
diff --git a/images/kube-api-proxy/Taskfile.dist.yaml b/images/kube-api-proxy/Taskfile.dist.yaml
@@ -8,7 +8,7 @@ includes:
     optional: true
 
 vars:
-  DevImage: "localhost:5000/$USER/kube-api-proxy:latest"
+  DevImage: "${DevImage:-localhost:5000/$USER/kube-api-proxy:latest}"
 
 tasks:
   default:
@@ -23,14 +23,42 @@ tasks:
 
   dev:deploy:
     desc: "apply manifest with proxy and test-controller"
+    cmds:
+      - task: dev:__deploy
+        vars:
+          CTR_COMMAND: "['./proxy']"
+
+  dev:deploy-with-dlv:
+    desc: "apply manifest with proxy with dlv and test-controller"
+    cmds:
+      - task: dev:__deploy
+        vars:
+          CTR_COMMAND: "['./dlv', '--listen=:2345', '--headless=true', '--continue', '--log=true', '--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc', '--accept-multiclient', '--api-version=2', 'exec', './proxy']"
+
+  dev:__deploy:
+    internal: true
     cmds:
       - |
         if ! kubectl get no 2>&1 >/dev/null ; then
           echo Restart cluster connection
           exit 1
         fi
       - |
-        cat local/proxy.yaml | IMAGE={{.DevImage}} envsubst | kubectl -n kproxy apply -f -
+        kubectl get ns kproxy &>/dev/null || kubectl create ns kproxy
+        kubectl apply -f - <<EOF
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: deckhouse-registry
+          namespace: kproxy
+        type: $(kubectl get secrets deckhouse-registry -n d8-system -ojsonpath='{.type}')
+        data: $(kubectl get secrets deckhouse-registry -n d8-system -ojsonpath='{.data}')
+        EOF
+        cat local/proxy.yaml | CTR_COMMAND="{{.CTR_COMMAND}}" IMAGE="{{.DevImage}}" envsubst | kubectl -n kproxy apply -f -
+
+  dev:undeploy:
+    desc: "delete manifest with proxy and test-controller"
+    cmd: kubectl delete ns kproxy
 
   dev:restart:
     desc: "restart deployment"

diff --git a/images/kube-api-proxy/local/Dockerfile b/images/kube-api-proxy/local/Dockerfile
@@ -1,5 +1,7 @@
 # Go builder.
-FROM golang:1.21-alpine3.19 AS builder
+FROM golang:1.22-alpine3.19 AS builder
+
+RUN go install github.com/go-delve/delve/cmd/dlv@latest
 
 # Cache-friendly download of go dependencies.
 ADD go.mod go.sum /app/
@@ -12,7 +14,7 @@ RUN GOOS=linux \
     go build -o proxy ./cmd/kube-api-proxy
 
 # Go builder.
-FROM golang:1.21-alpine3.19 AS builder-test-controller
+FROM golang:1.22-alpine3.19 AS builder-test-controller
 
 # Cache-friendly download of go dependencies.
 ADD local/test-controller/go.mod local/test-controller/go.sum /app/
@@ -28,8 +30,9 @@ FROM alpine:3.19
 RUN apk --no-cache add ca-certificates bash sed tini curl && \
     kubectlArch=linux/amd64 && \
     echo "Download kubectl for ${kubectlArch}" && \
-    wget https://storage.googleapis.com/kubernetes-release/release/v1.27.5/bin/${kubectlArch}/kubectl -O /bin/kubectl && \
+    wget https://storage.googleapis.com/kubernetes-release/release/v1.30.0/bin/${kubectlArch}/kubectl -O /bin/kubectl && \
     chmod +x /bin/kubectl
+COPY --from=builder /go/bin/dlv /
 COPY --from=builder /app/proxy /
 COPY --from=builder-test-controller /app/test-controller /
 ADD local/proxy.kubeconfig /

diff --git a/images/kube-api-proxy/local/proxy.yaml b/images/kube-api-proxy/local/proxy.yaml
@@ -23,19 +23,10 @@ spec:
       imagePullSecrets:
         - name: "deckhouse-registry"
       containers:
-        - name: proxy-only
-          image: "${IMAGE}"
-          imagePullPolicy: Always
-          command:
-            - /proxy
-          env:
-            - name: CLIENT_PROXY_PORT
-              value: "23916"
         - name: proxy
           image: "${IMAGE}"
           imagePullPolicy: Always
-          command:
-            - /proxy
+          command: ${CTR_COMMAND}
           env:
             - name: WEBHOOK_ADDRESS
               value: "https://127.0.0.1:9443"

diff --git a/images/kube-api-proxy/pkg/rewriter/admission_configuration.go b/images/kube-api-proxy/pkg/rewriter/admission_configuration.go
@@ -30,15 +30,15 @@ func RewriteValidatingOrList(rules *RewriteRules, obj []byte, action Action) ([]
 		return RewriteResourceOrList(obj, ValidatingWebhookConfigurationListKind, func(singleObj []byte) ([]byte, error) {
 			return RewriteArray(singleObj, "webhooks", func(webhook []byte) ([]byte, error) {
 				return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
-					return renameRoleRule(rules, item)
+					return RenameResourceRule(rules, item)
 				})
 			})
 		})
 	}
 	return RewriteResourceOrList(obj, ValidatingWebhookConfigurationListKind, func(singleObj []byte) ([]byte, error) {
 		return RewriteArray(singleObj, "webhooks", func(webhook []byte) ([]byte, error) {
 			return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
-				return restoreRoleRule(rules, item)
+				return RestoreResourceRule(rules, item)
 			})
 		})
 	})
@@ -49,15 +49,15 @@ func RewriteMutatingOrList(rules *RewriteRules, obj []byte, action Action) ([]by
 		return RewriteResourceOrList(obj, MutatingWebhookConfigurationListKind, func(singleObj []byte) ([]byte, error) {
 			return RewriteArray(singleObj, "webhooks", func(webhook []byte) ([]byte, error) {
 				return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
-					return renameRoleRule(rules, item)
+					return RenameResourceRule(rules, item)
 				})
 			})
 		})
 	}
 	return RewriteResourceOrList(obj, MutatingWebhookConfigurationListKind, func(singleObj []byte) ([]byte, error) {
 		return RewriteArray(singleObj, "webhooks", func(webhook []byte) ([]byte, error) {
 			return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
-				return restoreRoleRule(rules, item)
+				return RestoreResourceRule(rules, item)
 			})
 		})
 	})
@@ -72,15 +72,15 @@ func RenameWebhookConfigurationPatch(rules *RewriteRules, obj []byte) ([]byte, e
 	return TransformPatch(obj, func(mergePatch []byte) ([]byte, error) {
 		return RewriteArray(mergePatch, "webhooks", func(webhook []byte) ([]byte, error) {
 			return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
-				return restoreRoleRule(rules, item)
+				return RestoreResourceRule(rules, item)
 			})
 		})
 	}, func(jsonPatch []byte) ([]byte, error) {
 		path := gjson.GetBytes(jsonPatch, "path").String()
 		if path == "/webhooks" {
 			return RewriteArray(jsonPatch, "value", func(webhook []byte) ([]byte, error) {
 				return RewriteArray(webhook, "rules", func(item []byte) ([]byte, error) {
-					return renameRoleRule(rules, item)
+					return RenameResourceRule(rules, item)
 				})
 			})
 		}

diff --git a/images/kube-api-proxy/pkg/rewriter/admission_policy.go b/images/kube-api-proxy/pkg/rewriter/admission_policy.go
@@ -0,0 +1,67 @@
+/*
+Copyright 2024 Flant JSC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package rewriter
+
+const (
+	ValidatingAdmissionPolicyKind            = "ValidatingAdmissionPolicy"
+	ValidatingAdmissionPolicyListKind        = "ValidatingAdmissionPolicyList"
+	ValidatingAdmissionPolicyBindingKind     = "ValidatingAdmissionPolicyBinding"
+	ValidatingAdmissionPolicyBindingListKind = "ValidatingAdmissionPolicyBindingList"
+)
+
+// renames apiGroups and resources in a single resourceRule.
+// Rule examples:
+//	resourceRules:
+//	- apiGroups:
+//	    - ""
+//	  apiVersions:
+//      - '*'
+//    operations:
+//      - '*'
+//    resources:
+//      - nodes
+//    scope: '*'
+
+func RewriteValidatingAdmissionPolicyOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
+	if action == Rename {
+		return RewriteResourceOrList(obj, ValidatingAdmissionPolicyListKind, func(singleObj []byte) ([]byte, error) {
+			return RewriteArray(singleObj, "spec.matchConstraints.resourceRules", func(item []byte) ([]byte, error) {
+				return RenameResourceRule(rules, item)
+			})
+		})
+	}
+	return RewriteResourceOrList(obj, ValidatingAdmissionPolicyListKind, func(singleObj []byte) ([]byte, error) {
+		return RewriteArray(singleObj, "spec.matchConstraints.resourceRules", func(item []byte) ([]byte, error) {
+			return RestoreResourceRule(rules, item)
+		})
+	})
+}
+
+func RewriteValidatingAdmissionPolicyBindingOrList(rules *RewriteRules, obj []byte, action Action) ([]byte, error) {
+	if action == Rename {
+		return RewriteResourceOrList(obj, ValidatingAdmissionPolicyBindingListKind, func(singleObj []byte) ([]byte, error) {
+			return RewriteArray(singleObj, "spec.matchResources.resourceRules", func(item []byte) ([]byte, error) {
+				return RenameResourceRule(rules, item)
+			})
+		})
+	}
+	return RewriteResourceOrList(obj, ValidatingAdmissionPolicyBindingListKind, func(singleObj []byte) ([]byte, error) {
+		return RewriteArray(singleObj, "spec.matchResources.resourceRules", func(item []byte) ([]byte, error) {
+			return RestoreResourceRule(rules, item)
+		})
+	})
+}
diff --git a/images/kube-api-proxy/pkg/rewriter/discovery.go b/images/kube-api-proxy/pkg/rewriter/discovery.go
@@ -268,15 +268,57 @@ func RewriteAPIResourceList(rules *RewriteRules, obj []byte) ([]byte, error) {
 //
 // NOTE: Can't use RewriteArray here, because one APIGroupDiscovery with renamed
 // resource produces many APIGroupDiscovery objects with restored resource.
+
+func newSliceBytesBuilder() *sliceBytesBuilder {
+	return &sliceBytesBuilder{
+		buf: bytes.NewBuffer([]byte("[")),
+	}
+}
+
+type sliceBytesBuilder struct {
+	buf   *bytes.Buffer
+	begin bool
+}
+
+func (b *sliceBytesBuilder) WriteString(s string) {
+	if s == "" {
+		return
+	}
+	if b.begin {
+		b.buf.WriteString(",")
+	}
+	b.buf.WriteString(s)
+	b.begin = true
+}
+
+func (b *sliceBytesBuilder) Write(bytes []byte) {
+	if len(bytes) == 0 {
+		return
+	}
+	if b.begin {
+		b.buf.WriteString(",")
+	}
+	b.buf.Write(bytes)
+	b.begin = true
+}
+
+func (b *sliceBytesBuilder) Complete() *sliceBytesBuilder {
+	b.buf.WriteString("]")
+	return b
+}
+
+func (b *sliceBytesBuilder) Bytes() []byte {
+	return b.buf.Bytes()
+}
+
 func RewriteAPIGroupDiscoveryList(rules *RewriteRules, obj []byte) ([]byte, error) {
 	items := gjson.GetBytes(obj, "items").Array()
 	if len(items) == 0 {
 		return obj, nil
 	}
 
-	var rwrItems bytes.Buffer
-	rwrItems.WriteString("[")
-	first := true
+	rwrItems := newSliceBytesBuilder()
+
 	for _, item := range items {
 
 		itemBytes := []byte(item.Raw)
@@ -291,11 +333,6 @@ func RewriteAPIGroupDiscoveryList(rules *RewriteRules, obj []byte) ([]byte, erro
 			}
 
 			// No transform for non-renamed groups, add as-is.
-			if first {
-				first = false
-			} else {
-				rwrItems.WriteString(",")
-			}
 			rwrItems.Write(itemBytes)
 			continue
 		}
@@ -305,28 +342,16 @@ func RewriteAPIGroupDiscoveryList(rules *RewriteRules, obj []byte) ([]byte, erro
 			return nil, err
 		}
 		if newItems == nil {
-			// No transform for nil result.
-			if first {
-				first = false
-			} else {
-				rwrItems.WriteString(",")
-			}
 			rwrItems.Write(itemBytes)
 		} else {
 			// Replace renamed group with restored groups.
 			for _, newItem := range newItems {
-				if first {
-					first = false
-				} else {
-					rwrItems.WriteString(",")
-				}
 				rwrItems.Write(newItem)
 			}
 		}
 	}
 
-	rwrItems.WriteString("]")
-	return sjson.SetRawBytes(obj, "items", rwrItems.Bytes())
+	return sjson.SetRawBytes(obj, "items", rwrItems.Complete().Bytes())
 }
 
 // RestoreAggregatedGroupDiscovery returns an array of APIGroupDiscovery objects with restored resources.
@@ -405,31 +430,20 @@ func RestoreAggregatedGroupDiscovery(rules *RewriteRules, obj []byte) ([][]byte,
 		restoredGroupObj := []byte(fmt.Sprintf(`{"metadata":{"name":"%s", "creationTimestamp":null}}`, groupName))
 
 		// Construct an array of APIVersionDiscovery objects.
-		var restoredVersions bytes.Buffer
-		restoredVersions.WriteString("[")
-		first := true
+		restoredVersions := newSliceBytesBuilder()
 		for versionName, versionResources := range groupVersions {
-			if first {
-				first = false
-			} else {
-				restoredVersions.WriteString(",")
-			}
 			// Init restored APIVersionDiscovery object.
 			restoredVersionObj := []byte(fmt.Sprintf(`{"version":"%s"}`, versionName))
 
 			// Construct an array of APIResourceDiscovery objects.
 			{
-				var restoredVersionResources bytes.Buffer
-				restoredVersionResources.WriteString("[")
-				for i, resource := range versionResources {
-					if i > 0 {
-						restoredVersionResources.WriteString(",")
-					}
+
+				restoredVersionResources := newSliceBytesBuilder()
+				for _, resource := range versionResources {
 					restoredVersionResources.Write(resource)
 				}
-				restoredVersionResources.WriteString("]")
 				// Set resources field.
-				restoredVersionObj, err = sjson.SetRawBytes(restoredVersionObj, "resources", restoredVersionResources.Bytes())
+				restoredVersionObj, err = sjson.SetRawBytes(restoredVersionObj, "resources", restoredVersionResources.Complete().Bytes())
 				if err != nil {
 					return nil, err
 				}
@@ -438,8 +452,7 @@ func RestoreAggregatedGroupDiscovery(rules *RewriteRules, obj []byte) ([][]byte,
 			// Append restored APIVersionDiscovery object.
 			restoredVersions.Write(restoredVersionObj)
 		}
-		restoredVersions.WriteString("]")
-		restoredGroupObj, err := sjson.SetRawBytes(restoredGroupObj, "versions", restoredVersions.Bytes())
+		restoredGroupObj, err := sjson.SetRawBytes(restoredGroupObj, "versions", restoredVersions.Complete().Bytes())
 		if err != nil {
 			return nil, err
 		}