From b52af90f28ffbe2d02655ca326b135e4a4a88e81 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Fri, 15 Nov 2024 11:32:21 +0100 Subject: [PATCH 1/7] Ignition service for bootstrapping kubeadm and kubelet. --- config/samples/example-kubeadm.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/samples/example-kubeadm.yaml b/config/samples/example-kubeadm.yaml index 8642605..f8c6366 100644 --- a/config/samples/example-kubeadm.yaml +++ b/config/samples/example-kubeadm.yaml @@ -27,7 +27,7 @@ spec: size: v1-small-x86 image: firewall-ubuntu-3.0 networks: - - internet-mini-lab + - internet-mini-lab --- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 kind: MetalStackMachineTemplate From 1276a3ee056d70509e77ae82296a4b6984cc3f74 Mon Sep 17 00:00:00 2001 From: Valentin Knabel Date: Thu, 21 Nov 2024 12:51:20 +0100 Subject: [PATCH 2/7] feat: firewall controller manager --- .../defaults/main.yaml | 17 + .../firewall-controller-manager.yaml | 371 ++++++++++++++++++ 2 files changed, 388 insertions(+) create mode 100644 capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml diff --git a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml index 6bb9002..4644619 100644 --- a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml @@ -1,2 +1,19 @@ --- firewall_controller_manager_namespace: "firewall-controller-manager" + +firewall_controller_manager_image: ghcr.io/metal-stack/firewall-controller-manager +firewall_controller_manager_image_pull_policy: Always +firewall_controller_manager_replicas: 1 +# firewall_controller_manager_pod_annotations: + +firewall_controller_manager_seed_api_url: +firewall_controller_manager_shoot_api_url: +firewall_controller_manager_cluster_id: +firewall_controller_manager_metalapi_url: +firewall_controller_manager_generic_token_kubeconfig_secret_name: +firewall_controller_manager_ssh_key_secret_name: + +firewall_controller_manager_shoot_access_token_secret: "shoot-access-firewall-controller-manager" + +firewall_controller_manager_secrets_server: +firewall_controller_manager_ca_bundle: diff --git a/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml b/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml new file mode 100644 index 0000000..9b17a1e --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml @@ -0,0 +1,371 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} +rules: +- apiGroups: + - firewall.metal-stack.io + resources: + - firewalls + - firewalls/status + - firewallsets + - firewallsets/status + - firewalldeployments + - firewalldeployments/status + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - "" + resources: + - secrets + - serviceaccounts + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - extensions.gardener.cloud + resources: + - infrastructures + - extensions + verbs: + - get +- apiGroups: + - extensions.gardener.cloud + resources: + - infrastructures/status + verbs: + - patch +- apiGroups: + - extensions.gardener.cloud + resources: + - extensions + verbs: + - update +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: firewall-controller-manager +subjects: +- kind: ServiceAccount + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + labels: + app: firewall-controller-manager +spec: + selector: + matchLabels: + app: firewall-controller-manager + replicas: {{ firewall_controller_manager_replicas }} + template: + metadata: + labels: + app: firewall-controller-manager + networking.gardener.cloud/from-prometheus: "allowed" + networking.gardener.cloud/to-dns: "allowed" + networking.gardener.cloud/to-public-networks: "allowed" + networking.gardener.cloud/to-private-networks: "allowed" + networking.gardener.cloud/to-shoot-apiserver: "allowed" + networking.gardener.cloud/to-runtime-apiserver: "allowed" + networking.resources.gardener.cloud/to-kube-apiserver-tcp-443: "allowed" +{% if firewall_controller_manager_pod_annotations %} + annotations: +{{ firewall_controller_manager_pod_annotations | to_nice_yaml | indent(width=8, first=true) }} +{% end %} + spec: + serviceAccountName: firewall-controller-manager + containers: + - name: firewall-controller-manager + image: {{ firewall_controller_manager_image }} + imagePullPolicy: {{ firewall_controller_manager_image_pull_policy }} + args: + - -cert-dir=/certs + - -log-level=info + - -seed-api-url={{ firewall_controller_manager_seed_api_url }} + - -shoot-api-url={{ firewall_controller_manager_shootapiURL }} + - -internal-shoot-api-url=https://kube-apiserver + - -cluster-id={{ firewall_controller_manager_clusterID }} + - -enable-leader-election + - -metal-api-url={{ firewall_controller_manager_metalapi_url }} + - -namespace={{ firewall_controller_manager_namespace }} + - -shoot-kubeconfig-secret-name={{ firewall_controller_manager_generic_token_kubeconfig_secret_name }} + - -shoot-token-secret-name=shoot-access-firewall-controller-manager + - -ssh-key-secret-name={{ firewall_controller_manager_ssh_key_secret_name }} + - -shoot-token-path=/token + env: + - name: METAL_AUTH_HMAC + valueFrom: + secretKeyRef: + name: cloudprovider + key: metalAPIHMac + livenessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: webhook-certs + mountPath: "/certs" + readOnly: true + - name: token-dir + mountPath: "/token" + resources: + limits: + cpu: 400m + memory: 400Mi + requests: + cpu: 100m + memory: 20Mi + volumes: + - name: webhook-certs + secret: + secretName: {{ firewall_controller_manager_secrets_server }} + - name: token-dir + emptyDir: {} +--- +apiVersion: v1 +kind: Service +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + labels: + app: firewall-controller-manager + annotations: + networking.resources.gardener.cloud/from-world-to-ports: '[{"protocol":"TCP","port":9443}]' + networking.resources.gardener.cloud/from-all-webhook-targets-allowed-ports: '[{"protocol":"TCP","port":9443}]' + networking.resources.gardener.cloud/from-all-seed-scrape-targets-allowed-ports: '[{"protocol":"TCP","port":2112}]' +spec: + type: ClusterIP + ports: + - name: webhooks + port: 9443 + protocol: TCP + - name: metrics + port: 2112 + protocol: TCP + selector: + app: firewall-controller-manager +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewall + failurePolicy: Fail + name: firewall.metal-stack.io + objectSelector: + matchLabels: + gardener-shoot-namespace: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewalls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_caBundle | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewallset + failurePolicy: Fail + name: firewallset.metal-stack.io + objectSelector: + matchLabels: + gardener-shoot-namespace: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewallsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewalldeployment + failurePolicy: Fail + name: firewalldeployment.metal-stack.io + objectSelector: + matchLabels: + gardener-shoot-namespace: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewalldeployments + sideEffects: None +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewall + failurePolicy: Fail + name: firewall.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewalls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewallset + failurePolicy: Fail + name: firewallset.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewallsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewalldeployment + failurePolicy: Fail + name: firewalldeployment.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewalldeployments + sideEffects: None + From 27c7cefface1cc4ccba2e4b659b2f2411d3c13e8 Mon Sep 17 00:00:00 2001 From: Valentin Knabel Date: Thu, 21 Nov 2024 13:02:19 +0100 Subject: [PATCH 3/7] fix: camel case in variable names --- .../templates/firewall-controller-manager.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml b/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml index 9b17a1e..f9a1bb7 100644 --- a/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml +++ b/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml @@ -139,9 +139,9 @@ spec: - -cert-dir=/certs - -log-level=info - -seed-api-url={{ firewall_controller_manager_seed_api_url }} - - -shoot-api-url={{ firewall_controller_manager_shootapiURL }} + - -shoot-api-url={{ firewall_controller_manager_shoot_api_url }} - -internal-shoot-api-url=https://kube-apiserver - - -cluster-id={{ firewall_controller_manager_clusterID }} + - -cluster-id={{ firewall_controller_manager_cluster_id }} - -enable-leader-election - -metal-api-url={{ firewall_controller_manager_metalapi_url }} - -namespace={{ firewall_controller_manager_namespace }} From 71d4a8af97da5eaed5b91ffa06c00ebfe5b2189f Mon Sep 17 00:00:00 2001 From: Valentin Knabel Date: Thu, 21 Nov 2024 16:53:00 +0100 Subject: [PATCH 4/7] feat. --- .../defaults/main.yaml | 5 +++ .../tasks/main.yaml | 3 +- go.mod | 2 +- go.sum | 4 +- .../metalstackcluster_controller.go | 43 +++++++++++++++++++ 5 files changed, 52 insertions(+), 5 deletions(-) diff --git a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml index 4644619..ce47964 100644 --- a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml @@ -17,3 +17,8 @@ firewall_controller_manager_shoot_access_token_secret: "shoot-access-firewall-co firewall_controller_manager_secrets_server: firewall_controller_manager_ca_bundle: + +firewall_controller_manager_crd_fetch_base_url: "https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/heads/" +# TODO: +# firewall_controller_manager_crd_fetch_base_url: "https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/tags/" +firewall_controller_manager_image_tag: initial-firewall-ruleset diff --git a/capi-lab/roles/firewall-controller-manager/tasks/main.yaml b/capi-lab/roles/firewall-controller-manager/tasks/main.yaml index 0a35dd7..b4fac34 100644 --- a/capi-lab/roles/firewall-controller-manager/tasks/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/tasks/main.yaml @@ -14,14 +14,13 @@ - name: Deploy firewall-controller-manager CRDs k8s: - definition: "{{ lookup('url', 'https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/tags/' + firewall_controller_manager_image_tag + '/config/crds/' + item, split_lines=False) }}" + definition: "{{ lookup('url', firewall_controller_manager_crd_fetch_base_url + firewall_controller_manager_image_tag + '/config/crds/' + item, split_lines=False) }}" namespace: "{{ firewall_controller_manager_namespace }}" loop: - firewall.metal-stack.io_firewalldeployments.yaml - firewall.metal-stack.io_firewallmonitors.yaml - firewall.metal-stack.io_firewalls.yaml - firewall.metal-stack.io_firewallsets.yaml - # - name: Deploy firewall-controller-manager # k8s: # definition: diff --git a/go.mod b/go.mod index 40f9470..e5abc4d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.23.0 require ( github.com/go-logr/logr v1.4.2 - github.com/metal-stack/firewall-controller-manager v0.4.3 + github.com/metal-stack/firewall-controller-manager v0.4.4-0.20241121151352-d3362457f60b github.com/metal-stack/metal-go v0.37.2 github.com/metal-stack/metal-lib v0.18.4 github.com/onsi/ginkgo/v2 v2.20.2 diff --git a/go.sum b/go.sum index e6f1983..3ea7915 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNB github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/metal-stack/firewall-controller-manager v0.4.3 h1:WU5bqD710gUtzyA2NdWJuSveCbOhllQ7ybpgUg7aJW8= -github.com/metal-stack/firewall-controller-manager v0.4.3/go.mod h1:J/3LHcvfJCpEEC4yk+WD0exh3btaScCaFkzbnbOsqrY= +github.com/metal-stack/firewall-controller-manager v0.4.4-0.20241121151352-d3362457f60b h1:MKtYVt1QPVSd9LzTW532QzVz9c+hIUst7+8SmxhM8us= +github.com/metal-stack/firewall-controller-manager v0.4.4-0.20241121151352-d3362457f60b/go.mod h1:J/3LHcvfJCpEEC4yk+WD0exh3btaScCaFkzbnbOsqrY= github.com/metal-stack/metal-go v0.37.2 h1:SDIuV43y09kmwtHfsReOZoZ7c2F+lNP4iIhazfJL5tQ= github.com/metal-stack/metal-go v0.37.2/go.mod h1:3MJTYCS4YJz8D8oteTKhjpaAKNMMjMKYDrIy9awHGtQ= github.com/metal-stack/metal-lib v0.18.4 h1:7HnfSwSbrKNHU+i6i79YFk/eeuhBhwIEHWpGqS7pYCc= diff --git a/internal/controller/metalstackcluster_controller.go b/internal/controller/metalstackcluster_controller.go index ac33d47..2693cb8 100644 --- a/internal/controller/metalstackcluster_controller.go +++ b/internal/controller/metalstackcluster_controller.go @@ -433,6 +433,49 @@ func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcm tag.ClusterID: string(r.infraCluster.GetUID()), } + deploy.Spec.Template.Spec.InitialRuleSet = &fcmv2.InitialRuleSet{ + Egress: []fcmv2.EgressRule{ + { + Comment: "allow outgoing http", + Ports: []int32{80}, + Protocol: fcmv2.NetworkProtocolTCP, + To: []string{"0.0.0.0/0"}, + }, + { + Comment: "allow outgoing https", + Ports: []int32{443}, + Protocol: fcmv2.NetworkProtocolTCP, + To: []string{"0.0.0.0/0"}, + }, + { + Comment: "allow outgoing dns via tcp", + Ports: []int32{53}, + Protocol: fcmv2.NetworkProtocolTCP, + To: []string{"0.0.0.0/0"}, + }, + { + Comment: "allow outgoing dns and ntp via udp", + Ports: []int32{53, 123}, + Protocol: fcmv2.NetworkProtocolUDP, + To: []string{"0.0.0.0/0"}, + }, + }, + Ingress: []fcmv2.IngressRule{ + { + Comment: "allow incoming ssh", + Ports: []int32{22}, + Protocol: fcmv2.NetworkProtocolTCP, + From: []string{"0.0.0.0/0"}, // TODO: restrict cidr + }, + { + Comment: "allow incoming https to kube-apiserver", + Ports: []int32{443}, + Protocol: fcmv2.NetworkProtocolTCP, + From: []string{"0.0.0.0/0"}, // TODO: restrict cidr + }, + }, + } + if deploy.Spec.Template.Labels == nil { deploy.Spec.Template.Labels = map[string]string{} } From 0795041712aa679ee8022cafd0873e9d5695579a Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 21 Nov 2024 16:54:02 +0100 Subject: [PATCH 5/7] Provide cert and split into individual manifests for Ansible. # Conflicts: # capi-lab/roles/firewall-controller-manager/defaults/main.yaml # capi-lab/roles/firewall-controller-manager/tasks/main.yaml --- capi-lab/deploy.yaml | 5 +- capi-lab/fcm-certs/ca-config.json | 18 + capi-lab/fcm-certs/ca-csr.json | 14 + capi-lab/fcm-certs/ca-key.pem | 5 + capi-lab/fcm-certs/ca.pem | 12 + capi-lab/fcm-certs/roll.sh | 9 + capi-lab/fcm-certs/tls.crt | 16 + capi-lab/fcm-certs/tls.json | 19 + capi-lab/fcm-certs/tls.key | 5 + .../defaults/main.yaml | 15 +- .../tasks/main.yaml | 27 +- .../templates/cluster-role-binding.yaml | 13 + .../templates/cluster-role.yaml | 87 ++++ .../templates/deployment.yaml | 78 ++++ .../firewall-controller-manager.yaml | 371 ------------------ .../mutatingwebhookconfiguration.yaml | 72 ++++ .../templates/sa.yaml | 6 + .../templates/secret-ca.yaml | 9 + .../templates/secret.yaml | 7 + .../templates/service.yaml | 19 + .../validatingwebhookconfiguration.yaml | 81 ++++ 21 files changed, 505 insertions(+), 383 deletions(-) create mode 100644 capi-lab/fcm-certs/ca-config.json create mode 100644 capi-lab/fcm-certs/ca-csr.json create mode 100644 capi-lab/fcm-certs/ca-key.pem create mode 100644 capi-lab/fcm-certs/ca.pem create mode 100755 capi-lab/fcm-certs/roll.sh create mode 100644 capi-lab/fcm-certs/tls.crt create mode 100644 capi-lab/fcm-certs/tls.json create mode 100644 capi-lab/fcm-certs/tls.key create mode 100644 capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/deployment.yaml delete mode 100644 capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/sa.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/secret.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/service.yaml create mode 100644 capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml diff --git a/capi-lab/deploy.yaml b/capi-lab/deploy.yaml index 13c3165..01218c9 100644 --- a/capi-lab/deploy.yaml +++ b/capi-lab/deploy.yaml @@ -14,5 +14,8 @@ - name: prometheus - name: firewall-controller-manager vars: - firewall_controller_manager_namespace: cap-metal-stack + firewall_controller_manager_namespace: capms-system + firewall_controller_manager_ca: "{{ lookup('file', playbook_dir + '/fcm-certs/ca.pem') }}" + firewall_controller_manager_cert: "{{ lookup('file', playbook_dir + '/fcm-certs/tls.crt') }}" + firewall_controller_manager_cert_key: "{{ lookup('file', playbook_dir + '/fcm-certs/tls.key') }}" - name: cluster-api-provider-metal-stack diff --git a/capi-lab/fcm-certs/ca-config.json b/capi-lab/fcm-certs/ca-config.json new file mode 100644 index 0000000..e80812e --- /dev/null +++ b/capi-lab/fcm-certs/ca-config.json @@ -0,0 +1,18 @@ +{ + "signing": { + "default": { + "expiry": "168h" + }, + "profiles": { + "client-server": { + "expiry": "8760h", + "usages": [ + "signing", + "key encipherment", + "server auth", + "client auth" + ] + } + } + } +} diff --git a/capi-lab/fcm-certs/ca-csr.json b/capi-lab/fcm-certs/ca-csr.json new file mode 100644 index 0000000..1a3d24b --- /dev/null +++ b/capi-lab/fcm-certs/ca-csr.json @@ -0,0 +1,14 @@ +{ + "CN": "ca", + "key": { + "algo": "ecdsa", + "size": 256 + }, + "names": [ + { + "C": "DE", + "L": "Bavaria", + "ST": "Munich" + } + ] +} diff --git a/capi-lab/fcm-certs/ca-key.pem b/capi-lab/fcm-certs/ca-key.pem new file mode 100644 index 0000000..48fc579 --- /dev/null +++ b/capi-lab/fcm-certs/ca-key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIFUGS1Xbmf1C9NcitDjcU3yfM3JUSS8SAeIHAvkHgofhoAoGCCqGSM49 +AwEHoUQDQgAEYPaD8+nz3ffhuV3iq3958NFnO28pCIfXiZOCVLyQYsvlr88eFbrN +vjEHXAmvxTp5X2hlY5dbVh/CPC6FJbBFCw== +-----END EC PRIVATE KEY----- diff --git a/capi-lab/fcm-certs/ca.pem b/capi-lab/fcm-certs/ca.pem new file mode 100644 index 0000000..4c7ebb7 --- /dev/null +++ b/capi-lab/fcm-certs/ca.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIBvjCCAWSgAwIBAgIUQBnjRL2py37bbgxj2/pB9TYZdSMwCgYIKoZIzj0EAwIw +PTELMAkGA1UEBhMCREUxDzANBgNVBAgTBk11bmljaDEQMA4GA1UEBxMHQmF2YXJp +YTELMAkGA1UEAxMCY2EwHhcNMjQxMTIxMTIxMjAwWhcNMjkxMTIwMTIxMjAwWjA9 +MQswCQYDVQQGEwJERTEPMA0GA1UECBMGTXVuaWNoMRAwDgYDVQQHEwdCYXZhcmlh +MQswCQYDVQQDEwJjYTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABGD2g/Pp8933 +4bld4qt/efDRZztvKQiH14mTglS8kGLL5a/PHhW6zb4xB1wJr8U6eV9oZWOXW1Yf +wjwuhSWwRQujQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0G +A1UdDgQWBBT0JWN2t5PTJEOyBBbfGqjUdrsMXTAKBggqhkjOPQQDAgNIADBFAiEA +ojnyHUbtmkx1xnuon+VFZKjccZxyoMaU/0u2Sz0MhWwCICrpHbQTNLoL8Q48UfJK +33EilS1z6lxn/nM6+ql8WVfO +-----END CERTIFICATE----- diff --git a/capi-lab/fcm-certs/roll.sh b/capi-lab/fcm-certs/roll.sh new file mode 100755 index 0000000..b8293d8 --- /dev/null +++ b/capi-lab/fcm-certs/roll.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -eo pipefail + +echo "generating example certs" +cfssl genkey -initca ca-csr.json | cfssljson -bare ca +cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client-server tls.json | cfssljson -bare tls +rm *.csr +mv tls.pem tls.crt +mv tls-key.pem tls.key diff --git a/capi-lab/fcm-certs/tls.crt b/capi-lab/fcm-certs/tls.crt new file mode 100644 index 0000000..36f6d89 --- /dev/null +++ b/capi-lab/fcm-certs/tls.crt @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIChjCCAiugAwIBAgIUfIGP//S9eEv3UtQ/ZlfTc579jdowCgYIKoZIzj0EAwIw +PTELMAkGA1UEBhMCREUxDzANBgNVBAgTBk11bmljaDEQMA4GA1UEBxMHQmF2YXJp +YTELMAkGA1UEAxMCY2EwHhcNMjQxMTIxMTIxMjAwWhcNMjUxMTIxMTIxMjAwWjBE +MQswCQYDVQQGEwJERTEPMA0GA1UECBMGTXVuaWNoMRAwDgYDVQQHEwdCYXZhcmlh +MRIwEAYDVQQDEwlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAAQJ +MoYtsmZB2s53fzS+LXf/rFSI6sHiKJ4kbenK04agoarAsIGniCPgRb4MUj2LvhC5 +1xJJncCC21QVUZCXZb+lo4IBADCB/TAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYw +FAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFNDY +PWHrOS2a1CtLw91V3cKk+Y6NMB8GA1UdIwQYMBaAFPQlY3a3k9MkQ7IEFt8aqNR2 +uwxdMH4GA1UdEQR3MHWCCWxvY2FsaG9zdIIsZmlyZXdhbGwtY29udHJvbGxlci1t +YW5hZ2VyLmNhcG1zLXN5c3RlbS5zdmOCOmZpcmV3YWxsLWNvbnRyb2xsZXItbWFu +YWdlci5jYXBtcy1zeXN0ZW0uc3ZjLmNsdXN0ZXIubG9jYWwwCgYIKoZIzj0EAwID +SQAwRgIhAKlzsenMaiXH+IqONSjxL/Bk5Xk7HM+sWfbTyVoOHXnhAiEA0nd2f04Z +R36a+jGSXPxMgR2OOmScjfOUk3xnDDInMQE= +-----END CERTIFICATE----- diff --git a/capi-lab/fcm-certs/tls.json b/capi-lab/fcm-certs/tls.json new file mode 100644 index 0000000..ba00cd9 --- /dev/null +++ b/capi-lab/fcm-certs/tls.json @@ -0,0 +1,19 @@ +{ + "CN": "localhost", + "hosts": [ + "localhost", + "firewall-controller-manager.capms-system.svc", + "firewall-controller-manager.capms-system.svc.cluster.local" + ], + "key": { + "algo": "ecdsa", + "size": 256 + }, + "names": [ + { + "C": "DE", + "L": "Bavaria", + "ST": "Munich" + } + ] +} diff --git a/capi-lab/fcm-certs/tls.key b/capi-lab/fcm-certs/tls.key new file mode 100644 index 0000000..5678c7b --- /dev/null +++ b/capi-lab/fcm-certs/tls.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEICCR8PczdJo8Tjpum62cO2hrlS0irQRVAgYhzcAr9raXoAoGCCqGSM49 +AwEHoUQDQgAECTKGLbJmQdrOd380vi13/6xUiOrB4iieJG3pytOGoKGqwLCBp4gj +4EW+DFI9i74QudcSSZ3AgttUFVGQl2W/pQ== +-----END EC PRIVATE KEY----- diff --git a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml index ce47964..3e8570b 100644 --- a/capi-lab/roles/firewall-controller-manager/defaults/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/defaults/main.yaml @@ -1,22 +1,27 @@ --- firewall_controller_manager_namespace: "firewall-controller-manager" -firewall_controller_manager_image: ghcr.io/metal-stack/firewall-controller-manager firewall_controller_manager_image_pull_policy: Always firewall_controller_manager_replicas: 1 # firewall_controller_manager_pod_annotations: -firewall_controller_manager_seed_api_url: +firewall_controller_manager_seed_api_url: https://kubernetes firewall_controller_manager_shoot_api_url: firewall_controller_manager_cluster_id: -firewall_controller_manager_metalapi_url: + +firewall_controller_manager_metalapi_url: http://metal-api.metal-control-plane.svc.cluster.local:8080 +firewall_controller_manager_metalapi_hmac: metal-admin + firewall_controller_manager_generic_token_kubeconfig_secret_name: firewall_controller_manager_ssh_key_secret_name: firewall_controller_manager_shoot_access_token_secret: "shoot-access-firewall-controller-manager" -firewall_controller_manager_secrets_server: -firewall_controller_manager_ca_bundle: +firewall_controller_manager_ca: +firewall_controller_manager_cert: +firewall_controller_manager_cert_key: + +firewall_controller_manager_pod_annotations: {} firewall_controller_manager_crd_fetch_base_url: "https://raw.githubusercontent.com/metal-stack/firewall-controller-manager/refs/heads/" # TODO: diff --git a/capi-lab/roles/firewall-controller-manager/tasks/main.yaml b/capi-lab/roles/firewall-controller-manager/tasks/main.yaml index b4fac34..15222ce 100644 --- a/capi-lab/roles/firewall-controller-manager/tasks/main.yaml +++ b/capi-lab/roles/firewall-controller-manager/tasks/main.yaml @@ -8,21 +8,36 @@ apiVersion: v1 kind: Namespace metadata: - name: "{{ firewall_controller_manager_namespace }}" + name: "{{ item }}" labels: - name: "{{ firewall_controller_manager_namespace }}" + name: "{{ item }}" + loop: + - "{{ firewall_controller_manager_namespace }}" + - firewall - name: Deploy firewall-controller-manager CRDs k8s: definition: "{{ lookup('url', firewall_controller_manager_crd_fetch_base_url + firewall_controller_manager_image_tag + '/config/crds/' + item, split_lines=False) }}" namespace: "{{ firewall_controller_manager_namespace }}" + apply: true loop: - firewall.metal-stack.io_firewalldeployments.yaml - firewall.metal-stack.io_firewallmonitors.yaml - firewall.metal-stack.io_firewalls.yaml - firewall.metal-stack.io_firewallsets.yaml -# - name: Deploy firewall-controller-manager -# k8s: -# definition: -# namespace: "{{ firewall_controller_manager_namespace }}" +- name: Deploy firewall-controller-manager + k8s: + definition: "{{ lookup('template', item) }}" + namespace: "{{ firewall_controller_manager_namespace }}" + apply: true + loop: + - sa.yaml + - cluster-role.yaml + - cluster-role-binding.yaml + - mutatingwebhookconfiguration.yaml + - validatingwebhookconfiguration.yaml + - secret.yaml + - secret-ca.yaml + - deployment.yaml + - service.yaml diff --git a/capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml b/capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml new file mode 100644 index 0000000..a6366ed --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/cluster-role-binding.yaml @@ -0,0 +1,13 @@ +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: firewall-controller-manager +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: firewall-controller-manager +subjects: +- kind: ServiceAccount + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml b/capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml new file mode 100644 index 0000000..cd1a73c --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/cluster-role.yaml @@ -0,0 +1,87 @@ +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: firewall-controller-manager +rules: +- apiGroups: + - firewall.metal-stack.io + resources: + - firewalls + - firewalls/status + - firewallsets + - firewallsets/status + - firewalldeployments + - firewalldeployments/status + - firewallmonitors + - firewallmonitors/status + verbs: + - '*' +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - "" + resources: + - secrets + - serviceaccounts + - namespaces + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + - clusterroles + - clusterrolebindings + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - "" + resources: + - events + verbs: + - get + - list + - watch + - update + - patch + - create +- apiGroups: + - extensions.gardener.cloud + resources: + - infrastructures + - extensions + verbs: + - get +- apiGroups: + - extensions.gardener.cloud + resources: + - infrastructures/status + verbs: + - patch +- apiGroups: + - extensions.gardener.cloud + resources: + - extensions + verbs: + - update diff --git a/capi-lab/roles/firewall-controller-manager/templates/deployment.yaml b/capi-lab/roles/firewall-controller-manager/templates/deployment.yaml new file mode 100644 index 0000000..53f5a75 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/deployment.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + labels: + app: firewall-controller-manager +spec: + selector: + matchLabels: + app: firewall-controller-manager + replicas: {{ firewall_controller_manager_replicas }} + template: + metadata: + labels: + app: firewall-controller-manager +{% if firewall_controller_manager_pod_annotations %} + annotations: +{{ firewall_controller_manager_pod_annotations | to_nice_yaml | indent(width=8, first=true) }} +{% endif %} + spec: + serviceAccountName: firewall-controller-manager + containers: + - name: firewall-controller-manager + image: {{ firewall_controller_manager_image_name }}:{{ firewall_controller_manager_image_tag }} + imagePullPolicy: {{ firewall_controller_manager_image_pull_policy }} + args: + - -cert-dir=/certs + - -log-level=info + - -seed-api-url={{ firewall_controller_manager_seed_api_url }} + # - -shoot-api-url={{ firewall_controller_manager_shoot_api_url }} + # - -internal-shoot-api-url=https://kube-apiserver + # - -cluster-id={{ firewall_controller_manager_cluster_id }} + - -enable-leader-election + - -metal-api-url={{ firewall_controller_manager_metalapi_url }} + - -namespace={{ firewall_controller_manager_namespace }} + - -shoot-kubeconfig-secret-name=none + - -shoot-token-secret-name=none + - -ssh-key-secret-name=none + # - -shoot-token-path=/token + env: + - name: METAL_AUTH_HMAC + valueFrom: + secretKeyRef: + name: firewall-controller-manager-config + key: api-hmac + livenessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + volumeMounts: + - name: webhook-certs + mountPath: "/certs" + readOnly: true + - name: token-dir + mountPath: "/token" + resources: + limits: + cpu: 400m + memory: 400Mi + requests: + cpu: 100m + memory: 20Mi + volumes: + - name: webhook-certs + secret: + secretName: firewall-controller-manager-certs + - name: token-dir + emptyDir: {} diff --git a/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml b/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml deleted file mode 100644 index f9a1bb7..0000000 --- a/capi-lab/roles/firewall-controller-manager/templates/firewall-controller-manager.yaml +++ /dev/null @@ -1,371 +0,0 @@ ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} -rules: -- apiGroups: - - firewall.metal-stack.io - resources: - - firewalls - - firewalls/status - - firewallsets - - firewallsets/status - - firewalldeployments - - firewalldeployments/status - verbs: - - '*' -- apiGroups: - - coordination.k8s.io - resources: - - leases - verbs: - - get - - list - - watch - - update - - patch - - create -- apiGroups: - - "" - resources: - - secrets - - serviceaccounts - verbs: - - get - - list - - watch - - update - - patch - - create -- apiGroups: - - rbac.authorization.k8s.io - resources: - - roles - - rolebindings - verbs: - - get - - list - - watch - - update - - patch - - create -- apiGroups: - - "" - resources: - - events - verbs: - - get - - list - - watch - - update - - patch - - create -- apiGroups: - - extensions.gardener.cloud - resources: - - infrastructures - - extensions - verbs: - - get -- apiGroups: - - extensions.gardener.cloud - resources: - - infrastructures/status - verbs: - - patch -- apiGroups: - - extensions.gardener.cloud - resources: - - extensions - verbs: - - update ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: firewall-controller-manager -subjects: -- kind: ServiceAccount - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - labels: - app: firewall-controller-manager -spec: - selector: - matchLabels: - app: firewall-controller-manager - replicas: {{ firewall_controller_manager_replicas }} - template: - metadata: - labels: - app: firewall-controller-manager - networking.gardener.cloud/from-prometheus: "allowed" - networking.gardener.cloud/to-dns: "allowed" - networking.gardener.cloud/to-public-networks: "allowed" - networking.gardener.cloud/to-private-networks: "allowed" - networking.gardener.cloud/to-shoot-apiserver: "allowed" - networking.gardener.cloud/to-runtime-apiserver: "allowed" - networking.resources.gardener.cloud/to-kube-apiserver-tcp-443: "allowed" -{% if firewall_controller_manager_pod_annotations %} - annotations: -{{ firewall_controller_manager_pod_annotations | to_nice_yaml | indent(width=8, first=true) }} -{% end %} - spec: - serviceAccountName: firewall-controller-manager - containers: - - name: firewall-controller-manager - image: {{ firewall_controller_manager_image }} - imagePullPolicy: {{ firewall_controller_manager_image_pull_policy }} - args: - - -cert-dir=/certs - - -log-level=info - - -seed-api-url={{ firewall_controller_manager_seed_api_url }} - - -shoot-api-url={{ firewall_controller_manager_shoot_api_url }} - - -internal-shoot-api-url=https://kube-apiserver - - -cluster-id={{ firewall_controller_manager_cluster_id }} - - -enable-leader-election - - -metal-api-url={{ firewall_controller_manager_metalapi_url }} - - -namespace={{ firewall_controller_manager_namespace }} - - -shoot-kubeconfig-secret-name={{ firewall_controller_manager_generic_token_kubeconfig_secret_name }} - - -shoot-token-secret-name=shoot-access-firewall-controller-manager - - -ssh-key-secret-name={{ firewall_controller_manager_ssh_key_secret_name }} - - -shoot-token-path=/token - env: - - name: METAL_AUTH_HMAC - valueFrom: - secretKeyRef: - name: cloudprovider - key: metalAPIHMac - livenessProbe: - httpGet: - path: /readyz - port: 8081 - initialDelaySeconds: 15 - periodSeconds: 20 - readinessProbe: - httpGet: - path: /healthz - port: 8081 - initialDelaySeconds: 5 - periodSeconds: 10 - volumeMounts: - - name: webhook-certs - mountPath: "/certs" - readOnly: true - - name: token-dir - mountPath: "/token" - resources: - limits: - cpu: 400m - memory: 400Mi - requests: - cpu: 100m - memory: 20Mi - volumes: - - name: webhook-certs - secret: - secretName: {{ firewall_controller_manager_secrets_server }} - - name: token-dir - emptyDir: {} ---- -apiVersion: v1 -kind: Service -metadata: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - labels: - app: firewall-controller-manager - annotations: - networking.resources.gardener.cloud/from-world-to-ports: '[{"protocol":"TCP","port":9443}]' - networking.resources.gardener.cloud/from-all-webhook-targets-allowed-ports: '[{"protocol":"TCP","port":9443}]' - networking.resources.gardener.cloud/from-all-seed-scrape-targets-allowed-ports: '[{"protocol":"TCP","port":2112}]' -spec: - type: ClusterIP - ports: - - name: webhooks - port: 9443 - protocol: TCP - - name: metrics - port: 2112 - protocol: TCP - selector: - app: firewall-controller-manager ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: MutatingWebhookConfiguration -metadata: - name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} - service: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - port: 9443 - path: /mutate-firewall-metal-stack-io-v2-firewall - failurePolicy: Fail - name: firewall.metal-stack.io - objectSelector: - matchLabels: - gardener-shoot-namespace: {{ firewall_controller_manager_namespace }} - rules: - - apiGroups: - - firewall.metal-stack.io - apiVersions: - - v2 - operations: - - CREATE - resources: - - firewalls - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ firewall_controller_manager_caBundle | b64encode }} - service: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - port: 9443 - path: /mutate-firewall-metal-stack-io-v2-firewallset - failurePolicy: Fail - name: firewallset.metal-stack.io - objectSelector: - matchLabels: - gardener-shoot-namespace: {{ firewall_controller_manager_namespace }} - rules: - - apiGroups: - - firewall.metal-stack.io - apiVersions: - - v2 - operations: - - CREATE - resources: - - firewallsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} - service: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - port: 9443 - path: /mutate-firewall-metal-stack-io-v2-firewalldeployment - failurePolicy: Fail - name: firewalldeployment.metal-stack.io - objectSelector: - matchLabels: - gardener-shoot-namespace: {{ firewall_controller_manager_namespace }} - rules: - - apiGroups: - - firewall.metal-stack.io - apiVersions: - - v2 - operations: - - CREATE - resources: - - firewalldeployments - sideEffects: None ---- -apiVersion: admissionregistration.k8s.io/v1 -kind: ValidatingWebhookConfiguration -metadata: - name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} -webhooks: -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} - service: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - port: 9443 - path: /validate-firewall-metal-stack-io-v2-firewall - failurePolicy: Fail - name: firewall.metal-stack.io - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} - rules: - - apiGroups: - - firewall.metal-stack.io - apiVersions: - - v2 - operations: - - CREATE - - UPDATE - resources: - - firewalls - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} - service: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - port: 9443 - path: /validate-firewall-metal-stack-io-v2-firewallset - failurePolicy: Fail - name: firewallset.metal-stack.io - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} - rules: - - apiGroups: - - firewall.metal-stack.io - apiVersions: - - v2 - operations: - - CREATE - - UPDATE - resources: - - firewallsets - sideEffects: None -- admissionReviewVersions: - - v1 - clientConfig: - caBundle: {{ firewall_controller_manager_ca_bundle | b64encode }} - service: - name: firewall-controller-manager - namespace: {{ firewall_controller_manager_namespace }} - port: 9443 - path: /validate-firewall-metal-stack-io-v2-firewalldeployment - failurePolicy: Fail - name: firewalldeployment.metal-stack.io - namespaceSelector: - matchLabels: - kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} - rules: - - apiGroups: - - firewall.metal-stack.io - apiVersions: - - v2 - operations: - - CREATE - - UPDATE - resources: - - firewalldeployments - sideEffects: None - diff --git a/capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml b/capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml new file mode 100644 index 0000000..1a9bfaf --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/mutatingwebhookconfiguration.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewall + failurePolicy: Fail + name: firewall.metal-stack.io + objectSelector: {} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewalls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewallset + failurePolicy: Fail + name: firewallset.metal-stack.io + objectSelector: {} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewallsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /mutate-firewall-metal-stack-io-v2-firewalldeployment + failurePolicy: Fail + name: firewalldeployment.metal-stack.io + objectSelector: {} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + resources: + - firewalldeployments + sideEffects: None diff --git a/capi-lab/roles/firewall-controller-manager/templates/sa.yaml b/capi-lab/roles/firewall-controller-manager/templates/sa.yaml new file mode 100644 index 0000000..0ba4ccf --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/sa.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml b/capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml new file mode 100644 index 0000000..aa09697 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/secret-ca.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: firewall-controller-manager-certs +data: + ca.crt: {{ firewall_controller_manager_ca | b64encode }} + tls.crt: {{ firewall_controller_manager_cert | b64encode }} + tls.key: {{ firewall_controller_manager_cert_key | b64encode }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/secret.yaml b/capi-lab/roles/firewall-controller-manager/templates/secret.yaml new file mode 100644 index 0000000..88eff24 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/secret.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: firewall-controller-manager-config +stringData: + api-hmac: {{ firewall_controller_manager_metalapi_hmac }} diff --git a/capi-lab/roles/firewall-controller-manager/templates/service.yaml b/capi-lab/roles/firewall-controller-manager/templates/service.yaml new file mode 100644 index 0000000..abd4628 --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/service.yaml @@ -0,0 +1,19 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + labels: + app: firewall-controller-manager +spec: + type: ClusterIP + ports: + - name: webhooks + port: 9443 + protocol: TCP + - name: metrics + port: 2112 + protocol: TCP + selector: + app: firewall-controller-manager diff --git a/capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml b/capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml new file mode 100644 index 0000000..7be581a --- /dev/null +++ b/capi-lab/roles/firewall-controller-manager/templates/validatingwebhookconfiguration.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: firewall-controller-manager-{{ firewall_controller_manager_namespace }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewall + failurePolicy: Fail + name: firewall.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewalls + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewallset + failurePolicy: Fail + name: firewallset.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewallsets + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + caBundle: {{ firewall_controller_manager_ca | b64encode }} + service: + name: firewall-controller-manager + namespace: {{ firewall_controller_manager_namespace }} + port: 9443 + path: /validate-firewall-metal-stack-io-v2-firewalldeployment + failurePolicy: Fail + name: firewalldeployment.metal-stack.io + namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: {{ firewall_controller_manager_namespace }} + rules: + - apiGroups: + - firewall.metal-stack.io + apiVersions: + - v2 + operations: + - CREATE + - UPDATE + resources: + - firewalldeployments + sideEffects: None From 2207a80f95601e50f23eec478c860fee0087095c Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 21 Nov 2024 16:54:47 +0100 Subject: [PATCH 6/7] Generate SSH key for firewall and machine. --- config/rbac/role.yaml | 3 + config/samples/example-kubeadm.yaml | 12 +-- .../metalstackcluster_controller.go | 101 ++++++++++++++++-- .../metalstackmachine_controller.go | 18 +++- 4 files changed, 118 insertions(+), 16 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index b8514d4..2c66e33 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -9,8 +9,11 @@ rules: resources: - secrets verbs: + - create + - delete - get - list + - update - watch - apiGroups: - cluster.x-k8s.io diff --git a/config/samples/example-kubeadm.yaml b/config/samples/example-kubeadm.yaml index f8c6366..4f71599 100644 --- a/config/samples/example-kubeadm.yaml +++ b/config/samples/example-kubeadm.yaml @@ -101,17 +101,17 @@ spec: CNI_PLUGINS_VERSION="v1.3.0" DEST="/opt/cni/bin" mkdir -p "$DEST" - curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" | sudo tar -C "$DEST" -xz + curl -L "https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-amd64-${CNI_PLUGINS_VERSION}.tgz" | tar -C "$DEST" -xz RELEASE="v1.30.6" cd /usr/local/bin - sudo curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} - sudo chmod +x {kubeadm,kubelet,kubectl} + curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/amd64/{kubeadm,kubelet,kubectl} + chmod +x {kubeadm,kubelet,kubectl} RELEASE_VERSION="v0.16.2" - curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | sudo tee /usr/lib/systemd/system/kubelet.service - sudo mkdir -p /usr/lib/systemd/system/kubelet.service.d - curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | sudo tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf + curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubelet/kubelet.service" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service + mkdir -p /usr/lib/systemd/system/kubelet.service.d + curl -sSL "https://raw.githubusercontent.com/kubernetes/release/${RELEASE_VERSION}/cmd/krel/templates/latest/kubeadm/10-kubeadm.conf" | sed "s:/usr/bin:/usr/local/bin:g" | tee /usr/lib/systemd/system/kubelet.service.d/10-kubeadm.conf systemctl enable kubelet.service - path: /etc/containerd/config.toml diff --git a/internal/controller/metalstackcluster_controller.go b/internal/controller/metalstackcluster_controller.go index 2693cb8..d5e0120 100644 --- a/internal/controller/metalstackcluster_controller.go +++ b/internal/controller/metalstackcluster_controller.go @@ -18,10 +18,15 @@ package controller import ( "context" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" "errors" "fmt" "strconv" + "golang.org/x/crypto/ssh" "golang.org/x/sync/errgroup" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -73,6 +78,7 @@ type clusterReconciler struct { // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=metalstackclusters/status,verbs=get;update;patch // +kubebuilder:rbac:groups=infrastructure.cluster.x-k8s.io,resources=metalstackclusters/finalizers,verbs=update // +kubebuilder:rbac:groups=firewall.metal-stack.io,resources=firewalldeployments,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups="",resources=secrets,verbs=get;list;watch;create;update;delete // Reconcile reconciles the reconciled cluster to be reconciled. func (r *MetalStackClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -166,6 +172,13 @@ func (r *MetalStackClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { } func (r *clusterReconciler) reconcile() error { + sshPubKey, err := r.ensureSshKeyPair(r.ctx) + if err != nil { + return fmt.Errorf("unable to ensure ssh key pair: %w", err) + } + + r.log.Info("reconciled ssh key pair") + nodeNetworkID, err := r.ensureNodeNetwork() if err != nil { return fmt.Errorf("unable to ensure node network: %w", err) @@ -197,7 +210,7 @@ func (r *clusterReconciler) reconcile() error { return fmt.Errorf("failed to update infra cluster control plane endpoint: %w", err) } - fwdeploy, err := r.ensureFirewallDeployment(nodeNetworkID) + fwdeploy, err := r.ensureFirewallDeployment(nodeNetworkID, sshPubKey) if err != nil { return fmt.Errorf("unable to ensure firewall deployment: %w", err) } @@ -237,9 +250,83 @@ func (r *clusterReconciler) delete() error { r.log.Info("deleted node network") + err = r.deleteSshKeyPair(r.ctx) + if err != nil { + return fmt.Errorf("unable to delete ssh key pair: %w", err) + } + + r.log.Info("deleted ssh key pair") + return err } +func (r *clusterReconciler) ensureSshKeyPair(ctx context.Context) (string, error) { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.cluster.Name + "-ssh-keypair", + Namespace: r.cluster.Namespace, + }, + } + + err := r.client.Get(ctx, client.ObjectKeyFromObject(secret), secret) + if err == nil { + if key, ok := secret.Data["id_rsa.pub"]; ok { + return string(key), nil + } + } + if err != nil && !apierrors.IsNotFound(err) { + return "", err + } + + privateKey, err := rsa.GenerateKey(rand.Reader, 4096) + if err != nil { + return "", err + } + + privateKeyBlock := &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(privateKey), + } + + // generate and write public key + pubKey, err := ssh.NewPublicKey(&privateKey.PublicKey) + if err != nil { + return "", err + } + + secret.Data = map[string][]byte{ + "id_rsa": pem.EncodeToMemory(privateKeyBlock), + "id_rsa.pub": ssh.MarshalAuthorizedKey(pubKey), + } + + err = r.client.Create(ctx, secret) + if err != nil { + return "", err + } + + return string(ssh.MarshalAuthorizedKey(pubKey)), nil +} + +func (r *clusterReconciler) deleteSshKeyPair(ctx context.Context) error { + secret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ssh-keypair-" + r.cluster.Name, + Namespace: r.cluster.Namespace, + }, + } + + err := r.client.Delete(ctx, secret) + if err != nil { + if apierrors.IsNotFound(err) { + return nil + } + + return err + } + + return nil +} + func (r *clusterReconciler) ensureNodeNetwork() (string, error) { nws, err := r.findNodeNetwork() if err != nil { @@ -399,7 +486,7 @@ func (r *clusterReconciler) findControlPlaneIP() ([]*models.V1IPResponse, error) return resp.Payload, nil } -func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcmv2.FirewallDeployment, error) { +func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID, sshPubKey string) (*fcmv2.FirewallDeployment, error) { deploy := &fcmv2.FirewallDeployment{ ObjectMeta: metav1.ObjectMeta{ Name: r.infraCluster.Name, @@ -420,14 +507,12 @@ func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcm deploy.Annotations = map[string]string{} } deploy.Annotations[fcmv2.ReconcileAnnotation] = strconv.FormatBool(true) + deploy.Annotations[fcmv2.FirewallNoControllerConnectionAnnotation] = strconv.FormatBool(true) if deploy.Labels == nil { deploy.Labels = map[string]string{} } - // TODO: this is the selector for the mutating webhook, without it the mutation will not happen, but do we need mutation? - // deploy.Labels[MutatingWebhookObjectSelectorLabel] = cluster.ObjectMeta.Name - deploy.Spec.Replicas = 1 deploy.Spec.Selector = map[string]string{ tag.ClusterID: string(r.infraCluster.GetUID()), @@ -497,10 +582,8 @@ func (r *clusterReconciler) ensureFirewallDeployment(nodeNetworkID string) (*fcm // TODO: we need to allow internet connection for the nodes before the firewall-controller can connect to the control-plane // the FCM currently does not support this - deploy.Spec.Template.Spec.Userdata = "" - - // TODO: do we need to generate ssh keys for the machines and the firewall in this controller? - deploy.Spec.Template.Spec.SSHPublicKeys = nil + deploy.Spec.Template.Spec.Userdata = "{}" + deploy.Spec.Template.Spec.SSHPublicKeys = []string{sshPubKey} // TODO: consider auto update machine image feature diff --git a/internal/controller/metalstackmachine_controller.go b/internal/controller/metalstackmachine_controller.go index 60cc1f9..296e29b 100644 --- a/internal/controller/metalstackmachine_controller.go +++ b/internal/controller/metalstackmachine_controller.go @@ -274,6 +274,22 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) { return nil, fmt.Errorf("unable to fetch bootstrap secret: %w", err) } + sshSecret := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: r.infraCluster.Name + "-ssh-keypair", + Namespace: r.infraCluster.Namespace, + }, + } + err = r.client.Get(r.ctx, client.ObjectKeyFromObject(sshSecret), sshSecret) + if err != nil { + return nil, fmt.Errorf("unable to fetch ssh secret: %w", err) + } + + sshPubKey, ok := sshSecret.Data["id_rsa.pub"] + if !ok { + return nil, errors.New("ssh secret does not contain public key") + } + var ( ips []string nws = []*models.V1MachineAllocationNetwork{ @@ -311,7 +327,7 @@ func (r *machineReconciler) create() (*models.V1MachineResponse, error) { Networks: nws, Ips: ips, UserData: string(bootstrapSecret.Data["value"]), - // TODO: SSHPubKeys, ... + SSHPubKeys: []string{string(sshPubKey)}, }), nil) if err != nil { return nil, fmt.Errorf("failed to allocate machine: %w", err) From 75f939dd7c5d3c527b40f3a70d734f8d89dae017 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 21 Nov 2024 17:01:39 +0100 Subject: [PATCH 7/7] Go mod tidy. --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index e5abc4d..ed37ffd 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/metal-stack/metal-lib v0.18.4 github.com/onsi/ginkgo/v2 v2.20.2 github.com/onsi/gomega v1.34.2 + golang.org/x/crypto v0.28.0 golang.org/x/sync v0.8.0 k8s.io/api v0.31.0 k8s.io/apimachinery v0.31.0 @@ -102,7 +103,6 @@ require ( go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - golang.org/x/crypto v0.28.0 // indirect golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c // indirect golang.org/x/net v0.30.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect