From cef9795c031c728e8526f357aaaa1b33555a20ef Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 13 Jul 2024 19:17:28 +0200 Subject: [PATCH] Combines two node setup daemonsets into one: avoid race conditions Equivalent to hosts + containerd but drops support for containerd 1.4 Tried a distroless golang image first but failed to find a practical way to do nsenter. If we need k8s api access we could possibly revisit that. Switched to busybox to make startup faster. --- .../daemonset.yaml | 86 ++++++++++--------- .../kustomization.yaml | 0 registry/node-update-hosts/README.md | 7 -- registry/node-update-hosts/daemonset.yaml | 54 ------------ registry/node-update-hosts/kustomization.yaml | 2 - 5 files changed, 44 insertions(+), 105 deletions(-) rename registry/{node-update-containerd => containerd-config}/daemonset.yaml (51%) rename registry/{node-update-containerd => containerd-config}/kustomization.yaml (100%) delete mode 100644 registry/node-update-hosts/README.md delete mode 100644 registry/node-update-hosts/daemonset.yaml delete mode 100644 registry/node-update-hosts/kustomization.yaml diff --git a/registry/node-update-containerd/daemonset.yaml b/registry/containerd-config/daemonset.yaml similarity index 51% rename from registry/node-update-containerd/daemonset.yaml rename to registry/containerd-config/daemonset.yaml index b45cae83..9c8df74f 100644 --- a/registry/node-update-containerd/daemonset.yaml +++ b/registry/containerd-config/daemonset.yaml @@ -1,15 +1,16 @@ +# yaml-language-server: $schema=https://github.com/yannh/kubernetes-json-schema/raw/master/v1.27.9/daemonset.json apiVersion: apps/v1 kind: DaemonSet metadata: - name: containerd-registry-http + name: containerd-config spec: selector: matchLabels: - app: containerd-registry-http + app: containerd-config template: metadata: labels: - app: containerd-registry-http + app: containerd-config annotations: "cluster-autoscaler.kubernetes.io/safe-to-evict": "true" spec: @@ -18,63 +19,64 @@ spec: operator: Exists hostPID: true hostNetwork: true + containers: + - name: init-container-did-the-work + image: registry.k8s.io/pause:3.9@sha256:7031c1b283388d2c2e09b57badb803c05ebed362dc88d84b480cc47f72a21097 initContainers: - - name: update - image: ubuntu:22.04@sha256:27cb6e6ccef575a4698b66f5de06c7ecd61589132d5a91d098f7f3f9285415a9 - volumeMounts: - - name: etc-containerd - mountPath: /etc/containerd - readOnly: false - - name: results - mountPath: /results - env: - - name: CONFIG_PATH - value: /etc/containerd/config.toml - securityContext: - privileged: true + - name: config + image: ghcr.io/yolean/busybox:1.36.1-glibc@sha256:25e9fcbd3799fce9c0ec978303d35dbb18a6ffb1fc76fc9b181dd4e657e2cd13 command: - - bash + - sh #to troubleshoot use: - -cex - -ce - | - CONTAINERD_VERSION=$(nsenter --mount=/proc/1/ns/mnt -- containerd -v) - case "$CONTAINERD_VERSION" in - *"containerd 1.4."*) CRI_PLUGIN='plugins.cri' ;; - *) CRI_PLUGIN='plugins."io.containerd.grpc.v1.cri"' ;; - esac - echo "CONTAINERD_VERSION=$CONTAINERD_VERSION CRI_PLUGIN=$CRI_PLUGIN" + echo "" | tee -a /host-etc/hosts + grep builds-registry.ystack.svc.cluster.local /host-etc/hosts || \ + echo "$BUILDS_REGISTRY_PORT_80_TCP_ADDR builds-registry.ystack.svc.cluster.local" | tee -a /host-etc/hosts + grep prod-registry.ystack.svc.cluster.local /host-etc/hosts || \ + echo "$PROD_REGISTRY_PORT_80_TCP_ADDR prod-registry.ystack.svc.cluster.local" | tee -a /host-etc/hosts + + CONFIG_PATH=/etc/containerd/config.toml + CRI_PLUGIN='plugins."io.containerd.grpc.v1.cri"' if ! grep builds-registry.ystack.svc.cluster.local $CONFIG_PATH; then - echo "[$CRI_PLUGIN.registry.mirrors.\"builds-registry.ystack.svc.cluster.local\"]" >> $CONFIG_PATH - echo ' endpoint = ["http://builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH - echo "[$CRI_PLUGIN.registry.mirrors.\"builds-registry.ystack.svc.cluster.local:80\"]" >> $CONFIG_PATH - echo ' endpoint = ["http://builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH + echo "[$CRI_PLUGIN.registry.mirrors.\"builds-registry.ystack.svc.cluster.local\"]" >> $CONFIG_PATH + echo ' endpoint = ["http://builds-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH touch /results/config-updated fi if ! grep prod-registry.ystack.svc.cluster.local $CONFIG_PATH; then - echo "[$CRI_PLUGIN.registry.mirrors.\"prod-registry.ystack.svc.cluster.local\"]" >> $CONFIG_PATH - echo ' endpoint = ["http://prod-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH - echo "[$CRI_PLUGIN.registry.mirrors.\"prod-registry.ystack.svc.cluster.local:80\"]" >> $CONFIG_PATH - echo ' endpoint = ["http://prod-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH + echo "[$CRI_PLUGIN.registry.mirrors.\"prod-registry.ystack.svc.cluster.local\"]" >> $CONFIG_PATH + echo ' endpoint = ["http://prod-registry.ystack.svc.cluster.local"]' >> $CONFIG_PATH touch /results/config-updated fi - echo "-------------------- file --------------------" - cat $CONFIG_PATH - echo "-------------------- dump --------------------" - nsenter --mount=/proc/1/ns/mnt -- containerd config dump - echo "-------------------- --- --------------------" if [ -f /results/config-updated ]; then echo "Configuration has been updated; restarting containerd (not expecting any output) ..." nsenter --mount=/proc/1/ns/mnt -- systemctl restart containerd echo "Restart triggered" - else + else echo "No configration change" - fi - echo "Done" - containers: - - name: init-container-did-the-work - image: k8s.gcr.io/pause:3.6@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db + fi + echo "----------------------- /etc/hosts ------------------------" + cat /host-etc/hosts + echo "-------------------- containerd config --------------------" + nsenter --mount=/proc/1/ns/mnt -- containerd config dump + echo "-------------------------- done ---------------------------" + securityContext: + runAsUser: 0 + privileged: true + volumeMounts: + - name: etc-hosts + mountPath: /host-etc/hosts + readOnly: false + - name: etc-containerd + mountPath: /etc/containerd + readOnly: false + - name: results + mountPath: /results terminationGracePeriodSeconds: 30 volumes: + - name: etc-hosts + hostPath: + path: /etc/hosts - name: etc-containerd hostPath: path: /etc/containerd diff --git a/registry/node-update-containerd/kustomization.yaml b/registry/containerd-config/kustomization.yaml similarity index 100% rename from registry/node-update-containerd/kustomization.yaml rename to registry/containerd-config/kustomization.yaml diff --git a/registry/node-update-hosts/README.md b/registry/node-update-hosts/README.md deleted file mode 100644 index dbe9dcc4..00000000 --- a/registry/node-update-hosts/README.md +++ /dev/null @@ -1,7 +0,0 @@ - - -You need this, or any other form of name resolution update, -if your nodes's Docker don't resolve cluster addresses already. - -With Containerd it might be preferrable to configure registry mirrors, -pointing to the service IPs. diff --git a/registry/node-update-hosts/daemonset.yaml b/registry/node-update-hosts/daemonset.yaml deleted file mode 100644 index 22cf5b37..00000000 --- a/registry/node-update-hosts/daemonset.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: registry-etc-hosts-update -spec: - selector: - matchLabels: - app: registry-etc-hosts-update - template: - metadata: - labels: - app: registry-etc-hosts-update - annotations: - "cluster-autoscaler.kubernetes.io/safe-to-evict": "true" - spec: - tolerations: - - effect: NoSchedule - operator: Exists - initContainers: - - name: update - image: ubuntu:22.04@sha256:27cb6e6ccef575a4698b66f5de06c7ecd61589132d5a91d098f7f3f9285415a9 - volumeMounts: - - name: etchosts - mountPath: /host-etc/hosts - readOnly: false - env: - - name: REGISTRY_SERVICE_HOSTS - value: >- - builds-registry.ystack.svc.cluster.local - prod-registry.ystack.svc.cluster.local - command: - - bash - - -ce - - | - NL=$'\n' - HOSTS="$(cat /host-etc/hosts)" - for H in $REGISTRY_SERVICE_HOSTS; do - ent="" - [ ! -z "$FIXED_IP" ] && ent="$FIXED_IP $H"; - [ -z "$ent" ] && ent=$(getent hosts $H); - [ -z "$ent" ] && echo "Failed to get hosts entry for $H" && exit 1; - echo "$HOSTS" | grep " $H" || HOSTS="$HOSTS$NL$ent"; - echo "$HOSTS" | grep "$ent" || HOSTS=$(echo "$HOSTS" | sed "s|.* $H|$ent|"); - done; - echo "$HOSTS" | diff -u /host-etc/hosts - || echo "$HOSTS" > /host-etc/hosts - echo "Done." - containers: - - name: init-container-did-the-work - image: k8s.gcr.io/pause:3.6@sha256:3d380ca8864549e74af4b29c10f9cb0956236dfb01c40ca076fb6c37253234db - terminationGracePeriodSeconds: 30 - volumes: - - name: etchosts - hostPath: - path: /etc/hosts diff --git a/registry/node-update-hosts/kustomization.yaml b/registry/node-update-hosts/kustomization.yaml deleted file mode 100644 index 2ff0ac3b..00000000 --- a/registry/node-update-hosts/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: -- daemonset.yaml