Merge pull request #1435 from GiganticMinecraft/wip

k8s verup, IP変更対応, kubeadm joinに必要なconfigをansibleに作らせる
GiganticMinecraft · Dec 3, 2023 · 624a1ef · 624a1ef
2 parents 8f8f6f2 + 1d183b0
commit 624a1ef
Show file tree

Hide file tree

Showing 9 changed files with 114 additions and 79 deletions.
diff --git a/.github/workflows/pluto.yaml b/.github/workflows/pluto.yaml
@@ -18,8 +18,8 @@ jobs:
 
       - name: Use pluto
         run: |
-          pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/apps --target-versions k8s=v1.27.5
+          pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/apps --target-versions k8s=v1.27.6
 
       - name: Use pluto
         run: |
-          pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/app-templates --target-versions k8s=v1.27.5
+          pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/app-templates --target-versions k8s=v1.27.6
diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md
@@ -64,7 +64,7 @@ KubernetesノードのVMは cloudinit イメージで作成されています。
 2022/05/23現在、クラスタは (3 control plane nodes + 3 worker nodes) の構成で[作成されています](https://github.com/GiganticMinecraft/seichi_infra/blob/9b6a9346371b8f2add3a786b6badbe4e13d4464c/seichi-onp-k8s/cluster-boot-scripts/deploy-vm.sh#L14-L19)。
 
 クラスタの作成は以下のツール群で行っています。
-  - kubeadm, kubectl, kubelet v1.27.5
+  - kubeadm, kubectl, kubelet v1.27.6
 
 CNI には Cilium を利用しています。
 

diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml
@@ -0,0 +1,5 @@
+---
+ansible_python_interpreter: /usr/bin/python3
+ansible_port: 22
+ansible_user: cloudinit
+kube_api_server_vip: 192.168.18.100
diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory
@@ -1,56 +1,74 @@
-[k8s-servers]
-seichi-onp-k8s-cp-1 ansible_host=192.168.8.11
-seichi-onp-k8s-cp-2 ansible_host=192.168.8.12
-seichi-onp-k8s-cp-3 ansible_host=192.168.8.13
-seichi-onp-k8s-wk-1 ansible_host=192.168.8.21
-seichi-onp-k8s-wk-2 ansible_host=192.168.8.22
-seichi-onp-k8s-wk-3 ansible_host=192.168.8.23
+[seichi-onp-k8s-cp-1]
+cp-1 ansible_host=192.168.0.11 host_addr_srv=192.168.0.11 host_addr_san=192.168.18.11 host_addr_cls=192.168.32.11
+
+[seichi-onp-k8s-cp-2]
+cp-2 ansible_host=192.168.0.12 host_addr_srv=192.168.0.12 host_addr_san=192.168.18.12 host_addr_cls=192.168.32.12
+
+[seichi-onp-k8s-cp-3]
+cp-3 ansible_host=192.168.0.13 host_addr_srv=192.168.0.13 host_addr_san=192.168.18.13 host_addr_cls=192.168.32.13
+
+[seichi-onp-k8s-wk-1]
+wk-1 ansible_host=192.168.0.21 host_addr_srv=192.168.0.21 host_addr_san=192.168.18.21 host_addr_cls=192.168.32.21
+
+[seichi-onp-k8s-wk-2]
+wk-2 ansible_host=192.168.0.22 host_addr_srv=192.168.0.22 host_addr_san=192.168.18.22 host_addr_cls=192.168.32.22
+
+[seichi-onp-k8s-wk-3]
+wk-3 ansible_host=192.168.0.23 host_addr_srv=192.168.0.23 host_addr_san=192.168.18.23 host_addr_cls=192.168.32.23
+
+
+[k8s-servers:children]
+seichi-onp-k8s-cp-1
+seichi-onp-k8s-cp-2
+seichi-onp-k8s-cp-3
+seichi-onp-k8s-wk-1
+seichi-onp-k8s-wk-2
+seichi-onp-k8s-wk-3
 
 [k8s-servers:vars]
 ansible_ssh_pass=zaq12wsx
 
 
 [k8s-servers-with-ssh:children]
-k8s-servers
+seichi-onp-k8s-cp-1
+seichi-onp-k8s-cp-2
+seichi-onp-k8s-cp-3
+seichi-onp-k8s-wk-1
+seichi-onp-k8s-wk-2
+seichi-onp-k8s-wk-3
 
 [k8s-servers-with-ssh:vars]
 ansible_ssh_private_key_file=/root/.ssh/id_ed25519
 
 
-[k8s-servers-cp-with-ssh]
-seichi-onp-k8s-cp-1 ansible_host=192.168.8.11
-seichi-onp-k8s-cp-2 ansible_host=192.168.8.12
-seichi-onp-k8s-cp-3 ansible_host=192.168.8.13
+[k8s-servers-cp-with-ssh:children]
+seichi-onp-k8s-cp-1
+seichi-onp-k8s-cp-2
+seichi-onp-k8s-cp-3
 
 [k8s-servers-cp-with-ssh:vars]
 ansible_ssh_private_key_file=/root/.ssh/id_ed25519
 
 
-[k8s-servers-cp-leader-with-ssh]
-seichi-onp-k8s-cp-1 ansible_host=192.168.8.11
+[k8s-servers-cp-leader-with-ssh:children]
+seichi-onp-k8s-cp-1
 
 [k8s-servers-cp-leader-with-ssh:vars]
 ansible_ssh_private_key_file=/root/.ssh/id_ed25519
 
 
-[k8s-servers-cp-follower-with-ssh]
-seichi-onp-k8s-cp-2 ansible_host=192.168.8.12
-seichi-onp-k8s-cp-3 ansible_host=192.168.8.13
+[k8s-servers-cp-follower-with-ssh:children]
+seichi-onp-k8s-cp-2
+seichi-onp-k8s-cp-3
 
 [k8s-servers-cp-follower-with-ssh:vars]
 ansible_ssh_private_key_file=/root/.ssh/id_ed25519
 
 
-[k8s-servers-wk-with-ssh]
-seichi-onp-k8s-wk-1 ansible_host=192.168.8.21
-seichi-onp-k8s-wk-2 ansible_host=192.168.8.22
-seichi-onp-k8s-wk-3 ansible_host=192.168.8.23
+[k8s-servers-wk-with-ssh:children]
+seichi-onp-k8s-wk-1
+seichi-onp-k8s-wk-2
+seichi-onp-k8s-wk-3
 
 [k8s-servers-wk-with-ssh:vars]
 ansible_ssh_private_key_file=/root/.ssh/id_ed25519
-
-
-[all:vars]
-ansible_python_interpreter=/usr/bin/python3
-ansible_port=22
-ansible_user=cloudinit
diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml
@@ -1,10 +1,9 @@
-- name: Copy file
+- name: Deploy kubeadm join config file
   become: yes
-  ansible.builtin.copy:
-    src: /root/join_kubeadm_cp.yaml
+  template:
+    src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml
     dest: /root/join_kubeadm_cp.yaml
 
 - name: Execute kubeadm join command
   become: yes
   shell: "kubeadm join --config /root/join_kubeadm_cp.yaml"
-
diff --git a/...i-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml b/...i-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml
@@ -0,0 +1,21 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: "systemd"
+protectKernelDefaults: true
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: JoinConfiguration
+nodeRegistration:
+  criSocket: "unix:///var/run/containerd/containerd.sock"
+  kubeletExtraArgs:
+    node-ip: "{{ host_addr_cls }}"
+localAPIEndpoint:
+  advertiseAddress: "{{ host_addr_cls }}"
+  bindPort: 6443
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: "{{ kube_api_server_vip }}:8443"
+    token: "{{ kubeadm_bootstrap_token }}"
+    unsafeSkipCAVerification: true
+controlPlane:
+  certificateKey: "{{ kubeadm_uploaded_certs }}"
diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml
@@ -1,7 +1,7 @@
-- name: Copy file
+- name: Deploy kubeadm join config file
   become: yes
-  ansible.builtin.copy:
-    src: /root/join_kubeadm_wk.yaml
+  template:
+    src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/templates/join_kubeadm_wk.yaml
     dest: /root/join_kubeadm_wk.yaml
 
 - name: Execute kubeadm join command

diff --git a/...k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml b/...k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml
@@ -0,0 +1,16 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+cgroupDriver: "systemd"
+protectKernelDefaults: true
+---
+apiVersion: kubeadm.k8s.io/v1beta4
+kind: JoinConfiguration
+nodeRegistration:
+  criSocket: "unix:///var/run/containerd/containerd.sock"
+  kubeletExtraArgs:
+    node-ip: "{{ host_addr_cls }}"
+discovery:
+  bootstrapToken:
+    apiServerEndpoint: "{{ kube_api_server_vip }}:8443"
+    token: "{{ kubeadm_bootstrap_token }}"
+    unsafeSkipCAVerification: true
diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh
@@ -118,8 +118,8 @@ sysctl --system
 curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg
 echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list
 apt-get update
-apt-get install -y kubelet=1.27.5-00 kubeadm=1.27.5-00 kubectl=1.27.5-00
-apt-mark hold kubelet kubeadm kubectl
+apt-get install -y kubeadm kubelet=1.27.6-00 kubectl=1.27.6-00
+apt-mark hold kubelet kubectl
 
 # Disable swap
 swapoff -a
@@ -265,24 +265,33 @@ esac
 
 # Set kubeadm bootstrap token using openssl
 KUBEADM_BOOTSTRAP_TOKEN=$(openssl rand -hex 3).$(openssl rand -hex 8)
+KUBEADM_LOCAL_ENDPOINT=$(ip -4 addr show ens19 | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | awk 'NR==1{print $1}')
 
 # Set init configuration for the first control plane
 cat > "$HOME"/init_kubeadm.yaml <<EOF
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta4
 kind: InitConfiguration
 bootstrapTokens:
 - token: "$KUBEADM_BOOTSTRAP_TOKEN"
   description: "kubeadm bootstrap token"
   ttl: "24h"
 nodeRegistration:
   criSocket: "unix:///var/run/containerd/containerd.sock"
+  kubeletExtraArgs:
+    node-ip: "$KUBEADM_LOCAL_ENDPOINT"
+    imagePullPolicy: "IfNotPresent"
+localAPIEndpoint:
+  advertiseAddress: "$KUBEADM_LOCAL_ENDPOINT"
+  bindPort: 6443
+skipPhases:
+  - addon/kube-proxy
 ---
-apiVersion: kubeadm.k8s.io/v1beta3
+apiVersion: kubeadm.k8s.io/v1beta4
 kind: ClusterConfiguration
 networking:
   serviceSubnet: "10.96.0.0/16"
   podSubnet: "10.128.0.0/16"
-kubernetesVersion: "v1.27.5"
+kubernetesVersion: "v1.27.6"
 controlPlaneEndpoint: "${KUBE_API_SERVER_VIP}:8443"
 apiServer:
   certSANs:
@@ -304,6 +313,7 @@ controllerManager:
 scheduler:
   extraArgs:
     bind-address: "0.0.0.0"
+clusterName: "unchama-cloud"
 
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
@@ -333,43 +343,9 @@ helm install cilium cilium/cilium \
 # Generate control plane certificate
 KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1)
 
-# Set join configuration for other control plane nodes
-cat > "$HOME"/join_kubeadm_cp.yaml <<EOF
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-cgroupDriver: "systemd"
-protectKernelDefaults: true
----
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: JoinConfiguration
-nodeRegistration:
-  criSocket: "unix:///var/run/containerd/containerd.sock"
-discovery:
-  bootstrapToken:
-    apiServerEndpoint: "${KUBE_API_SERVER_VIP}:8443"
-    token: "$KUBEADM_BOOTSTRAP_TOKEN"
-    unsafeSkipCAVerification: true
-controlPlane:
-  certificateKey: "$KUBEADM_UPLOADED_CERTS"
-EOF
-
-# Set join configuration for worker nodes
-cat > "$HOME"/join_kubeadm_wk.yaml <<EOF
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-cgroupDriver: "systemd"
-protectKernelDefaults: true
----
-apiVersion: kubeadm.k8s.io/v1beta3
-kind: JoinConfiguration
-nodeRegistration:
-  criSocket: "unix:///var/run/containerd/containerd.sock"
-discovery:
-  bootstrapToken:
-    apiServerEndpoint: "${KUBE_API_SERVER_VIP}:8443"
-    token: "$KUBEADM_BOOTSTRAP_TOKEN"
-    unsafeSkipCAVerification: true
-EOF
+# add join information to ansible hosts variable
+echo "kubeadm_bootstrap_token: $KUBEADM_BOOTSTRAP_TOKEN" >> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml
+echo "kubeadm_uploaded_certs: $KUBEADM_UPLOADED_CERTS" >> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml
 
 # install ansible
 sudo apt-get install -y ansible git sshpass