diff --git a/.github/workflows/pluto.yaml b/.github/workflows/pluto.yaml index 152bb558b..216bf5242 100644 --- a/.github/workflows/pluto.yaml +++ b/.github/workflows/pluto.yaml @@ -18,8 +18,8 @@ jobs: - name: Use pluto run: | - pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/apps --target-versions k8s=v1.27.5 + pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/apps --target-versions k8s=v1.27.6 - name: Use pluto run: | - pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/app-templates --target-versions k8s=v1.27.5 + pluto detect-files -d seichi-onp-k8s/manifests/seichi-kubernetes/app-templates --target-versions k8s=v1.27.6 diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md index 2dfb2b886..a3cbfb24c 100644 --- a/seichi-onp-k8s/cluster-boot-up/README.md +++ b/seichi-onp-k8s/cluster-boot-up/README.md @@ -64,7 +64,7 @@ KubernetesノードのVMは cloudinit イメージで作成されています。 2022/05/23現在、クラスタは (3 control plane nodes + 3 worker nodes) の構成で[作成されています](https://github.com/GiganticMinecraft/seichi_infra/blob/9b6a9346371b8f2add3a786b6badbe4e13d4464c/seichi-onp-k8s/cluster-boot-scripts/deploy-vm.sh#L14-L19)。 クラスタの作成は以下のツール群で行っています。 - - kubeadm, kubectl, kubelet v1.27.5 + - kubeadm, kubectl, kubelet v1.27.6 CNI には Cilium を利用しています。 diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml new file mode 100644 index 000000000..03e870245 --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml @@ -0,0 +1,5 @@ +--- +ansible_python_interpreter: /usr/bin/python3 +ansible_port: 22 +ansible_user: cloudinit +kube_api_server_vip: 192.168.18.100 diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory index ce812b376..0fc0ad0f8 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory @@ -1,56 +1,74 @@ -[k8s-servers] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +[seichi-onp-k8s-cp-1] +cp-1 ansible_host=192.168.0.11 host_addr_srv=192.168.0.11 host_addr_san=192.168.18.11 host_addr_cls=192.168.32.11 + +[seichi-onp-k8s-cp-2] +cp-2 ansible_host=192.168.0.12 host_addr_srv=192.168.0.12 host_addr_san=192.168.18.12 host_addr_cls=192.168.32.12 + +[seichi-onp-k8s-cp-3] +cp-3 ansible_host=192.168.0.13 host_addr_srv=192.168.0.13 host_addr_san=192.168.18.13 host_addr_cls=192.168.32.13 + +[seichi-onp-k8s-wk-1] +wk-1 ansible_host=192.168.0.21 host_addr_srv=192.168.0.21 host_addr_san=192.168.18.21 host_addr_cls=192.168.32.21 + +[seichi-onp-k8s-wk-2] +wk-2 ansible_host=192.168.0.22 host_addr_srv=192.168.0.22 host_addr_san=192.168.18.22 host_addr_cls=192.168.32.22 + +[seichi-onp-k8s-wk-3] +wk-3 ansible_host=192.168.0.23 host_addr_srv=192.168.0.23 host_addr_san=192.168.18.23 host_addr_cls=192.168.32.23 + + +[k8s-servers:children] +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers:vars] ansible_ssh_pass=zaq12wsx [k8s-servers-with-ssh:children] -k8s-servers +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +[k8s-servers-cp-with-ssh:children] +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 [k8s-servers-cp-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-leader-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 +[k8s-servers-cp-leader-with-ssh:children] +seichi-onp-k8s-cp-1 [k8s-servers-cp-leader-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-follower-with-ssh] -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +[k8s-servers-cp-follower-with-ssh:children] +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 [k8s-servers-cp-follower-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-wk-with-ssh] -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +[k8s-servers-wk-with-ssh:children] +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers-wk-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 - - -[all:vars] -ansible_python_interpreter=/usr/bin/python3 -ansible_port=22 -ansible_user=cloudinit \ No newline at end of file diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml index 121469655..dd8af67c0 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml @@ -1,10 +1,9 @@ -- name: Copy file +- name: Deploy kubeadm join config file become: yes - ansible.builtin.copy: - src: /root/join_kubeadm_cp.yaml + template: + src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml dest: /root/join_kubeadm_cp.yaml - name: Execute kubeadm join command become: yes shell: "kubeadm join --config /root/join_kubeadm_cp.yaml" - \ No newline at end of file diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml new file mode 100644 index 000000000..331655b2d --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml @@ -0,0 +1,21 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: "systemd" +protectKernelDefaults: true +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +nodeRegistration: + criSocket: "unix:///var/run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ host_addr_cls }}" +localAPIEndpoint: + advertiseAddress: "{{ host_addr_cls }}" + bindPort: 6443 +discovery: + bootstrapToken: + apiServerEndpoint: "{{ kube_api_server_vip }}:8443" + token: "{{ kubeadm_bootstrap_token }}" + unsafeSkipCAVerification: true +controlPlane: + certificateKey: "{{ kubeadm_uploaded_certs }}" diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml index 06da29d94..99a8841ed 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml @@ -1,7 +1,7 @@ -- name: Copy file +- name: Deploy kubeadm join config file become: yes - ansible.builtin.copy: - src: /root/join_kubeadm_wk.yaml + template: + src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/templates/join_kubeadm_wk.yaml dest: /root/join_kubeadm_wk.yaml - name: Execute kubeadm join command diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml new file mode 100644 index 000000000..1db412aa0 --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml @@ -0,0 +1,16 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: "systemd" +protectKernelDefaults: true +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +nodeRegistration: + criSocket: "unix:///var/run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ host_addr_cls }}" +discovery: + bootstrapToken: + apiServerEndpoint: "{{ kube_api_server_vip }}:8443" + token: "{{ kubeadm_bootstrap_token }}" + unsafeSkipCAVerification: true diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh index 41e8eb974..c4b89518c 100644 --- a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh +++ b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh @@ -118,8 +118,8 @@ sysctl --system curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo gpg --dearmor -o /etc/apt/keyrings/kubernetes-archive-keyring.gpg echo "deb [signed-by=/etc/apt/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | sudo tee /etc/apt/sources.list.d/kubernetes.list apt-get update -apt-get install -y kubelet=1.27.5-00 kubeadm=1.27.5-00 kubectl=1.27.5-00 -apt-mark hold kubelet kubeadm kubectl +apt-get install -y kubeadm kubelet=1.27.6-00 kubectl=1.27.6-00 +apt-mark hold kubelet kubectl # Disable swap swapoff -a @@ -265,10 +265,11 @@ esac # Set kubeadm bootstrap token using openssl KUBEADM_BOOTSTRAP_TOKEN=$(openssl rand -hex 3).$(openssl rand -hex 8) +KUBEADM_LOCAL_ENDPOINT=$(ip -4 addr show ens19 | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | awk 'NR==1{print $1}') # Set init configuration for the first control plane cat > "$HOME"/init_kubeadm.yaml < "$HOME"/join_kubeadm_cp.yaml < "$HOME"/join_kubeadm_wk.yaml <> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml +echo "kubeadm_uploaded_certs: $KUBEADM_UPLOADED_CERTS" >> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml # install ansible sudo apt-get install -y ansible git sshpass