diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml new file mode 100644 index 000000000..03e870245 --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml @@ -0,0 +1,5 @@ +--- +ansible_python_interpreter: /usr/bin/python3 +ansible_port: 22 +ansible_user: cloudinit +kube_api_server_vip: 192.168.18.100 diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory index ce812b376..0fc0ad0f8 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory @@ -1,56 +1,74 @@ -[k8s-servers] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +[seichi-onp-k8s-cp-1] +cp-1 ansible_host=192.168.0.11 host_addr_srv=192.168.0.11 host_addr_san=192.168.18.11 host_addr_cls=192.168.32.11 + +[seichi-onp-k8s-cp-2] +cp-2 ansible_host=192.168.0.12 host_addr_srv=192.168.0.12 host_addr_san=192.168.18.12 host_addr_cls=192.168.32.12 + +[seichi-onp-k8s-cp-3] +cp-3 ansible_host=192.168.0.13 host_addr_srv=192.168.0.13 host_addr_san=192.168.18.13 host_addr_cls=192.168.32.13 + +[seichi-onp-k8s-wk-1] +wk-1 ansible_host=192.168.0.21 host_addr_srv=192.168.0.21 host_addr_san=192.168.18.21 host_addr_cls=192.168.32.21 + +[seichi-onp-k8s-wk-2] +wk-2 ansible_host=192.168.0.22 host_addr_srv=192.168.0.22 host_addr_san=192.168.18.22 host_addr_cls=192.168.32.22 + +[seichi-onp-k8s-wk-3] +wk-3 ansible_host=192.168.0.23 host_addr_srv=192.168.0.23 host_addr_san=192.168.18.23 host_addr_cls=192.168.32.23 + + +[k8s-servers:children] +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers:vars] ansible_ssh_pass=zaq12wsx [k8s-servers-with-ssh:children] -k8s-servers +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +[k8s-servers-cp-with-ssh:children] +seichi-onp-k8s-cp-1 +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 [k8s-servers-cp-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-leader-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 +[k8s-servers-cp-leader-with-ssh:children] +seichi-onp-k8s-cp-1 [k8s-servers-cp-leader-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-cp-follower-with-ssh] -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +[k8s-servers-cp-follower-with-ssh:children] +seichi-onp-k8s-cp-2 +seichi-onp-k8s-cp-3 [k8s-servers-cp-follower-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 -[k8s-servers-wk-with-ssh] -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +[k8s-servers-wk-with-ssh:children] +seichi-onp-k8s-wk-1 +seichi-onp-k8s-wk-2 +seichi-onp-k8s-wk-3 [k8s-servers-wk-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 - - -[all:vars] -ansible_python_interpreter=/usr/bin/python3 -ansible_port=22 -ansible_user=cloudinit \ No newline at end of file diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml index 121469655..dd8af67c0 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/tasks/main.yaml @@ -1,10 +1,9 @@ -- name: Copy file +- name: Deploy kubeadm join config file become: yes - ansible.builtin.copy: - src: /root/join_kubeadm_cp.yaml + template: + src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml dest: /root/join_kubeadm_cp.yaml - name: Execute kubeadm join command become: yes shell: "kubeadm join --config /root/join_kubeadm_cp.yaml" - \ No newline at end of file diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml new file mode 100644 index 000000000..331655b2d --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/11-kubeadm-join-cp/templates/join_kubeadm_cp.yaml @@ -0,0 +1,21 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: "systemd" +protectKernelDefaults: true +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +nodeRegistration: + criSocket: "unix:///var/run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ host_addr_cls }}" +localAPIEndpoint: + advertiseAddress: "{{ host_addr_cls }}" + bindPort: 6443 +discovery: + bootstrapToken: + apiServerEndpoint: "{{ kube_api_server_vip }}:8443" + token: "{{ kubeadm_bootstrap_token }}" + unsafeSkipCAVerification: true +controlPlane: + certificateKey: "{{ kubeadm_uploaded_certs }}" diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml index 06da29d94..99a8841ed 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/main.yaml @@ -1,7 +1,7 @@ -- name: Copy file +- name: Deploy kubeadm join config file become: yes - ansible.builtin.copy: - src: /root/join_kubeadm_wk.yaml + template: + src: /home/cloudinit/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/templates/join_kubeadm_wk.yaml dest: /root/join_kubeadm_wk.yaml - name: Execute kubeadm join command diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml new file mode 100644 index 000000000..1db412aa0 --- /dev/null +++ b/seichi-onp-k8s/cluster-boot-up/ansible/roles/12-kubeadm-join-wk/tasks/templates/join_kubeadm_wk.yaml @@ -0,0 +1,16 @@ +apiVersion: kubelet.config.k8s.io/v1beta1 +kind: KubeletConfiguration +cgroupDriver: "systemd" +protectKernelDefaults: true +--- +apiVersion: kubeadm.k8s.io/v1beta4 +kind: JoinConfiguration +nodeRegistration: + criSocket: "unix:///var/run/containerd/containerd.sock" + kubeletExtraArgs: + node-ip: "{{ host_addr_cls }}" +discovery: + bootstrapToken: + apiServerEndpoint: "{{ kube_api_server_vip }}:8443" + token: "{{ kubeadm_bootstrap_token }}" + unsafeSkipCAVerification: true diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh index 0295f4bed..c4b89518c 100644 --- a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh +++ b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh @@ -343,50 +343,9 @@ helm install cilium cilium/cilium \ # Generate control plane certificate KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1) -# Set join configuration for other control plane nodes -cat > "$HOME"/join_kubeadm_cp.yaml < "$HOME"/join_kubeadm_wk.yaml <> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml +echo "kubeadm_uploaded_certs: $KUBEADM_UPLOADED_CERTS" >> "$HOME"/seichi_infra/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/group_vars/all.yaml # install ansible sudo apt-get install -y ansible git sshpass