From c9c9bc7f7e426d8930bb678dec530a5690ea787c Mon Sep 17 00:00:00 2001 From: unchama <11990197+unchama@users.noreply.github.com> Date: Wed, 29 Nov 2023 23:27:31 +0900 Subject: [PATCH 1/3] =?UTF-8?q?=E6=9C=AC=E7=95=AAservice-seg=E3=81=AE?= =?UTF-8?q?=E7=B8=AE=E5=B0=8F=E3=81=AB=E4=BC=B4=E3=81=86ip=E3=82=A2?= =?UTF-8?q?=E3=83=89=E3=83=AC=E3=82=B9=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- seichi-onp-k8s/cluster-boot-up/README.md | 4 +-- .../ansible/hosts/k8s-servers/inventory | 30 +++++++++---------- .../snippets/seichi-onp-k8s-cp-1-network.yaml | 8 ++--- .../snippets/seichi-onp-k8s-cp-2-network.yaml | 8 ++--- .../snippets/seichi-onp-k8s-cp-3-network.yaml | 8 ++--- .../snippets/seichi-onp-k8s-wk-1-network.yaml | 8 ++--- .../snippets/seichi-onp-k8s-wk-2-network.yaml | 8 ++--- .../snippets/seichi-onp-k8s-wk-3-network.yaml | 8 ++--- .../manifests/seichi-kubernetes/README.md | 12 ++++---- .../cloudflared-tunnel-exits/http-exits.yaml | 2 +- .../metallb/metallb-address-pool.yaml | 2 +- .../bungeecord/bungeesemaphore-redis.yaml | 2 +- .../bungeecord/redisbungee-redis.yaml | 2 +- .../service-bungeecord-loadbalancer.yaml | 2 +- .../service-bungeecord-loadbalancer.yaml | 2 +- .../redis/bungeesemaphore-redis.yaml | 2 +- .../redis/redisbungee-redis.yaml | 2 +- 17 files changed, 55 insertions(+), 55 deletions(-) diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md index 2dfb2b886..cf3615992 100644 --- a/seichi-onp-k8s/cluster-boot-up/README.md +++ b/seichi-onp-k8s/cluster-boot-up/README.md @@ -54,10 +54,10 @@ KubernetesノードのVMは cloudinit イメージで作成されています。 - Service Network (10.96.0.0/16) - External - Node IP - - Service Network (192.168.8.0-192.168.8.127) + - Service Network (192.168.0.0-192.168.0.127) - Storage Network (192.168.18.0-192.168.18.127) - API Endpoint (192.168.18.100) - - LoadBalancer VIP (192.168.8.128-192.168.8.255) + - LoadBalancer VIP (192.168.0.128-192.168.0.255) ## Kubernetesクラスタの構成 diff --git a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory index ce812b376..e26ae2067 100644 --- a/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory +++ b/seichi-onp-k8s/cluster-boot-up/ansible/hosts/k8s-servers/inventory @@ -1,10 +1,10 @@ [k8s-servers] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +seichi-onp-k8s-cp-1 ansible_host=192.168.0.11 +seichi-onp-k8s-cp-2 ansible_host=192.168.0.12 +seichi-onp-k8s-cp-3 ansible_host=192.168.0.13 +seichi-onp-k8s-wk-1 ansible_host=192.168.0.21 +seichi-onp-k8s-wk-2 ansible_host=192.168.0.22 +seichi-onp-k8s-wk-3 ansible_host=192.168.0.23 [k8s-servers:vars] ansible_ssh_pass=zaq12wsx @@ -18,33 +18,33 @@ ansible_ssh_private_key_file=/root/.ssh/id_ed25519 [k8s-servers-cp-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +seichi-onp-k8s-cp-1 ansible_host=192.168.0.11 +seichi-onp-k8s-cp-2 ansible_host=192.168.0.12 +seichi-onp-k8s-cp-3 ansible_host=192.168.0.13 [k8s-servers-cp-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 [k8s-servers-cp-leader-with-ssh] -seichi-onp-k8s-cp-1 ansible_host=192.168.8.11 +seichi-onp-k8s-cp-1 ansible_host=192.168.0.11 [k8s-servers-cp-leader-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 [k8s-servers-cp-follower-with-ssh] -seichi-onp-k8s-cp-2 ansible_host=192.168.8.12 -seichi-onp-k8s-cp-3 ansible_host=192.168.8.13 +seichi-onp-k8s-cp-2 ansible_host=192.168.0.12 +seichi-onp-k8s-cp-3 ansible_host=192.168.0.13 [k8s-servers-cp-follower-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 [k8s-servers-wk-with-ssh] -seichi-onp-k8s-wk-1 ansible_host=192.168.8.21 -seichi-onp-k8s-wk-2 ansible_host=192.168.8.22 -seichi-onp-k8s-wk-3 ansible_host=192.168.8.23 +seichi-onp-k8s-wk-1 ansible_host=192.168.0.21 +seichi-onp-k8s-wk-2 ansible_host=192.168.0.22 +seichi-onp-k8s-wk-3 ansible_host=192.168.0.23 [k8s-servers-wk-with-ssh:vars] ansible_ssh_private_key_file=/root/.ssh/id_ed25519 diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-1-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-1-network.yaml index 93d69b36a..066c82de4 100644 --- a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-1-network.yaml +++ b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-1-network.yaml @@ -4,9 +4,9 @@ config: name: ens18 subnets: - type: static - address: '192.168.8.11' - netmask: '255.255.240.0' - gateway: '192.168.1.1' + address: '192.168.0.11' + netmask: '255.255.252.0' + gateway: '192.168.3.254' - type: physical name: ens19 subnets: @@ -15,6 +15,6 @@ config: netmask: '255.255.252.0' - type: nameserver address: - - '192.168.1.1' + - '192.168.100.1' search: - 'local' diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-2-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-2-network.yaml index f640052f3..09310f9c9 100644 --- a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-2-network.yaml +++ b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-2-network.yaml @@ -4,9 +4,9 @@ config: name: ens18 subnets: - type: static - address: '192.168.8.12' - netmask: '255.255.240.0' - gateway: '192.168.1.1' + address: '192.168.0.12' + netmask: '255.255.252.0' + gateway: '192.168.3.254' - type: physical name: ens19 subnets: @@ -15,6 +15,6 @@ config: netmask: '255.255.252.0' - type: nameserver address: - - '192.168.1.1' + - '192.168.100.1' search: - 'local' diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-3-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-3-network.yaml index e38a40572..aa6777b2d 100644 --- a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-3-network.yaml +++ b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-3-network.yaml @@ -4,9 +4,9 @@ config: name: ens18 subnets: - type: static - address: '192.168.8.13' - netmask: '255.255.240.0' - gateway: '192.168.1.1' + address: '192.168.0.13' + netmask: '255.255.252.0' + gateway: '192.168.3.254' - type: physical name: ens19 subnets: @@ -15,6 +15,6 @@ config: netmask: '255.255.252.0' - type: nameserver address: - - '192.168.1.1' + - '192.168.100.1' search: - 'local' diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-1-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-1-network.yaml index b4240e8b7..10d60cf27 100644 --- a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-1-network.yaml +++ b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-1-network.yaml @@ -4,9 +4,9 @@ config: name: ens18 subnets: - type: static - address: '192.168.8.21' - netmask: '255.255.240.0' - gateway: '192.168.1.1' + address: '192.168.0.21' + netmask: '255.255.252.0' + gateway: '192.168.3.254' - type: physical name: ens19 subnets: @@ -15,6 +15,6 @@ config: netmask: '255.255.252.0' - type: nameserver address: - - '192.168.1.1' + - '192.168.100.1' search: - 'local' diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-2-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-2-network.yaml index efcaba304..13e453908 100644 --- a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-2-network.yaml +++ b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-2-network.yaml @@ -4,9 +4,9 @@ config: name: ens18 subnets: - type: static - address: '192.168.8.22' - netmask: '255.255.240.0' - gateway: '192.168.1.1' + address: '192.168.0.22' + netmask: '255.255.252.0' + gateway: '192.168.3.254' - type: physical name: ens19 subnets: @@ -15,6 +15,6 @@ config: netmask: '255.255.252.0' - type: nameserver address: - - '192.168.1.1' + - '192.168.100.1' search: - 'local' diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-3-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-3-network.yaml index eff77df14..6dccf6f2c 100644 --- a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-3-network.yaml +++ b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-3-network.yaml @@ -4,9 +4,9 @@ config: name: ens18 subnets: - type: static - address: '192.168.8.23' - netmask: '255.255.240.0' - gateway: '192.168.1.1' + address: '192.168.0.23' + netmask: '255.255.252.0' + gateway: '192.168.3.254' - type: physical name: ens19 subnets: @@ -15,6 +15,6 @@ config: netmask: '255.255.252.0' - type: nameserver address: - - '192.168.1.1' + - '192.168.100.1' search: - 'local' diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md index be5b6eda8..5395ddbbd 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md @@ -42,8 +42,8 @@ TCP パケットをそのまま送り届ける必要があります。 | サービス | `Service` の VIP | | ---------------------------- | ----------------------------------------------------------- | -| BungeeCord (本番環境用) | [`192.168.8.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | -| BungeeCord (デバッグ環境用) | [`192.168.8.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | +| BungeeCord (本番環境用) | [`192.168.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | +| BungeeCord (デバッグ環境用) | [`192.168.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | | 投票受付サーバー | (まだ k8s 上に乗っていないので、 `Service` の VIP ではない) | ### オンプレネットワーク内からのトラフィックを受ける `Service` @@ -54,10 +54,10 @@ TCP パケットをそのまま送り届ける必要があります。 | サービス | `Service` の VIP | | ------------------------------ | ----------------------------------------------------------- | -| 本番 RedisBungee 用 Redis | [`192.168.8.132`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redisbungee-redis.yaml#L24) | -| 本番 BungeeSemaphore 用 Redis | [`192.168.8.133`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/bungeesemaphore-redis.yaml#L24) | -| Debug RedisBungee 用 Redis | [`192.168.8.134`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/redisbungee-redis.yaml#L24) | -| Debug BungeeSemaphore 用 Redis | [`192.168.8.135`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/bungeesemaphore-redis.yaml#L24) | +| 本番 RedisBungee 用 Redis | [`192.168.0.132`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redisbungee-redis.yaml#L24) | +| 本番 BungeeSemaphore 用 Redis | [`192.168.0.133`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/bungeesemaphore-redis.yaml#L24) | +| Debug RedisBungee 用 Redis | [`192.168.0.134`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/redisbungee-redis.yaml#L24) | +| Debug BungeeSemaphore 用 Redis | [`192.168.0.135`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/bungeesemaphore-redis.yaml#L24) | ## Kubernetes クラスタのブートストラップについて diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cloudflared-tunnel-exits/http-exits.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cloudflared-tunnel-exits/http-exits.yaml index 2e59ba223..14b9218dc 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cloudflared-tunnel-exits/http-exits.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cloudflared-tunnel-exits/http-exits.yaml @@ -50,7 +50,7 @@ spec: # Sentry - name: sentry external-hostname: sentry.onp.admin.seichi.click - internal-authority: "192.168.8.19:9000" + internal-authority: "192.168.3.19:9000" # 各サーバーの Dynmap ウェブサーバー - name: dynmap-s1 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml index cb8242188..c0e4e5d9b 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml @@ -5,7 +5,7 @@ metadata: namespace: metallb-system spec: addresses: - - 192.168.8.128-192.168.8.255 + - 192.168.0.128-192.168.0.255 --- apiVersion: metallb.io/v1beta1 kind: L2Advertisement diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/bungeesemaphore-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/bungeesemaphore-redis.yaml index d91dc4c42..cfa2af797 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/bungeesemaphore-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/bungeesemaphore-redis.yaml @@ -20,7 +20,7 @@ spec: master: service: type: LoadBalancer - loadBalancerIP: 192.168.8.135 + loadBalancerIP: 192.168.0.135 resources: requests: cpu: "250m" diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml index a01ba4d9a..68e0f5018 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml @@ -20,7 +20,7 @@ spec: master: service: type: LoadBalancer - loadBalancerIP: 192.168.8.134 + loadBalancerIP: 192.168.0.134 resources: requests: cpu: "250m" diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml index 4e6810eea..bf66790cd 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml @@ -5,4 +5,4 @@ metadata: name: bungeecord spec: type: LoadBalancer - loadBalancerIP: 192.168.8.131 + loadBalancerIP: 192.168.0.131 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml index ffcccbc9f..560d90aa1 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml @@ -5,4 +5,4 @@ metadata: name: bungeecord spec: type: LoadBalancer - loadBalancerIP: 192.168.8.130 + loadBalancerIP: 192.168.0.130 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml index b36c3959f..166331fab 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml @@ -20,7 +20,7 @@ spec: master: service: type: LoadBalancer - loadBalancerIP: 192.168.8.133 + loadBalancerIP: 192.168.0.133 resources: requests: cpu: "250m" diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml index 4e710f9c1..2d65d9036 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml @@ -20,7 +20,7 @@ spec: master: service: type: LoadBalancer - loadBalancerIP: 192.168.8.132 + loadBalancerIP: 192.168.0.132 resources: requests: cpu: "250m" From d5105e9a2da97d79176f0ee021a7731d42f7e4ac Mon Sep 17 00:00:00 2001 From: unchama <11990197+unchama@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:18:22 +0900 Subject: [PATCH 2/3] delete metallb --- .../metallb/kustomization.yaml | 7 ------- .../metallb/metallb-address-pool.yaml | 19 ------------------- 2 files changed, 26 deletions(-) delete mode 100644 seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/kustomization.yaml delete mode 100644 seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/kustomization.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/kustomization.yaml deleted file mode 100644 index a317597ed..000000000 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/kustomization.yaml +++ /dev/null @@ -1,7 +0,0 @@ -apiVersion: kustomize.config.k8s.io/v1beta1 -kind: Kustomization -namespace: metallb-system - -resources: - - github.com/metallb/metallb/config/native?ref=v0.13.12 - - ./metallb-address-pool.yaml diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml deleted file mode 100644 index c0e4e5d9b..000000000 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: metallb.io/v1beta1 -kind: IPAddressPool -metadata: - name: default-pool - namespace: metallb-system -spec: - addresses: - - 192.168.0.128-192.168.0.255 ---- -apiVersion: metallb.io/v1beta1 -kind: L2Advertisement -metadata: - name: default-advertisement - namespace: metallb-system -spec: - ipAddressPools: - - default-pool - interfaces: - - ens18 From 0621756ac5a9e54c6a2324607888575700b706d1 Mon Sep 17 00:00:00 2001 From: unchama <11990197+unchama@users.noreply.github.com> Date: Sat, 2 Dec 2023 12:19:47 +0900 Subject: [PATCH 3/3] enable bgp control plane / change ip-addr --- seichi-onp-k8s/cluster-boot-up/README.md | 8 +- .../scripts/nodes/k8s-node-setup.sh | 7 +- .../manifests/seichi-kubernetes/README.md | 4 +- .../app-of-other-apps/cilium.yaml | 103 ++++++++++++++++++ .../bungeecord/redisbungee-redis.yaml | 3 + .../service-bungeecord-loadbalancer.yaml | 3 +- .../service-bungeecord-loadbalancer.yaml | 3 +- .../redis/bungeesemaphore-redis.yaml | 3 + .../redis/redisbungee-redis.yaml | 3 + 9 files changed, 127 insertions(+), 10 deletions(-) diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md index cf3615992..fbf5bc599 100644 --- a/seichi-onp-k8s/cluster-boot-up/README.md +++ b/seichi-onp-k8s/cluster-boot-up/README.md @@ -50,14 +50,16 @@ KubernetesノードのVMは cloudinit イメージで作成されています。 - Storage Network (192.168.16.0/22) - Kubernetes - Internal - - Pod Network (10.128.0.0/16) - - Service Network (10.96.0.0/16) + - Pod Network (10.96.128.0/18) + - Service Network (10.96.64.0/18) - External - Node IP - Service Network (192.168.0.0-192.168.0.127) + - 192.168.0.0/22 の一部を使用 - Storage Network (192.168.18.0-192.168.18.127) + - 192.168.16.0/22 の一部を使用 - API Endpoint (192.168.18.100) - - LoadBalancer VIP (192.168.0.128-192.168.0.255) + - LoadBalancer VIP (10.96.0.0/22) ## Kubernetesクラスタの構成 diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh index 41e8eb974..cd036622e 100644 --- a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh +++ b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh @@ -280,8 +280,8 @@ nodeRegistration: apiVersion: kubeadm.k8s.io/v1beta3 kind: ClusterConfiguration networking: - serviceSubnet: "10.96.0.0/16" - podSubnet: "10.128.0.0/16" + serviceSubnet: "10.96.64.0/18" + podSubnet: "10.96.128.0/18" kubernetesVersion: "v1.27.5" controlPlaneEndpoint: "${KUBE_API_SERVER_VIP}:8443" apiServer: @@ -328,7 +328,8 @@ helm install cilium cilium/cilium \ --namespace kube-system \ --set kubeProxyReplacement=strict \ --set k8sServiceHost=${KUBE_API_SERVER_VIP} \ - --set k8sServicePort=8443 + --set k8sServicePort=8443 \ + --set bgpControlPlane.enabled=true # Generate control plane certificate KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1) diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md index 5395ddbbd..14536395e 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md @@ -42,8 +42,8 @@ TCP パケットをそのまま送り届ける必要があります。 | サービス | `Service` の VIP | | ---------------------------- | ----------------------------------------------------------- | -| BungeeCord (本番環境用) | [`192.168.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | -| BungeeCord (デバッグ環境用) | [`192.168.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | +| BungeeCord (本番環境用) | [`10.96.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | +| BungeeCord (デバッグ環境用) | [`10.96.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | | 投票受付サーバー | (まだ k8s 上に乗っていないので、 `Service` の VIP ではない) | ### オンプレネットワーク内からのトラフィックを受ける `Service` diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml index 1867f5a1b..eacfa5fac 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml @@ -15,6 +15,8 @@ spec: kubeProxyReplacement: strict k8sServiceHost: 192.168.18.100 # modify it if necessary k8sServicePort: 8443 + bgpControlPlane: + enabled: true pprof: enabled: true loadBalancer: @@ -95,3 +97,104 @@ spec: automated: prune: true selfHeal: true +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumLoadBalancerIPPool +metadata: + name: "lb-pool" +spec: + cidrs: + # 10.96.0.0-10.96.3.255 をloadBalancerのIPに割当可能 + - cidr: "10.96.0.0/22" +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-cp-1 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-cp-1 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-cp-2 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-cp-2 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + serviceSelector: + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-cp-3 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-cp-3 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-wk-1 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-wk-1 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-wk-2 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-wk-2 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- +apiVersion: "cilium.io/v2alpha1" +kind: CiliumBGPPeeringPolicy +metadata: + name: peerpolicy--seichi-onp-k8s-wk-3 +spec: + nodeSelector: + matchLabels: + kubernetes.io/hostname: seichi-onp-k8s-wk-3 + virtualRouters: + - localASN: 65184 + exportPodCIDR: true + neighbors: + - peerAddress: "192.168.3.254/32" + peerASN: 65184 +--- diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml index 68e0f5018..5c757e8c3 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml @@ -19,6 +19,9 @@ spec: notify-keyspace-events "Eg$x" master: service: + # loadBalancerIP has been deprecated in k8s v1.24 + # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある + # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips type: LoadBalancer loadBalancerIP: 192.168.0.134 resources: diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml index bf66790cd..72a68e463 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Service metadata: name: bungeecord + annotations: + "io.cilium/lb-ipam-ips": "10.96.0.131" spec: type: LoadBalancer - loadBalancerIP: 192.168.0.131 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml index 560d90aa1..ad849760e 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml @@ -3,6 +3,7 @@ apiVersion: v1 kind: Service metadata: name: bungeecord + annotations: + "io.cilium/lb-ipam-ips": "10.96.0.130" spec: type: LoadBalancer - loadBalancerIP: 192.168.0.130 diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml index 166331fab..f850cbf60 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml @@ -19,6 +19,9 @@ spec: notify-keyspace-events "Eg$x" master: service: + # loadBalancerIP has been deprecated in k8s v1.24 + # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある + # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips type: LoadBalancer loadBalancerIP: 192.168.0.133 resources: diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml index 2d65d9036..6f643b563 100644 --- a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml +++ b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/redisbungee-redis.yaml @@ -19,6 +19,9 @@ spec: notify-keyspace-events "Eg$x" master: service: + # loadBalancerIP has been deprecated in k8s v1.24 + # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある + # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips type: LoadBalancer loadBalancerIP: 192.168.0.132 resources: