Merge pull request #1431 from GiganticMinecraft/change-ip

本番service-sege縮小対応/cilium bgp control plane導入
GiganticMinecraft · Dec 3, 2023 · 7ab0a53 · 7ab0a53
2 parents 624a1ef + 2e8a167
commit 7ab0a53
Show file tree

Hide file tree

Showing 19 changed files with 161 additions and 70 deletions.
diff --git a/seichi-onp-k8s/cluster-boot-up/README.md b/seichi-onp-k8s/cluster-boot-up/README.md
@@ -50,14 +50,16 @@ KubernetesノードのVMは cloudinit イメージで作成されています。
  - Storage Network (192.168.16.0/22)
  - Kubernetes
    - Internal
-     - Pod Network (10.128.0.0/16)
-     - Service Network (10.96.0.0/16)
+    - Pod Network (10.96.128.0/18)
+    - Service Network (10.96.64.0/18)
    - External
      - Node IP
-       - Service Network (192.168.8.0-192.168.8.127)
+       - Service Network (192.168.0.0-192.168.0.127)
+         - 192.168.0.0/22 の一部を使用
        - Storage Network (192.168.18.0-192.168.18.127)
+         - 192.168.16.0/22 の一部を使用
      - API Endpoint (192.168.18.100)
-     - LoadBalancer VIP (192.168.8.128-192.168.8.255)
+    - LoadBalancer VIP (10.96.0.0/22)
 
 ## Kubernetesクラスタの構成
 

diff --git a/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh b/seichi-onp-k8s/cluster-boot-up/scripts/nodes/k8s-node-setup.sh
@@ -289,8 +289,8 @@ skipPhases:
 apiVersion: kubeadm.k8s.io/v1beta4
 kind: ClusterConfiguration
 networking:
-  serviceSubnet: "10.96.0.0/16"
-  podSubnet: "10.128.0.0/16"
+  serviceSubnet: "10.96.64.0/18"
+  podSubnet: "10.96.128.0/18"
 kubernetesVersion: "v1.27.6"
 controlPlaneEndpoint: "${KUBE_API_SERVER_VIP}:8443"
 apiServer:
@@ -338,7 +338,8 @@ helm install cilium cilium/cilium \
     --namespace kube-system \
     --set kubeProxyReplacement=strict \
     --set k8sServiceHost=${KUBE_API_SERVER_VIP} \
-    --set k8sServicePort=8443
+    --set k8sServicePort=8443 \
+    --set bgpControlPlane.enabled=true
 
 # Generate control plane certificate
 KUBEADM_UPLOADED_CERTS=$(kubeadm init phase upload-certs --upload-certs | tail -n 1)

diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-1-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-1-network.yaml
@@ -4,9 +4,9 @@ config:
     name: ens18
     subnets:
     - type: static
-      address: '192.168.8.11'
-      netmask: '255.255.240.0'
-      gateway: '192.168.1.1'
+      address: '192.168.0.11'
+      netmask: '255.255.252.0'
+      gateway: '192.168.3.254'
   - type: physical
     name: ens19
     subnets:
@@ -15,6 +15,6 @@ config:
       netmask: '255.255.252.0'
   - type: nameserver
     address:
-    - '192.168.1.1'
+    - '192.168.100.1'
     search:
     - 'local'
diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-2-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-2-network.yaml
@@ -4,9 +4,9 @@ config:
     name: ens18
     subnets:
     - type: static
-      address: '192.168.8.12'
-      netmask: '255.255.240.0'
-      gateway: '192.168.1.1'
+      address: '192.168.0.12'
+      netmask: '255.255.252.0'
+      gateway: '192.168.3.254'
   - type: physical
     name: ens19
     subnets:
@@ -15,6 +15,6 @@ config:
       netmask: '255.255.252.0'
   - type: nameserver
     address:
-    - '192.168.1.1'
+    - '192.168.100.1'
     search:
     - 'local'
diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-3-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-cp-3-network.yaml
@@ -4,9 +4,9 @@ config:
     name: ens18
     subnets:
     - type: static
-      address: '192.168.8.13'
-      netmask: '255.255.240.0'
-      gateway: '192.168.1.1'
+      address: '192.168.0.13'
+      netmask: '255.255.252.0'
+      gateway: '192.168.3.254'
   - type: physical
     name: ens19
     subnets:
@@ -15,6 +15,6 @@ config:
       netmask: '255.255.252.0'
   - type: nameserver
     address:
-    - '192.168.1.1'
+    - '192.168.100.1'
     search:
     - 'local'
diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-1-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-1-network.yaml
@@ -4,9 +4,9 @@ config:
     name: ens18
     subnets:
     - type: static
-      address: '192.168.8.21'
-      netmask: '255.255.240.0'
-      gateway: '192.168.1.1'
+      address: '192.168.0.21'
+      netmask: '255.255.252.0'
+      gateway: '192.168.3.254'
   - type: physical
     name: ens19
     subnets:
@@ -15,6 +15,6 @@ config:
       netmask: '255.255.252.0'
   - type: nameserver
     address:
-    - '192.168.1.1'
+    - '192.168.100.1'
     search:
     - 'local'
diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-2-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-2-network.yaml
@@ -4,9 +4,9 @@ config:
     name: ens18
     subnets:
     - type: static
-      address: '192.168.8.22'
-      netmask: '255.255.240.0'
-      gateway: '192.168.1.1'
+      address: '192.168.0.22'
+      netmask: '255.255.252.0'
+      gateway: '192.168.3.254'
   - type: physical
     name: ens19
     subnets:
@@ -15,6 +15,6 @@ config:
       netmask: '255.255.252.0'
   - type: nameserver
     address:
-    - '192.168.1.1'
+    - '192.168.100.1'
     search:
     - 'local'
diff --git a/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-3-network.yaml b/seichi-onp-k8s/cluster-boot-up/snippets/seichi-onp-k8s-wk-3-network.yaml
@@ -4,9 +4,9 @@ config:
     name: ens18
     subnets:
     - type: static
-      address: '192.168.8.23'
-      netmask: '255.255.240.0'
-      gateway: '192.168.1.1'
+      address: '192.168.0.23'
+      netmask: '255.255.252.0'
+      gateway: '192.168.3.254'
   - type: physical
     name: ens19
     subnets:
@@ -15,6 +15,6 @@ config:
       netmask: '255.255.252.0'
   - type: nameserver
     address:
-    - '192.168.1.1'
+    - '192.168.100.1'
     search:
     - 'local'
diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/README.md b/seichi-onp-k8s/manifests/seichi-kubernetes/README.md
@@ -42,8 +42,8 @@ TCP パケットをそのまま送り届ける必要があります。
 
 | サービス                     | `Service` の VIP                                            | 
 | ---------------------------- | ----------------------------------------------------------- | 
-|  BungeeCord (本番環境用)     | [`192.168.8.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
-|  BungeeCord (デバッグ環境用) | [`192.168.8.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
+|  BungeeCord (本番環境用)     | [`10.96.0.130`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
+|  BungeeCord (デバッグ環境用) | [`10.96.0.131`](https://github.com/GiganticMinecraft/seichi_infra/blob/83e996ec845ea2cd73d9cea391cd02a03435dbd8/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml#L8) | 
 |  投票受付サーバー            | (まだ k8s 上に乗っていないので、 `Service` の VIP ではない) |
 
 ### オンプレネットワーク内からのトラフィックを受ける `Service`
@@ -54,10 +54,10 @@ TCP パケットをそのまま送り届ける必要があります。
 
 | サービス                       | `Service` の VIP                                            | 
 | ------------------------------ | ----------------------------------------------------------- | 
-| 本番 RedisBungee 用 Redis      | [`192.168.8.132`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redisbungee-redis.yaml#L24) |
-| 本番 BungeeSemaphore 用 Redis  | [`192.168.8.133`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/bungeesemaphore-redis.yaml#L24) |
-| Debug RedisBungee 用 Redis     | [`192.168.8.134`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/redisbungee-redis.yaml#L24) |
-| Debug BungeeSemaphore 用 Redis | [`192.168.8.135`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/bungeesemaphore-redis.yaml#L24) |
+| 本番 RedisBungee 用 Redis      | [`192.168.0.132`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redisbungee-redis.yaml#L24) |
+| 本番 BungeeSemaphore 用 Redis  | [`192.168.0.133`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/bungeesemaphore-redis.yaml#L24) |
+| Debug RedisBungee 用 Redis     | [`192.168.0.134`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/redisbungee-redis.yaml#L24) |
+| Debug BungeeSemaphore 用 Redis | [`192.168.0.135`](https://github.com/GiganticMinecraft/seichi_infra/blob/fc00e4f9b755798ed2fcd80c76b68dac49c3dc16/seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/bungeesemaphore-redis.yaml#L24) |
 
 
 ## Kubernetes クラスタのブートストラップについて

diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cloudflared-tunnel-exits/http-exits.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cloudflared-tunnel-exits/http-exits.yaml
@@ -50,7 +50,7 @@ spec:
           # Sentry
           - name: sentry
             external-hostname: sentry.onp.admin.seichi.click
-            internal-authority: "192.168.8.19:9000"
+            internal-authority: "192.168.3.19:9000"
 
           # 各サーバーの Dynmap ウェブサーバー
           - name: dynmap-s1

diff --git a/...-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml b/...-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/app-of-other-apps/cilium.yaml
@@ -15,6 +15,8 @@ spec:
         kubeProxyReplacement: strict
         k8sServiceHost: 192.168.18.100 # modify it if necessary
         k8sServicePort: 8443
+        bgpControlPlane:
+          enabled: true
         pprof:
           enabled: true
         loadBalancer:
@@ -95,3 +97,104 @@ spec:
     automated:
       prune: true
       selfHeal: true
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumLoadBalancerIPPool
+metadata:
+  name: "lb-pool"
+spec:
+  cidrs:
+  # 10.96.0.0-10.96.3.255 をloadBalancerのIPに割当可能
+  - cidr: "10.96.0.0/22"
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-cp-1
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-cp-1
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-cp-2
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-cp-2
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    serviceSelector:
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-cp-3
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-cp-3
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-wk-1
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-wk-1
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-wk-2
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-wk-2
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
+apiVersion: "cilium.io/v2alpha1"
+kind: CiliumBGPPeeringPolicy
+metadata:
+  name: peerpolicy--seichi-onp-k8s-wk-3
+spec:
+  nodeSelector:
+    matchLabels:
+      kubernetes.io/hostname: seichi-onp-k8s-wk-3
+  virtualRouters:
+  - localASN: 65184
+    exportPodCIDR: true
+    neighbors:
+    - peerAddress: "192.168.3.254/32"
+      peerASN: 65184
+---
diff --git a/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/kustomization.yaml b/seichi-onp-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/kustomization.yaml
diff --git a/...-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml b/...-k8s/manifests/seichi-kubernetes/apps/cluster-wide-apps/metallb/metallb-address-pool.yaml
diff --git a/...nifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/bungeesemaphore-redis.yaml b/...nifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/bungeesemaphore-redis.yaml
@@ -20,7 +20,7 @@ spec:
         master:
           service:
             type: LoadBalancer
-            loadBalancerIP: 192.168.8.135
+            loadBalancerIP: 192.168.0.135
           resources:
             requests:
               cpu: "250m"

diff --git a/...s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml b/...s/manifests/seichi-kubernetes/apps/seichi-debug-gateway/bungeecord/redisbungee-redis.yaml
@@ -19,8 +19,11 @@ spec:
           notify-keyspace-events "Eg$x"
         master:
           service:
+            # loadBalancerIP has been deprecated in k8s v1.24
+            # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある
+            # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips
             type: LoadBalancer
-            loadBalancerIP: 192.168.8.134
+            loadBalancerIP: 192.168.0.134
           resources:
             requests:
               cpu: "250m"

diff --git a/...ichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/...ichi-kubernetes/apps/seichi-debug-gateway/bungeecord/service-bungeecord-loadbalancer.yaml
@@ -3,6 +3,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: bungeecord
+  annotations:
+    "io.cilium/lb-ipam-ips": "10.96.0.131"
 spec:
   type: LoadBalancer
-  loadBalancerIP: 192.168.8.131
diff --git a/...sts/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml b/...sts/seichi-kubernetes/apps/seichi-gateway/bungeecord/service-bungeecord-loadbalancer.yaml
@@ -3,6 +3,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: bungeecord
+  annotations:
+    "io.cilium/lb-ipam-ips": "10.96.0.130"
 spec:
   type: LoadBalancer
-  loadBalancerIP: 192.168.8.130
diff --git a/...np-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml b/...np-k8s/manifests/seichi-kubernetes/apps/seichi-minecraft/redis/bungeesemaphore-redis.yaml
@@ -19,8 +19,11 @@ spec:
           notify-keyspace-events "Eg$x"
         master:
           service:
+            # loadBalancerIP has been deprecated in k8s v1.24
+            # ciliumのlb-ipamを使用しているので今後それに沿った記載に改める必要がある
+            # https://docs.cilium.io/en/stable/network/lb-ipam/#requesting-ips
             type: LoadBalancer
-            loadBalancerIP: 192.168.8.133
+            loadBalancerIP: 192.168.0.133
           resources:
             requests:
               cpu: "250m"