From 886375a0ff2693baf490e93b11bebe39678062a8 Mon Sep 17 00:00:00 2001 From: TrekkieCoder Date: Sun, 30 Jul 2023 13:24:59 +0900 Subject: [PATCH] cicd for incluster-lb with calico --- cicd/k3s-calico-incluster/Vagrantfile | 62 ++++++++++ cicd/k3s-calico-incluster/config.sh | 4 + cicd/k3s-calico-incluster/host.sh | 5 + cicd/k3s-calico-incluster/k3s.yaml | 19 ++++ cicd/k3s-calico-incluster/kube-loxilb.yml | 132 ++++++++++++++++++++++ cicd/k3s-calico-incluster/loxilb-peer.yml | 64 +++++++++++ cicd/k3s-calico-incluster/loxilb.yml | 67 +++++++++++ cicd/k3s-calico-incluster/master-ip | 1 + cicd/k3s-calico-incluster/master1.sh | 13 +++ cicd/k3s-calico-incluster/master2.sh | 9 ++ cicd/k3s-calico-incluster/nginx.yml | 37 ++++++ cicd/k3s-calico-incluster/node-token | 1 + cicd/k3s-calico-incluster/rmconfig.sh | 7 ++ cicd/k3s-calico-incluster/sctp.yml | 41 +++++++ cicd/k3s-calico-incluster/sctp_client | Bin 0 -> 17480 bytes cicd/k3s-calico-incluster/udp.yml | 30 +++++ cicd/k3s-calico-incluster/udp_client | Bin 0 -> 17192 bytes cicd/k3s-calico-incluster/validation.sh | 58 ++++++++++ cicd/k3s-calico-incluster/wait_ready.sh | 37 ++++++ cicd/k3s-calico-incluster/worker.sh | 12 ++ 20 files changed, 599 insertions(+) create mode 100644 cicd/k3s-calico-incluster/Vagrantfile create mode 100755 cicd/k3s-calico-incluster/config.sh create mode 100755 cicd/k3s-calico-incluster/host.sh create mode 100644 cicd/k3s-calico-incluster/k3s.yaml create mode 100644 cicd/k3s-calico-incluster/kube-loxilb.yml create mode 100644 cicd/k3s-calico-incluster/loxilb-peer.yml create mode 100644 cicd/k3s-calico-incluster/loxilb.yml create mode 100644 cicd/k3s-calico-incluster/master-ip create mode 100755 cicd/k3s-calico-incluster/master1.sh create mode 100644 cicd/k3s-calico-incluster/master2.sh create mode 100644 cicd/k3s-calico-incluster/nginx.yml create mode 100644 cicd/k3s-calico-incluster/node-token create mode 100755 cicd/k3s-calico-incluster/rmconfig.sh create mode 100644 cicd/k3s-calico-incluster/sctp.yml create mode 100755 cicd/k3s-calico-incluster/sctp_client create mode 100644 cicd/k3s-calico-incluster/udp.yml create mode 100755 cicd/k3s-calico-incluster/udp_client create mode 100755 cicd/k3s-calico-incluster/validation.sh create mode 100755 cicd/k3s-calico-incluster/wait_ready.sh create mode 100644 cicd/k3s-calico-incluster/worker.sh diff --git a/cicd/k3s-calico-incluster/Vagrantfile b/cicd/k3s-calico-incluster/Vagrantfile new file mode 100644 index 000000000..ce731e274 --- /dev/null +++ b/cicd/k3s-calico-incluster/Vagrantfile @@ -0,0 +1,62 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +workers = (ENV['WORKERS'] || "2").to_i +#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64") +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + config.vm.define "host" do |host| + host.vm.hostname = 'host1' + #loxilb.vm.network "forwarded_port", guest: 55002, host: 5502, protocol: "tcp" + host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0" + host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0" + host.vm.provision :shell, :path => "host.sh" + host.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 2048] + vbox.customize ["modifyvm", :id, "--cpus", 1] + end + end + + config.vm.define "master1" do |master| + master.vm.hostname = 'master1' + master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0" + master.vm.network :private_network, ip: "192.168.90.10", :netmask => "255.255.255.0" + master.vm.provision :shell, :path => "master1.sh" + master.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 2048] + vbox.customize ["modifyvm", :id, "--cpus", 2] + end + end + + config.vm.define "master2" do |master| + master.vm.hostname = 'master2' + master.vm.network :private_network, ip: "192.168.80.11", :netmask => "255.255.255.0" + master.vm.network :private_network, ip: "192.168.90.11", :netmask => "255.255.255.0" + master.vm.provision :shell, :path => "master2.sh" + master.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 2048] + vbox.customize ["modifyvm", :id, "--cpus", 2] + end + end + + (1..workers).each do |node_number| + config.vm.define "worker#{node_number}" do |worker| + worker.vm.hostname = "worker#{node_number}" + ip = node_number + 100 + worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0" + worker.vm.provision :shell, :path => "worker.sh" + worker.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 2048] + vbox.customize ["modifyvm", :id, "--cpus", 1] + end + end + end +end diff --git a/cicd/k3s-calico-incluster/config.sh b/cicd/k3s-calico-incluster/config.sh new file mode 100755 index 000000000..89381649c --- /dev/null +++ b/cicd/k3s-calico-incluster/config.sh @@ -0,0 +1,4 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up +sudo ip route add 123.123.123.1 via 192.168.90.10 || true diff --git a/cicd/k3s-calico-incluster/host.sh b/cicd/k3s-calico-incluster/host.sh new file mode 100755 index 000000000..adfc5c77d --- /dev/null +++ b/cicd/k3s-calico-incluster/host.sh @@ -0,0 +1,5 @@ +sudo su +echo "123.123.123.1 k8s-svc" >> /etc/hosts +ifconfig eth2 mtu 1450 +ip route add 123.123.123.0/24 via 192.168.90.10 +echo "Host is up" diff --git a/cicd/k3s-calico-incluster/k3s.yaml b/cicd/k3s-calico-incluster/k3s.yaml new file mode 100644 index 000000000..90bc03ca3 --- /dev/null +++ b/cicd/k3s-calico-incluster/k3s.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUyT1RBMk9EWTJNakl3SGhjTk1qTXdOek13TURNeE1ESXlXaGNOTXpNd056STNNRE14TURJeQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUyT1RBMk9EWTJNakl3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTYTM4cFIrbEt5NFZEZ1AxckkzbW5KaXVPeVcwN3ZTczl2cjJYWEsyK3gKRGE0MGZxTWJmUC9rbnhYdGVxZS9RbVpSN2Z4L3psNmlYVkxPZ1BHdUxzbDFvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVUIyRXAwL2xaeUxjUlVTQ2lKYkpQCmVEeWc3Tk13Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQVBKRlJxcFRKY2ZFZU9zbFgwdW80ckN0S2F0TEZZb0QKa1U4Q1YyY2hGYXpQQWlCSEY0MFNQQU5mRU5DVkRtNlAyUUxqV2p1VmFoa3duUTNjMUk2OGdoZlRoUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + server: https://192.168.80.10:6443 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrVENDQVRlZ0F3SUJBZ0lJSkUrNHR1b2grVzB3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOamt3TmpnMk5qSXlNQjRYRFRJek1EY3pNREF6TVRBeU1sb1hEVEkwTURjeQpPVEF6TVRBeU1sb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJMdGI1NG9BNHJXbjVXc2oKVEFpN2hFamJIZmZBS29FRTNaMHErN0lIcjVYaCt4c0tBOEVLeHNLUHBKaHhRd1VQaXpmUEtYdWFZZlZ4Y0lpdQprdjVjNVRhalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCVGd6Vm8xUkRHV2VROHJyNU5KMnlSeVduQzJQekFLQmdncWhrak9QUVFEQWdOSUFEQkYKQWlFQXhqa2d1dC9QcVEwSEpVR2FmNVRGWjRBWWlwRjdGN2x0ZGVla1lMNHdPaHdDSUQ0eDFtd2VBWnJNNDlTUQpYTlhuTHVSVVROczdERmJab1ExZkNwOHdxZ1R6Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUyT1RBMk9EWTJNakl3SGhjTk1qTXdOek13TURNeE1ESXlXaGNOTXpNd056STNNRE14TURJeQpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUyT1RBMk9EWTJNakl3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFTZWVEc09QcjhJQXBoWlBIRG1ZZUpoRjZyM0h6K0NDTWFoaTdkeWk1MkoKRW5GaHkrNzdyVnQ2SUJSQnpNYzZEa1NlZnZpWTFDaS9CZk1sM2R5RDRrUHBvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVTRNMWFOVVF4bG5rUEs2K1RTZHNrCmNscHd0ajh3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnU3d0MllPYkNVNW1UekhEQ3JCUk5jT01Db1NIWEZTR3kKWlZaVXlhUVZPZ1lDSURFK244bjJFSjJiZGt3SDlKQ3ZtNGZMQkYya3dlNjZSZUxTcG5IYVloeVcKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSUp5bi9neUhvRXlmc3dQamRLcTBCeUpxT0M3TlczYWtHMGRyTFJBUzdGY3BvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFdTF2bmlnRGl0YWZsYXlOTUNMdUVTTnNkOThBcWdRVGRuU3I3c2dldmxlSDdHd29Ed1FyRwp3bytrbUhGREJRK0xOODhwZTVwaDlYRndpSzZTL2x6bE5nPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/cicd/k3s-calico-incluster/kube-loxilb.yml b/cicd/k3s-calico-incluster/kube-loxilb.yml new file mode 100644 index 000000000..71b84997d --- /dev/null +++ b/cicd/k3s-calico-incluster/kube-loxilb.yml @@ -0,0 +1,132 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + #- --loxiURL=http://192.168.80.10:11111 + - --externalCIDR=123.123.123.1/24 + - --setBGP=64512 + - --setRoles + #- --monitor + #- --setBGP + #- --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/k3s-calico-incluster/loxilb-peer.yml b/cicd/k3s-calico-incluster/loxilb-peer.yml new file mode 100644 index 000000000..1cb89deb8 --- /dev/null +++ b/cicd/k3s-calico-incluster/loxilb-peer.yml @@ -0,0 +1,64 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loxilb-peer + namespace: kube-system +spec: + selector: + matchLabels: + app: loxilb-peer-app + template: + metadata: + name: loxilb-peer + labels: + app: loxilb-peer-app + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: DoesNotExist + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist + containers: + - name: loxilb-peer-app + image: "ghcr.io/loxilb-io/loxilb:latest" + command: [ "/root/loxilb-io/loxilb/loxilb", "--peer" ] + ports: + - containerPort: 11111 + - containerPort: 179 + - containerPort: 50051 + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN +--- +apiVersion: v1 +kind: Service +metadata: + name: loxilb-peer-service + namespace: kube-system +spec: + clusterIP: None + selector: + app: loxilb-peer-app + ports: + - name: loxilb-peer-app + port: 11111 + targetPort: 11111 + protocol: TCP + - name: loxilb-peer-bgp + port: 179 + targetPort: 179 + protocol: TCP + - name: loxilb-peer-gobgp + port: 50051 + targetPort: 50051 + protocol: TCP + + diff --git a/cicd/k3s-calico-incluster/loxilb.yml b/cicd/k3s-calico-incluster/loxilb.yml new file mode 100644 index 000000000..51b47116c --- /dev/null +++ b/cicd/k3s-calico-incluster/loxilb.yml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loxilb-lb + namespace: kube-system +spec: + selector: + matchLabels: + app: loxilb-app + template: + metadata: + name: loxilb-lb + labels: + app: loxilb-app + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + containers: + - name: loxilb-app + image: "ghcr.io/loxilb-io/loxilb:latest" + command: [ "/root/loxilb-io/loxilb/loxilb", "--bgp", "--egr-hooks", "--blacklist=cali.|tunl.|vxlan[.]calico|veth." ] + ports: + - containerPort: 11111 + - containerPort: 179 + - containerPort: 50051 + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN +--- +apiVersion: v1 +kind: Service +metadata: + name: loxilb-lb-service + namespace: kube-system +spec: + clusterIP: None + selector: + app: loxilb-app + ports: + - name: loxilb-app + port: 11111 + targetPort: 11111 + protocol: TCP + - name: loxilb-app-bgp + port: 179 + targetPort: 179 + protocol: TCP + - name: loxilb-app-gobgp + port: 50051 + targetPort: 50051 + protocol: TCP diff --git a/cicd/k3s-calico-incluster/master-ip b/cicd/k3s-calico-incluster/master-ip new file mode 100644 index 000000000..69e9dfa30 --- /dev/null +++ b/cicd/k3s-calico-incluster/master-ip @@ -0,0 +1 @@ +192.168.80.10 diff --git a/cicd/k3s-calico-incluster/master1.sh b/cicd/k3s-calico-incluster/master1.sh new file mode 100755 index 000000000..3592b42b9 --- /dev/null +++ b/cicd/k3s-calico-incluster/master1.sh @@ -0,0 +1,13 @@ +sudo su +export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.90' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') +curl -fL https://get.k3s.io | sh -s - server --node-ip=192.168.80.10 --disable servicelb --disable traefik --cluster-init external-hostname=192.168.80.10 --node-external-ip=192.168.80.10 --disable-cloud-controller --kubelet-arg cloud-provider=external --flannel-backend=none --disable-network-policy --cluster-cidr=10.42.0.0/16 +sleep 60 +echo $MASTER_IP > /vagrant/master-ip +cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token +sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /etc/rancher/k3s/k3s.yaml +cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml +#sudo kubectl apply -f /vagrant/loxilb.yml +#sudo kubectl apply -f /vagrant/kube-loxilb.yml +sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/tigera-operator.yaml +sudo kubectl create -f https://raw.githubusercontent.com/projectcalico/calico/v3.26.0/manifests/custom-resources.yaml +/vagrant/wait_ready.sh diff --git a/cicd/k3s-calico-incluster/master2.sh b/cicd/k3s-calico-incluster/master2.sh new file mode 100644 index 000000000..6a34fcf71 --- /dev/null +++ b/cicd/k3s-calico-incluster/master2.sh @@ -0,0 +1,9 @@ +sudo su +export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') +export MASTER_ADDR=$(cat /vagrant/master-ip) +export NODE_TOKEN=$(cat /vagrant/node-token) + +#curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 --disable-cloud-controller -t ${NODE_TOKEN} +curl -fL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - server --server https://192.168.80.10:6443 --disable traefik --disable servicelb --node-ip=192.168.80.11 external-hostname=192.168.80.11 --node-external-ip=192.168.80.11 -t ${NODE_TOKEN} + +/vagrant/wait_ready.sh diff --git a/cicd/k3s-calico-incluster/nginx.yml b/cicd/k3s-calico-incluster/nginx.yml new file mode 100644 index 000000000..92e55c68c --- /dev/null +++ b/cicd/k3s-calico-incluster/nginx.yml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Service +metadata: + name: nginx-lb1 + annotations: + loxilb.io/lbmode: "fullnat" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: nginx-test + ports: + - port: 55002 + targetPort: 80 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: nginx-test + labels: + what: nginx-test +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + # - key: "node-role.kubernetes.io/control-plane" + # operator: DoesNotExist + containers: + - name: nginx-test + image: ghcr.io/loxilb-io/nginx:stable + ports: + - containerPort: 80 diff --git a/cicd/k3s-calico-incluster/node-token b/cicd/k3s-calico-incluster/node-token new file mode 100644 index 000000000..21511654c --- /dev/null +++ b/cicd/k3s-calico-incluster/node-token @@ -0,0 +1 @@ +K10ab8b1703d58898a7788c4e1937780086a37fba44dc4f825a3cbcbbb0b2b67f92::server:dc8b583f69d4955cfe3183a5a8b560f5 diff --git a/cicd/k3s-calico-incluster/rmconfig.sh b/cicd/k3s-calico-incluster/rmconfig.sh new file mode 100755 index 000000000..bd4b79e81 --- /dev/null +++ b/cicd/k3s-calico-incluster/rmconfig.sh @@ -0,0 +1,7 @@ +#!/bin/bash +sudo ip route del 123.123.123.1 via 192.168.90.10 || true +vagrant destroy -f worker1 +vagrant destroy -f worker2 +vagrant destroy -f master1 +vagrant destroy -f master2 +vagrant destroy -f host diff --git a/cicd/k3s-calico-incluster/sctp.yml b/cicd/k3s-calico-incluster/sctp.yml new file mode 100644 index 000000000..befccd4d4 --- /dev/null +++ b/cicd/k3s-calico-incluster/sctp.yml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Service +metadata: + name: sctp-lb1 + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "fullnat" +spec: + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + selector: + what: sctp-test + ports: + - port: 55004 + protocol: SCTP + targetPort: 9999 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: sctp-test + labels: + what: sctp-test +spec: + containers: + - name: sctp-test + image: ghcr.io/loxilb-io/alpine-socat:latest + command: [ "sh", "-c"] + args: + - while true; do + socat -v -T2 sctp-l:9999,reuseaddr,fork system:"echo 'server1'; cat"; + sleep 20; + done; + ports: + - containerPort: 9999 + env: + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/cicd/k3s-calico-incluster/sctp_client b/cicd/k3s-calico-incluster/sctp_client new file mode 100755 index 0000000000000000000000000000000000000000..7e1cfcb764c67d24ec71c82a526a7dbe4028c06a GIT binary patch literal 17480 zcmeHPe{5XEouBpEi4($l6O!TvN?uYTg@8A9Flj;w>^gSxd~zV<2c#H{-wR>D$>52nY^Aakn6DVug)F zRl>2mR0Jd{wu)=OcvU9FP&F@Sx}Vq zh$X#trPr?XmJFQp=+!1r=My_qoB!OA^ds=m7bq=D|x1r2ZeoaScyL;zdcH? zQXiFv2W8-b(Teq}r z>5SwukWC+Q?b)tYsLsXjMTnXXMOuVk9>tB$G776vm};8aFc z$&q+6Eq3+wcI_}WM>?I@=Exo5fu6nlj1D-nLEgDR{5Ilp|Is~d|51$O=pGLsMmH{v zA?>(UCX?6F2)-8WOxW1!7*V%d-)cnD? zrts#doz6J(uiRtfG}iD~M#v=>evK_6e`j4djU623UAW3pCM~$|>sc67SNLMYRh4Aw z9vAK&7Z18{o+C)%unU*-y|g04unTw3Bcm>y^5!t^!r4cZHsQk2Ku$R3!qp>)Ogrwv zmpRfxJmJF8@J@Kjg||4U5Z`j))IT_!bm7Yx1U==#-Q&Zw3#a*s!+96Jib2o|F8m`d zydJH4pzeXX2kIX9zwLo{TW2f9>!z zL6_!^Iy_CxrMb^JJWa@@xwymAL|mG?-{ENjF3rUpo*r0Aa~%#(6LD#7t;5pd!p9_4NpOt6NBaf&XQjm;b4c|FMt%fsa4! zIh&3F91w{Y<@df_+vLPRe%?Ip5p!CHl`8=^S6?vnDQmNgR) z7iSfP{4G-cj&($CxuO^5^)qkZqn}v_>Vb3m#VgiI7o-WlJ+mFt`b<9jez^=@Alh1qYL(~)6`jfdGYm-4;)!;E zrzrkCdREg5KYv&M4lK;G#7usn5)~C=&6VhHmB38?O%*$re}`hQ^NWh|rHaNZM~f%s z5w-KDb+42ErT=g$Gs{s3BkC8P$)7?Vbbmef7=E6ok>v#%PG--3SS}ax(@52)^5ipp zayqO}A(NTe2W8re00(FCf# z_8_nqs3x<|BZeZ99|P#aCn#dukDa1HO)p%0j%xcnRUX!U1s8EmrlPHSA%6iHj7&~z zs%{U!ZXti3(u%U{==7cf_GY4FRv=I6Pz_>A3fVP&8pcu86;#zJ+c2_x6KEm7(yl)# zT~+@j$ZRJ63^F@OSrm&DMP;b-*}}br^HdWWM!6=BQB8_@>NYcC*i5C)MH@3O`~)}n zY%k?_RwYt%F8mR)h41UdXGz>P`B`EA#R{*+>`p`Vg7DW!c=AQ9uPD2@UOY`b2*shY zlL=V2O~`4kSUid3`*#=4^cEJVkhI!{nf+A0$rrk|O#O=N@a49h*u{+OpMOgTL)o??=Io+d0F&1y17n)S=mge*2B`cDI^lZ=mu| z9a{JyMvZQJzKA{8SKL0iUWnKOy@j8}4)hlO5ZfOsd~iT7wpoDp_1*FzRc7{{1?Zmn zFlepmcukFWeT7nA;qC6izr@Nbf2~iR4d~l`IrgtKpMUyL?4j79*k@wK*}=g`%Npup zc78O|)9@r6lpTZTeBg^M0)x zI9<`zit@fKS_@H7%MkI0E9DIFPpQpWyoP92o0@oU(|MK7dy(!|1?Ih5syM>E*N5yX zHPNp2{n)ERAxzFDaoc~PgYWE*F+_lHaM)|6TE14`u#;j5zzSeHPnQdq)-Bqv&BpM-`n=^thr=DSA@T zX+sc| z@x{&Uk@n6=$1OJgCrzR+1)Tl;ZfV*D$@|m2cu?^Eb1&W~9<9=#LF~Tf`@k!|NO1r6 z;)?~}Ctkc+aR2e*O9b~jFTPZ8|Mue72)+-zI7S6-KUYE!qk~oz4~ach+A%0v#Nn!V z*g3D^G7tYR7d-BGaqI}x+=v|yitC*H`0mso-kywCm0#tYYw*f{MDY0K#jh9D?w_D& zb;cX7JVwi?8?oa-v08+y;yjvGbZFRW6qn@(@(L;hjnfTil#}kwk-+GiCE=HR_*#YY zJOEaPPvAG;Di7sHVJEJDxY`diPmq0{S4ft^5nKU`#yl@b5I;}gs#gA5_4+$$XQgVB21Vdp6DFy#IHZ6(P!Tz!AhIUrxxkr_W( z13y&*|4|M6*ER4`4ZI2UtyZ3!Yv6YQr}~Ph*Ll4)MXzYm=J;4e=Q^O@$#kE0(0uByb4q_+cK=Rs`dz8dn08n_t9 zS~+VBI{*em<$#A_jTkr(kv4NU`H;vML#a%EJY^)TOg3l4$Hs9UV`MaCT4o}$wS5cD zhg1`yV-!gvp3TOO7-rha9ub4t_=ssF#zsbtK*hxwki@wRm#91+GLT9_5hXK>oqJ=u zdyJlYyXlCDS3Dt%?oZtt+uggvn<7u70K!o!qeoX3^zOaF*wuGmSFF#tZ|BZ^J^PLQ zv97)zvMP^w4CKaS!R-?&>bOcYst$Je512%wd=7+8fBaX+WGd%Pd^%<#ZpGCB7GIL@ zuuCPi(>@kc0VsF-D2x}AXK1{TJm}(ut~%Y~O|G1u@#1K86x}`)Q#o1Vm+_tsG7`Cr zF&s}P=;ToEeNe_{C1WgSq7^F*K-~d+e=eu2%Of}R4a#esPVRUSIyvM;)X^a?Xc#!d z#NCMwH&s8eq>lgib=_x#ylL)ZO-`q=4>frOMI?7*#ESQWT3MS8a||b%%~ZL{J-LW&Q^bHgH%IFd#dHnp;L%ID2&j*dl1 zF~t~2%bKY;DX7?J$`TRT1`$LfLm99X$(aKpVwvM$>E4EPMt0taIjrvHVVtaTINOGO zTiY5OM^^EXB#hbWXlD_@kTQb1y~>#O=Rox2hba)x6n1FE_(y>$ucK|$mgP9nkK>{x zBmOA4Go|m0?!faNW4(yD^?AL+RAWI!{rV3A$0tRac4gpo z5mU~e-w*i9ABPsb8(@82KQWD}1S%u>ne}-cp9Dr#W_?~~G3ECU5RrjwxWnhb(Au8$ zc|FFoT?upfS&k{~H>Gz4%=5a9=^mxeKIZZ(SQHTfnr~w;#hlMue)!{ioKsc9!?&|C&#KT(c#ahJF6?=l=%8Y3-_p9cLZM=idGD%rm_V z-4>TVuLn1@0YqRqo2&4Qe~1L?yPQ9-8~Ohjuw&@{qriIHj^6=BRdn>#`^R?mV1I#> zT!HnO()nn&zKAN}cAIiOY=ZTeUj}8uIDdX#4=a7YJmuvM%T@kL+(}~nTOGuXu)dpe z-*JmZmBEwBaUVls1Q(Zw#~*sH%4K)!&(jMEgmGUY=WD0ZnykA1@F#5vT7SEPj}`w3 Ds0}*e literal 0 HcmV?d00001 diff --git a/cicd/k3s-calico-incluster/udp.yml b/cicd/k3s-calico-incluster/udp.yml new file mode 100644 index 000000000..1d7f647ec --- /dev/null +++ b/cicd/k3s-calico-incluster/udp.yml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + name: udp-lb1 + annotations: + loxilb.io/liveness: "yes" + loxilb.io/lbmode: "fullnat" +spec: + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + selector: + what: udp-test + ports: + - port: 55003 + protocol: UDP + targetPort: 33333 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: udp-test + labels: + what: udp-test +spec: + containers: + - name: udp-test + image: ghcr.io/loxilb-io/udp-echo:latest + ports: + - containerPort: 33333 diff --git a/cicd/k3s-calico-incluster/udp_client b/cicd/k3s-calico-incluster/udp_client new file mode 100755 index 0000000000000000000000000000000000000000..b70cd81fccd77645a3ecd1f5fc7944f3dc1b399a GIT binary patch literal 17192 zcmeHOZ)_aJ6`%9Pi9?dJLt`*WAq!0;r^pvOB&h)ze74V7BRhm(18ox47vDNQasRm6 z0~=9raDzCRqppzJNUek;QMHIlbwAK1RDjda5GoZVAE>CHB1J%b6$NvIDlH_}_h#Q) z@2&4jh1xIGj&=8&_nY_L%)XhO+wsmk9PI6`@^}O%ueeE|G`rel+NogVM#=!Ri&n7; zzSoPZ#8Qx#NX{&8Hvvk`^hQ-F))4OnM7svel)v|!{GBBEWTG({CKiaJvh?W!?L zvJ+n?r;J=~)u*5_Bt?E(IEo6880~f#o53boEC$<3afae@l#uOEM7t!}CCQF)p6X-d zcw+PM(@ODlq5@@@6!|TL)9x;^b7B|iGh!W>Zhy}Rf0_DrkzJ!A-3TXgJ&au6ZD5CS z`A-vf^KPoIQa?;n{23{{VskXIfBV+W(Qs2V63>h_jka!Y+P>AFPWZRTCSbd;4jfZE zb`A(|Gm9LBQTD|elCk{RJsY1M*b{p8msdB{zrW$r6)*2T@HYFvI#CA`+VhY=8S{86 zqKLpYqai*#b>9^DWlMk1aT z$&8T}nwB;~L-%MyBll>-p-5Cj;<}+FK|v%_k+?A|(ncz($3;w!rNK0%58XGMO2kCS zNJPX?G?CUtT91c~1h^U+4Qa!XcqkfqKnD@)MfWK~i-jU_v7@)Aqf^`B-)iNy__qnI zr@v1N>nZ&}ByH%a{=Uv=BChv`_D9jjfmk9=J~UIctRh}!75sVSH|FrqKNl1rn zynVi$w((dur=Qw*jvbcG*m&pwD;>4*D=ii#KN}AnXr)s&-e<8woVM}Y_qJj;>qfwh zfExif0&WD{2>cHt@Ja1e|5PVFs8J`Y|F~HQ_0X(QRXnXuyj*ia)>_=w0d%3bu@j(n zy+D0@rWJ+!pNqxfl$7xSs*pcp$@pMX$RD?4e4r`he__e^AXCU6v1ELJDdfLr$@t(> z$RD(1d|)Z$@3mxnP$}d?mW&T5h5T)nj1O*we87_Nfvu2lVHvLG0;~sW*LMkdJs`&% zvhI-gIphI{+~<&k4!O-CZ*#~^4tb+PzS1Gr+vS$O^<>Z7t!Cd*C*Gak)!&<|ejj>3 z&8R z+ulLgbWzRDtH(dQSv`K8nK~3@Ou#79)B(#o}BTI_G5dw?X#o16!+pw9nrUg_s!tJN4iPGvFcn!d@WN>~p&S z!s_Ibjg8W8@O;axI(Z0Lb+Wxi%^ph1tjSEw)~SR>-_WlL1$Oa#I9$mbMDyH5} zS<492GVua_i?PcfppV0Y^V|fO9K{3r<*0;P#R<#0XG6}1l$O?5wJx7-@gncGSRRSPZ-KjN)H*jLT{v0JsM4jjcHW(tn=Oo z%VmK}@$(-Qi=%+2=8MH?z&8MKRdD#@VsQ}gV4+w%4cK_DSe!>5u$5l2;5$)Fo(Fad z&uE=z-O6P(Q&5IHzUyxWAMl?BA-a~~0!M6s&sO-1LY?SS`0BcS4Y$;;xW8sx+`RUN zuV1&}8l=&FFMJeei!?+M67mLo8i1b#0ZUo#2xDjw8;6}iW!2eeScz+u2 zN26tswdV}pvzUn%Evra8g$ox*SD0Y!ds{~`z5>Ox)&#-(-*8=niOWB|SWKWiO`FPi z4Khj@-XnI3%6ZS*yJ!T7_qsKhqNpciATu#Y`?>Jl7beBA-s3%Imz%{RNd)sV)DUj> zEa~yyGp_$C;#m*gAj#{4>Ldj(+@w5C{@{HWmrF}n214v%W8qd0*s?&dwW@#)18rxRFt=_iyz#H??G> z)bh}lW`FZmf6FG5|00jLi_>H6@3(8?eo)?@Udnp~?>{f)mxu?;bdVQzU-P&w)n6)j zJeTsz1dsbtzDDqPDCL(69!I78C4$FmDZj!R*QGq%R)EfN2UDbqTK!9 z6%E$>QK}EKU%Q<#^In+s>dNvwJ1*GZG_*u~TwGwx9L~E{5Vv{z=3L->#4EIa*whx{ zYT|hwXZ{wZGDJ9g%+42m>4JNag(e zT>aSVgJz{Zxy-NDL4_k`x^_I*m9zP!e53|6c{_Te!iYl@AzNm%^yA;PW{r>Uq zRXy;EtfTbng5(#ElNRZxR2QXA;OpR5)j57@QNL>O{c=kMKX+H)AE?0RD)2`t@XuA? zU#-BuRe}Euc)VWceN+SSR~CmyTn_x|sw)Kl9;e>V|n%EzTYH(m9 zmDWO;QTUg9EE&}eJ?w97-riDP2~TxIv`{J)8q@T+ks1@jsZdPU!kJiX3`}g22AW2R z>QFQSHqa)svc3u?igT^bzbDAf!Lt?QdR1ART6r6uzG2vBfpNeimvLG9Wtv>m;- zb_9C0Tf4h&5B6*QfsWoFdX=Y6hSC|?=6`#-1P{QpxAUnKc@{+vhl~&%d2tju4$3T) zZZ*%-ED-3)JUmm%nMY_!IrHpHDJM_Wlv0Zh(>RN<-R3zN2tnzg87-VnXd|I`7!TC+ z+zMLZNL^1}$T+6aOuMZe!Fu0Jg zjhN-@=XGZUIJ{)G=XC_*yHI9Nvu+lE*S06#u>jOz@ZBJhaYb<5jvmTv90#> zBM|KN3fVE9wkt#4;a>d2VbAL;M%HIHEHlnH?BQX^OoX!}GO!)vF(|ONpPy$KtLc7n z+OzzU!=Bf5jQoEi)_2D5w`9-#FBH-w+(;sW)Bg8RfbD1dM#a=*>@r2`+iCwMl-TW) zimA$2gN3%l+$oBmIvO#*ForW@dVtdA=U~IP+?WEmoBGbx h-EKdK3vftzhjXsiEH(daQqI2T22*3L!@wbme*v)VBOL$$ literal 0 HcmV?d00001 diff --git a/cicd/k3s-calico-incluster/validation.sh b/cicd/k3s-calico-incluster/validation.sh new file mode 100755 index 000000000..c96f4ff18 --- /dev/null +++ b/cicd/k3s-calico-incluster/validation.sh @@ -0,0 +1,58 @@ +#!/bin/bash +source ../common.sh +echo k3s-flannel-cluster + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +sleep 45 +extIP="123.123.123.1" +echo $extIP + +echo "Service Info" +vagrant ssh master -c 'sudo kubectl get svc' +echo "LB Info" +vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' +echo "EP Info" +vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' + +print_debug_info() { + echo "llb1 route-info" + vagrant ssh loxilb -c 'ip route' + vagrant ssh master -c 'sudo kubectl get pods -A' + vagrant ssh master -c 'sudo kubectl get svc' + vagrant ssh master -c 'sudo kubectl get nodes' +} + +out=$(curl -s --connect-timeout 10 http://$extIP:55002) +if [[ ${out} == *"Welcome to nginx"* ]]; then + echo "k3s-flannel-cluster (kube-loxilb) tcp [OK]" +else + echo "k3s-flannel-cluster (kube-loxilb) tcp [FAILED]" + print_debug_info + exit 1 +fi + +out=$(timeout 10 ../common/udp_client $extIP 55003) +if [[ ${out} == *"Client"* ]]; then + echo "k3s-flannel-cluster (kube-loxilb) udp [OK]" +else + echo "k3s-flannel-cluster (kube-loxilb) udp [FAILED]" + print_debug_info + exit 1 +fi + +out=$(timeout 10 ../common/sctp_client 192.168.90.1 41291 $extIP 55004) +if [[ ${out} == *"server1"* ]]; then + echo "k3s-flannel-cluster (kube-loxilb) sctp [OK]" +else + echo "k3s-flannel-cluster (kube-loxilb) sctp [FAILED]" + print_debug_info + exit 1 +fi + +exit diff --git a/cicd/k3s-calico-incluster/wait_ready.sh b/cicd/k3s-calico-incluster/wait_ready.sh new file mode 100755 index 000000000..5ff06e373 --- /dev/null +++ b/cicd/k3s-calico-incluster/wait_ready.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +function wait_cluster_ready { + Res=$(sudo kubectl get pods -A | + while IFS= read -r line; do + if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then + echo "not ready" + return + fi + done) + if [[ $Res == *"not ready"* ]]; then + return 1 + fi + return 0 +} + +function wait_cluster_ready_full { + i=1 + nr=0 + for ((;;)) do + wait_cluster_ready + nr=$? + if [[ $nr == 0 ]]; then + echo "Cluster is ready" + break + fi + i=$(( $i + 1 )) + if [[ $i -ge 40 ]]; then + echo "Cluster is not ready.Giving up" + exit 1 + fi + echo "Cluster is not ready...." + sleep 10 + done +} + +wait_cluster_ready_full diff --git a/cicd/k3s-calico-incluster/worker.sh b/cicd/k3s-calico-incluster/worker.sh new file mode 100644 index 000000000..0e9350d89 --- /dev/null +++ b/cicd/k3s-calico-incluster/worker.sh @@ -0,0 +1,12 @@ +sudo su +export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') +export MASTER_ADDR=$(cat /vagrant/master-ip) +export NODE_TOKEN=$(cat /vagrant/node-token) +mkdir -p /etc/rancher/k3s +cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml +curl -sfL https://get.k3s.io | K3S_TOKEN=${NODE_TOKEN} sh -s - agent --server https://192.168.80.10:6443 --node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} -t ${NODE_TOKEN} +#sudo kubectl apply -f /vagrant/loxilb-peer.yml +sudo kubectl apply -f /vagrant/nginx.yml +#sudo kubectl apply -f /vagrant/udp.yml +#sudo kubectl apply -f /vagrant/sctp.yml +/vagrant/wait_ready.sh