From 7d8e161a2820f9112fb9cc564cbd690d7fedd6a3 Mon Sep 17 00:00:00 2001 From: Hanna Date: Wed, 24 Jul 2024 13:46:13 +0900 Subject: [PATCH] Added CICD for the Rabbitmq on the K3s incluster --- cicd/k3s-rabbitmq-incluster/Vagrantfile | 51 + cicd/k3s-rabbitmq-incluster/config.sh | 3 + cicd/k3s-rabbitmq-incluster/grafana.yaml | 84 + cicd/k3s-rabbitmq-incluster/host.sh | 10 + cicd/k3s-rabbitmq-incluster/install_cilium.sh | 13 + cicd/k3s-rabbitmq-incluster/iperf-service.yml | 32 + cicd/k3s-rabbitmq-incluster/k3s.yaml | 19 + cicd/k3s-rabbitmq-incluster/kube-loxilb.yml | 130 ++ cicd/k3s-rabbitmq-incluster/loxilb.sh | 13 + cicd/k3s-rabbitmq-incluster/loxilb.yml | 77 + .../manifests/cilium-ippool.yaml | 7 + .../manifests/cilium-policy.yaml | 9 + .../manifests/iperf-service-loxilb.yml | 32 + .../manifests/iperf-service-metallb.yml | 29 + .../manifests/kube-loxilb.yml | 130 ++ .../manifests/loxilb.yml | 67 + .../manifests/metallb-addr-pool.yml | 8 + .../manifests/metallb-native.yaml | 2042 +++++++++++++++++ cicd/k3s-rabbitmq-incluster/master.sh | 14 + .../master_with_cilium.sh | 29 + cicd/k3s-rabbitmq-incluster/prom_rbac.yaml | 42 + cicd/k3s-rabbitmq-incluster/prom_svc.yaml | 17 + cicd/k3s-rabbitmq-incluster/prometheus.yaml | 21 + .../prometheus_servicemonitor.yaml | 13 + cicd/k3s-rabbitmq-incluster/rmconfig.sh | 4 + .../rmq/client-service.yaml | 24 + .../k3s-rabbitmq-incluster/rmq/configmap.yaml | 15 + .../rmq/headless-service.yaml | 20 + .../k3s-rabbitmq-incluster/rmq/namespace.yaml | 6 + cicd/k3s-rabbitmq-incluster/rmq/rabbitmq.yaml | 24 + cicd/k3s-rabbitmq-incluster/rmq/rbac.yaml | 33 + .../rmq/statefulset.yaml | 142 ++ cicd/k3s-rabbitmq-incluster/validation.sh | 58 + cicd/k3s-rabbitmq-incluster/wait_ready.sh | 38 + cicd/k3s-rabbitmq-incluster/worker.sh | 13 + .../worker_with_cilium.sh | 14 + 36 files changed, 3283 insertions(+) create mode 100644 cicd/k3s-rabbitmq-incluster/Vagrantfile create mode 100755 cicd/k3s-rabbitmq-incluster/config.sh create mode 100644 cicd/k3s-rabbitmq-incluster/grafana.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/host.sh create mode 100755 cicd/k3s-rabbitmq-incluster/install_cilium.sh create mode 100644 cicd/k3s-rabbitmq-incluster/iperf-service.yml create mode 100644 cicd/k3s-rabbitmq-incluster/k3s.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/kube-loxilb.yml create mode 100644 cicd/k3s-rabbitmq-incluster/loxilb.sh create mode 100644 cicd/k3s-rabbitmq-incluster/loxilb.yml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/cilium-ippool.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/cilium-policy.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/iperf-service-loxilb.yml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/iperf-service-metallb.yml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/kube-loxilb.yml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/loxilb.yml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/metallb-addr-pool.yml create mode 100644 cicd/k3s-rabbitmq-incluster/manifests/metallb-native.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/master.sh create mode 100644 cicd/k3s-rabbitmq-incluster/master_with_cilium.sh create mode 100644 cicd/k3s-rabbitmq-incluster/prom_rbac.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/prom_svc.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/prometheus.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/prometheus_servicemonitor.yaml create mode 100755 cicd/k3s-rabbitmq-incluster/rmconfig.sh create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/client-service.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/configmap.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/headless-service.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/namespace.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/rabbitmq.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/rbac.yaml create mode 100644 cicd/k3s-rabbitmq-incluster/rmq/statefulset.yaml create mode 100755 cicd/k3s-rabbitmq-incluster/validation.sh create mode 100755 cicd/k3s-rabbitmq-incluster/wait_ready.sh create mode 100644 cicd/k3s-rabbitmq-incluster/worker.sh create mode 100644 cicd/k3s-rabbitmq-incluster/worker_with_cilium.sh diff --git a/cicd/k3s-rabbitmq-incluster/Vagrantfile b/cicd/k3s-rabbitmq-incluster/Vagrantfile new file mode 100644 index 000000000..fc1347f48 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/Vagrantfile @@ -0,0 +1,51 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +workers = (ENV['WORKERS'] || "3").to_i +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + config.vm.define "host" do |host| + host.vm.hostname = 'host' + host.vm.network :private_network, ip: "192.168.80.9", :netmask => "255.255.255.0" + host.vm.network :private_network, ip: "192.168.90.9", :netmask => "255.255.255.0" + host.vm.provision :shell, :path => "host.sh" + host.vm.provider :virtualbox do |vbox| + vbox.memory = "4096" + vbox.cpus = "8" + vbox.default_nic_type = "virtio" + end + end + + config.vm.define "master" do |master| + master.vm.hostname = 'master' + master.vm.network :private_network, ip: "192.168.80.10", :netmask => "255.255.255.0" + master.vm.provision :shell, :path => "master.sh" + master.vm.provider :virtualbox do |vbox| + vbox.memory = "4096" + vbox.cpus = "4" + vbox.default_nic_type = "virtio" + end + end + + (1..workers).each do |node_number| + config.vm.define "worker#{node_number}" do |worker| + worker.vm.hostname = "worker#{node_number}" + ip = node_number + 100 + worker.vm.network :private_network, ip: "192.168.80.#{ip}", :netmask => "255.255.255.0" + worker.vm.provision :shell, :path => "worker.sh" + worker.vm.provider :virtualbox do |vbox| + vbox.memory = "4096" + vbox.cpus = "4" + vbox.default_nic_type = "virtio" + end + end + end +end diff --git a/cicd/k3s-rabbitmq-incluster/config.sh b/cicd/k3s-rabbitmq-incluster/config.sh new file mode 100755 index 000000000..6b8ee48ef --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/config.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up diff --git a/cicd/k3s-rabbitmq-incluster/grafana.yaml b/cicd/k3s-rabbitmq-incluster/grafana.yaml new file mode 100644 index 000000000..b41ec5405 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/grafana.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: grafana + name: grafana +spec: + selector: + matchLabels: + app: grafana + template: + metadata: + labels: + app: grafana + spec: + securityContext: + fsGroup: 472 + supplementalGroups: + - 0 + containers: + - name: grafana + image: grafana/grafana:9.1.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 3000 + name: http-grafana + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /robots.txt + port: 3000 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 30 + successThreshold: 1 + timeoutSeconds: 2 + livenessProbe: + failureThreshold: 3 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: 3000 + timeoutSeconds: 1 + resources: + requests: + cpu: 250m + memory: 750Mi + volumeMounts: + - mountPath: /var/lib/grafana + name: grafana-pv + volumes: + - name: grafana-pv + persistentVolumeClaim: + claimName: grafana-pvc +--- +apiVersion: v1 +kind: Service +metadata: + name: grafana +spec: + ports: + - port: 3000 + protocol: TCP + targetPort: http-grafana + selector: + app: grafana + sessionAffinity: None + type: LoadBalancer + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb diff --git a/cicd/k3s-rabbitmq-incluster/host.sh b/cicd/k3s-rabbitmq-incluster/host.sh new file mode 100644 index 000000000..f29d40e71 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/host.sh @@ -0,0 +1,10 @@ +apt-get update +apt-get install -y software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce + +#curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +#add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get install -y iperf iperf3 diff --git a/cicd/k3s-rabbitmq-incluster/install_cilium.sh b/cicd/k3s-rabbitmq-incluster/install_cilium.sh new file mode 100755 index 000000000..7206c022d --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/install_cilium.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +#Install Cilium +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) +CLI_ARCH=amd64 +if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi +curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum +sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin +rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +mkdir -p ~/.kube/ +sudo cat /etc/rancher/k3s/k3s.yaml > ~/.kube/config +cilium install diff --git a/cicd/k3s-rabbitmq-incluster/iperf-service.yml b/cicd/k3s-rabbitmq-incluster/iperf-service.yml new file mode 100644 index 000000000..1d23f273c --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/iperf-service.yml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: iperf-service + annotations: + loxilb.io/lbmode: "onearm" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: perf-test + ports: + - port: 55001 + targetPort: 5001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: iperf1 + labels: + what: perf-test +spec: + containers: + - name: iperf + image: eyes852/ubuntu-iperf-test:0.5 + command: + - iperf + - "-s" + ports: + - containerPort: 5001 + diff --git a/cicd/k3s-rabbitmq-incluster/k3s.yaml b/cicd/k3s-rabbitmq-incluster/k3s.yaml new file mode 100644 index 000000000..546f1c3fd --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/k3s.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +clusters: +- cluster: + certificate-authority-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkakNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdGMyVnkKZG1WeUxXTmhRREUzTVRBNE1qWXhOVEl3SGhjTk1qUXdNekU1TURVeU9URXlXaGNOTXpRd016RTNNRFV5T1RFeQpXakFqTVNFd0h3WURWUVFEREJock0zTXRjMlZ5ZG1WeUxXTmhRREUzTVRBNE1qWXhOVEl3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFRcjhwZm83akZnSUN1WkZlVkdCZVRrc01PdElZWjZidWVVYVFBc1BLbU0KTGVYbm9Uc1JzRFJ5Wi92Vmw1NzNZZHNHeTYxSHh1WFN3bTNPUFhvS25DKzFvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVVltWTVuMnhPdUpmNEgvUmt5aDJwCnpPaDE4K3d3Q2dZSUtvWkl6ajBFQXdJRFJ3QXdSQUlnSnUvbVRlUk9qeC8rdGdNckxQdC9NMmF0a1RqRUw3NkkKU2xQV0N3eEticlVDSUhRZFZEQVJGVWtPd1ZNNEppdVhTaG5JYkt1OXJBNzdocXZBdlZ0ZFEzWEMKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= + server: https://192.168.80.10:6443 + name: default +contexts: +- context: + cluster: default + user: default + name: default +current-context: default +kind: Config +preferences: {} +users: +- name: default + user: + client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJrRENDQVRlZ0F3SUJBZ0lJRUk5Tm02SXBsdzB3Q2dZSUtvWkl6ajBFQXdJd0l6RWhNQjhHQTFVRUF3d1kKYXpOekxXTnNhV1Z1ZEMxallVQXhOekV3T0RJMk1UVXlNQjRYRFRJME1ETXhPVEExTWpreE1sb1hEVEkxTURNeApPVEExTWpreE1sb3dNREVYTUJVR0ExVUVDaE1PYzNsemRHVnRPbTFoYzNSbGNuTXhGVEFUQmdOVkJBTVRESE41CmMzUmxiVHBoWkcxcGJqQlpNQk1HQnlxR1NNNDlBZ0VHQ0NxR1NNNDlBd0VIQTBJQUJONlJVVGVwcnZBRkNuN2EKcDZndmRUMmxPN1MrWXd3bTZ3em45T2xXcWw4ZnJqNTgwcktEWVNxVmFCdkxUL2IrZytBL0pQRUV6TXFscWdYTwpGYWo0TElTalNEQkdNQTRHQTFVZER3RUIvd1FFQXdJRm9EQVRCZ05WSFNVRUREQUtCZ2dyQmdFRkJRY0RBakFmCkJnTlZIU01FR0RBV2dCVGRNaERoRUZQOEdlMVBSblh4d2hWcGltUHZYakFLQmdncWhrak9QUVFEQWdOSEFEQkUKQWlBaGNVd1d3WE1iRGZaVkE2NHVSemhweDR5dmg0UUNEM0ZZa2YwQkwwQ2FlZ0lnTmZ4enArenUxWk5PZWpoTAo2d3ZXVXhuekZpQ2xZYUpzNDVrcCt6ZFJuME09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0KLS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUJkekNDQVIyZ0F3SUJBZ0lCQURBS0JnZ3Foa2pPUFFRREFqQWpNU0V3SHdZRFZRUUREQmhyTTNNdFkyeHAKWlc1MExXTmhRREUzTVRBNE1qWXhOVEl3SGhjTk1qUXdNekU1TURVeU9URXlXaGNOTXpRd016RTNNRFV5T1RFeQpXakFqTVNFd0h3WURWUVFEREJock0zTXRZMnhwWlc1MExXTmhRREUzTVRBNE1qWXhOVEl3V1RBVEJnY3Foa2pPClBRSUJCZ2dxaGtqT1BRTUJCd05DQUFUTUoxd1ZOcEIwS0N6MWx5bWhRRGc3UDhRSGxGcHBUOHc5blFCWGYyeGQKMWtTb2RyS3RvSzlQYTJtelNiWFNtei9acTBpQk94SkY3aTdyT3BhQzZXUHdvMEl3UURBT0JnTlZIUThCQWY4RQpCQU1DQXFRd0R3WURWUjBUQVFIL0JBVXdBd0VCL3pBZEJnTlZIUTRFRmdRVTNUSVE0UkJUL0JudFQwWjE4Y0lWCmFZcGo3MTR3Q2dZSUtvWkl6ajBFQXdJRFNBQXdSUUloQUlRN3ZNVldZNkMxaVdLakIzNEYzdVZFQS9GSVpKVVAKRWM1bEFLS0JSWW8vQWlBbUFVVnQzRkRrSEYreFhJWUlzenBscWVDNWZ0Y0g1azJDaFFrbXFZaThXQT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K + client-key-data: LS0tLS1CRUdJTiBFQyBQUklWQVRFIEtFWS0tLS0tCk1IY0NBUUVFSU9KamZxUm90eWRLd0poQkY5SHJlTG1RNExSYVp2NFFiRXp0K0I0WnBTeWNvQW9HQ0NxR1NNNDkKQXdFSG9VUURRZ0FFM3BGUk42bXU4QVVLZnRxbnFDOTFQYVU3dEw1akRDYnJET2YwNlZhcVh4K3VQbnpTc29OaApLcFZvRzh0UDl2NkQ0RDhrOFFUTXlxV3FCYzRWcVBnc2hBPT0KLS0tLS1FTkQgRUMgUFJJVkFURSBLRVktLS0tLQo= diff --git a/cicd/k3s-rabbitmq-incluster/kube-loxilb.yml b/cicd/k3s-rabbitmq-incluster/kube-loxilb.yml new file mode 100644 index 000000000..f6d54ad8f --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/kube-loxilb.yml @@ -0,0 +1,130 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + #- --loxiURL=http://192.168.80.9:11111 + - --externalCIDR=192.168.80.20/32 + - --setRoles=0.0.0.0 + #- --monitor + #- --setBGP + - --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/k3s-rabbitmq-incluster/loxilb.sh b/cicd/k3s-rabbitmq-incluster/loxilb.sh new file mode 100644 index 000000000..74e66ae9b --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/loxilb.sh @@ -0,0 +1,13 @@ +export LOXILB_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep -v '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +apt-get update +apt-get install -y software-properties-common +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce +docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit -v /dev/log:/dev/log --net=host --name loxilb ghcr.io/loxilb-io/loxilb:latest +echo alias loxicmd=\"sudo docker exec -it loxilb loxicmd\" >> ~/.bashrc +echo alias loxilb=\"sudo docker exec -it loxilb \" >> ~/.bashrc + +echo $LOXILB_IP > /vagrant/loxilb-ip diff --git a/cicd/k3s-rabbitmq-incluster/loxilb.yml b/cicd/k3s-rabbitmq-incluster/loxilb.yml new file mode 100644 index 000000000..30cf81d96 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/loxilb.yml @@ -0,0 +1,77 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loxilb-lb + namespace: kube-system + labels: + app: loxilb-app +spec: + selector: + matchLabels: + app: loxilb-app + template: + metadata: + labels: + app: loxilb-app + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + #affinity: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: "node-role.kubernetes.io/master" + # operator: Exists + # - key: "node-role.kubernetes.io/control-plane" + # operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + containers: + - name: loxilb-lb + image: "ghcr.io/loxilb-io/loxilb:latest" + imagePullPolicy: Always + #command: [ "/root/loxilb-io/loxilb/loxilb", "--egr-hooks", "--blacklist=cni.|veth.|flannel.|cali.|tunl.|vxlan[.]calico", "--ipvs-compat", "--k8s-api=cluster" ] + command: [ "/root/loxilb-io/loxilb/loxilb", "--ipvs-compat"] + ports: + - containerPort: 11111 + - containerPort: 179 + - containerPort: 50051 + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN + env: + - name: MY_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +--- +apiVersion: v1 +kind: Service +metadata: + name: loxilb-lb-service + namespace: kube-system +spec: + clusterIP: None + selector: + app: loxilb-app + ports: + - name: loxilb-app + port: 11111 + targetPort: 11111 + protocol: TCP + - name: loxilb-app-bgp + port: 179 + targetPort: 179 + protocol: TCP + - name: loxilb-app-gobgp + port: 50051 + targetPort: 50051 + protocol: TCP diff --git a/cicd/k3s-rabbitmq-incluster/manifests/cilium-ippool.yaml b/cicd/k3s-rabbitmq-incluster/manifests/cilium-ippool.yaml new file mode 100644 index 000000000..5dd565002 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/cilium-ippool.yaml @@ -0,0 +1,7 @@ +apiVersion: "cilium.io/v2alpha1" +kind: CiliumLoadBalancerIPPool +metadata: + name: "lb-pool" +spec: + cidrs: + - cidr: "192.168.80.20/24" diff --git a/cicd/k3s-rabbitmq-incluster/manifests/cilium-policy.yaml b/cicd/k3s-rabbitmq-incluster/manifests/cilium-policy.yaml new file mode 100644 index 000000000..b155ad31e --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/cilium-policy.yaml @@ -0,0 +1,9 @@ +apiVersion: "cilium.io/v2alpha1" +kind: CiliumL2AnnouncementPolicy +metadata: + name: basic-policy +spec: + interfaces: + - eth1 + externalIPs: true + loadBalancerIPs: true diff --git a/cicd/k3s-rabbitmq-incluster/manifests/iperf-service-loxilb.yml b/cicd/k3s-rabbitmq-incluster/manifests/iperf-service-loxilb.yml new file mode 100644 index 000000000..1d23f273c --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/iperf-service-loxilb.yml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Service +metadata: + name: iperf-service + annotations: + loxilb.io/lbmode: "onearm" +spec: + externalTrafficPolicy: Local + loadBalancerClass: loxilb.io/loxilb + selector: + what: perf-test + ports: + - port: 55001 + targetPort: 5001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: iperf1 + labels: + what: perf-test +spec: + containers: + - name: iperf + image: eyes852/ubuntu-iperf-test:0.5 + command: + - iperf + - "-s" + ports: + - containerPort: 5001 + diff --git a/cicd/k3s-rabbitmq-incluster/manifests/iperf-service-metallb.yml b/cicd/k3s-rabbitmq-incluster/manifests/iperf-service-metallb.yml new file mode 100644 index 000000000..066c796e2 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/iperf-service-metallb.yml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Service +metadata: + name: iperf-service +spec: + externalTrafficPolicy: Local + selector: + what: perf-test + ports: + - port: 55001 + targetPort: 5001 + type: LoadBalancer +--- +apiVersion: v1 +kind: Pod +metadata: + name: iperf1 + labels: + what: perf-test +spec: + containers: + - name: iperf + image: eyes852/ubuntu-iperf-test:0.5 + command: + - iperf + - "-s" + ports: + - containerPort: 5001 + diff --git a/cicd/k3s-rabbitmq-incluster/manifests/kube-loxilb.yml b/cicd/k3s-rabbitmq-incluster/manifests/kube-loxilb.yml new file mode 100644 index 000000000..9357b17b0 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/kube-loxilb.yml @@ -0,0 +1,130 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-loxilb + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + - list + - patch + - apiGroups: + - "" + resources: + - endpoints + - services + - services/status + verbs: + - get + - watch + - list + - patch + - update + - apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - watch + - list + - apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create + - apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: kube-loxilb +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-loxilb +subjects: + - kind: ServiceAccount + name: kube-loxilb + namespace: kube-system +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-loxilb + namespace: kube-system + labels: + app: loxilb +spec: + replicas: 1 + selector: + matchLabels: + app: loxilb + template: + metadata: + labels: + app: loxilb + spec: + hostNetwork: true + tolerations: + - effect: NoSchedule + operator: Exists + # Mark the pod as a critical add-on for rescheduling. + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + terminationGracePeriodSeconds: 0 + containers: + - name: kube-loxilb + image: ghcr.io/loxilb-io/kube-loxilb:latest + imagePullPolicy: Always + command: + - /bin/kube-loxilb + args: + #- --loxiURL=http://192.168.80.9:11111 + - --externalCIDR=192.168.80.20/32 + - --setRoles=0.0.0.0 + #- --monitor + #- --setBGP + #- --setLBMode=1 + #- --config=/opt/loxilb/agent/kube-loxilb.conf + resources: + requests: + cpu: "100m" + memory: "50Mi" + limits: + cpu: "100m" + memory: "50Mi" + securityContext: + privileged: true + capabilities: + add: ["NET_ADMIN", "NET_RAW"] diff --git a/cicd/k3s-rabbitmq-incluster/manifests/loxilb.yml b/cicd/k3s-rabbitmq-incluster/manifests/loxilb.yml new file mode 100644 index 000000000..cd783e487 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/loxilb.yml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loxilb-lb + namespace: kube-system + labels: + app: loxilb-app +spec: + selector: + matchLabels: + app: loxilb-app + template: + metadata: + labels: + app: loxilb-app + spec: + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + tolerations: + - key: "node-role.kubernetes.io/master" + operator: Exists + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + priorityClassName: system-node-critical + serviceAccountName: kube-loxilb + containers: + - name: loxilb-lb + image: "ghcr.io/loxilb-io/loxilb:latest" + imagePullPolicy: Always + command: [ "/root/loxilb-io/loxilb/loxilb", "--egr-hooks", "--blacklist=cni.|veth.|flannel.|cali.|tunl.|vxlan[.]calico", "--ipvs-compat", "--k8s-api=cluster" ] + ports: + - containerPort: 11111 + - containerPort: 179 + - containerPort: 50051 + securityContext: + privileged: true + capabilities: + add: + - SYS_ADMIN + env: + - name: MY_NODE_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP +--- +apiVersion: v1 +kind: Service +metadata: + name: loxilb-lb-service + namespace: kube-system +spec: + clusterIP: None + selector: + app: loxilb-app + ports: + - name: loxilb-app + port: 11111 + targetPort: 11111 + protocol: TCP + - name: loxilb-app-bgp + port: 179 + targetPort: 179 + protocol: TCP + - name: loxilb-app-gobgp + port: 50051 + targetPort: 50051 + protocol: TCP diff --git a/cicd/k3s-rabbitmq-incluster/manifests/metallb-addr-pool.yml b/cicd/k3s-rabbitmq-incluster/manifests/metallb-addr-pool.yml new file mode 100644 index 000000000..ea7d2ba4f --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/metallb-addr-pool.yml @@ -0,0 +1,8 @@ +apiVersion: metallb.io/v1beta1 +kind: IPAddressPool +metadata: + name: cheap + namespace: metallb-system +spec: + addresses: + - 192.168.80.20/32 diff --git a/cicd/k3s-rabbitmq-incluster/manifests/metallb-native.yaml b/cicd/k3s-rabbitmq-incluster/manifests/metallb-native.yaml new file mode 100644 index 000000000..f8a80308f --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/manifests/metallb-native.yaml @@ -0,0 +1,2042 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/warn: privileged + name: metallb-system +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: addresspools.metallb.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + service: + name: webhook-service + namespace: metallb-system + path: /convert + conversionReviewVersions: + - v1alpha1 + - v1beta1 + group: metallb.io + names: + kind: AddressPool + listKind: AddressPoolList + plural: addresspools + singular: addresspool + scope: Namespaced + versions: + - deprecated: true + deprecationWarning: metallb.io v1alpha1 AddressPool is deprecated + name: v1alpha1 + schema: + openAPIV3Schema: + description: AddressPool is the Schema for the addresspools API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AddressPoolSpec defines the desired state of AddressPool. + properties: + addresses: + description: A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share + the same settings. Each range can be either a CIDR prefix, or an + explicit start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: AutoAssign flag used to prevent MetallB from automatic + allocation for a pool. + type: boolean + bgpAdvertisements: + description: When an IP is allocated from this pool, how should it + be translated into BGP announcements? + items: + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets + you “roll up” the /32s into a larger prefix. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: Optional, defaults to 128 (i.e. no aggregation) + if not specified. + format: int32 + type: integer + communities: + description: BGP communities + items: + type: string + type: array + localPref: + description: BGP LOCAL_PREF attribute which is used by BGP best + path algorithm, Path with higher localpref is preferred over + one with lower localpref. + format: int32 + type: integer + type: object + type: array + protocol: + description: Protocol can be used to select how the announcement is + done. + enum: + - layer2 + - bgp + type: string + required: + - addresses + - protocol + type: object + status: + description: AddressPoolStatus defines the observed state of AddressPool. + type: object + required: + - spec + type: object + served: true + storage: false + subresources: + status: {} + - deprecated: true + deprecationWarning: metallb.io v1beta1 AddressPool is deprecated, consider using + IPAddressPool + name: v1beta1 + schema: + openAPIV3Schema: + description: AddressPool represents a pool of IP addresses that can be allocated + to LoadBalancer services. AddressPool is deprecated and being replaced by + IPAddressPool. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AddressPoolSpec defines the desired state of AddressPool. + properties: + addresses: + description: A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share + the same settings. Each range can be either a CIDR prefix, or an + explicit start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: AutoAssign flag used to prevent MetallB from automatic + allocation for a pool. + type: boolean + bgpAdvertisements: + description: Drives how an IP allocated from this pool should translated + into BGP announcements. + items: + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets + you “roll up” the /32s into a larger prefix. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: Optional, defaults to 128 (i.e. no aggregation) + if not specified. + format: int32 + type: integer + communities: + description: BGP communities to be associated with the given + advertisement. + items: + type: string + type: array + localPref: + description: BGP LOCAL_PREF attribute which is used by BGP best + path algorithm, Path with higher localpref is preferred over + one with lower localpref. + format: int32 + type: integer + type: object + type: array + protocol: + description: Protocol can be used to select how the announcement is + done. + enum: + - layer2 + - bgp + type: string + required: + - addresses + - protocol + type: object + status: + description: AddressPoolStatus defines the observed state of AddressPool. + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: bfdprofiles.metallb.io +spec: + group: metallb.io + names: + kind: BFDProfile + listKind: BFDProfileList + plural: bfdprofiles + singular: bfdprofile + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.passiveMode + name: Passive Mode + type: boolean + - jsonPath: .spec.transmitInterval + name: Transmit Interval + type: integer + - jsonPath: .spec.receiveInterval + name: Receive Interval + type: integer + - jsonPath: .spec.detectMultiplier + name: Multiplier + type: integer + name: v1beta1 + schema: + openAPIV3Schema: + description: BFDProfile represents the settings of the bfd session that can + be optionally associated with a BGP session. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BFDProfileSpec defines the desired state of BFDProfile. + properties: + detectMultiplier: + description: Configures the detection multiplier to determine packet + loss. The remote transmission interval will be multiplied by this + value to determine the connection loss detection timer. + format: int32 + maximum: 255 + minimum: 2 + type: integer + echoInterval: + description: Configures the minimal echo receive transmission interval + that this system is capable of handling in milliseconds. Defaults + to 50ms + format: int32 + maximum: 60000 + minimum: 10 + type: integer + echoMode: + description: Enables or disables the echo transmission mode. This + mode is disabled by default, and not supported on multi hops setups. + type: boolean + minimumTtl: + description: 'For multi hop sessions only: configure the minimum expected + TTL for an incoming BFD control packet.' + format: int32 + maximum: 254 + minimum: 1 + type: integer + passiveMode: + description: 'Mark session as passive: a passive session will not + attempt to start the connection and will wait for control packets + from peer before it begins replying.' + type: boolean + receiveInterval: + description: The minimum interval that this system is capable of receiving + control packets in milliseconds. Defaults to 300ms. + format: int32 + maximum: 60000 + minimum: 10 + type: integer + transmitInterval: + description: The minimum transmission interval (less jitter) that + this system wants to use to send BFD control packets in milliseconds. + Defaults to 300ms + format: int32 + maximum: 60000 + minimum: 10 + type: integer + type: object + status: + description: BFDProfileStatus defines the observed state of BFDProfile. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: bgpadvertisements.metallb.io +spec: + group: metallb.io + names: + kind: BGPAdvertisement + listKind: BGPAdvertisementList + plural: bgpadvertisements + singular: bgpadvertisement + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.ipAddressPools + name: IPAddressPools + type: string + - jsonPath: .spec.ipAddressPoolSelectors + name: IPAddressPool Selectors + type: string + - jsonPath: .spec.peers + name: Peers + type: string + - jsonPath: .spec.nodeSelectors + name: Node Selectors + priority: 10 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BGPAdvertisement allows to advertise the IPs coming from the + selected IPAddressPools via BGP, setting the parameters of the BGP Advertisement. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPAdvertisementSpec defines the desired state of BGPAdvertisement. + properties: + aggregationLength: + default: 32 + description: The aggregation-length advertisement option lets you + “roll up” the /32s into a larger prefix. Defaults to 32. Works for + IPv4 addresses. + format: int32 + minimum: 1 + type: integer + aggregationLengthV6: + default: 128 + description: The aggregation-length advertisement option lets you + “roll up” the /128s into a larger prefix. Defaults to 128. Works + for IPv6 addresses. + format: int32 + type: integer + communities: + description: The BGP communities to be associated with the announcement. + Each item can be a standard community of the form 1234:1234, a large + community of the form large:1234:1234:1234 or the name of an alias + defined in the Community CRD. + items: + type: string + type: array + ipAddressPoolSelectors: + description: A selector for the IPAddressPools which would get advertised + via this advertisement. If no IPAddressPool is selected by this + or by the list, the advertisement is applied to all the IPAddressPools. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + ipAddressPools: + description: The list of IPAddressPools to advertise via this advertisement, + selected by name. + items: + type: string + type: array + localPref: + description: The BGP LOCAL_PREF attribute which is used by BGP best + path algorithm, Path with higher localpref is preferred over one + with lower localpref. + format: int32 + type: integer + nodeSelectors: + description: NodeSelectors allows to limit the nodes to announce as + next hops for the LoadBalancer IP. When empty, all the nodes having are + announced as next hops. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + peers: + description: Peers limits the bgppeer to advertise the ips of the + selected pools to. When empty, the loadbalancer IP is announced + to all the BGPPeers configured. + items: + type: string + type: array + type: object + status: + description: BGPAdvertisementStatus defines the observed state of BGPAdvertisement. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: bgppeers.metallb.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + caBundle: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tDQpNSUlGWlRDQ0EwMmdBd0lCQWdJVU5GRW1XcTM3MVpKdGkrMmlSQzk1WmpBV1MxZ3dEUVlKS29aSWh2Y05BUUVMDQpCUUF3UWpFTE1Ba0dBMVVFQmhNQ1dGZ3hGVEFUQmdOVkJBY01ERVJsWm1GMWJIUWdRMmwwZVRFY01Cb0dBMVVFDQpDZ3dUUkdWbVlYVnNkQ0JEYjIxd1lXNTVJRXgwWkRBZUZ3MHlNakEzTVRrd09UTXlNek5hRncweU1qQTRNVGd3DQpPVE15TXpOYU1FSXhDekFKQmdOVkJBWVRBbGhZTVJVd0V3WURWUVFIREF4RVpXWmhkV3gwSUVOcGRIa3hIREFhDQpCZ05WQkFvTUUwUmxabUYxYkhRZ1EyOXRjR0Z1ZVNCTWRHUXdnZ0lpTUEwR0NTcUdTSWIzRFFFQkFRVUFBNElDDQpEd0F3Z2dJS0FvSUNBUUNxVFpxMWZRcC9vYkdlenhES0o3OVB3Ny94azJwellualNzMlkzb1ZYSm5sRmM4YjVlDQpma2ZZQnY2bndscW1keW5PL2phWFBaQmRQSS82aFdOUDBkdVhadEtWU0NCUUpyZzEyOGNXb3F0MGNTN3pLb1VpDQpvcU1tQ0QvRXVBeFFNZjhRZDF2c1gvVllkZ0poVTZBRXJLZEpIaXpFOUJtUkNkTDBGMW1OVW55Rk82UnRtWFZUDQpidkxsTDVYeTc2R0FaQVBLOFB4aVlDa0NtbDdxN0VnTWNiOXlLWldCYmlxQ3VkTXE5TGJLNmdKNzF6YkZnSXV4DQo1L1pXK2JraTB2RlplWk9ZODUxb1psckFUNzJvMDI4NHNTWW9uN0pHZVZkY3NoUnh5R1VpSFpSTzdkaXZVTDVTDQpmM2JmSDFYbWY1ZDQzT0NWTWRuUUV2NWVaOG8zeWVLa3ZrbkZQUGVJMU9BbjdGbDlFRVNNR2dhOGFaSG1URSttDQpsLzlMSmdDYjBnQmtPT0M0WnV4bWh2aERKV1EzWnJCS3pMQlNUZXN0NWlLNVlwcXRWVVk2THRyRW9FelVTK1lsDQpwWndXY2VQWHlHeHM5ZURsR3lNVmQraW15Y3NTU1UvVno2Mmx6MnZCS21NTXBkYldDQWhud0RsRTVqU2dyMjRRDQp0eGNXLys2N3d5KzhuQlI3UXdqVTFITndVRjBzeERWdEwrZ1NHVERnSEVZSlhZelYvT05zMy94TkpoVFNPSkxNDQpoeXNVdyttaGdackdhbUdXcHVIVU1DUitvTWJzMTc1UkcrQjJnUFFHVytPTjJnUTRyOXN2b0ZBNHBBQm8xd1dLDQpRYjRhY3pmeVVscElBOVFoSmFsZEY3S3dPSHVlV3gwRUNrNXg0T2tvVDBvWVp0dzFiR0JjRGtaSmF3SURBUUFCDQpvMU13VVRBZEJnTlZIUTRFRmdRVW90UlNIUm9IWTEyRFZ4R0NCdEhpb1g2ZmVFQXdId1lEVlIwakJCZ3dGb0FVDQpvdFJTSFJvSFkxMkRWeEdDQnRIaW9YNmZlRUF3RHdZRFZSMFRBUUgvQkFVd0F3RUIvekFOQmdrcWhraUc5dzBCDQpBUXNGQUFPQ0FnRUFSbkpsWWRjMTFHd0VxWnh6RDF2R3BDR2pDN2VWTlQ3aVY1d3IybXlybHdPYi9aUWFEa0xYDQpvVStaOVVXT1VlSXJTdzUydDdmQUpvVVAwSm5iYkMveVIrU1lqUGhvUXNiVHduOTc2ZldBWTduM3FMOXhCd1Y0DQphek41OXNjeUp0dlhMeUtOL2N5ak1ReDRLajBIMFg0bWJ6bzVZNUtzWWtYVU0vOEFPdWZMcEd0S1NGVGgrSEFDDQpab1Q5YnZHS25adnNHd0tYZFF0Wnh0akhaUjVqK3U3ZGtQOTJBT051RFNabS8rWVV4b2tBK09JbzdSR3BwSHNXDQo1ZTdNY0FTVXRtb1FORXd6dVFoVkJaRWQ1OGtKYjUrV0VWbGNzanlXNnRTbzErZ25tTWNqR1BsMWgxR2hVbjV4DQpFY0lWRnBIWXM5YWo1NmpBSjk1MVQvZjhMaWxmTlVnanBLQ0c1bnl0SUt3emxhOHNtdGlPdm1UNEpYbXBwSkI2DQo4bmdHRVluVjUrUTYwWFJ2OEhSSGp1VG9CRHVhaERrVDA2R1JGODU1d09FR2V4bkZpMXZYWUxLVllWb1V2MXRKDQo4dVdUR1pwNllDSVJldlBqbzg5ZytWTlJSaVFYUThJd0dybXE5c0RoVTlqTjA0SjdVL1RvRDFpNHE3VnlsRUc5DQorV1VGNkNLaEdBeTJIaEhwVncyTGFoOS9lUzdZMUZ1YURrWmhPZG1laG1BOCtqdHNZamJadnR5Mm1SWlF0UUZzDQpUU1VUUjREbUR2bVVPRVRmeStpRHdzK2RkWXVNTnJGeVVYV2dkMnpBQU4ydVl1UHFGY2pRcFNPODFzVTJTU3R3DQoxVzAyeUtYOGJEYmZFdjBzbUh3UzliQnFlSGo5NEM1Mjg0YXpsdTBmaUdpTm1OUEM4ckJLRmhBPQ0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== + service: + name: webhook-service + namespace: metallb-system + path: /convert + conversionReviewVersions: + - v1beta1 + - v1beta2 + group: metallb.io + names: + kind: BGPPeer + listKind: BGPPeerList + plural: bgppeers + singular: bgppeer + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.peerAddress + name: Address + type: string + - jsonPath: .spec.peerASN + name: ASN + type: string + - jsonPath: .spec.bfdProfile + name: BFD Profile + type: string + - jsonPath: .spec.ebgpMultiHop + name: Multi Hops + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: BGPPeer is the Schema for the peers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec defines the desired state of Peer. + properties: + bfdProfile: + type: string + ebgpMultiHop: + description: EBGP peer is multi-hops away + type: boolean + holdTime: + description: Requested BGP hold time, per RFC4271. + type: string + keepaliveTime: + description: Requested BGP keepalive time, per RFC4271. + type: string + myASN: + description: AS number to use for the local end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + nodeSelectors: + description: Only connect to this peer on nodes that match one of + these selectors. + items: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + minItems: 1 + type: array + required: + - key + - operator + - values + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + type: array + password: + description: Authentication password for routers enforcing TCP MD5 + authenticated sessions + type: string + peerASN: + description: AS number to expect from the remote end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + peerAddress: + description: Address to dial when establishing the session. + type: string + peerPort: + description: Port to dial when establishing the session. + maximum: 16384 + minimum: 0 + type: integer + routerID: + description: BGP router ID to advertise to the peer + type: string + sourceAddress: + description: Source address to use when establishing the session. + type: string + required: + - myASN + - peerASN + - peerAddress + type: object + status: + description: BGPPeerStatus defines the observed state of Peer. + type: object + type: object + served: true + storage: false + subresources: + status: {} + - additionalPrinterColumns: + - jsonPath: .spec.peerAddress + name: Address + type: string + - jsonPath: .spec.peerASN + name: ASN + type: string + - jsonPath: .spec.bfdProfile + name: BFD Profile + type: string + - jsonPath: .spec.ebgpMultiHop + name: Multi Hops + type: string + name: v1beta2 + schema: + openAPIV3Schema: + description: BGPPeer is the Schema for the peers API. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BGPPeerSpec defines the desired state of Peer. + properties: + bfdProfile: + description: The name of the BFD Profile to be used for the BFD session + associated to the BGP session. If not set, the BFD session won't + be set up. + type: string + ebgpMultiHop: + description: To set if the BGPPeer is multi-hops away. Needed for + FRR mode only. + type: boolean + holdTime: + description: Requested BGP hold time, per RFC4271. + type: string + keepaliveTime: + description: Requested BGP keepalive time, per RFC4271. + type: string + myASN: + description: AS number to use for the local end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + nodeSelectors: + description: Only connect to this peer on nodes that match one of + these selectors. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + password: + description: Authentication password for routers enforcing TCP MD5 + authenticated sessions + type: string + passwordSecret: + description: passwordSecret is name of the authentication secret for + BGP Peer. the secret must be of type "kubernetes.io/basic-auth", + and created in the same namespace as the MetalLB deployment. The + password is stored in the secret as the key "password". + properties: + name: + description: name is unique within a namespace to reference a + secret resource. + type: string + namespace: + description: namespace defines the space within which the secret + name must be unique. + type: string + type: object + x-kubernetes-map-type: atomic + peerASN: + description: AS number to expect from the remote end of the session. + format: int32 + maximum: 4294967295 + minimum: 0 + type: integer + peerAddress: + description: Address to dial when establishing the session. + type: string + peerPort: + default: 179 + description: Port to dial when establishing the session. + maximum: 16384 + minimum: 0 + type: integer + routerID: + description: BGP router ID to advertise to the peer + type: string + sourceAddress: + description: Source address to use when establishing the session. + type: string + vrf: + description: To set if we want to peer with the BGPPeer using an interface + belonging to a host vrf + type: string + required: + - myASN + - peerASN + - peerAddress + type: object + status: + description: BGPPeerStatus defines the observed state of Peer. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: communities.metallb.io +spec: + group: metallb.io + names: + kind: Community + listKind: CommunityList + plural: communities + singular: community + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: Community is a collection of aliases for communities. Users can + define named aliases to be used in the BGPPeer CRD. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CommunitySpec defines the desired state of Community. + properties: + communities: + items: + properties: + name: + description: The name of the alias for the community. + type: string + value: + description: The BGP community value corresponding to the given + name. Can be a standard community of the form 1234:1234 or + a large community of the form large:1234:1234:1234. + type: string + type: object + type: array + type: object + status: + description: CommunityStatus defines the observed state of Community. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: ipaddresspools.metallb.io +spec: + group: metallb.io + names: + kind: IPAddressPool + listKind: IPAddressPoolList + plural: ipaddresspools + singular: ipaddresspool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.autoAssign + name: Auto Assign + type: boolean + - jsonPath: .spec.avoidBuggyIPs + name: Avoid Buggy IPs + type: boolean + - jsonPath: .spec.addresses + name: Addresses + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: IPAddressPool represents a pool of IP addresses that can be allocated + to LoadBalancer services. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: IPAddressPoolSpec defines the desired state of IPAddressPool. + properties: + addresses: + description: A list of IP address ranges over which MetalLB has authority. + You can list multiple ranges in a single pool, they will all share + the same settings. Each range can be either a CIDR prefix, or an + explicit start-end range of IPs. + items: + type: string + type: array + autoAssign: + default: true + description: AutoAssign flag used to prevent MetallB from automatic + allocation for a pool. + type: boolean + avoidBuggyIPs: + default: false + description: AvoidBuggyIPs prevents addresses ending with .0 and .255 + to be used by a pool. + type: boolean + serviceAllocation: + description: AllocateTo makes ip pool allocation to specific namespace + and/or service. The controller will use the pool with lowest value + of priority in case of multiple matches. A pool with no priority + set will be used only if the pools with priority can't be used. + If multiple matching IPAddressPools are available it will check + for the availability of IPs sorting the matching IPAddressPools + by priority, starting from the highest to the lowest. If multiple + IPAddressPools have the same priority, choice will be random. + properties: + namespaceSelectors: + description: NamespaceSelectors list of label selectors to select + namespace(s) for ip pool, an alternative to using namespace + list. + items: + description: A label selector is a label query over a set of + resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. A + null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + namespaces: + description: Namespaces list of namespace(s) on which ip pool + can be attached. + items: + type: string + type: array + priority: + description: Priority priority given for ip pool while ip allocation + on a service. + type: integer + serviceSelectors: + description: ServiceSelectors list of label selector to select + service(s) for which ip pool can be used for ip allocation. + items: + description: A label selector is a label query over a set of + resources. The result of matchLabels and matchExpressions + are ANDed. An empty label selector matches all objects. A + null label selector matches no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, + Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values array + must be non-empty. If the operator is Exists or + DoesNotExist, the values array must be empty. This + array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field is + "key", the operator is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + type: object + required: + - addresses + type: object + status: + description: IPAddressPoolStatus defines the observed state of IPAddressPool. + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.1 + creationTimestamp: null + name: l2advertisements.metallb.io +spec: + group: metallb.io + names: + kind: L2Advertisement + listKind: L2AdvertisementList + plural: l2advertisements + singular: l2advertisement + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.ipAddressPools + name: IPAddressPools + type: string + - jsonPath: .spec.ipAddressPoolSelectors + name: IPAddressPool Selectors + type: string + - jsonPath: .spec.interfaces + name: Interfaces + type: string + - jsonPath: .spec.nodeSelectors + name: Node Selectors + priority: 10 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: L2Advertisement allows to advertise the LoadBalancer IPs provided + by the selected pools via L2. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: L2AdvertisementSpec defines the desired state of L2Advertisement. + properties: + interfaces: + description: A list of interfaces to announce from. The LB IP will + be announced only from these interfaces. If the field is not set, + we advertise from all the interfaces on the host. + items: + type: string + type: array + ipAddressPoolSelectors: + description: A selector for the IPAddressPools which would get advertised + via this advertisement. If no IPAddressPool is selected by this + or by the list, the advertisement is applied to all the IPAddressPools. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + ipAddressPools: + description: The list of IPAddressPools to advertise via this advertisement, + selected by name. + items: + type: string + type: array + nodeSelectors: + description: NodeSelectors allows to limit the nodes to announce as + next hops for the LoadBalancer IP. When empty, all the nodes having are + announced as next hops. + items: + description: A label selector is a label query over a set of resources. + The result of matchLabels and matchExpressions are ANDed. An empty + label selector matches all objects. A null label selector matches + no objects. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that + contains values, a key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of string values. If the + operator is In or NotIn, the values array must be non-empty. + If the operator is Exists or DoesNotExist, the values + array must be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator + is "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: array + type: object + status: + description: L2AdvertisementStatus defines the observed state of L2Advertisement. + type: object + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app: metallb + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resourceNames: + - memberlist + resources: + - secrets + verbs: + - list +- apiGroups: + - apps + resourceNames: + - controller + resources: + - deployments + verbs: + - get +- apiGroups: + - metallb.io + resources: + - bgppeers + verbs: + - get + - list +- apiGroups: + - metallb.io + resources: + - addresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bfdprofiles + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - ipaddresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bgpadvertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - l2advertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - communities + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - list +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - addresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bfdprofiles + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bgppeers + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - l2advertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - bgpadvertisements + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - ipaddresspools + verbs: + - get + - list + - watch +- apiGroups: + - metallb.io + resources: + - communities + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:controller +rules: +- apiGroups: + - "" + resources: + - services + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - list +- apiGroups: + - "" + resources: + - services/status + verbs: + - update +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resourceNames: + - controller + resources: + - podsecuritypolicies + verbs: + - use +- apiGroups: + - admissionregistration.k8s.io + resourceNames: + - metallb-webhook-configuration + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + - mutatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resourceNames: + - addresspools.metallb.io + - bfdprofiles.metallb.io + - bgpadvertisements.metallb.io + - bgppeers.metallb.io + - ipaddresspools.metallb.io + - l2advertisements.metallb.io + - communities.metallb.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app: metallb + name: metallb-system:speaker +rules: +- apiGroups: + - "" + resources: + - services + - endpoints + - nodes + - namespaces + verbs: + - get + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - policy + resourceNames: + - speaker + resources: + - podsecuritypolicies + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: controller + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + labels: + app: metallb + name: pod-lister + namespace: metallb-system +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pod-lister +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:controller +subjects: +- kind: ServiceAccount + name: controller + namespace: metallb-system +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app: metallb + name: metallb-system:speaker +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: metallb-system:speaker +subjects: +- kind: ServiceAccount + name: speaker + namespace: metallb-system +--- +apiVersion: v1 +data: + excludel2.yaml: | + announcedInterfacesToExclude: ["^docker.*", "^cbr.*", "^dummy.*", "^virbr.*", "^lxcbr.*", "^veth.*", "^lo$", "^cali.*", "^tunl.*", "^flannel.*", "^kube-ipvs.*", "^cni.*", "^nodelocaldns.*"] +kind: ConfigMap +metadata: + name: metallb-excludel2 + namespace: metallb-system +--- +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: metallb-system +--- +apiVersion: v1 +kind: Service +metadata: + name: webhook-service + namespace: metallb-system +spec: + ports: + - port: 443 + targetPort: 9443 + selector: + component: controller +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: metallb + component: controller + name: controller + namespace: metallb-system +spec: + revisionHistoryLimit: 3 + selector: + matchLabels: + app: metallb + component: controller + template: + metadata: + annotations: + prometheus.io/port: "7472" + prometheus.io/scrape: "true" + labels: + app: metallb + component: controller + spec: + containers: + - args: + - --port=7472 + - --log-level=info + env: + - name: METALLB_ML_SECRET_NAME + value: memberlist + - name: METALLB_DEPLOYMENT + value: controller + image: quay.io/metallb/controller:v0.13.12 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: controller + ports: + - containerPort: 7472 + name: monitoring + - containerPort: 9443 + name: webhook-server + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - all + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + nodeSelector: + kubernetes.io/os: linux + securityContext: + fsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: controller + terminationGracePeriodSeconds: 0 + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app: metallb + component: speaker + name: speaker + namespace: metallb-system +spec: + selector: + matchLabels: + app: metallb + component: speaker + template: + metadata: + annotations: + prometheus.io/port: "7472" + prometheus.io/scrape: "true" + labels: + app: metallb + component: speaker + spec: + containers: + - args: + - --port=7472 + - --log-level=info + env: + - name: METALLB_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: METALLB_HOST + valueFrom: + fieldRef: + fieldPath: status.hostIP + - name: METALLB_ML_BIND_ADDR + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: METALLB_ML_LABELS + value: app=metallb,component=speaker + - name: METALLB_ML_SECRET_KEY_PATH + value: /etc/ml_secret_key + image: quay.io/metallb/speaker:v0.13.12 + livenessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: speaker + ports: + - containerPort: 7472 + name: monitoring + - containerPort: 7946 + name: memberlist-tcp + - containerPort: 7946 + name: memberlist-udp + protocol: UDP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /metrics + port: monitoring + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_RAW + drop: + - ALL + readOnlyRootFilesystem: true + volumeMounts: + - mountPath: /etc/ml_secret_key + name: memberlist + readOnly: true + - mountPath: /etc/metallb + name: metallb-excludel2 + readOnly: true + hostNetwork: true + nodeSelector: + kubernetes.io/os: linux + serviceAccountName: speaker + terminationGracePeriodSeconds: 2 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + volumes: + - name: memberlist + secret: + defaultMode: 420 + secretName: memberlist + - configMap: + defaultMode: 256 + name: metallb-excludel2 + name: metallb-excludel2 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + creationTimestamp: null + name: metallb-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta2-bgppeer + failurePolicy: Fail + name: bgppeersvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta2 + operations: + - CREATE + - UPDATE + resources: + - bgppeers + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-addresspool + failurePolicy: Fail + name: addresspoolvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - addresspools + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-bfdprofile + failurePolicy: Fail + name: bfdprofilevalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - DELETE + resources: + - bfdprofiles + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-bgpadvertisement + failurePolicy: Fail + name: bgpadvertisementvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - bgpadvertisements + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-community + failurePolicy: Fail + name: communityvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - communities + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-ipaddresspool + failurePolicy: Fail + name: ipaddresspoolvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ipaddresspools + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: metallb-system + path: /validate-metallb-io-v1beta1-l2advertisement + failurePolicy: Fail + name: l2advertisementvalidationwebhook.metallb.io + rules: + - apiGroups: + - metallb.io + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - l2advertisements + sideEffects: None diff --git a/cicd/k3s-rabbitmq-incluster/master.sh b/cicd/k3s-rabbitmq-incluster/master.sh new file mode 100644 index 000000000..7f6f91718 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/master.sh @@ -0,0 +1,14 @@ +export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable-cloud-controller --kube-proxy-arg proxy-mode=ipvs --flannel-iface=eth1 \ +--disable-network-policy \ +--node-ip=${MASTER_IP} --node-external-ip=${MASTER_IP} \ +--bind-address=${MASTER_IP}" sh - + +echo $MASTER_IP > /vagrant/master-ip +sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token +sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml +sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml +#sudo kubectl apply -f "https://github.com/rabbitmq/cluster-operator/releases/latest/download/cluster-operator.yml" +#sudo kubectl apply -f /vagrant/rmq/rabbitmq.yaml +/vagrant/wait_ready.sh diff --git a/cicd/k3s-rabbitmq-incluster/master_with_cilium.sh b/cicd/k3s-rabbitmq-incluster/master_with_cilium.sh new file mode 100644 index 000000000..01ad60b71 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/master_with_cilium.sh @@ -0,0 +1,29 @@ +export MASTER_IP=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') + +#curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable-cloud-controller --kube-proxy-arg proxy-mode=ipvs --flannel-iface=eth1 \ + +curl -sfL https://get.k3s.io | INSTALL_K3S_EXEC="--disable traefik --disable servicelb --disable-cloud-controller \ +--flannel-backend=none \ +--disable-network-policy \ +--node-ip=${MASTER_IP} --node-external-ip=${MASTER_IP} \ +--bind-address=${MASTER_IP}" sh - + +#Install Cilium +CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/master/stable.txt) +CLI_ARCH=amd64 +if [ "$(uname -m)" = "aarch64" ]; then CLI_ARCH=arm64; fi +curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum +sudo tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin +rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} +mkdir -p ~/.kube/ +sudo cat /etc/rancher/k3s/k3s.yaml > ~/.kube/config +cilium install + +echo $MASTER_IP > /vagrant/master-ip +sudo cp /var/lib/rancher/k3s/server/node-token /vagrant/node-token +sudo cp /etc/rancher/k3s/k3s.yaml /vagrant/k3s.yaml +sudo sed -i -e "s/127.0.0.1/${MASTER_IP}/g" /vagrant/k3s.yaml +#sudo kubectl apply -f "https://github.com/rabbitmq/cluster-operator/releases/latest/download/cluster-operator.yml" +#sudo kubectl apply -f /vagrant/rmq/rabbitmq.yaml +/vagrant/wait_ready.sh diff --git a/cicd/k3s-rabbitmq-incluster/prom_rbac.yaml b/cicd/k3s-rabbitmq-incluster/prom_rbac.yaml new file mode 100644 index 000000000..967e14368 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/prom_rbac.yaml @@ -0,0 +1,42 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - nodes/metrics + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: ["get", "list", "watch"] +- nonResourceURLs: ["/metrics"] + verbs: ["get"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: prometheus +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus + namespace: default diff --git a/cicd/k3s-rabbitmq-incluster/prom_svc.yaml b/cicd/k3s-rabbitmq-incluster/prom_svc.yaml new file mode 100644 index 000000000..c6eebe80d --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/prom_svc.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: prometheus + labels: + app: prometheus +spec: + #externalTrafficPolicy: Local + #loadBalancerClass: loxilb.io/loxilb + #type: LoadBalancer + ports: + - name: web + port: 9090 + targetPort: web + selector: + app.kubernetes.io/name: prometheus + sessionAffinity: ClientIP diff --git a/cicd/k3s-rabbitmq-incluster/prometheus.yaml b/cicd/k3s-rabbitmq-incluster/prometheus.yaml new file mode 100644 index 000000000..c8a2c55b1 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/prometheus.yaml @@ -0,0 +1,21 @@ +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: prometheus + labels: + app: prometheus +spec: + image: quay.io/prometheus/prometheus:v2.22.1 + nodeSelector: + kubernetes.io/os: linux + replicas: 3 + resources: + requests: + memory: 400Mi + securityContext: + fsGroup: 2000 + runAsNonRoot: true + runAsUser: 1000 + serviceAccountName: prometheus + version: v2.22.1 + serviceMonitorSelector: {} diff --git a/cicd/k3s-rabbitmq-incluster/prometheus_servicemonitor.yaml b/cicd/k3s-rabbitmq-incluster/prometheus_servicemonitor.yaml new file mode 100644 index 000000000..ea252e6d1 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/prometheus_servicemonitor.yaml @@ -0,0 +1,13 @@ +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: prometheus-self + labels: + app: prometheus +spec: + endpoints: + - interval: 5s + port: web + selector: + matchLabels: + app: prometheus diff --git a/cicd/k3s-rabbitmq-incluster/rmconfig.sh b/cicd/k3s-rabbitmq-incluster/rmconfig.sh new file mode 100755 index 000000000..096e37b75 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmconfig.sh @@ -0,0 +1,4 @@ +#!/bin/bash +vagrant destroy -f worker1 +vagrant destroy -f master +vagrant destroy -f host diff --git a/cicd/k3s-rabbitmq-incluster/rmq/client-service.yaml b/cicd/k3s-rabbitmq-incluster/rmq/client-service.yaml new file mode 100644 index 000000000..c3dfc6bc7 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/client-service.yaml @@ -0,0 +1,24 @@ +kind: Service +apiVersion: v1 +metadata: + namespace: test-rabbitmq + name: rabbitmq-client + labels: + app: rabbitmq + type: LoadBalancer +spec: + type: LoadBalancer + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + ports: + - name: http + protocol: TCP + port: 15672 + - name: prometheus + protocol: TCP + port: 15692 + - name: amqp + protocol: TCP + port: 5672 + selector: + app: rabbitmq diff --git a/cicd/k3s-rabbitmq-incluster/rmq/configmap.yaml b/cicd/k3s-rabbitmq-incluster/rmq/configmap.yaml new file mode 100644 index 000000000..5360b145b --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: rabbitmq-config + namespace: test-rabbitmq +data: + enabled_plugins: | + [rabbitmq_peer_discovery_k8s, rabbitmq_management, rabbitmq_prometheus]. + rabbitmq.conf: | + cluster_formation.peer_discovery_backend = k8s + cluster_formation.k8s.host = kubernetes.default.svc.cluster.local + cluster_formation.k8s.address_type = hostname + cluster_formation.k8s.service_name = rabbitmq-headless + + queue_master_locator=min-masters diff --git a/cicd/k3s-rabbitmq-incluster/rmq/headless-service.yaml b/cicd/k3s-rabbitmq-incluster/rmq/headless-service.yaml new file mode 100644 index 000000000..37b1222df --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/headless-service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: rabbitmq-headless + namespace: test-rabbitmq +spec: + clusterIP: None + ports: + - name: epmd + port: 4369 + protocol: TCP + targetPort: 4369 + - name: cluster-rpc + port: 25672 + protocol: TCP + targetPort: 25672 + selector: + app: rabbitmq + type: ClusterIP + sessionAffinity: None diff --git a/cicd/k3s-rabbitmq-incluster/rmq/namespace.yaml b/cicd/k3s-rabbitmq-incluster/rmq/namespace.yaml new file mode 100644 index 000000000..e2ef8f8bc --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/namespace.yaml @@ -0,0 +1,6 @@ +## All resources will be created in this namespace +## To delete all resources created by this example, simply delete this namespace: +apiVersion: v1 +kind: Namespace +metadata: + name: test-rabbitmq diff --git a/cicd/k3s-rabbitmq-incluster/rmq/rabbitmq.yaml b/cicd/k3s-rabbitmq-incluster/rmq/rabbitmq.yaml new file mode 100644 index 000000000..4e84bb508 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/rabbitmq.yaml @@ -0,0 +1,24 @@ +apiVersion: rabbitmq.com/v1beta1 +kind: RabbitmqCluster +metadata: + name: hello-world +spec: + replicas: 3 + service: + type: LoadBalancer + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: DoesNotExist + - key: "node-role.kubernetes.io/control-plane" + operator: DoesNotExist + override: + service: + spec: + loadBalancerClass: loxilb.io/loxilb + externalTrafficPolicy: Local + ports: + - port: 5672 diff --git a/cicd/k3s-rabbitmq-incluster/rmq/rbac.yaml b/cicd/k3s-rabbitmq-incluster/rmq/rbac.yaml new file mode 100644 index 000000000..6403f39c7 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/rbac.yaml @@ -0,0 +1,33 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rabbitmq + namespace: test-rabbitmq +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rabbitmq + namespace: test-rabbitmq +rules: +- apiGroups: [""] + resources: ["endpoints"] + verbs: ["get"] +- apiGroups: [""] + resources: ["events"] + verbs: ["create"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rabbitmq + namespace: test-rabbitmq +subjects: +- kind: ServiceAccount + name: rabbitmq + namespace: test-rabbitmq +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rabbitmq diff --git a/cicd/k3s-rabbitmq-incluster/rmq/statefulset.yaml b/cicd/k3s-rabbitmq-incluster/rmq/statefulset.yaml new file mode 100644 index 000000000..6422406ed --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/rmq/statefulset.yaml @@ -0,0 +1,142 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: rabbitmq + namespace: test-rabbitmq +spec: + selector: + matchLabels: + app: "rabbitmq" + # headless service that gives network identity to the RMQ nodes, and enables them to cluster + serviceName: rabbitmq-headless # serviceName is the name of the service that governs this StatefulSet. This service must exist before the StatefulSet, and is responsible for the network identity of the set. Pods get DNS/hostnames that follow the pattern: pod-specific-string.serviceName.default.svc.cluster.local where "pod-specific-string" is managed by the StatefulSet controller. + volumeClaimTemplates: + - metadata: + name: rabbitmq-data + namespace: test-rabbitmq + spec: + storageClassName: standard + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "3Gi" + template: + metadata: + name: rabbitmq + namespace: test-rabbitmq + labels: + app: rabbitmq + spec: + initContainers: + # Since k8s 1.9.4, config maps mount read-only volumes. Since the Docker image also writes to the config file, + # the file must be mounted as read-write. We use init containers to copy from the config map read-only + # path, to a read-write path + - name: "rabbitmq-config" + image: busybox:1.32.0 + volumeMounts: + - name: rabbitmq-config + mountPath: /tmp/rabbitmq + - name: rabbitmq-config-rw + mountPath: /etc/rabbitmq + command: + - sh + - -c + # the newline is needed since the Docker image entrypoint scripts appends to the config file + - cp /tmp/rabbitmq/rabbitmq.conf /etc/rabbitmq/rabbitmq.conf && echo '' >> /etc/rabbitmq/rabbitmq.conf; + cp /tmp/rabbitmq/enabled_plugins /etc/rabbitmq/enabled_plugins + volumes: + - name: rabbitmq-config + configMap: + name: rabbitmq-config + optional: false + items: + - key: enabled_plugins + path: "enabled_plugins" + - key: rabbitmq.conf + path: "rabbitmq.conf" + # read-write volume into which to copy the rabbitmq.conf and enabled_plugins files + # this is needed since the docker image writes to the rabbitmq.conf file + # and Kubernetes Config Maps are mounted as read-only since Kubernetes 1.9.4 + - name: rabbitmq-config-rw + emptyDir: {} + - name: rabbitmq-data + persistentVolumeClaim: + claimName: rabbitmq-data + serviceAccount: rabbitmq + # The Docker image runs as the `rabbitmq` user with uid 999 + # and writes to the `rabbitmq.conf` file + # The security context is needed since the image needs + # permission to write to this file. Without the security + # context, `rabbitmq.conf` is owned by root and inaccessible + # by the `rabbitmq` user + securityContext: + fsGroup: 999 + runAsUser: 999 + runAsGroup: 999 + containers: + - name: rabbitmq + # Community Docker Image + image: rabbitmq:latest + volumeMounts: + # mounting rabbitmq.conf and enabled_plugins + # this should have writeable access, this might be a problem + - name: rabbitmq-config-rw + mountPath: "/etc/rabbitmq" + # mountPath: "/etc/rabbitmq/conf.d/" + # rabbitmq data directory + - name: rabbitmq-data + mountPath: "/var/lib/rabbitmq/mnesia" + env: + - name: RABBITMQ_DEFAULT_PASS + valueFrom: + secretKeyRef: + name: rabbitmq-admin + key: pass + - name: RABBITMQ_DEFAULT_USER + valueFrom: + secretKeyRef: + name: rabbitmq-admin + key: user + - name: RABBITMQ_ERLANG_COOKIE + valueFrom: + secretKeyRef: + name: erlang-cookie + key: cookie + ports: + - name: amqp + containerPort: 5672 + protocol: TCP + - name: management + containerPort: 15672 + protocol: TCP + - name: prometheus + containerPort: 15692 + protocol: TCP + - name: epmd + containerPort: 4369 + protocol: TCP + livenessProbe: + exec: + # This is just an example. There is no "one true health check" but rather + # several rabbitmq-diagnostics commands that can be combined to form increasingly comprehensive + # and intrusive health checks. + # Learn more at https://www.rabbitmq.com/monitoring.html#health-checks. + # + # Stage 2 check: + command: ["rabbitmq-diagnostics", "status"] + initialDelaySeconds: 60 + # See https://www.rabbitmq.com/monitoring.html for monitoring frequency recommendations. + periodSeconds: 60 + timeoutSeconds: 15 + readinessProbe: # probe to know when RMQ is ready to accept traffic + exec: + # This is just an example. There is no "one true health check" but rather + # several rabbitmq-diagnostics commands that can be combined to form increasingly comprehensive + # and intrusive health checks. + # Learn more at https://www.rabbitmq.com/monitoring.html#health-checks. + # + # Stage 1 check: + command: ["rabbitmq-diagnostics", "ping"] + initialDelaySeconds: 20 + periodSeconds: 60 + timeoutSeconds: 10 diff --git a/cicd/k3s-rabbitmq-incluster/validation.sh b/cicd/k3s-rabbitmq-incluster/validation.sh new file mode 100755 index 000000000..c96f4ff18 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/validation.sh @@ -0,0 +1,58 @@ +#!/bin/bash +source ../common.sh +echo k3s-flannel-cluster + +if [ "$1" ]; then + KUBECONFIG="$1" +fi + +# Set space as the delimiter +IFS=' ' + +sleep 45 +extIP="123.123.123.1" +echo $extIP + +echo "Service Info" +vagrant ssh master -c 'sudo kubectl get svc' +echo "LB Info" +vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get lb -o wide' +echo "EP Info" +vagrant ssh loxilb -c 'sudo docker exec -i loxilb loxicmd get ep -o wide' + +print_debug_info() { + echo "llb1 route-info" + vagrant ssh loxilb -c 'ip route' + vagrant ssh master -c 'sudo kubectl get pods -A' + vagrant ssh master -c 'sudo kubectl get svc' + vagrant ssh master -c 'sudo kubectl get nodes' +} + +out=$(curl -s --connect-timeout 10 http://$extIP:55002) +if [[ ${out} == *"Welcome to nginx"* ]]; then + echo "k3s-flannel-cluster (kube-loxilb) tcp [OK]" +else + echo "k3s-flannel-cluster (kube-loxilb) tcp [FAILED]" + print_debug_info + exit 1 +fi + +out=$(timeout 10 ../common/udp_client $extIP 55003) +if [[ ${out} == *"Client"* ]]; then + echo "k3s-flannel-cluster (kube-loxilb) udp [OK]" +else + echo "k3s-flannel-cluster (kube-loxilb) udp [FAILED]" + print_debug_info + exit 1 +fi + +out=$(timeout 10 ../common/sctp_client 192.168.90.1 41291 $extIP 55004) +if [[ ${out} == *"server1"* ]]; then + echo "k3s-flannel-cluster (kube-loxilb) sctp [OK]" +else + echo "k3s-flannel-cluster (kube-loxilb) sctp [FAILED]" + print_debug_info + exit 1 +fi + +exit diff --git a/cicd/k3s-rabbitmq-incluster/wait_ready.sh b/cicd/k3s-rabbitmq-incluster/wait_ready.sh new file mode 100755 index 000000000..b0e69d01e --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/wait_ready.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +function wait_cluster_ready { + Res=$(sudo kubectl get pods -A | + while IFS= read -r line; do + if [[ "$line" != *"Running"* && "$line" != *"READY"* ]]; then + echo "$line: not ready" + return + fi + done) + if [[ $Res == *"not ready"* ]]; then + echo $Res + return 1 + fi + return 0 +} + +function wait_cluster_ready_full { + i=1 + nr=0 + for ((;;)) do + wait_cluster_ready + nr=$? + if [[ $nr == 0 ]]; then + echo "Cluster is ready" + break + fi + i=$(( $i + 1 )) + if [[ $i -ge 40 ]]; then + echo "Cluster is not ready.Giving up" + exit 1 + fi + echo "Cluster is not ready...." + sleep 10 + done +} + +wait_cluster_ready_full diff --git a/cicd/k3s-rabbitmq-incluster/worker.sh b/cicd/k3s-rabbitmq-incluster/worker.sh new file mode 100644 index 000000000..5da2836b4 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/worker.sh @@ -0,0 +1,13 @@ +export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') +export MASTER_ADDR=$(cat /vagrant/master-ip) +export NODE_TOKEN=$(cat /vagrant/node-token) + +sudo mkdir -p /etc/rancher/k3s +sudo cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml + +curl -sfL https://get.k3s.io | K3S_URL="https://${MASTER_ADDR}:6443" K3S_TOKEN="${NODE_TOKEN}" INSTALL_K3S_EXEC="--node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} --kube-proxy-arg proxy-mode=ipvs --flannel-iface=eth1" sh - +#sudo kubectl apply -f /vagrant/nginx.yml +#sudo kubectl apply -f /vagrant/udp.yml +#sudo kubectl apply -f /vagrant/iperf-service.yml +#sudo kubectl apply -f /vagrant/loxilb.yml +/vagrant/wait_ready.sh diff --git a/cicd/k3s-rabbitmq-incluster/worker_with_cilium.sh b/cicd/k3s-rabbitmq-incluster/worker_with_cilium.sh new file mode 100644 index 000000000..0e7be7c64 --- /dev/null +++ b/cicd/k3s-rabbitmq-incluster/worker_with_cilium.sh @@ -0,0 +1,14 @@ +export WORKER_ADDR=$(ip a |grep global | grep -v '10.0.2.15' | grep '192.168.80' | awk '{print $2}' | cut -f1 -d '/') +export MASTER_ADDR=$(cat /vagrant/master-ip) +export NODE_TOKEN=$(cat /vagrant/node-token) + +sudo mkdir -p /etc/rancher/k3s +sudo cp -f /vagrant/k3s.yaml /etc/rancher/k3s/k3s.yaml +curl -sfL https://get.k3s.io | K3S_URL="https://${MASTER_ADDR}:6443" K3S_TOKEN="${NODE_TOKEN}" INSTALL_K3S_EXEC="--node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR}" sh - + +#curl -sfL https://get.k3s.io | K3S_URL="https://${MASTER_ADDR}:6443" K3S_TOKEN="${NODE_TOKEN}" INSTALL_K3S_EXEC="--node-ip=${WORKER_ADDR} --node-external-ip=${WORKER_ADDR} --kube-proxy-arg proxy-mode=ipvs --flannel-iface=eth1" sh - +#sudo kubectl apply -f /vagrant/nginx.yml +#sudo kubectl apply -f /vagrant/udp.yml +#sudo kubectl apply -f /vagrant/iperf-service.yml +#sudo kubectl apply -f /vagrant/loxilb.yml +/vagrant/wait_ready.sh