Merge pull request #519 from loxilb-io/bfd-fix1

PR : BFD Implementation - changes for cicd scripts
loxilb-io · Feb 7, 2024 · ba2d708 · ba2d708
2 parents 97f4e41 + 6ec3366
commit ba2d708
Show file tree

Hide file tree

Showing 9 changed files with 74 additions and 36 deletions.
diff --git a/cicd/common.sh b/cicd/common.sh
@@ -13,6 +13,7 @@ hns="sudo ip netns "
 hexist="$vrn$hn"
 lxdocker="ghcr.io/loxilb-io/loxilb:latest"
 cluster_opts=""
+ka_opts=""
 var=$(lsb_release -r | cut -f2)
 if [[ $var == *"22.04"* ]];then
   lxdocker="ghcr.io/loxilb-io/loxilb:latestu22"
@@ -104,24 +105,9 @@ spawn_docker_host() {
     fi
     if [[ ! -z ${ka+x} ]]; then
       sudo mkdir -p /etc/shared/$dname/
-      if [[ "$ka" == "in" ]];then
-        ka_opts="-k in"
-        if [[ ! -z "$kpath" ]]; then
-            ka_conf="-v $kpath:/etc/keepalived/" 
-        fi
-      fi
-      docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config $ka_conf --name $dname $lxdocker
+      docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config --name $dname $lxdocker
       get_llb_peerIP $dname
       docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts
-
-      if [[ "$ka" == "out" ]];then
-        ka_opts="-k out"
-        if [[ ! -z "$kpath" ]]; then
-            ka_conf="-v $kpath:/container/service/keepalived/assets/" 
-        fi
-
-        docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dit --network=container:$dname $ka_conf -v /etc/shared/$dname:/etc/shared --name ka_$dname osixia/keepalived:2.0.20
-      fi
     else
       docker run -u root --cap-add SYS_ADMIN   --restart unless-stopped --privileged -dt --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts
       docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts
@@ -170,6 +156,7 @@ get_llb_peerIP() {
         llb2IP="$A.$B.$C.$((D+1))"
       fi
       cluster_opts=" --cluster=$llb2IP --self=0"
+      ka_opts=" --ka=$llb2IP"
     elif [[ "$1" == "llb2" ]]; then
       llb2IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' llb2)
       if [[ "lb$llb2IP" == "lb" ]];then
@@ -179,6 +166,7 @@ get_llb_peerIP() {
         llb1IP="$A.$B.$C.$((D-1))"
       fi
       cluster_opts=" --cluster=$llb1IP --self=1"
+      ka_opts=" --ka=$llb1IP"
     fi
 }
 

diff --git a/cicd/k3s-calico/config.sh b/cicd/k3s-calico/config.sh
@@ -159,10 +159,6 @@ sleep 10
 # Wait for cluster to be ready
 wait_cluster_ready_full
 
-# Start nginx pods and services for test
-kubectl $KUBECONFIG apply -f nginx.yml
-kubectl $KUBECONFIG apply -f nginx-svc-lb.yml
-
 sleep 5 
 
 # Start nginx pods and services for test(using kube-loxilb)
@@ -179,7 +175,7 @@ sleep 30
 kubectl $KUBECONFIG get svc
 
 # Route back to user
-sudo ip route add 1.1.1.1/32 via 12.12.12.1
+sudo ip route add 1.1.1.1/32 via 14.14.14.1
 
 # Wait for cluster to be ready
 wait_cluster_ready_full
diff --git a/cicd/k3s-calico/nginx-svc-lb1.yml b/cicd/k3s-calico/nginx-svc-lb1.yml
@@ -2,6 +2,8 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginx-lb1
+  annotations:
+    loxilb.io/staticIP : "11.11.11.11"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb

diff --git a/cicd/k3s-calico/validation.sh b/cicd/k3s-calico/validation.sh
@@ -11,7 +11,7 @@ IFS=' '
 
 for((i=0; i<120; i++))
 do
-  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb")
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1")
   read -a strarr <<< "$extLB"
   len=${#strarr[*]}
   if [[ $((len)) -lt 6 ]]; then
@@ -60,14 +60,6 @@ print_debug_info() {
 code=0
 print_debug_info
 
-out=$($hexec user curl -s --connect-timeout 10 http://$extIP:80) 
-if [[ ${out} == *"Welcome to nginx"* ]]; then
-  echo "calico-k3s (ccm) [OK]"
-else
-  echo "calico-k3s (ccm) [FAILED]"
-  code=1
-fi
-
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002) 
 
 if [[ ${out} == *"Welcome to nginx"* ]]; then
@@ -77,6 +69,24 @@ else
   code=1
 fi
 
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "udp-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find nginx-lb service"
+    sleep 1
+    continue
+  fi 
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
 out=$($hexec user timeout 30 ../common/udp_client $extIP 55003)
 if [[ ${out} == *"Client"* ]]; then
   echo "calico-k3s (kube-loxillb) udp [OK]"
@@ -85,6 +95,24 @@ else
   code=1
 fi
 
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "sctp-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find nginx-lb service"
+    sleep 1
+    continue
+  fi
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
 out=$($hexec user timeout 30 ../common/sctp_socat_client 1.1.1.1 41291 $extIP 55004)
 if [[ ${out} == *"server1"* ]]; then
   echo "calico-k3s (kube-loxillb) sctp [OK]"

diff --git a/cicd/k3s-sctpmh/config.sh b/cicd/k3s-sctpmh/config.sh
@@ -7,8 +7,8 @@ echo "#########################################"
 echo "Spawning all hosts"
 echo "#########################################"
 
-spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config1
-spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in --ka-config $(pwd)/keepalived_config2
+spawn_docker_host --dock-type loxilb --dock-name llb1 --with-bgp yes --bgp-config $(pwd)/llb1_gobgp_config --with-ka in
+spawn_docker_host --dock-type loxilb --dock-name llb2 --with-bgp yes --bgp-config $(pwd)/llb2_gobgp_config --with-ka in
 spawn_docker_host --dock-type host --dock-name ep1
 spawn_docker_host --dock-type host --dock-name ep2
 spawn_docker_host --dock-type host --dock-name ep3
@@ -99,7 +99,8 @@ $hexec r1 ip route add 20.20.20.1/32 via 11.11.11.11
 #add_route llb2 1.1.1.0/24 11.11.11.254
 
 # Route back to user
-sudo ip route add 11.11.11.0/24 via 12.12.12.1
+sudo ip route add 11.11.11.0/24 via 14.14.14.1
+sudo ip route add 123.123.123.0/24 via 14.14.14.1
 
 # Change default route in llb1
 $hexec llb1 ip route del default 
@@ -195,11 +196,9 @@ $dexec llb1 loxicmd get lb -o wide
 echo "llb1: loxicmd get ep -o wide"
 echo "****************************"
 $dexec llb1 loxicmd get ep -o wide
-$dexec llb1 cat /etc/shared/keepalive.state
 echo "llb2: loxicmd get lb -o wide"
 echo "****************************"
 $dexec llb2 loxicmd get lb -o wide
 echo "llb2: loxicmd get ep -o wide"
 echo "****************************"
 $dexec llb2 loxicmd get ep -o wide
-$dexec llb2 cat /etc/shared/keepalive.state
diff --git a/cicd/k3s-sctpmh/nginx-svc-lb1.yml b/cicd/k3s-sctpmh/nginx-svc-lb1.yml
@@ -2,6 +2,9 @@ apiVersion: v1
 kind: Service
 metadata:
   name: nginx-lb1
+  annotations:
+    loxilb.io/lbmode: "fullnat"
+    loxilb.io/staticIP : "11.11.11.11"
 spec:
   externalTrafficPolicy: Local
   loadBalancerClass: loxilb.io/loxilb

diff --git a/cicd/k3s-sctpmh/nginx.yml b/cicd/k3s-sctpmh/nginx.yml
@@ -2,6 +2,9 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: nginx
+  annotations:
+    loxilb.io/lbmode: "fullnat"
+    loxilb.io/staticIP : "11.11.11.11"
   labels:
     app.kubernetes.io/name: proxy
 spec:

diff --git a/cicd/k3s-sctpmh/validation.sh b/cicd/k3s-sctpmh/validation.sh
@@ -125,6 +125,25 @@ sleep 10
 # Wait for cluster to be ready
 wait_cluster_ready_full
 
+for((i=0; i<120; i++))
+do
+  extLB=$(sudo kubectl $KUBECONFIG get svc | grep "nginx-lb1")
+  read -a strarr <<< "$extLB"
+  len=${#strarr[*]}
+  if [[ $((len)) -lt 6 ]]; then
+    echo "Can't find nginx-lb service"
+    sleep 1
+    continue
+  fi
+  if [[ ${strarr[3]} != *"none"* ]]; then
+    extIP="$(cut -d'-' -f2 <<<${strarr[3]})"
+    port=${strarr[4]}
+    break
+  fi
+  echo "No external LB allocated"
+  sleep 1
+done
+
 out=$($hexec user curl -s --connect-timeout 10 http://$extIP:55002)
 if [[ ${out} == *"Welcome to nginx"* ]]; then
   echo "cluster-k3s TCP service nginx-lb del+add (kube-loxilb) [OK]"

diff --git a/loxinet/cluster.go b/loxinet/cluster.go
@@ -90,7 +90,7 @@ func (ci *CIStateH) startBFDProto() {
 	time.Sleep(KAInitTiVal * time.Second)
 
 	bs := bfd.StructNew(3784)
-	err := bs.BFDAddRemote(ci.RemoteIP.String(), 3784, bfd.BFDMinSysTXIntervalUs, 3, "Default", ci)
+	err := bs.BFDAddRemote(ci.RemoteIP.String(), 3784, bfd.BFDMinSysTXIntervalUs, 3, cmn.CIDefault, ci)
 	if err != nil {
 		tk.LogIt(tk.LogCritical, "KA - Cant add BFD remote\n")
 	}