Skip to content

Commit

Permalink
loxilb-iogh-87 : sctpmh with sctp_test test suite cicd updated
Browse files Browse the repository at this point in the history
  • Loading branch information
nik-netlox committed Sep 10, 2024
1 parent 83a1e61 commit 3c905f5
Show file tree
Hide file tree
Showing 9 changed files with 363 additions and 135 deletions.
165 changes: 117 additions & 48 deletions cicd/sctpmh/check_ha.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,62 +4,131 @@ master="llb1"
backup="llb2"

function check_ha() {
while : ; do
status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state')
status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state')
count=0
if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]];
then
master="llb1"
backup="llb2"
break
elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]];
then
master="llb2"
backup="llb1"
break
else
count=$(( $count + 1 ))
if [[ $count -ge 20 ]]; then
while : ; do
status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state')
status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state')
count=0
if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]];
then
master="llb1"
backup="llb2"
break
elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]];
then
master="llb2"
backup="llb1"
break
else
count=$(( $count + 1 ))
if [[ $count -ge 20 ]]; then
echo "KA llb1-$status1, llb2-$status2 [NOK] - Exiting" >&2
exit 1;
fi
echo "KA llb1-$status1, llb2-$status2 [NOK]" >&2
exit 1;
sleep 5
fi
sleep 5
fi
done
done
}

function checkSync() {
count=1
sync=0
while [[ $count -le 5 ]] ; do
echo -e "\nStatus at MASTER:$master\n" >&2
$dexec $master loxicmd get ct | grep est >&2
count=1
sync=0
while [[ $count -le 5 ]] ; do
echo -e "\nStatus at MASTER:$master\n" >&2
ct=`$dexec $master loxicmd get ct | grep est`
echo "${ct//'\n'/$'\n'}" >&2

echo -e "\nStatus at BACKUP:$backup\n" >&2
$dexec $backup loxicmd get ct | grep est >&2
echo -e "\nStatus at BACKUP:$backup\n" >&2
ct=`$dexec $backup loxicmd get ct | grep est`
echo "${ct//'\n'/$'\n'}" >&2

nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)
nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l)

if [[ $nres1 == 0 ]]; then
echo -e "No active connections in Master:$master. Exiting!" >&2
return 2
fi
if [[ $nres1 == 0 ]]; then
echo -e "No active connections in Master:$master. Exiting!" >&2
return 2
fi

if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then
echo -e "\nConnections sync successful!!!\n" >&2
sync=1
break;
fi
echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2
count=$(( $count + 1 ))
sleep 2
done
if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then
echo -e "\nConnections sync successful!!!\n" >&2
sync=1
break;
fi
echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2
count=$(( $count + 1 ))
sleep 2
done

if [[ $sync == 0 ]]; then
echo -e "\nConnection Sync failed\n" >&2
return 0
fi
echo "$sync"
if [[ $sync == 0 ]]; then
echo -e "\nConnection Sync failed\n" >&2
return 0
fi
echo "$sync"
}

function restart_mloxilb() {
if [[ $master == "llb1" ]]; then
pat="cluster=172.17.0.3"
copts=" --cluster=172.17.0.3"
self=" --self=0"
ka=" --ka=172.17.0.3:172.17.0.2"
else
pat="cluster=172.17.0.2"
copts=" --cluster=172.17.0.2"
self=" --self=1"
ka=" --ka=172.17.0.2:172.17.0.3"
fi
echo "Restarting MASTER: $master"
pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2)
echo "Killing $pid" >&2
docker exec -dt $master kill -9 $pid
docker exec -dt $master ip link del llb0
docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null &
pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2)
echo "New loxilb pid: $pid" >&2
}

function restart_loxilbs() {
if [[ $master == "llb1" ]]; then
mpat="cluster=172.17.0.3"
mcopts=" --cluster=172.17.0.3"
mself=" --self=0"
mka=" --ka=172.17.0.3:172.17.0.2"

bpat="cluster=172.17.0.2"
bcopts=" --cluster=172.17.0.2"
bself=" --self=1"
bka=" --ka=172.17.0.2:172.17.0.3"
else
mpat="cluster=172.17.0.2"
mcopts=" --cluster=172.17.0.2"
mself=" --self=1"
mka=" --ka=172.17.0.2:172.17.0.3"

bpat="cluster=172.17.0.3"
bcopts=" --cluster=172.17.0.3"
bself=" --self=0"
bka=" --ka=172.17.0.3:172.17.0.2"
fi
echo "Restarting $master"
pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2)
echo "Killing $mpid" >&2
docker exec -dt $master kill -9 $pid
docker exec -dt $master ip link del llb0
docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $mcopts $mself $mka > /dev/null &
pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2)
echo "New loxilb pid: $pid" >&2

echo "Restarting $backup"
pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2)
echo "Killing $pid" >&2
docker exec -dt $backup kill -9 $pid
docker exec -dt $backup ip link del llb0
docker exec -dt $backup nohup /root/loxilb-io/loxilb/loxilb $bcopts $bself $bka > /dev/null &
pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2)
echo "New loxilb pid: $pid" >&2

}


43 changes: 22 additions & 21 deletions cicd/sctpmh/config.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,18 @@ echo "#########################################"
echo "Connecting and configuring hosts"
echo "#########################################"

connect_docker_hosts user r1
connect_docker_hosts user r2
connect_docker_hosts r1 sw1
connect_docker_hosts r2 sw1
connect_docker_hosts sw1 llb1
connect_docker_hosts sw1 llb2
connect_docker_hosts llb1 sw2
connect_docker_hosts llb2 sw2
connect_docker_hosts sw2 r3
connect_docker_hosts sw2 r4
connect_docker_hosts r3 ep1
connect_docker_hosts r4 ep1
connect_docker_hosts user r1 1500
connect_docker_hosts user r2 1500
connect_docker_hosts r1 sw1 1500
connect_docker_hosts r2 sw1 1500
connect_docker_hosts sw1 llb1 1500
connect_docker_hosts sw1 llb2 1500
connect_docker_hosts llb1 sw2 1500
connect_docker_hosts llb2 sw2 1500
connect_docker_hosts sw2 r3 1500
connect_docker_hosts sw2 r4 1500
connect_docker_hosts r3 ep1 1500
connect_docker_hosts r4 ep1 1500

create_docker_host_cnbridge --host1 sw1 --host2 llb1
create_docker_host_cnbridge --host1 sw1 --host2 llb2
Expand Down Expand Up @@ -79,14 +79,15 @@ $hexec ep1 ip route change default via 31.31.31.254

# Backup paths in user
$hexec user ip route add 124.124.124.1/32 via 2.2.2.254
$hexec user ip route add 125.125.125.1/32 via 2.2.2.254
#$hexec user ip route add 125.125.125.1/32 via 2.2.2.254
#$hexec user ip route add 32.32.32.1/32 via 2.2.2.254
$hexec user ip route add 134.134.134.1/32 via 2.2.2.254
$hexec user ip route add 135.135.135.1/32 via 2.2.2.254
#$hexec user ip route add 135.135.135.1/32 via 2.2.2.254

$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254
$hexec ep1 ip route add 125.125.125.1/32 via 32.32.32.254
#$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254
#$hexec ep1 ip route add 125.125.125.1/32 via 31.31.31.254
$hexec ep1 ip route add 134.134.134.1/32 via 32.32.32.254
$hexec ep1 ip route add 135.135.135.1/32 via 32.32.32.254
$hexec ep1 ip route add 135.135.135.1/32 via 31.31.31.254

$hexec llb1 ip route add 1.1.1.0/24 via 11.11.11.253
$hexec llb1 ip route add 2.2.2.0/24 via 11.11.11.254
Expand All @@ -107,10 +108,10 @@ create_lb_rule llb2 123.123.123.1 --name=sctpmh1 --secips=124.124.124.1,125.125.
create_lb_rule llb1 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat
create_lb_rule llb2 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat

$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none
$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none
$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none
$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none
$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping
$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping
$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping
$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping


create_lb_rule llb1 11.11.11.11 --tcp=80:8080 --endpoints=31.31.31.1:1
Expand Down
83 changes: 49 additions & 34 deletions cicd/sctpmh/validation.sh
Original file line number Diff line number Diff line change
@@ -1,40 +1,55 @@
#!/bin/bash
code=0
echo "SCTP Multihoming - Test case #1"
echo -e "*********************************************************************************"
./validation1.sh
if [[ $? == 1 ]]; then
code=1
fi
echo -e "\n\n\nSCTP Multihoming - Test case #2"
echo -e "*********************************************************************************"
./validation2.sh
if [[ $? == 1 ]]; then
code=1
fi
echo -e "\n\n\nSCTP Multihoming - Test case #3"
echo -e "*********************************************************************************"
./validation3.sh
if [[ $? == 1 ]]; then
code=1
fi
echo -e "\n\n\nSCTP Multihoming - Test case #4"
echo -e "*********************************************************************************"
./validation4.sh
if [[ $? == 1 ]]; then
code=1
fi
echo -e "\n\n\nSCTP Multihoming - Test case #5"
echo -e "*********************************************************************************"
sleep 60
./validation5.sh
if [[ $? == 1 ]]; then
code=1
fi
echo -e "\n\n\n*********************************************************************************"
tc=( "Basic Test - Client & EP Uni-homed and LB is Multi-homed" "Multipath Test, Client and LB Multihomed, EP is uni-homed" "C2LB Multipath Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E Multipath Failover Test - Client, LB and EP all Multihomed" "C2LB HA Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E HA Failover Test. Client, LB and EP all Multihomed" )
padding="............................................................................................................."
border="**************************************************************************************************************************************************"

for((j=0,i=1; i<=6; i++, j++)); do
echo "SCTP Multihoming - Test case #$i"
echo -e "\n\n\n$border\n"
./validation$i.sh
echo -e "\n\n"
file=status$i.txt
status=`cat $file`
title=${tc[j]}
echo -e "\n\n"

if [[ $status == "NOK" ]]; then
code=1
printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]";
else
printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]";
fi
echo -e "\n\n\n$border\n\n"

sleep 30
done

echo -e "\n\n\n$border\n"
printf "================================================== SCTP MULTIHOMING CONSOLIDATED RESULT ==========================================================\n"
for((j=0,i=1; i<=6; i++, j++)); do
file=status$i.txt
status=`cat $file`
title=${tc[j]}
echo -e "\n\n"

if [[ $status == "NOK" ]]; then
code=1
printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]";
else
printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]";
fi
done

echo -e "\n$border"

echo -e "\n\n\n$border\n"
if [[ $code == 0 ]]; then
echo -e "\n\n SCTP Multihoming CICD [OK]"
echo -e "SCTP Multihoming with sctp_test CICD [OK]"
else
echo -e "\n\n SCTP Multihoming CICD [NOK]"
echo -e "SCTP Multihoming with sctp_test CICD [NOK]"
fi
echo -e "\n$border\n"

sudo rm -rf statu*.txt
exit $code
9 changes: 8 additions & 1 deletion cicd/sctpmh/validation1.sh
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
#!/bin/bash
source ../common.sh
source check_ha.sh

echo -e "sctpmh: SCTP Multihoming Basic Test - Client & EP Uni-homed and LB is Multi-homed\n"
extIP="123.123.123.1"
port=2020

check_ha

echo "SCTP Multihoming service sctp-lb -> $extIP:$port"
echo -e "------------------------------------------------------------------------------------\n"

Expand All @@ -26,7 +29,10 @@ sudo pkill sctp_darn
if [[ "$res" == "$exp" ]]; then
echo $res
echo -e "\nsctpmh SCTP Multihoming service Basic Test [OK]\n"
echo "OK" > status1.txt
restart_loxilbs
else
echo "NOK" > status1.txt
echo "sctpmh SCTP Multihoming service Basic Test [NOK]"
echo "Expected : $exp"
echo "Received : $res"
Expand All @@ -51,11 +57,12 @@ else
echo "llb1 ep-info"
$dexec llb1 loxicmd get ep
echo "llb1 bpf-info"
$dexec llb1 ntc filter show dev eth0 ingress
$dexec llb1 tc filter show dev eth0 ingress
echo "BFP trace -- "
sudo timeout 5 cat /sys/kernel/debug/tracing/trace_pipe
sudo killall -9 cat
echo "BFP trace -- "
restart_loxilbs
exit 1
fi
echo -e "------------------------------------------------------------------------------------\n\n\n"
Loading

0 comments on commit 3c905f5

Please sign in to comment.