From 3c905f51da1560006f7c1cd69923653ba2a20c0d Mon Sep 17 00:00:00 2001 From: Nikhil Malik Date: Tue, 10 Sep 2024 17:39:42 +0900 Subject: [PATCH] gh-87 : sctpmh with sctp_test test suite cicd updated --- cicd/sctpmh/check_ha.sh | 165 ++++++++++++++++++++++++++----------- cicd/sctpmh/config.sh | 43 +++++----- cicd/sctpmh/validation.sh | 83 +++++++++++-------- cicd/sctpmh/validation1.sh | 9 +- cicd/sctpmh/validation2.sh | 6 +- cicd/sctpmh/validation3.sh | 6 +- cicd/sctpmh/validation4.sh | 8 +- cicd/sctpmh/validation5.sh | 38 +++------ cicd/sctpmh/validation6.sh | 140 +++++++++++++++++++++++++++++++ 9 files changed, 363 insertions(+), 135 deletions(-) create mode 100755 cicd/sctpmh/validation6.sh diff --git a/cicd/sctpmh/check_ha.sh b/cicd/sctpmh/check_ha.sh index b04922194..b31574fb0 100644 --- a/cicd/sctpmh/check_ha.sh +++ b/cicd/sctpmh/check_ha.sh @@ -4,62 +4,131 @@ master="llb1" backup="llb2" function check_ha() { -while : ; do - status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') - status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') - count=0 - if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]]; - then - master="llb1" - backup="llb2" - break - elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]]; - then - master="llb2" - backup="llb1" - break - else - count=$(( $count + 1 )) - if [[ $count -ge 20 ]]; then + while : ; do + status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') + status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') + count=0 + if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]]; + then + master="llb1" + backup="llb2" + break + elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]]; + then + master="llb2" + backup="llb1" + break + else + count=$(( $count + 1 )) + if [[ $count -ge 20 ]]; then + echo "KA llb1-$status1, llb2-$status2 [NOK] - Exiting" >&2 + exit 1; + fi echo "KA llb1-$status1, llb2-$status2 [NOK]" >&2 - exit 1; + sleep 5 fi - sleep 5 - fi -done + done } function checkSync() { -count=1 -sync=0 -while [[ $count -le 5 ]] ; do -echo -e "\nStatus at MASTER:$master\n" >&2 -$dexec $master loxicmd get ct | grep est >&2 + count=1 + sync=0 + while [[ $count -le 5 ]] ; do + echo -e "\nStatus at MASTER:$master\n" >&2 + ct=`$dexec $master loxicmd get ct | grep est` + echo "${ct//'\n'/$'\n'}" >&2 -echo -e "\nStatus at BACKUP:$backup\n" >&2 -$dexec $backup loxicmd get ct | grep est >&2 + echo -e "\nStatus at BACKUP:$backup\n" >&2 + ct=`$dexec $backup loxicmd get ct | grep est` + echo "${ct//'\n'/$'\n'}" >&2 -nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) -nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) -if [[ $nres1 == 0 ]]; then - echo -e "No active connections in Master:$master. Exiting!" >&2 - return 2 -fi + if [[ $nres1 == 0 ]]; then + echo -e "No active connections in Master:$master. Exiting!" >&2 + return 2 + fi -if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then - echo -e "\nConnections sync successful!!!\n" >&2 - sync=1 - break; -fi -echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2 -count=$(( $count + 1 )) -sleep 2 -done + if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then + echo -e "\nConnections sync successful!!!\n" >&2 + sync=1 + break; + fi + echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2 + count=$(( $count + 1 )) + sleep 2 + done -if [[ $sync == 0 ]]; then - echo -e "\nConnection Sync failed\n" >&2 - return 0 -fi -echo "$sync" + if [[ $sync == 0 ]]; then + echo -e "\nConnection Sync failed\n" >&2 + return 0 + fi + echo "$sync" } + +function restart_mloxilb() { + if [[ $master == "llb1" ]]; then + pat="cluster=172.17.0.3" + copts=" --cluster=172.17.0.3" + self=" --self=0" + ka=" --ka=172.17.0.3:172.17.0.2" + else + pat="cluster=172.17.0.2" + copts=" --cluster=172.17.0.2" + self=" --self=1" + ka=" --ka=172.17.0.2:172.17.0.3" + fi + echo "Restarting MASTER: $master" + pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) + echo "Killing $pid" >&2 + docker exec -dt $master kill -9 $pid + docker exec -dt $master ip link del llb0 + docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null & + pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 +} + +function restart_loxilbs() { + if [[ $master == "llb1" ]]; then + mpat="cluster=172.17.0.3" + mcopts=" --cluster=172.17.0.3" + mself=" --self=0" + mka=" --ka=172.17.0.3:172.17.0.2" + + bpat="cluster=172.17.0.2" + bcopts=" --cluster=172.17.0.2" + bself=" --self=1" + bka=" --ka=172.17.0.2:172.17.0.3" + else + mpat="cluster=172.17.0.2" + mcopts=" --cluster=172.17.0.2" + mself=" --self=1" + mka=" --ka=172.17.0.2:172.17.0.3" + + bpat="cluster=172.17.0.3" + bcopts=" --cluster=172.17.0.3" + bself=" --self=0" + bka=" --ka=172.17.0.3:172.17.0.2" + fi + echo "Restarting $master" + pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2) + echo "Killing $mpid" >&2 + docker exec -dt $master kill -9 $pid + docker exec -dt $master ip link del llb0 + docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $mcopts $mself $mka > /dev/null & + pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 + + echo "Restarting $backup" + pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2) + echo "Killing $pid" >&2 + docker exec -dt $backup kill -9 $pid + docker exec -dt $backup ip link del llb0 + docker exec -dt $backup nohup /root/loxilb-io/loxilb/loxilb $bcopts $bself $bka > /dev/null & + pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 + +} + + diff --git a/cicd/sctpmh/config.sh b/cicd/sctpmh/config.sh index 411d4e5d4..2436deedb 100755 --- a/cicd/sctpmh/config.sh +++ b/cicd/sctpmh/config.sh @@ -21,18 +21,18 @@ echo "#########################################" echo "Connecting and configuring hosts" echo "#########################################" -connect_docker_hosts user r1 -connect_docker_hosts user r2 -connect_docker_hosts r1 sw1 -connect_docker_hosts r2 sw1 -connect_docker_hosts sw1 llb1 -connect_docker_hosts sw1 llb2 -connect_docker_hosts llb1 sw2 -connect_docker_hosts llb2 sw2 -connect_docker_hosts sw2 r3 -connect_docker_hosts sw2 r4 -connect_docker_hosts r3 ep1 -connect_docker_hosts r4 ep1 +connect_docker_hosts user r1 1500 +connect_docker_hosts user r2 1500 +connect_docker_hosts r1 sw1 1500 +connect_docker_hosts r2 sw1 1500 +connect_docker_hosts sw1 llb1 1500 +connect_docker_hosts sw1 llb2 1500 +connect_docker_hosts llb1 sw2 1500 +connect_docker_hosts llb2 sw2 1500 +connect_docker_hosts sw2 r3 1500 +connect_docker_hosts sw2 r4 1500 +connect_docker_hosts r3 ep1 1500 +connect_docker_hosts r4 ep1 1500 create_docker_host_cnbridge --host1 sw1 --host2 llb1 create_docker_host_cnbridge --host1 sw1 --host2 llb2 @@ -79,14 +79,15 @@ $hexec ep1 ip route change default via 31.31.31.254 # Backup paths in user $hexec user ip route add 124.124.124.1/32 via 2.2.2.254 -$hexec user ip route add 125.125.125.1/32 via 2.2.2.254 +#$hexec user ip route add 125.125.125.1/32 via 2.2.2.254 +#$hexec user ip route add 32.32.32.1/32 via 2.2.2.254 $hexec user ip route add 134.134.134.1/32 via 2.2.2.254 -$hexec user ip route add 135.135.135.1/32 via 2.2.2.254 +#$hexec user ip route add 135.135.135.1/32 via 2.2.2.254 -$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254 -$hexec ep1 ip route add 125.125.125.1/32 via 32.32.32.254 +#$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254 +#$hexec ep1 ip route add 125.125.125.1/32 via 31.31.31.254 $hexec ep1 ip route add 134.134.134.1/32 via 32.32.32.254 -$hexec ep1 ip route add 135.135.135.1/32 via 32.32.32.254 +$hexec ep1 ip route add 135.135.135.1/32 via 31.31.31.254 $hexec llb1 ip route add 1.1.1.0/24 via 11.11.11.253 $hexec llb1 ip route add 2.2.2.0/24 via 11.11.11.254 @@ -107,10 +108,10 @@ create_lb_rule llb2 123.123.123.1 --name=sctpmh1 --secips=124.124.124.1,125.125. create_lb_rule llb1 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat create_lb_rule llb2 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat -$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none -$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none -$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none -$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none +$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping +$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping +$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping +$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping create_lb_rule llb1 11.11.11.11 --tcp=80:8080 --endpoints=31.31.31.1:1 diff --git a/cicd/sctpmh/validation.sh b/cicd/sctpmh/validation.sh index 34cd2749d..8a62cfff2 100755 --- a/cicd/sctpmh/validation.sh +++ b/cicd/sctpmh/validation.sh @@ -1,40 +1,55 @@ #!/bin/bash code=0 -echo "SCTP Multihoming - Test case #1" -echo -e "*********************************************************************************" -./validation1.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #2" -echo -e "*********************************************************************************" -./validation2.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #3" -echo -e "*********************************************************************************" -./validation3.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #4" -echo -e "*********************************************************************************" -./validation4.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #5" -echo -e "*********************************************************************************" -sleep 60 -./validation5.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\n*********************************************************************************" +tc=( "Basic Test - Client & EP Uni-homed and LB is Multi-homed" "Multipath Test, Client and LB Multihomed, EP is uni-homed" "C2LB Multipath Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E Multipath Failover Test - Client, LB and EP all Multihomed" "C2LB HA Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E HA Failover Test. Client, LB and EP all Multihomed" ) +padding="............................................................................................................." +border="**************************************************************************************************************************************************" + +for((j=0,i=1; i<=6; i++, j++)); do + echo "SCTP Multihoming - Test case #$i" + echo -e "\n\n\n$border\n" + ./validation$i.sh + echo -e "\n\n" + file=status$i.txt + status=`cat $file` + title=${tc[j]} + echo -e "\n\n" + + if [[ $status == "NOK" ]]; then + code=1 + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]"; + else + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]"; + fi + echo -e "\n\n\n$border\n\n" + + sleep 30 +done + +echo -e "\n\n\n$border\n" +printf "================================================== SCTP MULTIHOMING CONSOLIDATED RESULT ==========================================================\n" +for((j=0,i=1; i<=6; i++, j++)); do + file=status$i.txt + status=`cat $file` + title=${tc[j]} + echo -e "\n\n" + + if [[ $status == "NOK" ]]; then + code=1 + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]"; + else + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]"; + fi +done + +echo -e "\n$border" + +echo -e "\n\n\n$border\n" if [[ $code == 0 ]]; then - echo -e "\n\n SCTP Multihoming CICD [OK]" + echo -e "SCTP Multihoming with sctp_test CICD [OK]" else - echo -e "\n\n SCTP Multihoming CICD [NOK]" + echo -e "SCTP Multihoming with sctp_test CICD [NOK]" fi +echo -e "\n$border\n" + +sudo rm -rf statu*.txt exit $code diff --git a/cicd/sctpmh/validation1.sh b/cicd/sctpmh/validation1.sh index 6043bf114..f41fffce4 100755 --- a/cicd/sctpmh/validation1.sh +++ b/cicd/sctpmh/validation1.sh @@ -1,10 +1,13 @@ #!/bin/bash source ../common.sh +source check_ha.sh echo -e "sctpmh: SCTP Multihoming Basic Test - Client & EP Uni-homed and LB is Multi-homed\n" extIP="123.123.123.1" port=2020 +check_ha + echo "SCTP Multihoming service sctp-lb -> $extIP:$port" echo -e "------------------------------------------------------------------------------------\n" @@ -26,7 +29,10 @@ sudo pkill sctp_darn if [[ "$res" == "$exp" ]]; then echo $res echo -e "\nsctpmh SCTP Multihoming service Basic Test [OK]\n" + echo "OK" > status1.txt + restart_loxilbs else + echo "NOK" > status1.txt echo "sctpmh SCTP Multihoming service Basic Test [NOK]" echo "Expected : $exp" echo "Received : $res" @@ -51,11 +57,12 @@ else echo "llb1 ep-info" $dexec llb1 loxicmd get ep echo "llb1 bpf-info" - $dexec llb1 ntc filter show dev eth0 ingress + $dexec llb1 tc filter show dev eth0 ingress echo "BFP trace -- " sudo timeout 5 cat /sys/kernel/debug/tracing/trace_pipe sudo killall -9 cat echo "BFP trace -- " + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation2.sh b/cicd/sctpmh/validation2.sh index c9dbfd192..3829422b0 100755 --- a/cicd/sctpmh/validation2.sh +++ b/cicd/sctpmh/validation2.sh @@ -12,7 +12,7 @@ echo -e "----------------------------------------------------------------------- echo -e "\nHA state Master:$master BACKUP-$backup\n" -$hexec ep1 sctp_test -H 0.0.0.0 -P 9999 -l > ep1.out & +$hexec ep1 sctp_test -H 31.31.31.1 -P 9999 -l > ep1.out & sleep 2 $hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 50000 > user.out & @@ -72,7 +72,10 @@ sudo pkill sctp_test sudo rm *.out if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 ]]; then echo "sctpmh SCTP Multihoming Multipath [OK]" + echo "OK" > status2.txt + restart_loxilbs else + echo "NOK" > status2.txt echo "sctpmh SCTP Multihoming Multipath [NOK]" echo -e "\nuser" sudo ip netns exec user ip route @@ -99,6 +102,7 @@ else $dexec llb2 loxicmd get lb echo "llb2 ep-info" $dexec llb2 loxicmd get ep + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation3.sh b/cicd/sctpmh/validation3.sh index 9231b03d0..f2d3a8939 100755 --- a/cicd/sctpmh/validation3.sh +++ b/cicd/sctpmh/validation3.sh @@ -15,7 +15,7 @@ echo -e "\nHA state Master:$master BACKUP-$backup\n" echo -e "\nTraffic Flow: User -> LB -> EP " -$hexec ep1 sctp_test -H 0.0.0.0 -P 9999 -l > ep1.out & +$hexec ep1 sctp_test -H 31.31.31.1 -P 9999 -l > ep1.out & sleep 2 $hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 50000 > user.out & @@ -92,7 +92,10 @@ $hexec user ip route add default via 1.1.1.254 if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 ]]; then echo "sctpmh SCTP Multihoming Multipath Failover [OK]" + echo "OK" > status3.txt + restart_loxilbs else + echo "NOK" > status3.txt echo "sctpmh SCTP Multihoming Multipath Failover [NOK]" echo -e "\nuser" sudo ip netns exec user ip route @@ -119,6 +122,7 @@ else echo "llb2 ep-info" $dexec llb2 loxicmd get ep echo "-----------------------------" + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation4.sh b/cicd/sctpmh/validation4.sh index b5ce09f19..93f76e10a 100755 --- a/cicd/sctpmh/validation4.sh +++ b/cicd/sctpmh/validation4.sh @@ -1,7 +1,7 @@ #!/bin/bash source ../common.sh source check_ha.sh -echo -e "sctpmh: SCTP Multihoming - Multipath Failover Test. Client, LB and EP all Multihomed\n" +echo -e "sctpmh: SCTP Multihoming - E2E Multipath Failover Test. Client, LB and EP all Multihomed\n" extIP="133.133.133.1" port=2020 @@ -13,7 +13,7 @@ echo -e "----------------------------------------------------------------------- echo -e "\nHA state Master:$master BACKUP-$backup\n" echo -e "\nTraffic Flow: EP ---> LB ---> User" -$hexec user sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 9999 -l > user.out & +$hexec user sctp_test -H 0.0.0.0 -P 9999 -l > user.out & sleep 2 $hexec ep1 stdbuf -oL sctp_test -H 31.31.31.1 -B 32.32.32.1 -P 20000 -h $extIP -p $port -s -m 100 -x 50000 > ep1.out & @@ -91,7 +91,10 @@ $hexec user ip route add default via 1.1.1.254 if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 ]]; then echo "sctpmh SCTP Multihoming E2E Multipath Failover [OK]" + echo "OK" > status4.txt + restart_loxilbs else + echo "NOK" > status4.txt echo "sctpmh SCTP Multihoming E2E Multipath Failover [NOK]" echo -e "\nuser" sudo ip netns exec user ip route @@ -118,6 +121,7 @@ else $dexec llb2 loxicmd get lb echo "llb2 ep-info" $dexec llb2 loxicmd get ep + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation5.sh b/cicd/sctpmh/validation5.sh index a2d52bc61..61d4a40bd 100755 --- a/cicd/sctpmh/validation5.sh +++ b/cicd/sctpmh/validation5.sh @@ -14,11 +14,12 @@ echo -e "----------------------------------------------------------------------- echo -e "\nHA state Master:$master BACKUP-$backup\n" echo -e "\nTraffic Flow: User -> LB -> EP " +sudo pkill sctp_test -$hexec ep1 sctp_test -H 0.0.0.0 -P 9999 -l > ep1.out & +$hexec ep1 sctp_test -H 31.31.31.1 -P 9999 -l > ep1.out & sleep 2 -$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 200000 > user.out & +$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 1400 -x 300000 > user.out & #Path counters p1c_old=0 @@ -32,29 +33,8 @@ hadone=0 code=0 nsyncOk=0 -function restart_mloxilb() { - if [[ $master == "llb1" ]]; then - pat="cluster=172.17.0.3" - copts=" --cluster=172.17.0.3" - self=" --self=0" - ka=" --ka=172.17.0.3:172.17.0.2" - else - pat="cluster=172.17.0.2" - copts=" --cluster=172.17.0.2" - self=" --self=1" - ka=" --ka=172.17.0.2:172.17.0.3" - fi - pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) - echo Killing $pid >&2 - docker exec -dt $master kill -9 $pid - docker exec -dt $master ip link del llb0 - docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null & - pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) - echo "New loxilb pid: $pid" >&2 -} - for((i=0;i<200;i++)) do - fin=`tail -n 100 user.out | grep "Client: Sending packets.(200000/200000)"` + fin=`tail -n 100 user.out | grep "Client: Sending packets.(300000/300000)"` if [[ ! -z $fin ]]; then fin=1 echo "sctp_test done." @@ -65,7 +45,7 @@ for((i=0;i<200;i++)) do check_ha echo -e "\nHA state Master:$master BACKUP-$backup\n" nsyncOk=$(checkSync) - if [[ $nsyncOk == 2 ]]; then + if [[ $nsyncOk == 2 ]]; then #No active connections in Master, no need to continue. break; fi fi @@ -115,9 +95,12 @@ sudo rm -rf *.out sudo pkill sctp_test if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $syncOk == 1 ]]; then - echo "sctpmh SCTP Multihoming HA Failover [OK]" + echo "sctpmh SCTP Multihoming C2LB HA Failover [OK]" + echo "OK" > status5.txt + restart_loxilbs else - echo "sctpmh SCTP Multihoming HA Failover [NOK]" + echo "NOK" > status5.txt + echo "sctpmh SCTP Multihoming C2LB HA Failover [NOK]" echo -e "\nuser" sudo ip netns exec user ip route echo -e "\nr1" @@ -143,6 +126,7 @@ else echo "llb2 ep-info" $dexec llb2 loxicmd get ep echo "-----------------------------" + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation6.sh b/cicd/sctpmh/validation6.sh new file mode 100755 index 000000000..bbccecd3a --- /dev/null +++ b/cicd/sctpmh/validation6.sh @@ -0,0 +1,140 @@ +#!/bin/bash +source ../common.sh +source check_ha.sh + +echo -e "sctpmh: SCTP Multihoming - E2E HA Failover Test. Client, LB and EP all Multi-homed\n" +extIP="133.133.133.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" + +echo -e "\nTraffic Flow: EP ---> LB ---> User" + +sudo pkill sctp_test + +#$hexec user sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 9999 -l > user.out & +$hexec user sctp_test -H 0.0.0.0 -P 9999 -l > user.out & +sleep 2 + +$hexec ep1 stdbuf -oL sctp_test -H 31.31.31.1 -B 32.32.32.1 -P 20000 -h $extIP -p $port -s -m 1400 -x 500000 > ep1.out & + +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +checkha=0 +hadone=0 +code=0 +nsyncOk=0 + +for((i=0;i<200;i++)) do + fin=`tail -n 100 ep1.out | grep "Client: Sending packets.(500000/500000)"` + if [[ ! -z $fin ]]; then + fin=1 + echo "sctp_test done." + break; + fi + + syncOk=$nsyncOk + if [[ $checkha == 1 ]]; then + check_ha + echo -e "\nHA state Master:$master BACKUP-$backup\n" + nsyncOk=$(checkSync) + if [[ $nsyncOk == 2 ]]; then #No active connections in Master, no need to continue. + break; + fi + fi + echo -e "\n" + $dexec $master loxicmd get ct --servName=sctpmh2 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "133.133.133.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "134.134.134.1 | 32.32.32.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "135.135.135.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + + + echo "Counters: $p1c_new $p2c_new $p3c_new" + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [ACTIVE]" + p1=1 + else + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + echo -e "\n" + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + + if [[ $hadone == 0 ]]; then + nsyncOk=$(checkSync) + if [[ $nsyncOk == 1 ]]; then + restart_mloxilb + checkha=1 + hadone=1 + fi + fi + sleep 5 +done + + +sudo rm -rf *.out +sudo pkill sctp_test + +if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $nsyncOk == 1 ]]; then + echo "sctpmh SCTP Multihoming E2E HA Failover [OK]" + echo "OK" > status6.txt + restart_loxilbs +else + echo "NOK" > status6.txt + echo "sctpmh SCTP Multihoming E2E HA Failover [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + echo "-----------------------------" + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n"