diff --git a/.github/workflows/sctpmh-seagull-sanity.yml b/.github/workflows/sctpmh-seagull-sanity.yml new file mode 100644 index 000000000..4e49da07a --- /dev/null +++ b/.github/workflows/sctpmh-seagull-sanity.yml @@ -0,0 +1,36 @@ +name: SCTP-MH-LB-Seagull-Sanity-CI +on: + # schedule: + # Runs "At 11:00 UTC every day-of-week" + #- cron: '0 11 * * *' + workflow_dispatch: + inputs: + testName: + description: 'Test Run-Name' + required: true + default: 'sctpmh-seagull-sanity' +jobs: + test-runner: + name: k8s-calico-incluster-sanity + runs-on: [self-hosted, large] + if: github.repository == 'loxilb-io/loxilb' + && github.event.inputs.tagName == '' + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Run the test + run: | + cd cicd/sctpmh-seagull + ./config.sh + ./validation.sh + cd - + + - name: Clean test-bed + if: success() || failure() + run: | + cd cicd/sctpmh-seagull || true + ./rmconfig.sh + cd - diff --git a/cicd/common.sh b/cicd/common.sh index 619a1c36b..b96e2271f 100644 --- a/cicd/common.sh +++ b/cicd/common.sh @@ -561,7 +561,7 @@ function create_lb_rule() { return fi - hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func) + hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func) if [[ $hook != *"tc_packet_func"* ]]; then echo "ERROR : No hook point found"; exit 1 diff --git a/cicd/docker-k3s-calico/common.sh b/cicd/docker-k3s-calico/common.sh index ea1196029..1931a2309 100644 --- a/cicd/docker-k3s-calico/common.sh +++ b/cicd/docker-k3s-calico/common.sh @@ -536,7 +536,7 @@ function create_lb_rule() { echo "$1: loxicmd create lb ${args[*]}" $dexec $1 loxicmd create lb ${args[*]} - hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func) + hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func) if [[ $hook != *"tc_packet_func"* ]]; then echo "ERROR : No hook point found"; exit 1 diff --git a/cicd/docker-k3s-cilium/common.sh b/cicd/docker-k3s-cilium/common.sh index ea1196029..1931a2309 100644 --- a/cicd/docker-k3s-cilium/common.sh +++ b/cicd/docker-k3s-cilium/common.sh @@ -536,7 +536,7 @@ function create_lb_rule() { echo "$1: loxicmd create lb ${args[*]}" $dexec $1 loxicmd create lb ${args[*]} - hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func) + hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func) if [[ $hook != *"tc_packet_func"* ]]; then echo "ERROR : No hook point found"; exit 1 diff --git a/cicd/k0s-incluster/common.sh b/cicd/k0s-incluster/common.sh index ea1196029..1931a2309 100755 --- a/cicd/k0s-incluster/common.sh +++ b/cicd/k0s-incluster/common.sh @@ -536,7 +536,7 @@ function create_lb_rule() { echo "$1: loxicmd create lb ${args[*]}" $dexec $1 loxicmd create lb ${args[*]} - hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_func) + hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func) if [[ $hook != *"tc_packet_func"* ]]; then echo "ERROR : No hook point found"; exit 1 diff --git a/cicd/microk8s-incluster/common.sh b/cicd/microk8s-incluster/common.sh index c65963cd2..1931a2309 100755 --- a/cicd/microk8s-incluster/common.sh +++ b/cicd/microk8s-incluster/common.sh @@ -536,8 +536,8 @@ function create_lb_rule() { echo "$1: loxicmd create lb ${args[*]}" $dexec $1 loxicmd create lb ${args[*]} - hook=$($dexec llb1 tc filter show dev eth0 ingress | grep tc_packet_hook) - if [[ $hook != *"tc_packet_hook"* ]]; then + hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func) + if [[ $hook != *"tc_packet_func"* ]]; then echo "ERROR : No hook point found"; exit 1 fi diff --git a/cicd/sctpmh-seagull/Vagrantfile b/cicd/sctpmh-seagull/Vagrantfile new file mode 100644 index 000000000..f1d438ff8 --- /dev/null +++ b/cicd/sctpmh-seagull/Vagrantfile @@ -0,0 +1,27 @@ +# -*- mode: ruby -*- +# vi: set ft=ruby : + +#box_name = (ENV['VAGRANT_BOX'] || "ubuntu/focal64") +box_name = (ENV['VAGRANT_BOX'] || "sysnet4admin/Ubuntu-k8s") +box_version = "0.7.1" +Vagrant.configure("2") do |config| + config.vm.box = "#{box_name}" + config.vm.box_version = "#{box_version}" + + if Vagrant.has_plugin?("vagrant-vbguest") + config.vbguest.auto_update = false + end + + config.vm.define "bastion" do |bastion| + bastion.vm.hostname = 'bastion' + #bastion.vm.network :private_network, ip: "4.0.5.3", :netmask => "255.255.255.0" + bastion.vm.network :private_network, ip: "4.0.4.3", :netmask => "255.255.255.0" + bastion.vm.provision :shell, :path => "bastion.sh" + bastion.vm.provider :virtualbox do |vbox| + vbox.customize ["modifyvm", :id, "--memory", 8196] + vbox.customize ["modifyvm", :id, "--cpus", 12] + #vbox.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"] + #vbox.customize ["modifyvm", :id, "--nicpromisc3", "allow-all"] + end + end +end diff --git a/cicd/sctpmh-seagull/bastion.sh b/cicd/sctpmh-seagull/bastion.sh new file mode 100644 index 000000000..dc9a4a3a2 --- /dev/null +++ b/cicd/sctpmh-seagull/bastion.sh @@ -0,0 +1,12 @@ +apt-get update +apt-get install -y software-properties-common curl wget lksctp-tools jq +curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - +add-apt-repository -y "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" +apt-get update +apt-get install -y docker-ce +echo "blacklist sctp" >> /etc/modprobe.d/blacklist.conf +echo "install sctp /bin/false" >> /etc/modprobe.d/blacklist.conf + +echo "Rebooting Now!" +reboot +" diff --git a/cicd/sctpmh-seagull/check_ha.sh b/cicd/sctpmh-seagull/check_ha.sh new file mode 100644 index 000000000..b31574fb0 --- /dev/null +++ b/cicd/sctpmh-seagull/check_ha.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +master="llb1" +backup="llb2" + +function check_ha() { + while : ; do + status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') + status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') + count=0 + if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]]; + then + master="llb1" + backup="llb2" + break + elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]]; + then + master="llb2" + backup="llb1" + break + else + count=$(( $count + 1 )) + if [[ $count -ge 20 ]]; then + echo "KA llb1-$status1, llb2-$status2 [NOK] - Exiting" >&2 + exit 1; + fi + echo "KA llb1-$status1, llb2-$status2 [NOK]" >&2 + sleep 5 + fi + done +} + +function checkSync() { + count=1 + sync=0 + while [[ $count -le 5 ]] ; do + echo -e "\nStatus at MASTER:$master\n" >&2 + ct=`$dexec $master loxicmd get ct | grep est` + echo "${ct//'\n'/$'\n'}" >&2 + + echo -e "\nStatus at BACKUP:$backup\n" >&2 + ct=`$dexec $backup loxicmd get ct | grep est` + echo "${ct//'\n'/$'\n'}" >&2 + + nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + + if [[ $nres1 == 0 ]]; then + echo -e "No active connections in Master:$master. Exiting!" >&2 + return 2 + fi + + if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then + echo -e "\nConnections sync successful!!!\n" >&2 + sync=1 + break; + fi + echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2 + count=$(( $count + 1 )) + sleep 2 + done + + if [[ $sync == 0 ]]; then + echo -e "\nConnection Sync failed\n" >&2 + return 0 + fi + echo "$sync" +} + +function restart_mloxilb() { + if [[ $master == "llb1" ]]; then + pat="cluster=172.17.0.3" + copts=" --cluster=172.17.0.3" + self=" --self=0" + ka=" --ka=172.17.0.3:172.17.0.2" + else + pat="cluster=172.17.0.2" + copts=" --cluster=172.17.0.2" + self=" --self=1" + ka=" --ka=172.17.0.2:172.17.0.3" + fi + echo "Restarting MASTER: $master" + pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) + echo "Killing $pid" >&2 + docker exec -dt $master kill -9 $pid + docker exec -dt $master ip link del llb0 + docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null & + pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 +} + +function restart_loxilbs() { + if [[ $master == "llb1" ]]; then + mpat="cluster=172.17.0.3" + mcopts=" --cluster=172.17.0.3" + mself=" --self=0" + mka=" --ka=172.17.0.3:172.17.0.2" + + bpat="cluster=172.17.0.2" + bcopts=" --cluster=172.17.0.2" + bself=" --self=1" + bka=" --ka=172.17.0.2:172.17.0.3" + else + mpat="cluster=172.17.0.2" + mcopts=" --cluster=172.17.0.2" + mself=" --self=1" + mka=" --ka=172.17.0.2:172.17.0.3" + + bpat="cluster=172.17.0.3" + bcopts=" --cluster=172.17.0.3" + bself=" --self=0" + bka=" --ka=172.17.0.3:172.17.0.2" + fi + echo "Restarting $master" + pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2) + echo "Killing $mpid" >&2 + docker exec -dt $master kill -9 $pid + docker exec -dt $master ip link del llb0 + docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $mcopts $mself $mka > /dev/null & + pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 + + echo "Restarting $backup" + pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2) + echo "Killing $pid" >&2 + docker exec -dt $backup kill -9 $pid + docker exec -dt $backup ip link del llb0 + docker exec -dt $backup nohup /root/loxilb-io/loxilb/loxilb $bcopts $bself $bka > /dev/null & + pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 + +} + + diff --git a/cicd/sctpmh-seagull/common.sh b/cicd/sctpmh-seagull/common.sh new file mode 100644 index 000000000..d11adb0e3 --- /dev/null +++ b/cicd/sctpmh-seagull/common.sh @@ -0,0 +1,580 @@ +#!/bin/bash + +if [[ "$1" == "init" ]]; then + pull_dockers +fi + +hn="netns" +pid="" +vrn="/var/run/" +hexec="sudo ip netns exec " +dexec="sudo docker exec -i " +hns="sudo ip netns " +hexist="$vrn$hn" +lxdocker="ghcr.io/loxilb-io/loxilb:latest" +hostdocker="ghcr.io/loxilb-io/nettest:latest" +cluster_opts="" +extra_opts="" +ka_opts="" +#var=$(lsb_release -r | cut -f2) +#if [[ $var == *"22.04"* ]];then +# lxdocker="ghcr.io/loxilb-io/loxilb:latestu22" +#fi + +loxilbs=() + +## Given a docker name(arg1), return its pid +get_docker_pid() { + id=`docker ps -f name=$1| grep -w $1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + pid=`docker inspect -f '{{.State.Pid}}' $id` +} + +## Pull all necessary dockers for testbed +pull_dockers() { + ## loxilb docker + docker pull $lxdocker + ## Host docker + docker pull docker pull $hostdocker + ## BGP host docker + docker pull ewindisch/quagga +} + +## Creates a docker host +## arg1 - "loxilb"|"host" +## arg2 - instance-name +spawn_docker_host() { + POSITIONAL_ARGS=() + local bpath + local kpath + local ka + local bgp + while [[ $# -gt 0 ]]; do + case "$1" in + -t | --dock-type ) + dtype="$2" + shift 2 + ;; + -d | --dock-name ) + dname="$2" + shift 2 + ;; + -b | --with-bgp ) + if [[ "$2" == "yes" ]]; then + bgp=$2 + fi + shift 2 + ;; + -c | --bgp-config ) + bpath="$2" + bgp="yes" + shift 2 + ;; + -k | --with-ka ) + ka="in" + if [[ "$2" == "out" ]]; then + ka=$2 + fi + shift 2 + ;; + -n | --ka-config ) + kpath="$2" + if [[ -z ${ka+x} ]]; then + ka="in" + fi + shift 2 + ;; + -e | --extra-args) + extra_opts="$2" + shift 2 + ;; + -x | --docker-args) + docker_extra_opts="$2" + shift 2 + ;; + -*|--*) + echo "Unknown option $1" + exit + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + echo "Spawning $dname($dtype)" >&2 + if [[ "$dtype" == "loxilb" ]]; then + loxilbs+=("$dname") + if [[ "$pick_config" == "yes" ]]; then + echo "$dname will pick config from $(pwd)/${dname}_config" + loxilb_config="-v $(pwd)/${dname}_config:/etc/loxilb/" + fi + if [[ "$bgp" == "yes" ]]; then + bgp_opts="-b" + if [[ ! -z "$bpath" ]]; then + bgp_conf="-v $bpath:/etc/gobgp/" + fi + fi + if [[ ! -z ${ka+x} ]]; then + sudo mkdir -p /etc/shared/$dname/ + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt $docker_extra_opts --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log -v /etc/shared/$dname:/etc/shared $loxilb_config --name $dname $lxdocker + get_llb_peerIP $dname + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $ka_opts $extra_opts + else + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dt $docker_extra_opts --entrypoint /bin/bash $bgp_conf -v /dev/log:/dev/log $loxilb_config --name $dname $lxdocker $bgp_opts + docker exec -dt $dname /root/loxilb-io/loxilb/loxilb $bgp_opts $cluster_opts $extra_opts + fi + elif [[ "$dtype" == "host" ]]; then + if [[ ! -z "$bpath" ]]; then + bgp_conf="--volume $bpath:/etc/quagga" + fi + if [[ "$bgp" == "yes" || ! -z "$bpath" ]]; then + docker run -u root --cap-add SYS_ADMIN --restart unless-stopped --privileged -dit $bgp_conf --name $dname ewindisch/quagga + else + docker run -u root --cap-add SYS_ADMIN -dit --name $dname $hostdocker + fi + elif [[ "$dtype" == "seahost" ]]; then + docker run -u root --cap-add SYS_ADMIN -i -t --rm --detach --entrypoint /bin/bash --name $dname ghcr.io/loxilb-io/seagull:ubuntu1804 + docker exec -dit $dname ifconfig eth0 0 + fi + + pid="" + + sleep 2 + get_docker_pid $dname + echo $pid + if [ ! -f "$hexist/$dname" -a "$pid" != "" ]; then + sudo mkdir -p /var/run/netns + sudo touch /var/run/netns/$dname + #echo "sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$2" + sudo mount -o bind /proc/$pid/ns/net /var/run/netns/$dname + fi + + $hexec $dname ifconfig lo up + $hexec $dname sysctl net.ipv6.conf.all.disable_ipv6=1 2>&1 >> /dev/null + #$hexec $dname sysctl net.ipv4.conf.all.arp_accept=1 2>&1 >> /dev/null + if [ -f /proc/sys/net/ipv4/conf/eth0/arp_ignore ]; then + $hexec $dname sysctl net.ipv4.conf.eth0.arp_ignore=2 2>&1 >> /dev/null + fi +} + +## Get loxilb peer docker IP +get_llb_peerIP() { + if [[ "$1" == "llb1" ]]; then + llb1IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' llb1) + if [[ "lb$llb1IP" == "lb" ]];then + llb2IP="172.17.0.3" + else + read A B C D <<<"${llb1IP//./ }" + llb2IP="$A.$B.$C.$((D+1))" + fi + cluster_opts=" --cluster=$llb2IP --self=0" + ka_opts=" --ka=$llb2IP:$llb1IP" + elif [[ "$1" == "llb2" ]]; then + llb2IP=$(docker inspect --format='{{.NetworkSettings.IPAddress}}' llb2) + if [[ "lb$llb2IP" == "lb" ]];then + llb1IP="172.17.0.2" + else + read A B C D <<<"${llb2IP//./ }" + llb1IP="$A.$B.$C.$((D-1))" + fi + cluster_opts=" --cluster=$llb1IP --self=1" + ka_opts=" --ka=$llb1IP:$llb2IP" + fi +} + +## Deletes a docker host +## arg1 - hostname +delete_docker_host() { + dcmd="kill" + if [[ $1 == "llb"* ]] || [[ $1 == "loxilb"* ]]; then + dcmd="stop" + fi + if docker $dcmd $1 2>&1 >> /dev/null + then + hd="true" + ka=`docker ps -f name=ka_$1| grep -w ka_$1 | cut -d " " -f 1 | grep -iv "CONTAINER"` + loxilbs=( "${loxilbs[@]/$1}" ) + if [ "$ka" != "" ]; then + docker kill ka_$1 2>&1 >> /dev/null + docker rm ka_$1 2>&1 >> /dev/null + fi + fi + if [ -f "$hexist/$1" ]; then + $hns del $1 + sudo rm -fr "$hexist/$1" 2>&1 >> /dev/null + fi + docker rm $1 2>&1 >> /dev/null || true +} + +## Connects two docker hosts +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - mtu +connect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + + mtu="9000" + if [[ $# -gt 2 ]]; then + mtu=$3 + fi + + #echo $link1 $link2 + sudo ip -n $1 link add $link1 type veth peer name $link2 netns $2 + sudo ip -n $1 link set $link1 mtu $mtu up + sudo ip -n $2 link set $link2 mtu $mtu up +} + +## arg1 - hostname1 +## arg2 - hostname2 +disconnect_docker_hosts() { + link1=e$1$2 + link2=e$2$1 + # echo $link1 $link2 + if [ -f "$hexist/$1" ]; then + ifexist1=`sudo ip -n $1 link show $link1 | grep -w $link1` + if [ "chk$ifexist1" != "chk" ]; then + sudo ip -n $1 link set $link1 down 2>&1 >> /dev/null + sudo ip -n $1 link del $link1 2>&1 >> /dev/null + fi + fi + + if [ -f "$hexist/$2" ]; then + ifexist2=`sudo ip -n $2 link show | grep -w $link2` + if [ "chk$ifexist2" != "chk" ]; then + sudo ip -n $2 link set $link2 down 2>&1 >> /dev/null + sudo ip -n $2 link del $link2 2>&1 >> /dev/null + fi + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - ip_addr +## arg4 - gw +config_docker_host() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local xid="$2" + shift + shift + ;; + --addr) + local addr="$2" + shift + shift + ;; + --gw) + local gw="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + + link1=e$h1$h2 + link2=e$h2$h1 + #echo "$h1:$link1->$h2:$link2" + + #if [[ -n "${loxilbs[$h1]}" && "$pick_config" == "yes" ]]; then + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + if [[ "$ptype" == "phy" ]]; then + sudo ip -n $h1 addr add $addr dev $link1 + elif [[ "$ptype" == "vlan" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$xid + elif [[ "$ptype" == "vxlan" ]]; then + sudo ip -n $h1 addr add $addr dev vxlan$xid + elif [[ "$ptype" == "trunk" ]]; then + trunk="bond$xid" + sudo ip -n $h1 link set $link1 down + sudo ip -n $h1 link add $trunk type bond + sudo ip -n $h1 link set $link1 master $trunk + sudo ip -n $h1 link set $link1 up + sudo ip -n $h1 link set $trunk up + + sudo ip -n $h2 link set $link2 down + sudo ip -n $h2 link add $trunk type bond + sudo ip -n $h2 link set $link2 master $trunk + sudo ip -n $h2 link set $link2 up + sudo ip -n $h2 link set $trunk up + + sudo ip -n $h1 addr add $addr dev bond$xid + if [[ "$gw" != "" ]]; then + sudo ip -n $h2 addr add $gw/24 dev bond$xid + sudo ip -n $h1 route add default via $gw proto static + fi + else + echo "Check port-type" + fi + + if [[ "$gw" != "" ]]; then + sudo ip -n $h1 route del default 2>&1 >> /dev/null + sudo ip -n $h1 route add default via $gw + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vlan +## arg4 - tagged/untagged +create_docker_host_vlan() { + local addr="" + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --ptype) + local ptype="$2" + shift + shift + ;; + --id) + local vid="$2" + shift + shift + ;; + --addr) + addr="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$ptype" == "tagged" ]]; then + brport="$link1.$vid" + sudo ip -n $h1 link add link $link1 name $brport type vlan id $vid + sudo ip -n $h1 link set $brport up + else + brport=$link1 + fi + + sudo ip -n $h1 link add vlan$vid type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master vlan$vid + sudo ip -n $h1 link set vlan$vid up + if [[ "$addr" != "" ]]; then + sudo ip -n $h1 addr add $addr dev vlan$vid + fi +} + +## arg1 - hostname1 +## arg2 - hostname2 +## arg3 - vxlan-id +## arg4 - phy/vlan +## arg5 - local ip if arg4 is phy/vlan-id if arg4 is vlan +## arg6 - local ip if arg4 is vlan +create_docker_host_vxlan() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + --uif) + local uifType="$2" + shift + shift + ;; + --vid) + local vid="$2" + shift + shift + ;; + --pvid) + local pvid="$2" + shift + shift + ;; + --id) + local vxid="$2" + shift + shift + ;; + --ep) + local ep="$2" + shift + shift + ;; + --lip) + local lip="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + if [[ "$uifType" == "phy" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev $link1 dstport 0 + sudo ip -n $h1 link set vxlan$vxid up + elif [[ "$uifType" == "vlan" ]]; then + sudo ip -n $h1 link add vxlan$vxid type vxlan id $vxid local $lip dev vlan$vid dstport 0 + sudo ip -n $h1 link set vxlan$vxid up + fi + + if [[ "$pvid" != "" ]]; then + sudo ip -n $h1 link add vlan$pvid type bridge 2>&1 | true + sudo ip -n $h1 link set vxlan$vxid master vlan$pvid + sudo ip -n $h1 link set vlan$pvid up + fi + + if [[ "$ep" != "" ]]; then + sudo bridge -n $h1 fdb append 00:00:00:00:00:00 dst $ep dev vxlan$vxid + fi + +} + +## arg1 - hostname1 +## arg2 - hostname2 +create_docker_host_cnbridge() { + POSITIONAL_ARGS=() + while [[ $# -gt 0 ]]; do + case $1 in + --host1) + local h1="$2" + shift + shift + ;; + --host2) + local h2="$2" + shift + shift + ;; + -*|--*) + echo "Unknown option $1" + exit 1 + ;; + *) + POSITIONAL_ARGS+=("$1") # save positional arg + shift # past argument + ;; + esac + done + + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$h1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + + set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters + link1=e$h1$h2 + link2=e$h2$h1 + + #echo "$h1:$link1->$h2:$link2" + + brport=$link1 + + sudo ip -n $h1 link add br$h1 type bridge 2>&1 | true + sudo ip -n $h1 link set $brport master br$h1 + sudo ip -n $h1 link set br$h1 up +} + +#Arg1: host name +#Arg2: --:: +#Arg3: --endpoints::,.. +function create_lb_rule() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + args=( "$@" ) + args=( "${args[@]/$1}" ) + echo "$1: loxicmd create lb ${args[*]}" + $dexec $1 loxicmd create lb ${args[*]} + + if [[ ${args[*]} == *"--mode=fullproxy"* ]]; then + return + fi + + hook=$($dexec $1 tc filter show dev eth0 ingress | grep tc_packet_func) + if [[ $hook != *"tc_packet_func"* ]]; then + echo "ERROR : No hook point found"; + exit 1 + fi +} + +#Arg1: host name +#Arg2: +#Arg3: +function add_route() { + if [[ ${loxilbs[*]} =~ (^|[[:space:]])$1($|[[:space:]]) && "$pick_config" == "yes" ]]; then + return + fi + echo "$1: ip route add $2 via $3 proto static" + $hexec $1 ip route add $2 via $3 proto static +} diff --git a/cicd/sctpmh-seagull/config.sh b/cicd/sctpmh-seagull/config.sh new file mode 100755 index 000000000..bc09793b2 --- /dev/null +++ b/cicd/sctpmh-seagull/config.sh @@ -0,0 +1,23 @@ +#!/bin/bash +vagrant global-status | grep -i virtualbox | cut -f 1 -d ' ' | xargs -L 1 vagrant destroy -f +vagrant up + +for i in {1..60} +do + ping 4.0.4.3 -c 1 -W 1 2>&1> /dev/null; + if [[ $? -eq 0 ]] + then + echo -e "Machine rebooted [OK]" + code=0 + break + else + echo -e "Waiting for machine to be UP" + sleep 1 + fi +done +if [[ $code == 0 ]]; +then + vagrant ssh bastion -c 'sudo /vagrant/setup.sh' +else + echo "VM not up" +fi diff --git a/cicd/sctpmh-seagull/rmconfig.sh b/cicd/sctpmh-seagull/rmconfig.sh new file mode 100755 index 000000000..88da3a557 --- /dev/null +++ b/cicd/sctpmh-seagull/rmconfig.sh @@ -0,0 +1,3 @@ +#!/bin/bash +vagrant destroy -f bastion +sudo rm -rf status*.txt diff --git a/cicd/sctpmh-seagull/rmsetup.sh b/cicd/sctpmh-seagull/rmsetup.sh new file mode 100755 index 000000000..a2bf9aaeb --- /dev/null +++ b/cicd/sctpmh-seagull/rmsetup.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +echo "#########################################" +echo "Removing testbed" +echo "#########################################" + +source /vagrant/common.sh + +disconnect_docker_hosts user r1 +disconnect_docker_hosts user r2 +disconnect_docker_hosts r1 sw1 +disconnect_docker_hosts r2 sw1 +disconnect_docker_hosts sw1 llb1 +disconnect_docker_hosts sw1 llb2 +disconnect_docker_hosts llb1 sw2 +disconnect_docker_hosts llb2 sw2 +disconnect_docker_hosts sw2 r3 +disconnect_docker_hosts sw2 r4 +disconnect_docker_hosts r3 ep1 +disconnect_docker_hosts r4 ep1 + +delete_docker_host user +delete_docker_host llb1 +delete_docker_host llb2 +delete_docker_host r1 +delete_docker_host r2 +delete_docker_host r3 +delete_docker_host r4 +delete_docker_host sw1 +delete_docker_host sw2 +delete_docker_host ep1 + +echo "#########################################" +echo "Removed testbed" +echo "#########################################" diff --git a/cicd/sctpmh-seagull/setup.sh b/cicd/sctpmh-seagull/setup.sh new file mode 100755 index 000000000..a19ff84d0 --- /dev/null +++ b/cicd/sctpmh-seagull/setup.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +source /vagrant/common.sh + +echo "#########################################" +echo "Spawning all hosts" +echo "#########################################" + +spawn_docker_host --dock-type loxilb --dock-name llb1 --with-ka in +spawn_docker_host --dock-type loxilb --dock-name llb2 --with-ka in +spawn_docker_host --dock-type seahost --dock-name ep1 +spawn_docker_host --dock-type host --dock-name r1 +spawn_docker_host --dock-type host --dock-name r2 +spawn_docker_host --dock-type host --dock-name r3 +spawn_docker_host --dock-type host --dock-name r4 +spawn_docker_host --dock-type host --dock-name sw1 +spawn_docker_host --dock-type host --dock-name sw2 +spawn_docker_host --dock-type seahost --dock-name user + +echo "#########################################" +echo "Connecting and configuring hosts" +echo "#########################################" + +connect_docker_hosts user r1 +connect_docker_hosts user r2 +connect_docker_hosts r1 sw1 +connect_docker_hosts r2 sw1 +connect_docker_hosts sw1 llb1 +connect_docker_hosts sw1 llb2 +connect_docker_hosts llb1 sw2 +connect_docker_hosts llb2 sw2 +connect_docker_hosts sw2 r3 +connect_docker_hosts sw2 r4 +connect_docker_hosts r3 ep1 +connect_docker_hosts r4 ep1 + +create_docker_host_cnbridge --host1 sw1 --host2 llb1 +create_docker_host_cnbridge --host1 sw1 --host2 llb2 +create_docker_host_cnbridge --host1 sw1 --host2 r1 +create_docker_host_cnbridge --host1 sw1 --host2 r2 + +create_docker_host_cnbridge --host1 sw2 --host2 llb1 +create_docker_host_cnbridge --host1 sw2 --host2 llb2 +create_docker_host_cnbridge --host1 sw2 --host2 r3 +create_docker_host_cnbridge --host1 sw2 --host2 r4 + +#node1 config +config_docker_host --host1 user --host2 r1 --ptype phy --addr 1.1.1.1/24 --gw 1.1.1.254 +config_docker_host --host1 user --host2 r2 --ptype phy --addr 2.2.2.1/24 +config_docker_host --host1 r1 --host2 user --ptype phy --addr 1.1.1.254/24 +config_docker_host --host1 r2 --host2 user --ptype phy --addr 2.2.2.254/24 + +create_docker_host_vlan --host1 llb1 --host2 sw1 --id 11 --ptype untagged +create_docker_host_vlan --host1 llb2 --host2 sw1 --id 11 --ptype untagged +create_docker_host_vlan --host1 r1 --host2 sw1 --id 11 --ptype untagged +create_docker_host_vlan --host1 r2 --host2 sw1 --id 11 --ptype untagged +config_docker_host --host1 r1 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.253/24 --gw 11.11.11.11 +config_docker_host --host1 r2 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.254/24 --gw 11.11.11.11 +config_docker_host --host1 llb1 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.1/24 +config_docker_host --host1 llb2 --host2 sw1 --ptype vlan --id 11 --addr 11.11.11.2/24 + +create_docker_host_vlan --host1 llb1 --host2 sw2 --id 10 --ptype untagged +create_docker_host_vlan --host1 llb2 --host2 sw2 --id 10 --ptype untagged +create_docker_host_vlan --host1 r3 --host2 sw2 --id 10 --ptype untagged +create_docker_host_vlan --host1 r4 --host2 sw2 --id 10 --ptype untagged + +config_docker_host --host1 r3 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.253/24 --gw 10.10.10.10 +config_docker_host --host1 r4 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.254/24 --gw 10.10.10.10 +config_docker_host --host1 llb1 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.1/24 +config_docker_host --host1 llb2 --host2 sw2 --ptype vlan --id 10 --addr 10.10.10.2/24 + +config_docker_host --host1 ep1 --host2 r3 --ptype phy --addr 31.31.31.1/24 --gw 31.31.31.254 +config_docker_host --host1 ep1 --host2 r4 --ptype phy --addr 32.32.32.1/24 +config_docker_host --host1 r3 --host2 ep1 --ptype phy --addr 31.31.31.254/24 +config_docker_host --host1 r4 --host2 ep1 --ptype phy --addr 32.32.32.254/24 + +$hexec user ip route change default via 1.1.1.254 +$hexec ep1 ip route change default via 31.31.31.254 + +# Backup paths in user +$hexec user ip route add 21.21.21.1/32 via 2.2.2.254 +$hexec user ip route add 134.134.134.1/32 via 2.2.2.254 + +$hexec ep1 ip route add 134.134.134.1/32 via 32.32.32.254 +$hexec ep1 ip route add 135.135.135.1/32 via 31.31.31.254 + +$hexec llb1 ip route add 1.1.1.0/24 via 11.11.11.253 +$hexec llb1 ip route add 2.2.2.0/24 via 11.11.11.254 +$hexec llb2 ip route add 1.1.1.0/24 via 11.11.11.253 +$hexec llb2 ip route add 2.2.2.0/24 via 11.11.11.254 + +$hexec llb1 ip route add 31.31.31.0/24 via 10.10.10.253 +$hexec llb1 ip route add 32.32.32.0/24 via 10.10.10.254 +$hexec llb2 ip route add 31.31.31.0/24 via 10.10.10.253 +$hexec llb2 ip route add 32.32.32.0/24 via 10.10.10.254 + +sleep 20 +##Create LB rule user->ep1 +create_lb_rule llb1 20.20.20.1 --name=sctpmh1 --secips=21.21.21.1,22.22.22.1 --sctp=2020:8080 --endpoints=31.31.31.1:1 --mode=fullnat +create_lb_rule llb2 20.20.20.1 --name=sctpmh1 --secips=21.21.21.1,22.22.22.1 --sctp=2020:8080 --endpoints=31.31.31.1:1 --mode=fullnat + +##Create LB rule ep1->user +create_lb_rule llb1 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:8080 --endpoints=1.1.1.1:1 --mode=fullnat +create_lb_rule llb2 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:8080 --endpoints=1.1.1.1:1 --mode=fullnat + +$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping +$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping +$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping +$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping + + +create_lb_rule llb1 11.11.11.11 --tcp=80:8080 --endpoints=31.31.31.1:1 +create_lb_rule llb2 11.11.11.11 --tcp=80:8080 --endpoints=31.31.31.1:1 +create_lb_rule llb1 10.10.10.10 --tcp=80:8080 --endpoints=31.31.31.1:1 +create_lb_rule llb2 10.10.10.10 --tcp=80:8080 --endpoints=31.31.31.1:1 + +$dexec llb1 loxicmd save --all +$dexec llb2 loxicmd save --all + +$hexec user ifconfig eth0 0 +$hexec ep1 ifconfig eth0 0 diff --git a/cicd/sctpmh-seagull/validation.sh b/cicd/sctpmh-seagull/validation.sh new file mode 100755 index 000000000..93d6ca8a4 --- /dev/null +++ b/cicd/sctpmh-seagull/validation.sh @@ -0,0 +1,56 @@ +#!/bin/bash +code=0 +tc=( "Basic Test - Client & EP Uni-homed and LB is Multi-homed" "Multipath Test, Client and LB Multihomed, EP is uni-homed" "C2LB Multipath Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E Multipath Failover Test - Client, LB and EP all Multihomed" "C2LB HA Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E HA Failover Test. Client, LB and EP all Multihomed" ) +padding="............................................................................................................." +border="**************************************************************************************************************************************************" + +for((j=0,i=1; i<=6; i++, j++)); do + echo "SCTP Multihoming - Test case #$i" + echo -e "\n\n\n$border\n" + cmd="sudo /vagrant/validation$i.sh" + vagrant ssh bastion -c "$cmd" + echo -e "\n\n" + file=status$i.txt + status=`cat $file` + title=${tc[j]} + echo -e "\n\n" + + if [[ $status == "NOK" ]]; then + code=1 + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]"; + else + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]"; + fi + echo -e "\n\n\n$border\n\n" + + sleep 30 +done + +echo -e "\n\n\n$border\n" +printf "================================================== SCTP MULTIHOMING CONSOLIDATED RESULT ==========================================================\n" +for((j=0,i=1; i<=6; i++, j++)); do + file=status$i.txt + status=`cat $file` + title=${tc[j]} + echo -e "\n\n" + + if [[ $status == "NOK" ]]; then + code=1 + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]"; + else + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]"; + fi +done + +echo -e "\n$border" + +echo -e "\n\n\n$border\n" +if [[ $code == 0 ]]; then + echo -e "SCTP multihoming with seagull CICD [OK]" +else + echo -e "SCTP Multihoming with seagull CICD [NOK]" +fi +echo -e "\n$border\n" + + +exit $code diff --git a/cicd/sctpmh-seagull/validation1.sh b/cicd/sctpmh-seagull/validation1.sh new file mode 100755 index 000000000..6bde78ab0 --- /dev/null +++ b/cicd/sctpmh-seagull/validation1.sh @@ -0,0 +1,67 @@ +#!/bin/bash +source /vagrant/common.sh +source /vagrant/check_ha.sh + +echo -e "sctpmh: SCTP Multihoming Basic Test - Client & EP Uni-homed and LB is Multi-homed\n" +extIP="20.20.20.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +sudo docker exec -dt ep1 ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 40 stdbuf -oL seagull -conf ../config/conf.server.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.server.xml > ep1.out' 2>&1 > /dev/null & +sleep 2 + +sudo docker exec -dt user ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 25 stdbuf -oL seagull -conf ../config/conf.client.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.client.xml > user.out' 2>&1 > /dev/null & + +sleep 2 + +for((i=0;i<5;i++)) do + $dexec user bash -c 'tail -n 25 /opt/seagull/diameter-env/run/user.out' + res=$(sudo docker exec -t user bash -c 'tail -n 10 /opt/seagull/diameter-env/run/user.out | grep "Successful calls"'| xargs | cut -d '|' -f 4) + $dexec $master loxicmd get ct --servName=sctpmh1 + echo -e "\n" + sleep 5 +done + +if [ "$res" -gt "0" ]; then + #echo -e $res + echo -e "\nsctpmh SCTP Multihoming service Basic Test [OK]\n" + echo "OK" > /vagrant/status1.txt + restart_loxilbs +else + echo "NOK" > /vagrant/status1.txt + echo "sctpmh SCTP Multihoming service Basic Test [NOK]" + echo "Calls : $res" + ## Dump some debug info + echo "system route-info" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo "llb1 bpf-info" + $dexec llb1 tc filter show dev eth0 ingress + echo "BFP trace -- " + sudo timeout 5 cat /sys/kernel/debug/tracing/trace_pipe + sudo killall -9 cat + echo "BFP trace -- " + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh-seagull/validation2.sh b/cicd/sctpmh-seagull/validation2.sh new file mode 100755 index 000000000..f001a497f --- /dev/null +++ b/cicd/sctpmh-seagull/validation2.sh @@ -0,0 +1,114 @@ +#!/bin/bash +source /vagrant/common.sh +source /vagrant/check_ha.sh +echo -e "sctpmh: SCTP Multihoming - Multipath Test, Client and LB Multihomed, EP is uni-homed\n" +extIP="20.20.20.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" + +sudo docker exec -dt ep1 ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 40 stdbuf -oL seagull -conf ../config/conf.server.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.server.xml > ep1.out' 2>&1 > /dev/null & +sleep 2 + +sudo docker exec -dt user ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 25 stdbuf -oL seagull -conf ../config/conf.client.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.client.xml > user.out' 2>&1 > /dev/null & + +sleep 2 +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +call_old=0 +call_new=0 + +for((i=0;i<5;i++)) do + + $dexec user bash -c 'tail -n 25 /opt/seagull/diameter-env/run/user.out' + call_new=$(sudo docker exec -t user bash -c 'tail -n 10 /opt/seagull/diameter-env/run/user.out | grep "Successful calls"'| xargs | cut -d '|' -f 4) + echo -e "\n\n" + $dexec $master loxicmd get ct --servName=sctpmh1 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "20.20.20.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "21.21.21.1 | 2.2.2.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "22.22.22.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + + echo "Counters: $p1c_new $p2c_new $p3c_new" + + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 1.1.1.1 -> 20.20.20.1 [ACTIVE]" + p1=1 + else + echo "Path 1: 1.1.1.1 -> 20.20.20.1 [NOT ACTIVE]" + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 2.2.2.1 -> 21.21.21.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 2.2.2.1 -> 21.21.21.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 1.1.1.1 -> 22.22.22.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 1.1.1.1 -> 22.22.22.1 [NOT ACTIVE]" + fi + if [[ $call_new -gt $call_old ]]; then + echo "\nSuccessful Calls: \t$call_new [ACTIVE]" + calls=1 + else + echo "\nSuccessful Calls: \t$call_new [NOT ACTIVE]" + fi + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + call_old=$call_new + echo -e "\n" + sleep 5 +done + +if [[ $p1 == 1 && $p2 == 1 && $p3 == 1 && $calls == 1 ]]; then + echo "sctpmh SCTP Multihoming Multipath [OK]" + echo "OK" > /vagrant/status2.txt + restart_loxilbs +else + echo "NOK" > /vagrant/status2.txt + echo "sctpmh SCTP Multihoming Multipath [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo "-----------------------------" + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh-seagull/validation3.sh b/cicd/sctpmh-seagull/validation3.sh new file mode 100755 index 000000000..5a31136d9 --- /dev/null +++ b/cicd/sctpmh-seagull/validation3.sh @@ -0,0 +1,171 @@ +#!/bin/bash +source /vagrant/common.sh +source /vagrant/check_ha.sh + +echo -e "sctpmh: SCTP Multihoming - Multipath Failover Test. Client and LB Multihomed, EP is uni-homed\n" +extIP="20.20.20.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" + +echo -e "\nTraffic Flow: User -> LB -> EP " + +sudo docker exec -dt user ksh -c "sed -i 's/\"call-rate\" value=\"5000\"/\"call-rate\" value=\"100\"/g' /opt/seagull/diameter-env/config/conf.client.xml" + +sudo docker exec -dt ep1 ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 220 stdbuf -oL seagull -conf ../config/conf.server.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.server.xml > ep1.out' 2>&1 > /dev/null & +sleep 2 + +sudo docker exec -dt user ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 210 stdbuf -oL seagull -conf ../config/conf.client.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.client.xml > user.out' 2>&1 > /dev/null & + +sleep 30 +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +down=0 +code=0 +call_old=0 +call_new=0 +fail_old=0 +fail_new=0 +recover=0 +frecover=0 + +for((i=0;i<35;i++)) do + $dexec user bash -c 'tail -n 25 /opt/seagull/diameter-env/run/user.out' + call_new=$(sudo docker exec -t user bash -c 'tail -n 10 /opt/seagull/diameter-env/run/user.out | grep "Successful calls"'| xargs | cut -d '|' -f 4) + fail_new=$(sudo docker exec -t user bash -c 'tail -n 10 /opt/seagull/diameter-env/run/user.out | grep "Failed calls"'| xargs | cut -d '|' -f 4) + echo -e "\n\n" + $dexec $master loxicmd get ct --servName=sctpmh1 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "20.20.20.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "21.21.21.1 | 2.2.2.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "22.22.22.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + + echo "Counters: $p1c_new $p2c_new $p3c_new" + + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 1.1.1.1 -> 20.20.20.1 [ACTIVE]" + p1=1 + #if [[ $down == 1 ]]; then + #echo "This path shouldn't be ACTIVE" + #code=1 + #fi + echo -e "Turning off this path from User->LB\n" + $hexec user ip link set euserr1 down; + down=1 + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "20.20.20.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + else + if [[ $down == 1 ]]; then + p1dok=1 + echo "Path 1: 1.1.1.1 -> 20.20.20.1 NOT ACTIVE - [OK]" + else + echo "Path 1: 1.1.1.1 -> 20.20.20.1 [NOT ACTIVE]" + fi + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 2.2.2.1 -> 21.21.21.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 2.2.2.1 -> 21.21.21.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 1.1.1.1 -> 22.22.22.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 1.1.1.1 -> 22.22.22.1 [NOT ACTIVE]" + fi + + echo -e "\n" + if [[ $recover == 1 ]]; then + printf "\t***Setup Recovered***" + fi + echo -e "\n\n" + + if [[ $fail_new -gt $fail_old && $down == 1 && $recover == 0 ]]; then + printf "Failed Calls: \t%10s \t[INCREASING]\n" $fail_new + fstart=1 + code=1 + calls=0 + else + if [[ $fail_new -eq $fail_old ]]; then + if [[ $down == 1 && $fstart == 1 ]]; then + printf "Failed Calls: \t%10s \t[STABLE]\n" $fail_new + frecover=1 + code=0 + else + printf "Failed Calls: \t%10s\n" $fail_new + fi + fi + fi + + if [[ $call_new -gt $call_old ]]; then + printf "Successful Calls: \t%10s \t[ACTIVE]\n" $call_new + calls=1 + if [[ $down == 1 && $frecover == 1 ]]; then + recover=1 + fi + else + printf "Successful Calls: \t%10s \t[NOT ACTIVE]\n" $call_new + fi + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + call_old=$call_new + fail_old=$fail_new + echo -e "\n" + sleep 5 +done + +#Restore +$hexec user ip link set euserr1 up +$hexec user ip route add default via 1.1.1.254 +sudo docker exec -dt user ksh -c "sed -i 's/\"call-rate\" value=\"100\"/\"call-rate\" value=\"5000\"/g' /opt/seagull/diameter-env/config/conf.client.xml" + +if [[ $calls == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $recover == 1 ]]; then + echo "sctpmh SCTP Multihoming Multipath Failover [OK]" + echo "OK" > /vagrant/status3.txt + restart_loxilbs +else + echo "NOK" > /vagrant/status3.txt + echo "sctpmh SCTP Multihoming Multipath Failover [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + echo "-----------------------------" + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh-seagull/validation4.sh b/cicd/sctpmh-seagull/validation4.sh new file mode 100755 index 000000000..0bf76671f --- /dev/null +++ b/cicd/sctpmh-seagull/validation4.sh @@ -0,0 +1,172 @@ +#!/bin/bash +source /vagrant/common.sh +source /vagrant/check_ha.sh +echo -e "sctpmh: SCTP Multihoming - E2E Multipath Failover Test. Client, LB and EP all Multihomed\n" +extIP="133.133.133.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" +echo -e "\nTraffic Flow: EP ---> LB ---> User" + +sudo docker exec -dt user ksh -c 'sed -i 's/source=31.31.31.1/source=0.0.0.0/g' /opt/seagull/diameter-env/config/conf.server.xml' + +sudo docker exec -dt user ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 220 stdbuf -oL seagull -conf ../config/conf.server.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.server.xml > user.out' 2>&1 > /dev/null & +sleep 2 + +sudo docker exec -dt ep1 ksh -c "sed -i 's/\"call-rate\" value=\"5000\"/\"call-rate\" value=\"100\"/g' /opt/seagull/diameter-env/config/conf.client.xml" +sudo docker exec -dt ep1 ksh -c 'sed -i 's/dest=20.20.20.1/dest=133.133.133.1/g' /opt/seagull/diameter-env/config/conf.client.xml' +sudo docker exec -dt ep1 ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 210 stdbuf -oL seagull -conf ../config/conf.client.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.client.xml > ep1.out' 2>&1 > /dev/null & + +sleep 20 + +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +down=0 +code=0 +call_old=0 +call_new=0 +fail_old=0 +fail_new=0 +recover=0 +frecover=0 +calls=0 +for((i=0;i<35;i++)) do + $dexec ep1 bash -c 'tail -n 25 /opt/seagull/diameter-env/run/ep1.out' + call_new=$(sudo docker exec -t ep1 bash -c 'tail -n 10 /opt/seagull/diameter-env/run/ep1.out | grep "Successful calls"'| xargs | cut -d '|' -f 4) + fail_new=$(sudo docker exec -t ep1 bash -c 'tail -n 10 /opt/seagull/diameter-env/run/ep1.out | grep "Failed calls"'| xargs | cut -d '|' -f 4) + echo -e "\n" + $dexec $master loxicmd get ct --servName=sctpmh2 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "133.133.133.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "134.134.134.1 | 32.32.32.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "135.135.135.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + + echo "Counters: $p1c_new $p2c_new $p3c_new" + + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [ACTIVE]" + p1=1 + echo -e "Turning off this path at User.\nEP----->LB--x-->User" + $hexec user ip link set euserr1 down; + down=1 + else + if [[ $down == 1 ]]; then + p1dok=1 + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 NOT ACTIVE - [OK]" + else + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + + echo -e "\n" + if [[ $recover == 1 ]]; then + printf "\t***Setup Recovered***" + fi + echo -e "\n\n" + + if [[ $fail_new -gt $fail_old && $down == 1 && $recover == 0 ]]; then + printf "Failed Calls: \t%10s \t[INCREASING]\n" $fail_new + fstart=1 + code=1 + calls=0 + else + if [[ $fail_new -eq $fail_old ]]; then + if [[ $down == 1 && $fstart == 1 ]]; then + printf "Failed Calls: \t%10s \t[STABLE]\n" $fail_new + frecover=1 + code=0 + else + printf "Failed Calls: \t%10s\n" $fail_new + fi + fi + fi + + if [[ $call_new -gt $call_old ]]; then + printf "Successful Calls: \t%10s \t[ACTIVE]\n" $call_new + calls=1 + if [[ $down == 1 && $frecover == 1 ]]; then + recover=1 + fi + else + printf "Successful Calls: \t%10s \t[NOT ACTIVE]\n" $call_new + fi + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + call_old=$call_new + fail_old=$fail_new + echo -e "\n" + sleep 5 +done + +#sudo rm -rf *.out +#sudo pkill sctp_test + +#Restore +$hexec user ip link set euserr1 up +$hexec user ip route add default via 1.1.1.254 +sudo docker exec -dt user ksh -c 'sed -i 's/source=0.0.0.0/source=31.31.31.1/g' /opt/seagull/diameter-env/config/conf.server.xml' +sudo docker exec -dt ep1 ksh -c 'sed -i 's/dest=133.133.133.1/dest=20.20.20.1/g' /opt/seagull/diameter-env/config/conf.client.xml' +sudo docker exec -dt ep1 ksh -c "sed -i 's/\"call-rate\" value=\"100\"/\"call-rate\" value=\"5000\"/g' /opt/seagull/diameter-env/config/conf.client.xml" + +if [[ $calls == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $recover == 1 ]]; then + echo "sctpmh SCTP Multihoming E2E Multipath Failover [OK]" + echo "OK" > /vagrant/status4.txt + restart_loxilbs +else + echo "NOK" > /vagrant/status4.txt + echo "sctpmh SCTP Multihoming E2E Multipath Failover [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo "-----------------------------" + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh-seagull/validation5.sh b/cicd/sctpmh-seagull/validation5.sh new file mode 100755 index 000000000..8bb74733d --- /dev/null +++ b/cicd/sctpmh-seagull/validation5.sh @@ -0,0 +1,168 @@ +#!/bin/bash +source /vagrant/common.sh +source /vagrant/check_ha.sh + +echo -e "sctpmh: SCTP Multihoming - C2LB HA Failover Test. Client and LB Multihomed, EP is uni-homed\n" +extIP="20.20.20.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" + +echo -e "\nTraffic Flow: User -> LB -> EP " + +sudo docker exec -dt ep1 ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 120 stdbuf -oL seagull -conf ../config/conf.server.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.server.xml > ep1.out' 2>&1 > /dev/null & +sleep 2 + +sudo docker exec -dt user ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 100 stdbuf -oL seagull -conf ../config/conf.client.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.client.xml > user.out' 2>&1 > /dev/null & + +sleep 20 + +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +checkha=0 +hadone=0 +code=0 +nsyncOk=0 +frecover=1 + +for((i=0;i<15;i++)) do + syncOk=$nsyncOk + if [[ $checkha == 1 ]]; then + check_ha + echo -e "\nHA state Master:$master BACKUP-$backup\n" + nsyncOk=$(checkSync) + if [[ $nsyncOk == 2 ]]; then #No active connections in Master, no need to continue. + break; + fi + fi + $dexec user bash -c 'tail -n 25 /opt/seagull/diameter-env/run/user.out' + call_new=$(sudo docker exec -t user bash -c 'tail -n 10 /opt/seagull/diameter-env/run/user.out | grep "Successful calls"'| xargs | cut -d '|' -f 4) + fail_new=$(sudo docker exec -t user bash -c 'tail -n 10 /opt/seagull/diameter-env/run/user.out | grep "Failed calls"'| xargs | cut -d '|' -f 4) + + $dexec $master loxicmd get ct --servName=sctpmh1 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "20.20.20.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "21.21.21.1 | 2.2.2.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh1 | grep "22.22.22.1 | 1.1.1.1" | xargs | cut -d '|' -f 10) + + echo "Counters: $p1c_new $p2c_new $p3c_new" + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 1.1.1.1 -> 20.20.20.1 [ACTIVE]" + p1=1 + else + echo "Path 1: 1.1.1.1 -> 20.20.20.1 [NOT ACTIVE]" + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 2.2.2.1 -> 21.21.21.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 2.2.2.1 -> 21.21.21.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 1.1.1.1 -> 22.22.22.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 1.1.1.1 -> 22.22.22.1 [NOT ACTIVE]" + fi + + echo -e "\n" + if [[ $recover == 1 ]]; then + printf "\t***Setup Recovered***" + fi + echo -e "\n\n" + + if [[ $fail_new -gt $fail_old && $hadone == 1 && $recover == 0 ]]; then + echo -e "Failed Calls: \t\t$fail_new \t\t[INCREASING]" + fstart=1 + frecover=0 + calls=0 + code=1 + else + if [[ $fail_new -eq $fail_old ]]; then + if [[ $hadone == 1 ]]; then + printf "Failed Calls: \t%10s \t[STABLE]\n" $fail_new + frecover=1 + code=0 + else + printf "Failed Calls: \t%10s\n" $fail_new + fi + + fi + fi + + if [[ $call_new -gt $call_old ]]; then + printf "Successful Calls: \t%10s \t[ACTIVE]\n" $call_new + calls=1 + if [[ $hadone == 1 && $frecover == 1 ]]; then + recover=1 + fi + else + printf "Successful Calls: \t%10s \t[NOT ACTIVE]\n" $call_new + fi + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + call_old=$call_new + fail_old=$fail_new + + if [[ $hadone == 0 ]]; then + nsyncOk=$(checkSync) + if [[ $nsyncOk == 1 ]]; then + restart_mloxilb + checkha=1 + hadone=1 + calls=0 + fi + fi + sleep 5 +done + +if [[ $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $nsyncOk == 1 && $recover == 1 ]]; then + echo "sctpmh SCTP Multihoming C2LB HA Failover [OK]" + echo "OK" > /vagrant/status5.txt + restart_loxilbs +else + echo "NOK" > /vagrant/status5.txt + echo "sctpmh SCTP Multihoming C2LB HA Failover [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + echo "-----------------------------" + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh-seagull/validation6.sh b/cicd/sctpmh-seagull/validation6.sh new file mode 100755 index 000000000..768d6a485 --- /dev/null +++ b/cicd/sctpmh-seagull/validation6.sh @@ -0,0 +1,175 @@ +#!/bin/bash +source /vagrant/common.sh +source /vagrant/check_ha.sh + +echo -e "sctpmh: SCTP Multihoming - E2E HA Failover Test. Client, LB Multihomed and EP all multi-homed\n" +extIP="133.133.133.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" + +echo -e "\nTraffic Flow: EP ---> LB ---> User" + +sudo docker exec -dt user ksh -c 'sed -i 's/source=31.31.31.1/source=0.0.0.0/g' /opt/seagull/diameter-env/config/conf.server.xml' + +sudo docker exec -dt user ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 220 stdbuf -oL seagull -conf ../config/conf.server.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.server.xml > user.out' 2>&1 > /dev/null & +sleep 2 + +sudo docker exec -dt ep1 ksh -c 'sed -i 's/dest=20.20.20.1/dest=133.133.133.1/g' /opt/seagull/diameter-env/config/conf.client.xml' +sudo docker exec -dt ep1 ksh -c 'export LD_PRELOAD=/usr/local/bin/libsctplib.so.1.0.8; export LD_LIBRARY_PATH=/usr/local/bin; cd /opt/seagull/diameter-env/run/; timeout 200 stdbuf -oL seagull -conf ../config/conf.client.xml -dico ../config/base_s6a.xml -scen ../scenario/ulr-ula.client.xml > ep1.out' 2>&1 > /dev/null & + + +sleep 20 + +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +checkha=0 +hadone=0 +code=0 +nsyncOk=0 +frecover=1 + +for((i=0;i<15;i++)) do + syncOk=$nsyncOk + if [[ $checkha == 1 ]]; then + check_ha + echo -e "\nHA state Master:$master BACKUP-$backup\n" + nsyncOk=$(checkSync) + if [[ $nsyncOk == 2 ]]; then #No active connections in Master, no need to continue. + break; + fi + fi + $dexec ep1 bash -c 'tail -n 25 /opt/seagull/diameter-env/run/ep1.out' + call_new=$(sudo docker exec -t ep1 bash -c 'tail -n 10 /opt/seagull/diameter-env/run/ep1.out | grep "Successful calls"'| xargs | cut -d '|' -f 4) + fail_new=$(sudo docker exec -t ep1 bash -c 'tail -n 10 /opt/seagull/diameter-env/run/ep1.out | grep "Failed calls"'| xargs | cut -d '|' -f 4) + echo -e "\n" + $dexec $master loxicmd get ct --servName=sctpmh2 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "133.133.133.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "134.134.134.1 | 32.32.32.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "135.135.135.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + + + echo "Counters: $p1c_new $p2c_new $p3c_new" + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [ACTIVE]" + p1=1 + else + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + echo -e "\n" + if [[ $recover == 1 ]]; then + printf "\t***Setup Recovered***" + fi + echo -e "\n\n" + + + if [[ $fail_new -gt $fail_old && $hadone == 1 && $recover == 0 ]]; then + echo -e "Failed Calls: \t\t$fail_new \t\t[INCREASING]" + fstart=1 + frecover=0 + calls=0 + code=1 + else + if [[ $fail_new -eq $fail_old ]]; then + if [[ $hadone == 1 ]]; then + printf "Failed Calls: \t%10s \t[STABLE]\n" $fail_new + frecover=1 + code=0 + else + printf "Failed Calls: \t%10s\n" $fail_new + fi + fi + fi + + if [[ $call_new -gt $call_old ]]; then + printf "Successful Calls: \t%10s \t[ACTIVE]\n" $call_new + calls=1 + if [[ $hadone == 1 && $frecover == 1 ]]; then + recover=1 + fi + else + printf "Successful Calls: \t%10s \t[NOT ACTIVE]\n" $call_new + fi + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + call_old=$call_new + fail_old=$fail_new + + if [[ $hadone == 0 ]]; then + nsyncOk=$(checkSync) + if [[ $nsyncOk == 1 ]]; then + restart_mloxilb + checkha=1 + hadone=1 + fi + fi + sleep 5 +done + +#Revert +sudo docker exec -dt user ksh -c 'sed -i 's/source=0.0.0.0/source=31.31.31.1/g' /opt/seagull/diameter-env/config/conf.server.xml' +sudo docker exec -dt ep1 ksh -c 'sed -i 's/dest=133.133.133.1/dest=20.20.20.1/g' /opt/seagull/diameter-env/config/conf.client.xml' + +if [[ $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $nsyncOk == 1 && $recover == 1 ]]; then + echo "sctpmh SCTP Multihoming E2E HA Failover [OK]" + echo "OK" > /vagrant/status6.txt + restart_loxilbs +else + echo "NOK" > /vagrant/status6.txt + echo "sctpmh SCTP Multihoming E2E HA Failover [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + echo "-----------------------------" + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/check_ha.sh b/cicd/sctpmh/check_ha.sh index b04922194..b31574fb0 100644 --- a/cicd/sctpmh/check_ha.sh +++ b/cicd/sctpmh/check_ha.sh @@ -4,62 +4,131 @@ master="llb1" backup="llb2" function check_ha() { -while : ; do - status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') - status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') - count=0 - if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]]; - then - master="llb1" - backup="llb2" - break - elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]]; - then - master="llb2" - backup="llb1" - break - else - count=$(( $count + 1 )) - if [[ $count -ge 20 ]]; then + while : ; do + status1=$($hexec llb1 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') + status2=$($hexec llb2 curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/cistate/all' -H 'accept: application/json' | jq -r '.Attr[0].state') + count=0 + if [[ $status1 == "MASTER" && $status2 == "BACKUP" ]]; + then + master="llb1" + backup="llb2" + break + elif [[ $status2 == "MASTER" && $status1 == "BACKUP" ]]; + then + master="llb2" + backup="llb1" + break + else + count=$(( $count + 1 )) + if [[ $count -ge 20 ]]; then + echo "KA llb1-$status1, llb2-$status2 [NOK] - Exiting" >&2 + exit 1; + fi echo "KA llb1-$status1, llb2-$status2 [NOK]" >&2 - exit 1; + sleep 5 fi - sleep 5 - fi -done + done } function checkSync() { -count=1 -sync=0 -while [[ $count -le 5 ]] ; do -echo -e "\nStatus at MASTER:$master\n" >&2 -$dexec $master loxicmd get ct | grep est >&2 + count=1 + sync=0 + while [[ $count -le 5 ]] ; do + echo -e "\nStatus at MASTER:$master\n" >&2 + ct=`$dexec $master loxicmd get ct | grep est` + echo "${ct//'\n'/$'\n'}" >&2 -echo -e "\nStatus at BACKUP:$backup\n" >&2 -$dexec $backup loxicmd get ct | grep est >&2 + echo -e "\nStatus at BACKUP:$backup\n" >&2 + ct=`$dexec $backup loxicmd get ct | grep est` + echo "${ct//'\n'/$'\n'}" >&2 -nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) -nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + nres1=$($hexec $master curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) + nres2=$($hexec $backup curl -sX 'GET' 'http://0.0.0.0:11111/netlox/v1/config/conntrack/all' -H 'accept: application/json' | grep -ow "\"conntrackState\":\"est\"" | wc -l) -if [[ $nres1 == 0 ]]; then - echo -e "No active connections in Master:$master. Exiting!" >&2 - return 2 -fi + if [[ $nres1 == 0 ]]; then + echo -e "No active connections in Master:$master. Exiting!" >&2 + return 2 + fi -if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then - echo -e "\nConnections sync successful!!!\n" >&2 - sync=1 - break; -fi -echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2 -count=$(( $count + 1 )) -sleep 2 -done + if [[ $nres1 == $nres2 && $nres1 != 0 ]]; then + echo -e "\nConnections sync successful!!!\n" >&2 + sync=1 + break; + fi + echo -e "\nConnections sync pending.. Let's wait a little more..\n" >&2 + count=$(( $count + 1 )) + sleep 2 + done -if [[ $sync == 0 ]]; then - echo -e "\nConnection Sync failed\n" >&2 - return 0 -fi -echo "$sync" + if [[ $sync == 0 ]]; then + echo -e "\nConnection Sync failed\n" >&2 + return 0 + fi + echo "$sync" } + +function restart_mloxilb() { + if [[ $master == "llb1" ]]; then + pat="cluster=172.17.0.3" + copts=" --cluster=172.17.0.3" + self=" --self=0" + ka=" --ka=172.17.0.3:172.17.0.2" + else + pat="cluster=172.17.0.2" + copts=" --cluster=172.17.0.2" + self=" --self=1" + ka=" --ka=172.17.0.2:172.17.0.3" + fi + echo "Restarting MASTER: $master" + pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) + echo "Killing $pid" >&2 + docker exec -dt $master kill -9 $pid + docker exec -dt $master ip link del llb0 + docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null & + pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 +} + +function restart_loxilbs() { + if [[ $master == "llb1" ]]; then + mpat="cluster=172.17.0.3" + mcopts=" --cluster=172.17.0.3" + mself=" --self=0" + mka=" --ka=172.17.0.3:172.17.0.2" + + bpat="cluster=172.17.0.2" + bcopts=" --cluster=172.17.0.2" + bself=" --self=1" + bka=" --ka=172.17.0.2:172.17.0.3" + else + mpat="cluster=172.17.0.2" + mcopts=" --cluster=172.17.0.2" + mself=" --self=1" + mka=" --ka=172.17.0.2:172.17.0.3" + + bpat="cluster=172.17.0.3" + bcopts=" --cluster=172.17.0.3" + bself=" --self=0" + bka=" --ka=172.17.0.3:172.17.0.2" + fi + echo "Restarting $master" + pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2) + echo "Killing $mpid" >&2 + docker exec -dt $master kill -9 $pid + docker exec -dt $master ip link del llb0 + docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $mcopts $mself $mka > /dev/null & + pid=$(docker exec -i $master ps -aef | grep $mpat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 + + echo "Restarting $backup" + pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2) + echo "Killing $pid" >&2 + docker exec -dt $backup kill -9 $pid + docker exec -dt $backup ip link del llb0 + docker exec -dt $backup nohup /root/loxilb-io/loxilb/loxilb $bcopts $bself $bka > /dev/null & + pid=$(docker exec -i $backup ps -aef | grep $bpat | xargs | cut -d ' ' -f 2) + echo "New loxilb pid: $pid" >&2 + +} + + diff --git a/cicd/sctpmh/config.sh b/cicd/sctpmh/config.sh index 411d4e5d4..b14694a07 100755 --- a/cicd/sctpmh/config.sh +++ b/cicd/sctpmh/config.sh @@ -79,14 +79,15 @@ $hexec ep1 ip route change default via 31.31.31.254 # Backup paths in user $hexec user ip route add 124.124.124.1/32 via 2.2.2.254 -$hexec user ip route add 125.125.125.1/32 via 2.2.2.254 +#$hexec user ip route add 125.125.125.1/32 via 2.2.2.254 +#$hexec user ip route add 32.32.32.1/32 via 2.2.2.254 $hexec user ip route add 134.134.134.1/32 via 2.2.2.254 -$hexec user ip route add 135.135.135.1/32 via 2.2.2.254 +#$hexec user ip route add 135.135.135.1/32 via 2.2.2.254 -$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254 -$hexec ep1 ip route add 125.125.125.1/32 via 32.32.32.254 +#$hexec ep1 ip route add 124.124.124.1/32 via 32.32.32.254 +#$hexec ep1 ip route add 125.125.125.1/32 via 31.31.31.254 $hexec ep1 ip route add 134.134.134.1/32 via 32.32.32.254 -$hexec ep1 ip route add 135.135.135.1/32 via 32.32.32.254 +$hexec ep1 ip route add 135.135.135.1/32 via 31.31.31.254 $hexec llb1 ip route add 1.1.1.0/24 via 11.11.11.253 $hexec llb1 ip route add 2.2.2.0/24 via 11.11.11.254 @@ -107,10 +108,10 @@ create_lb_rule llb2 123.123.123.1 --name=sctpmh1 --secips=124.124.124.1,125.125. create_lb_rule llb1 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat create_lb_rule llb2 133.133.133.1 --name=sctpmh2 --secips=134.134.134.1,135.135.135.1 --sctp=2020:9999 --endpoints=1.1.1.1:1 --mode=fullnat -$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none -$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none -$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_9999 --probetype=none -$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_9999 --probetype=none +$dexec llb1 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping +$dexec llb1 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping +$dexec llb2 loxicmd create ep 1.1.1.1 --name=1.1.1.1_sctp_8080 --probetype=ping +$dexec llb2 loxicmd create ep 31.31.31.1 --name=31.31.31.1_sctp_8080 --probetype=ping create_lb_rule llb1 11.11.11.11 --tcp=80:8080 --endpoints=31.31.31.1:1 diff --git a/cicd/sctpmh/validation.sh b/cicd/sctpmh/validation.sh index 34cd2749d..8a62cfff2 100755 --- a/cicd/sctpmh/validation.sh +++ b/cicd/sctpmh/validation.sh @@ -1,40 +1,55 @@ #!/bin/bash code=0 -echo "SCTP Multihoming - Test case #1" -echo -e "*********************************************************************************" -./validation1.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #2" -echo -e "*********************************************************************************" -./validation2.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #3" -echo -e "*********************************************************************************" -./validation3.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #4" -echo -e "*********************************************************************************" -./validation4.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\nSCTP Multihoming - Test case #5" -echo -e "*********************************************************************************" -sleep 60 -./validation5.sh -if [[ $? == 1 ]]; then - code=1 -fi -echo -e "\n\n\n*********************************************************************************" +tc=( "Basic Test - Client & EP Uni-homed and LB is Multi-homed" "Multipath Test, Client and LB Multihomed, EP is uni-homed" "C2LB Multipath Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E Multipath Failover Test - Client, LB and EP all Multihomed" "C2LB HA Failover Test - Client and LB Multihomed, EP is uni-homed" "E2E HA Failover Test. Client, LB and EP all Multihomed" ) +padding="............................................................................................................." +border="**************************************************************************************************************************************************" + +for((j=0,i=1; i<=6; i++, j++)); do + echo "SCTP Multihoming - Test case #$i" + echo -e "\n\n\n$border\n" + ./validation$i.sh + echo -e "\n\n" + file=status$i.txt + status=`cat $file` + title=${tc[j]} + echo -e "\n\n" + + if [[ $status == "NOK" ]]; then + code=1 + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]"; + else + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]"; + fi + echo -e "\n\n\n$border\n\n" + + sleep 30 +done + +echo -e "\n\n\n$border\n" +printf "================================================== SCTP MULTIHOMING CONSOLIDATED RESULT ==========================================================\n" +for((j=0,i=1; i<=6; i++, j++)); do + file=status$i.txt + status=`cat $file` + title=${tc[j]} + echo -e "\n\n" + + if [[ $status == "NOK" ]]; then + code=1 + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[FAILED]"; + else + printf "Test case #%2s - %s%s %s\n" "$i" "$title" "${padding:${#title}}" "[PASSED]"; + fi +done + +echo -e "\n$border" + +echo -e "\n\n\n$border\n" if [[ $code == 0 ]]; then - echo -e "\n\n SCTP Multihoming CICD [OK]" + echo -e "SCTP Multihoming with sctp_test CICD [OK]" else - echo -e "\n\n SCTP Multihoming CICD [NOK]" + echo -e "SCTP Multihoming with sctp_test CICD [NOK]" fi +echo -e "\n$border\n" + +sudo rm -rf statu*.txt exit $code diff --git a/cicd/sctpmh/validation1.sh b/cicd/sctpmh/validation1.sh index 6043bf114..f41fffce4 100755 --- a/cicd/sctpmh/validation1.sh +++ b/cicd/sctpmh/validation1.sh @@ -1,10 +1,13 @@ #!/bin/bash source ../common.sh +source check_ha.sh echo -e "sctpmh: SCTP Multihoming Basic Test - Client & EP Uni-homed and LB is Multi-homed\n" extIP="123.123.123.1" port=2020 +check_ha + echo "SCTP Multihoming service sctp-lb -> $extIP:$port" echo -e "------------------------------------------------------------------------------------\n" @@ -26,7 +29,10 @@ sudo pkill sctp_darn if [[ "$res" == "$exp" ]]; then echo $res echo -e "\nsctpmh SCTP Multihoming service Basic Test [OK]\n" + echo "OK" > status1.txt + restart_loxilbs else + echo "NOK" > status1.txt echo "sctpmh SCTP Multihoming service Basic Test [NOK]" echo "Expected : $exp" echo "Received : $res" @@ -51,11 +57,12 @@ else echo "llb1 ep-info" $dexec llb1 loxicmd get ep echo "llb1 bpf-info" - $dexec llb1 ntc filter show dev eth0 ingress + $dexec llb1 tc filter show dev eth0 ingress echo "BFP trace -- " sudo timeout 5 cat /sys/kernel/debug/tracing/trace_pipe sudo killall -9 cat echo "BFP trace -- " + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation2.sh b/cicd/sctpmh/validation2.sh index c9dbfd192..122fb03a8 100755 --- a/cicd/sctpmh/validation2.sh +++ b/cicd/sctpmh/validation2.sh @@ -12,10 +12,10 @@ echo -e "----------------------------------------------------------------------- echo -e "\nHA state Master:$master BACKUP-$backup\n" -$hexec ep1 sctp_test -H 0.0.0.0 -P 9999 -l > ep1.out & +$hexec ep1 sctp_test -H 31.31.31.1 -P 9999 -l > ep1.out & sleep 2 -$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 50000 > user.out & +$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -c 6 -x 10000 > user.out & #Path counters p1c_old=0 @@ -26,7 +26,7 @@ p3c_old=0 p3c_new=0 for((i=0;i<100;i++)) do - fin=`tail -n 100 user.out | grep "Client: Sending packets.(50000/50000)"` + fin=`tail -n 100 user.out | grep "Client: Sending packets.(10000/10000)"` if [[ ! -z $fin ]]; then fin=1 echo "sctp_test done." @@ -72,7 +72,10 @@ sudo pkill sctp_test sudo rm *.out if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 ]]; then echo "sctpmh SCTP Multihoming Multipath [OK]" + echo "OK" > status2.txt + restart_loxilbs else + echo "NOK" > status2.txt echo "sctpmh SCTP Multihoming Multipath [NOK]" echo -e "\nuser" sudo ip netns exec user ip route @@ -99,6 +102,7 @@ else $dexec llb2 loxicmd get lb echo "llb2 ep-info" $dexec llb2 loxicmd get ep + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation3.sh b/cicd/sctpmh/validation3.sh index 9231b03d0..b80ca3eed 100755 --- a/cicd/sctpmh/validation3.sh +++ b/cicd/sctpmh/validation3.sh @@ -15,10 +15,10 @@ echo -e "\nHA state Master:$master BACKUP-$backup\n" echo -e "\nTraffic Flow: User -> LB -> EP " -$hexec ep1 sctp_test -H 0.0.0.0 -P 9999 -l > ep1.out & +$hexec ep1 sctp_test -H 31.31.31.1 -P 9999 -l > ep1.out & sleep 2 -$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 50000 > user.out & +$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -c 6 -x 1000 > user.out & #Path counters p1c_old=0 p1c_new=0 @@ -28,8 +28,11 @@ p3c_old=0 p3c_new=0 down=0 code=0 + +sleep 5 + for((i=0;i<200;i++)) do - fin=`tail -n 100 user.out | grep "Client: Sending packets.(50000/50000)"` + fin=`tail -n 100 user.out | grep "Client: Sending packets.(1000/1000)"` if [[ ! -z $fin ]]; then fin=1 echo "sctp_test done." @@ -92,7 +95,10 @@ $hexec user ip route add default via 1.1.1.254 if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 ]]; then echo "sctpmh SCTP Multihoming Multipath Failover [OK]" + echo "OK" > status3.txt + restart_loxilbs else + echo "NOK" > status3.txt echo "sctpmh SCTP Multihoming Multipath Failover [NOK]" echo -e "\nuser" sudo ip netns exec user ip route @@ -119,6 +125,7 @@ else echo "llb2 ep-info" $dexec llb2 loxicmd get ep echo "-----------------------------" + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation4.sh b/cicd/sctpmh/validation4.sh index b5ce09f19..4bb191834 100755 --- a/cicd/sctpmh/validation4.sh +++ b/cicd/sctpmh/validation4.sh @@ -1,7 +1,7 @@ #!/bin/bash source ../common.sh source check_ha.sh -echo -e "sctpmh: SCTP Multihoming - Multipath Failover Test. Client, LB and EP all Multihomed\n" +echo -e "sctpmh: SCTP Multihoming - E2E Multipath Failover Test. Client, LB and EP all Multihomed\n" extIP="133.133.133.1" port=2020 @@ -13,10 +13,10 @@ echo -e "----------------------------------------------------------------------- echo -e "\nHA state Master:$master BACKUP-$backup\n" echo -e "\nTraffic Flow: EP ---> LB ---> User" -$hexec user sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 9999 -l > user.out & +$hexec user sctp_test -H 0.0.0.0 -P 9999 -l > user.out & sleep 2 -$hexec ep1 stdbuf -oL sctp_test -H 31.31.31.1 -B 32.32.32.1 -P 20000 -h $extIP -p $port -s -m 100 -x 50000 > ep1.out & +$hexec ep1 stdbuf -oL sctp_test -H 31.31.31.1 -B 32.32.32.1 -P 20000 -h $extIP -p $port -s -c 6 -x 1000 > ep1.out & #Path counters p1c_old=0 @@ -27,8 +27,10 @@ p3c_old=0 p3c_new=0 down=0 code=0 +sleep 2 + for((i=0;i<200;i++)) do - fin=`tail -n 100 ep1.out | grep "Client: Sending packets.(50000/50000)"` + fin=`tail -n 100 ep1.out | grep "Client: Sending packets.(1000/1000)"` if [[ ! -z $fin ]]; then fin=1 echo "sctp_test done." @@ -91,7 +93,10 @@ $hexec user ip route add default via 1.1.1.254 if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 ]]; then echo "sctpmh SCTP Multihoming E2E Multipath Failover [OK]" + echo "OK" > status4.txt + restart_loxilbs else + echo "NOK" > status4.txt echo "sctpmh SCTP Multihoming E2E Multipath Failover [NOK]" echo -e "\nuser" sudo ip netns exec user ip route @@ -118,6 +123,7 @@ else $dexec llb2 loxicmd get lb echo "llb2 ep-info" $dexec llb2 loxicmd get ep + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation5.sh b/cicd/sctpmh/validation5.sh index a2d52bc61..757a65d4b 100755 --- a/cicd/sctpmh/validation5.sh +++ b/cicd/sctpmh/validation5.sh @@ -14,11 +14,12 @@ echo -e "----------------------------------------------------------------------- echo -e "\nHA state Master:$master BACKUP-$backup\n" echo -e "\nTraffic Flow: User -> LB -> EP " +sudo pkill sctp_test -$hexec ep1 sctp_test -H 0.0.0.0 -P 9999 -l > ep1.out & +$hexec ep1 sctp_test -H 31.31.31.1 -P 9999 -l > ep1.out & sleep 2 -$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -m 100 -x 200000 > user.out & +$hexec user stdbuf -oL sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 20000 -h $extIP -p $port -s -c 6 -x 10000 > user.out & #Path counters p1c_old=0 @@ -32,29 +33,8 @@ hadone=0 code=0 nsyncOk=0 -function restart_mloxilb() { - if [[ $master == "llb1" ]]; then - pat="cluster=172.17.0.3" - copts=" --cluster=172.17.0.3" - self=" --self=0" - ka=" --ka=172.17.0.3:172.17.0.2" - else - pat="cluster=172.17.0.2" - copts=" --cluster=172.17.0.2" - self=" --self=1" - ka=" --ka=172.17.0.2:172.17.0.3" - fi - pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) - echo Killing $pid >&2 - docker exec -dt $master kill -9 $pid - docker exec -dt $master ip link del llb0 - docker exec -dt $master nohup /root/loxilb-io/loxilb/loxilb $copts $self $ka > /dev/null & - pid=$(docker exec -i $master ps -aef | grep $pat | xargs | cut -d ' ' -f 2) - echo "New loxilb pid: $pid" >&2 -} - -for((i=0;i<200;i++)) do - fin=`tail -n 100 user.out | grep "Client: Sending packets.(200000/200000)"` +for((i=0;i<400;i++)) do + fin=`tail -n 100 user.out | grep "Client: Sending packets.(10000/10000)"` if [[ ! -z $fin ]]; then fin=1 echo "sctp_test done." @@ -65,7 +45,7 @@ for((i=0;i<200;i++)) do check_ha echo -e "\nHA state Master:$master BACKUP-$backup\n" nsyncOk=$(checkSync) - if [[ $nsyncOk == 2 ]]; then + if [[ $nsyncOk == 2 ]]; then #No active connections in Master, no need to continue. break; fi fi @@ -115,9 +95,12 @@ sudo rm -rf *.out sudo pkill sctp_test if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $syncOk == 1 ]]; then - echo "sctpmh SCTP Multihoming HA Failover [OK]" + echo "sctpmh SCTP Multihoming C2LB HA Failover [OK]" + echo "OK" > status5.txt + restart_loxilbs else - echo "sctpmh SCTP Multihoming HA Failover [NOK]" + echo "NOK" > status5.txt + echo "sctpmh SCTP Multihoming C2LB HA Failover [NOK]" echo -e "\nuser" sudo ip netns exec user ip route echo -e "\nr1" @@ -143,6 +126,7 @@ else echo "llb2 ep-info" $dexec llb2 loxicmd get ep echo "-----------------------------" + restart_loxilbs exit 1 fi echo -e "------------------------------------------------------------------------------------\n\n\n" diff --git a/cicd/sctpmh/validation6.sh b/cicd/sctpmh/validation6.sh new file mode 100755 index 000000000..585ed17c3 --- /dev/null +++ b/cicd/sctpmh/validation6.sh @@ -0,0 +1,140 @@ +#!/bin/bash +source ../common.sh +source check_ha.sh + +echo -e "sctpmh: SCTP Multihoming - E2E HA Failover Test. Client, LB and EP all Multi-homed\n" +extIP="133.133.133.1" +port=2020 + +check_ha + +echo "SCTP Multihoming service sctp-lb(Multipath traffic) -> $extIP:$port" +echo -e "------------------------------------------------------------------------------------\n" + +echo -e "\nHA state Master:$master BACKUP-$backup\n" + +echo -e "\nTraffic Flow: EP ---> LB ---> User" + +sudo pkill sctp_test + +#$hexec user sctp_test -H 1.1.1.1 -B 2.2.2.1 -P 9999 -l > user.out & +$hexec user sctp_test -H 0.0.0.0 -P 9999 -l > user.out & +sleep 2 + +$hexec ep1 stdbuf -oL sctp_test -H 31.31.31.1 -B 32.32.32.1 -P 20000 -h $extIP -p $port -s -c 6 -x 10000 > ep1.out & + +#Path counters +p1c_old=0 +p1c_new=0 +p2c_old=0 +p2c_new=0 +p3c_old=0 +p3c_new=0 +checkha=0 +hadone=0 +code=0 +nsyncOk=0 + +for((i=0;i<500;i++)) do + fin=`tail -n 100 ep1.out | grep "Client: Sending packets.(10000/10000)"` + if [[ ! -z $fin ]]; then + fin=1 + echo "sctp_test done." + break; + fi + + syncOk=$nsyncOk + if [[ $checkha == 1 ]]; then + check_ha + echo -e "\nHA state Master:$master BACKUP-$backup\n" + nsyncOk=$(checkSync) + if [[ $nsyncOk == 2 ]]; then #No active connections in Master, no need to continue. + break; + fi + fi + echo -e "\n" + $dexec $master loxicmd get ct --servName=sctpmh2 + echo -e "\n" + p1c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "133.133.133.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + p2c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "134.134.134.1 | 32.32.32.1" | xargs | cut -d '|' -f 10) + p3c_new=$(sudo docker exec -i $master loxicmd get ct --servName=sctpmh2 | grep "135.135.135.1 | 31.31.31.1" | xargs | cut -d '|' -f 10) + + + echo "Counters: $p1c_new $p2c_new $p3c_new" + if [[ $p1c_new -gt $p1c_old ]]; then + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [ACTIVE]" + p1=1 + else + echo "Path 1: 31.31.31.1 -> 133.133.133.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + + if [[ $p2c_new -gt $p2c_old ]]; then + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [ACTIVE]" + p2=1 + else + echo "Path 2: 32.32.32.1 -> 134.134.134.1 -> 2.2.2.1 [NOT ACTIVE]" + fi + + if [[ $p3c_new -gt $p3c_old ]]; then + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [ACTIVE]" + p3=1 + else + echo "Path 3: 31.31.31.1 -> 135.135.135.1 -> 1.1.1.1 [NOT ACTIVE]" + fi + echo -e "\n" + + p1c_old=$p1c_new + p2c_old=$p1c_new + p2c_old=$p1c_new + + if [[ $hadone == 0 ]]; then + nsyncOk=$(checkSync) + if [[ $nsyncOk == 1 ]]; then + restart_mloxilb + checkha=1 + hadone=1 + fi + fi + sleep 5 +done + + +sudo rm -rf *.out +sudo pkill sctp_test + +if [[ $fin == 1 && $p1 == 1 && $p2 == 1 && $p3 == 1 && $code == 0 && $nsyncOk == 1 ]]; then + echo "sctpmh SCTP Multihoming E2E HA Failover [OK]" + echo "OK" > status6.txt + restart_loxilbs +else + echo "NOK" > status6.txt + echo "sctpmh SCTP Multihoming E2E HA Failover [NOK]" + echo -e "\nuser" + sudo ip netns exec user ip route + echo -e "\nr1" + sudo ip netns exec r1 ip route + echo -e "\nr2" + sudo ip netns exec r2 ip route + echo -e "\nllb1" + sudo ip netns exec llb1 ip route + echo -e "\nllb2" + sudo ip netns exec llb2 ip route + echo -e "\nr3" + sudo ip netns exec r3 ip route + echo -e "\nr4" + sudo ip netns exec r4 ip route + echo "-----------------------------" + + echo -e "\nllb1 lb-info" + $dexec llb1 loxicmd get lb + echo "llb1 ep-info" + $dexec llb1 loxicmd get ep + echo -e "\nllb2 lb-info" + $dexec llb2 loxicmd get lb + echo "llb2 ep-info" + $dexec llb2 loxicmd get ep + echo "-----------------------------" + restart_loxilbs + exit 1 +fi +echo -e "------------------------------------------------------------------------------------\n\n\n"