Auto advertise VIPs in same subnet as local addrs

cicd/udplb/udp_jumbo_app.go

@@ -76,7 +76,6 @@ func main() {
 	}
 }
 
-
 func UDPClient() {
 
 	if len(os.Args) < 2 {

loxinet/layer3.go

@@ -226,13 +226,13 @@ func (l3 *L3H) IfaDeleteAll(Obj string) (int, error) {
 // IfaSelect - Given any ip address, select optimal ip address from Obj's ifa list
 // This is useful to determine source ip address when sending traffic
 // to the given ip address
-func (l3 *L3H) IfaSelect(Obj string, addr net.IP, findAny bool) (int, net.IP) {
+func (l3 *L3H) IfaSelect(Obj string, addr net.IP, findAny bool) (int, net.IP, string) {
 
 	key := IfaKey{Obj}
 	ifa := l3.IfaMap[key]
 
 	if ifa == nil {
-		return L3ObjErr, net.IPv4(0, 0, 0, 0)
+		return L3ObjErr, net.IPv4(0, 0, 0, 0), ""
 	}
 
 	for _, ifaEnt := range ifa.Ifas {
@@ -245,32 +245,70 @@ func (l3 *L3H) IfaSelect(Obj string, addr net.IP, findAny bool) (int, net.IP) {
 		}
 
 		if ifaEnt.IfaNet.Contains(addr) {
-			return 0, ifaEnt.IfaAddr
+			return 0, ifaEnt.IfaAddr, Obj
 		}
 	}
 
 	if findAny == false {
-		return L3AddrErr, net.IPv4(0, 0, 0, 0)
+		return L3AddrErr, net.IPv4(0, 0, 0, 0), ""
 	}
 
 	// Select first IP
 	if len(ifa.Ifas) > 0 {
-		return 0, ifa.Ifas[0].IfaAddr
+		return 0, ifa.Ifas[0].IfaAddr, Obj
+	}
+
+	return L3AddrErr, net.IPv4(0, 0, 0, 0), ""
+}
+
+// IfaFindAny - Given any ip address, check if it matches ip address in any ifa list
+// This is useful to determine if ip address is already assigned to some interface
+func (l3 *L3H) IfaFindAny(addr net.IP) (int, net.IP) {
+	for ifName := range l3.IfaMap {
+		ret, rAddr := l3.IfaFind(ifName.Obj, addr)
+		if ret == 0 {
+			return 0, rAddr
+		}
+	}
+	return L3AddrErr, nil
+}
+
+// IfaFind - Given any ip address, check if it matches ip address from Obj's ifa list
+// This is useful to determine if ip address is already assigned to some interface
+func (l3 *L3H) IfaFind(Obj string, addr net.IP) (int, net.IP) {
+
+	key := IfaKey{Obj}
+	ifa := l3.IfaMap[key]
+
+	if ifa == nil {
+		return L3ObjErr, net.IPv4(0, 0, 0, 0)
+	}
+
+	for _, ifaEnt := range ifa.Ifas {
+
+		if tk.IsNetIPv6(addr.String()) && tk.IsNetIPv4(ifaEnt.IfaNet.IP.String()) {
+			continue
+		}
+
+		if ifaEnt.IfaNet.IP.Equal(addr) {
+			return 0, ifaEnt.IfaAddr
+		}
 	}
 
 	return L3AddrErr, net.IPv4(0, 0, 0, 0)
 }
 
 // IfaSelectAny - Given any dest ip address, select optimal interface source ip address
 // This is useful to determine source ip address when sending traffic to the given ip address
-func (l3 *L3H) IfaSelectAny(addr net.IP, findAny bool) (int, net.IP) {
+func (l3 *L3H) IfaSelectAny(addr net.IP, findAny bool) (int, net.IP, string) {
 	var err int
 	var tDat tk.TrieData
 	var firstIP *net.IP
 
 	v6 := false
 	IfObj := ""
 	firstIP = nil
+	firstIfObj := ""
 
 	if tk.IsNetIPv4(addr.String()) {
 		err, _, tDat = l3.Zone.Rt.Trie4.FindTrie(addr.String())
@@ -294,7 +332,7 @@ func (l3 *L3H) IfaSelectAny(addr net.IP, findAny bool) (int, net.IP) {
 		}
 	}
 
-	if IfObj != "" {
+	if IfObj != "" && IfObj != "lo" {
 		return l3.IfaSelect(IfObj, addr, findAny)
 	}
 
@@ -313,24 +351,25 @@ func (l3 *L3H) IfaSelectAny(addr net.IP, findAny bool) (int, net.IP) {
 			}
 
 			if ifaEnt.IfaNet.Contains(addr) {
-				return 0, ifaEnt.IfaAddr
+				return 0, ifaEnt.IfaAddr, ifa.Key.Obj
 			}
 
 			if firstIP == nil {
 				firstIP = &ifaEnt.IfaAddr
+				firstIfObj = ifa.Key.Obj
 			}
 		}
 	}
 
 	if findAny == false {
-		return L3AddrErr, net.IPv4(0, 0, 0, 0)
+		return L3AddrErr, net.IPv4(0, 0, 0, 0), ""
 	}
 
 	if firstIP != nil {
-		return 0, *firstIP
+		return 0, *firstIP, firstIfObj
 	}
 
-	return L3AddrErr, net.IPv4(0, 0, 0, 0)
+	return L3AddrErr, net.IPv4(0, 0, 0, 0), ""
 }
 
 // Ifa2String - Format an ifa to a string

loxinet/loxinet.go

@@ -145,6 +145,8 @@ func loxiNetTicker(bgpPeerMode bool) {
 				pprof.StopCPUProfile()
 			} else if sig == syscall.SIGINT || sig == syscall.SIGTERM {
 				tk.LogIt(tk.LogCritical, "Shutdown on sig %v\n", sig)
+				// TODO - More subsystem cleanup TBD
+				mh.zr.Rules.RuleDestructAll()
 				if !bgpPeerMode {
 					mh.dpEbpf.DpEbpfUnInit()
 				}

loxinet/neighbor.go

@@ -134,7 +134,7 @@ func (n *NeighH) Activate(ne *Neigh) {
 		return
 	}
 
-	ret, Sip := n.Zone.L3.IfaSelect(ne.OifPort.Name, ne.Addr, true)
+	ret, Sip, _ := n.Zone.L3.IfaSelect(ne.OifPort.Name, ne.Addr, true)
 	if ret != 0 {
 		tk.LogIt(tk.LogDebug, "Failed to select l3 ifa select\n")
 		return
@@ -162,7 +162,7 @@ func (n *NeighH) NeighAddTunEP(ne *Neigh, rIP net.IP, sIP net.IP, tunID uint32,
 	}
 	if sIP == nil {
 		e := 0
-		e, sIP = n.Zone.L3.IfaSelect(port.Name, rIP, false)
+		e, sIP, _ = n.Zone.L3.IfaSelect(port.Name, rIP, false)
 		if e != 0 {
 			tk.LogIt(tk.LogError, "%s:ifa select error\n", port.Name)
 			return -1, nil

loxinet/rules.go

@@ -17,11 +17,13 @@
 package loxinet
 
 import (
+	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/loxilb-io/loxilb/api/loxinlp"
 	cmn "github.com/loxilb-io/loxilb/common"
 	tk "github.com/loxilb-io/loxilib"
 	probing "github.com/prometheus-community/pro-bing"
@@ -1212,6 +1214,8 @@ func (R *RuleH) AddNatLbRule(serv cmn.LbServiceArg, servSecIPs []cmn.LbSecIPArg,
 		R.tables[RtLB].rArr[r.ruleNum] = r
 	}
 
+	R.AdvRuleVIPIfL2(sNetAddr.IP)
+
 	r.DP(DpCreate)
 
 	return 0, nil
@@ -1270,6 +1274,10 @@ func (R *RuleH) DeleteNatLbRule(serv cmn.LbServiceArg) (int, error) {
 		R.tables[RtLB].rArr[rule.ruleNum] = nil
 	}
 
+	if IsIPHostAddr(sNetAddr.IP.String()) {
+		loxinlp.DelAddrNoHook(sNetAddr.IP.String()+"/32", "lo")
+	}
+
 	tk.LogIt(tk.LogDebug, "nat lb-rule deleted %s-%s\n", rule.tuples.String(), rule.act.String())
 
 	rule.DP(DpRemove)
@@ -1733,13 +1741,13 @@ func (R *RuleH) epCheckNow(ep *epHost) {
 			sType = "tcp"
 		} else if ep.opts.probeType == HostProbeConnectUDP {
 			sType = "udp"
-			ret, sIP := R.zone.L3.IfaSelectAny(net.ParseIP(ep.hostName), true)
+			ret, sIP, _ := R.zone.L3.IfaSelectAny(net.ParseIP(ep.hostName), true)
 			if ret == 0 {
 				sHint = sIP.String()
 			}
 		} else {
 			sType = "sctp"
-			ret, sIP := R.zone.L3.IfaSelectAny(net.ParseIP(ep.hostName), true)
+			ret, sIP, _ := R.zone.L3.IfaSelectAny(net.ParseIP(ep.hostName), true)
 			if ret == 0 {
 				sHint = sIP.String()
 			}
@@ -1899,6 +1907,8 @@ func (R *RuleH) RulesSync() {
 			rule.ruleNum, ruleKeys, ruleActs,
 			rule.stat.packets, rule.stat.bytes)
 
+		R.AdvRuleVIPIfL2(rule.tuples.l3Dst.addr.IP)
+
 		if rule.hChk.actChk == false {
 			continue
 		}
@@ -1932,22 +1942,28 @@ func (R *RuleH) RulesTicker() {
 func (R *RuleH) RuleDestructAll() {
 	var lbs cmn.LbServiceArg
 	var fwr cmn.FwRuleArg
+	fmt.Printf("Deleting Rules\n")
+
 	for _, r := range R.tables[RtLB].eMap {
 		lbs.ServIP = r.tuples.l3Dst.addr.IP.String()
-		if r.tuples.l4Dst.val == 6 {
+		fmt.Printf("Deleting %s\n", r.tuples.l3Dst.addr.IP.String())
+
+		if r.tuples.l4Prot.val == 6 {
 			lbs.Proto = "tcp"
-		} else if r.tuples.l4Dst.val == 1 {
+		} else if r.tuples.l4Prot.val == 1 {
 			lbs.Proto = "icmp"
-		} else if r.tuples.l4Dst.val == 17 {
+		} else if r.tuples.l4Prot.val == 17 {
 			lbs.Proto = "udp"
-		} else if r.tuples.l4Dst.val == 132 {
+		} else if r.tuples.l4Prot.val == 132 {
 			lbs.Proto = "sctp"
 		} else {
 			continue
 		}
 
 		lbs.ServPort = r.tuples.l4Dst.val
 
+		fmt.Printf("Deleting fin %s\n", r.tuples.l3Dst.addr.IP.String())
+
 		R.DeleteNatLbRule(lbs)
 	}
 	for _, r := range R.tables[RtFw].eMap {
@@ -2102,7 +2118,7 @@ func (r *ruleEnt) Nat2DP(work DpWorkT) int {
 		for idx := range nWork.endPoints {
 			ep := &nWork.endPoints[idx]
 			if mode == cmn.LBModeOneArm {
-				e, sip := r.zone.L3.IfaSelectAny(ep.XIP, false)
+				e, sip, _ := r.zone.L3.IfaSelectAny(ep.XIP, false)
 				if e != 0 {
 					tk.LogIt(tk.LogDebug, "Failed to find suitable source for %s\n", ep.XIP.String())
 					r.sync = DpCreateErr
@@ -2131,7 +2147,7 @@ func (r *ruleEnt) Nat2DP(work DpWorkT) int {
 		for idx := range nWork.endPoints {
 			ep := &nWork.endPoints[idx]
 			if tk.IsNetIPv6(nWork.ServiceIP.String()) && tk.IsNetIPv4(ep.XIP.String()) {
-				e, sip := r.zone.L3.IfaSelectAny(ep.XIP, false)
+				e, sip, _ := r.zone.L3.IfaSelectAny(ep.XIP, false)
 				if e != 0 {
 					r.sync = DpCreateErr
 					return -1
@@ -2256,3 +2272,40 @@ func (r *ruleEnt) DP(work DpWorkT) int {
 	return r.Fw2DP(work)
 
 }
+
+func (R *RuleH) AdvRuleVIPIfL2(IP net.IP) error {
+	ciState, _ := mh.has.CIStateGetInst(cmn.CIDefault)
+	if ciState == "MASTER" {
+		ev, _, iface := R.zone.L3.IfaSelectAny(IP, false)
+		if ev == 0 {
+			if !IsIPHostAddr(IP.String()) {
+				if loxinlp.AddAddrNoHook(IP.String()+"/32", "lo") != 0 {
+					tk.LogIt(tk.LogError, "nat lb-rule vip %s:%s add failed\n", IP.String(), "lo")
+				} else {
+					tk.LogIt(tk.LogInfo, "nat lb-rule vip %s:%s added\n", IP.String(), "lo")
+				}
+			}
+			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+			defer cancel()
+			rCh := make(chan int)
+			go GratArpReqWithCtx(ctx, rCh, IP, iface)
+			select {
+			case <-rCh:
+				break
+			case <-ctx.Done():
+				tk.LogIt(tk.LogInfo, "nat lb-rule vip %s - iface %s : GratARP timeout\n", IP.String(), iface)
+			}
+		}
+
+	} else {
+		if IsIPHostAddr(IP.String()) {
+			if loxinlp.DelAddrNoHook(IP.String()+"/32", "lo") != 0 {
+				tk.LogIt(tk.LogError, "nat lb-rule vip %s:%s delete failed\n", IP.String(), "lo")
+			} else {
+				tk.LogIt(tk.LogInfo, "nat lb-rule vip %s:%s deleted\n", IP.String(), "lo")
+			}
+		}
+	}
+
+	return nil
+}