diff --git a/api/loxinlp/nlp.go b/api/loxinlp/nlp.go index 4bdce6cf9..5f85db543 100644 --- a/api/loxinlp/nlp.go +++ b/api/loxinlp/nlp.go @@ -1189,8 +1189,18 @@ func AddRoute(route nlp.Route) int { ipNet = *route.Dst } - ret, err := hooks.NetRouteAdd(&cmn.RouteMod{Protocol: int(route.Protocol), Flags: route.Flags, - Gw: route.Gw, LinkIndex: route.LinkIndex, Dst: ipNet}) + var gws []cmn.GWInfo + + if len(route.MultiPath) <= 0 { + gw := cmn.GWInfo{Gw: route.Gw, LinkIndex: route.LinkIndex} + gws = append(gws, gw) + } else { + for i := range route.MultiPath { + gws = append(gws, cmn.GWInfo{Gw: route.MultiPath[i].Gw, LinkIndex: route.MultiPath[i].LinkIndex}) + } + } + + ret, err := hooks.NetRouteAdd(&cmn.RouteMod{Protocol: int(route.Protocol), Flags: route.Flags, Dst: ipNet, GWs: gws}) if err != nil { if route.Gw != nil { tk.LogIt(tk.LogError, "[NLP] RT %s via %s proto %d add failed-%s\n", ipNet.String(), @@ -1350,14 +1360,28 @@ func NUWorkSingle(m nlp.NeighUpdate) int { func RUWorkSingle(m nlp.RouteUpdate) int { var ret int - link, err := nlp.LinkByIndex(m.LinkIndex) - if err != nil { - fmt.Println(err) - return -1 - } + if len(m.MultiPath) <= 0 { + link, err := nlp.LinkByIndex(m.LinkIndex) + if err != nil { + tk.LogIt(tk.LogError, "RUWorkSingle: link find error %s", err) + return -1 + } - if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) { - return -1 + if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) { + return -1 + } + } else { + for _, path := range m.MultiPath { + link, err := nlp.LinkByIndex(path.LinkIndex) + if err != nil { + tk.LogIt(tk.LogError, "RUWorkSingle: link find error %s", err) + return -1 + } + + if iSBlackListedIntf(link.Attrs().Name, link.Attrs().MasterIndex) { + return -1 + } + } } if skipIfRoute { @@ -1559,25 +1583,28 @@ func NlpGet(ch chan bool) int { AddNeigh(neigh, link) } } + } - /* Get Routes */ - routes, err := nlp.RouteList(link, nlp.FAMILY_ALL) - if err != nil { - tk.LogIt(tk.LogError, "[NLP] Error getting route list %v\n", err) - } + /* Get Routes */ + routes, err := nlp.RouteList(nil, nlp.FAMILY_ALL) + if err != nil { + tk.LogIt(tk.LogError, "[NLP] Error getting route list %v\n", err) + } - if len(routes) == 0 { - tk.LogIt(tk.LogDebug, "[NLP] No STATIC routes found for intf %s\n", link.Attrs().Name) - } else { - for _, route := range routes { - if skipIfRoute { - if route.Scope.String() == "link" && tk.IsNetIPv4(route.Dst.IP.String()) { - continue - } + if len(routes) == 0 { + tk.LogIt(tk.LogDebug, "[NLP] No STATIC routes found\n") + } else { + for _, route := range routes { + var m nlp.RouteUpdate + if skipIfRoute { + if route.Scope.String() == "link" && tk.IsNetIPv4(route.Dst.IP.String()) { + continue } - - AddRoute(route) } + m.Type = syscall.RTM_NEWROUTE + m.Route = route + + RUWorkSingle(m) } } tk.LogIt(tk.LogInfo, "[NLP] nlp get done\n") diff --git a/common/common.go b/common/common.go index e4395d4b0..687b892e0 100644 --- a/common/common.go +++ b/common/common.go @@ -369,16 +369,22 @@ type RouteGet struct { Sync DpStatusT } +// GWInfo - Info about gateway +type GWInfo struct { + // Gw - gateway information if any + Gw net.IP + // LinkIndex - OS allocated index + LinkIndex int +} + // RouteMod - Info about a route type RouteMod struct { // Protocol - Protocol type Protocol int // Flags - flag type Flags int - // Gw - gateway information if any - Gw net.IP - // LinkIndex - OS allocated index - LinkIndex int + // GWs - gateway information if any + GWs []GWInfo // Dst - ip addr Dst net.IPNet } diff --git a/loxilb-ebpf b/loxilb-ebpf index 599359117..d34f73f20 160000 --- a/loxilb-ebpf +++ b/loxilb-ebpf @@ -1 +1 @@ -Subproject commit 5993591178537c835398f6316a4958be9ddc3a1c +Subproject commit d34f73f20de8e062ab5a4af720da8b1f9637d11e diff --git a/pkg/loxinet/apiclient.go b/pkg/loxinet/apiclient.go index f82f8bfd5..9b6a8b94d 100644 --- a/pkg/loxinet/apiclient.go +++ b/pkg/loxinet/apiclient.go @@ -279,25 +279,32 @@ func (na *NetAPIStruct) NetRouteAdd(rm *cmn.RouteMod) (int, error) { var ret int var err error + if len(rm.GWs) <= 0 { + return RtNhErr, errors.New("invalid gws") + } if na.BgpPeerMode { return RtNhErr, errors.New("running in bgp only mode") } intfRt := false mlen, _ := rm.Dst.Mask.Size() - if rm.Gw == nil { + if rm.GWs[0].Gw == nil { // This is an interface route if (tk.IsNetIPv4(rm.Dst.IP.String()) && mlen == 32) || (tk.IsNetIPv6(rm.Dst.IP.String()) && mlen == 128) { intfRt = true - rm.Gw = rm.Dst.IP + rm.GWs[0].Gw = rm.Dst.IP } } mh.mtx.Lock() defer mh.mtx.Unlock() - ra := RtAttr{Protocol: rm.Protocol, OSFlags: rm.Flags, HostRoute: false, Ifi: rm.LinkIndex, IfRoute: intfRt} - if rm.Gw != nil { - na := []RtNhAttr{{rm.Gw, rm.LinkIndex}} + ra := RtAttr{Protocol: rm.Protocol, OSFlags: rm.Flags, HostRoute: false, Ifi: rm.GWs[0].LinkIndex, IfRoute: intfRt} + if rm.GWs[0].Gw != nil { + var na []RtNhAttr + for _, gw := range rm.GWs { + na = append(na, RtNhAttr{gw.Gw, gw.LinkIndex}) + } ret, err = mh.zr.Rt.RtAdd(rm.Dst, RootZone, ra, na) + } else { ret, err = mh.zr.Rt.RtAdd(rm.Dst, RootZone, ra, nil) } diff --git a/pkg/loxinet/dpbroker.go b/pkg/loxinet/dpbroker.go index 0818a20d6..c30f3294a 100644 --- a/pkg/loxinet/dpbroker.go +++ b/pkg/loxinet/dpbroker.go @@ -168,7 +168,8 @@ type RouteDpWorkQ struct { Dst net.IPNet RtType int RtMark int - NMark int + NMax int + NMark [8]int } // StatDpWorkQ - work queue entry for stat operation diff --git a/pkg/loxinet/dpebpf_linux.go b/pkg/loxinet/dpebpf_linux.go index 7f56c7275..4fe84e699 100644 --- a/pkg/loxinet/dpebpf_linux.go +++ b/pkg/loxinet/dpebpf_linux.go @@ -698,7 +698,7 @@ func DpRouterMacMod(w *RouterMacDpWorkQ) int { rtNhAct := (*rtNhAct)(getPtrOffset(unsafe.Pointer(dat), C.sizeof_struct_dp_cmn_act)) C.memset(unsafe.Pointer(rtNhAct), 0, C.sizeof_struct_dp_rt_nh_act) - rtNhAct.nh_num = 0 + rtNhAct.nh_num[0] = 0 rtNhAct.tid = 0 rtNhAct.bd = C.ushort(w.BD) } else { @@ -710,7 +710,7 @@ func DpRouterMacMod(w *RouterMacDpWorkQ) int { C.sizeof_struct_dp_cmn_act)) C.memset(unsafe.Pointer(rtNhAct), 0, C.sizeof_struct_dp_rt_nh_act) - rtNhAct.nh_num = C.ushort(w.NhNum) + rtNhAct.nh_num[0] = C.ushort(w.NhNum) tid := ((w.TunID << 8) & 0xffffff00) rtNhAct.tid = C.uint(tk.Htonl(tid)) } @@ -882,11 +882,16 @@ func DpRouteMod(w *RouteDpWorkQ) int { dat := new(rtDat) C.memset(unsafe.Pointer(dat), 0, C.sizeof_struct_dp_rt_tact) - if w.NMark >= 0 { + if w.NMax > 0 { dat.ca.act_type = C.DP_SET_RT_NHNUM act = (*rtL3NhAct)(getPtrOffset(unsafe.Pointer(dat), C.sizeof_struct_dp_cmn_act)) - act.nh_num = C.ushort(w.NMark) + act.naps = C.ushort(w.NMax) + for i := range w.NMark { + if i < C.DP_MAX_ACTIVE_PATHS { + act.nh_num[i] = C.ushort(w.NMark[i]) + } + } } else { dat.ca.act_type = C.DP_SET_TOCP } diff --git a/pkg/loxinet/route.go b/pkg/loxinet/route.go index 310279be0..7e7064a96 100644 --- a/pkg/loxinet/route.go +++ b/pkg/loxinet/route.go @@ -138,7 +138,7 @@ func (r *RtH) RtFind(Dst net.IPNet, Zone string) *Rt { key := RtKey{Dst.String(), Zone} rt, found := r.RtMap[key] - if found == true { + if found { return rt } return nil @@ -151,9 +151,12 @@ func (r *RtH) RouteGet() ([]cmn.RouteGet, error) { var tmpRt cmn.RouteGet tmpRt.Dst = rk.RtCidr tmpRt.Flags = GetFlagToString(r2.TFlags) - if len(r2.NhAttr) != 0 { - // TODO : Current multiple gw not showing. So I added as a static code - tmpRt.Gw = r2.NhAttr[0].NhAddr.String() + tmpRt.Gw = "" + for i, gw := range r2.NextHops { + if i != 0 { + tmpRt.Gw += "," + } + tmpRt.Gw += gw.Addr.String() } tmpRt.HardwareMark = int(r2.Mark) tmpRt.Protocol = r2.Attr.Protocol @@ -199,20 +202,15 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int, } } - if nhLen > 1 { - tk.LogIt(tk.LogError, "rt add - %s:%s ecmp not supported\n", Dst.String(), Zone) - return RtNhErr, errors.New("ecmp-rt error not supported") - } - rt, found := r.RtMap[key] - if found == true { + if found { rtMod := false if len(rt.NhAttr) != nhLen { rtMod = true } else { for i := 0; i < nhLen; i++ { // FIXME - Need to sort before comparing - if Na[i].NhAddr.Equal(rt.NhAttr[i].NhAddr) == false { + if !Na[i].NhAddr.Equal(rt.NhAttr[i].NhAddr) { rtMod = false break } @@ -275,7 +273,7 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int, hwmac, _ := net.ParseMAC("00:00:00:00:00:00") - for i := 0; i < len(Na); i++ { + for i := range Na { nh, _ := r.Zone.Nh.NeighFind(Na[i].NhAddr, Zone) if nh == nil { @@ -307,7 +305,7 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int, // Pair this route with appropriate neighbor //if rt.TFlags & RT_TYPE_HOST != RT_TYPE_HOST { - for i := 0; i < len(rt.NextHops); i++ { + for i := range rt.NextHops { r.Zone.Nh.NeighPairRt(rt.NextHops[i], rt) } //} @@ -334,14 +332,13 @@ func (r *RtH) RtAdd(Dst net.IPNet, Zone string, Ra RtAttr, Na []RtNhAttr) (int, } delete(r.RtMap, rt.Key) r.Mark.PutCounter(rt.Mark) - fmt.Printf("rt add - %s:%s lpm add fail\n", Dst.String(), Zone) tk.LogIt(tk.LogError, "rt add - %s:%s lpm add fail\n", Dst.String(), Zone) return RtTrieAddErr, errors.New("RT Trie Err") } rt.DP(DpCreate) - tk.LogIt(tk.LogDebug, "rt added - %s:%s mark:%v\n", Dst.String(), Zone, rt.RtGetNhMark()) + tk.LogIt(tk.LogDebug, "rt added - %s:%s mark:%s\n", Dst.String(), Zone, rt.RtNhMarkString()) return 0, nil } @@ -534,10 +531,22 @@ func (r *RtH) RoutesTicker() { r.RoutesSync() } +// RtNhMarkString - get the rt-entry's neighbor in string format +func (rt *Rt) RtNhMarkString() string { + str := "" + for i, nh := range rt.NextHops { + if i != 0 { + str += "," + } + str += fmt.Sprintf("%v", nh.Mark) + } + return str +} + // RtGetNhMark - get the rt-entry's neighbor identifier -func (rt *Rt) RtGetNhMark() uint64 { - if len(rt.NextHops) > 0 { - return rt.NextHops[0].Mark +func (rt *Rt) RtGetNhMark(n int) uint64 { + if len(rt.NextHops) > 0 && n < len(rt.NextHops) { + return rt.NextHops[n].Mark } return ^uint64(0) } @@ -570,7 +579,10 @@ func (rt *Rt) DP(work DpWorkT) int { rtWq.Dst = *rtNet rtWq.RtType = rt.TFlags rtWq.RtMark = int(rt.Mark) - rtWq.NMark = int(rt.RtGetNhMark()) + rtWq.NMax = len(rt.NextHops) + for i := range rt.NextHops { + rtWq.NMark[i] = int(rt.RtGetNhMark(i)) + } mh.dp.ToDpCh <- rtWq