From 8a8cf7a79e2adf0cb4348f70420cd073a3b3ad07 Mon Sep 17 00:00:00 2001 From: Matthias Frei Date: Tue, 23 Nov 2021 09:39:06 +0100 Subject: [PATCH] pan: add bundled selector Add first, primitive version for a SelectorBundle. SelectorBundle combines the selectors for multiple connections, attempting to use (and keep using, e.g under changing policies and paths expring or going down) disjoint paths. The idea is that the bandwidths of different paths can be accumulated and the load of the indivdual connections can be distributed over different network links. --- pkg/pan/bundle.go | 218 +++++++++++++++++++++++++++++++++++++++++ pkg/pan/bundle_test.go | 188 +++++++++++++++++++++++++++++++++++ pkg/pan/def.go | 7 ++ 3 files changed, 413 insertions(+) create mode 100644 pkg/pan/bundle.go create mode 100644 pkg/pan/bundle_test.go diff --git a/pkg/pan/bundle.go b/pkg/pan/bundle.go new file mode 100644 index 00000000..36d25a4e --- /dev/null +++ b/pkg/pan/bundle.go @@ -0,0 +1,218 @@ +// Copyright 2021 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pan + +import ( + "fmt" + "sync" +) + +// SelectorBundle combines the selectors for multiple connections, attempting +// to use (and keep using, e.g under changing policies and paths expring or +// going down) disjoint paths. +// The idea is that the bandwidths of different paths can be accumulated and +// the load of the indivdual connections can be distributed over different +// network links. +// +// First create a SelectorBundle object and then use New() to create selectors +// for each individual Conn (i.e. for each Dial call). +// The bundle can be used for connections to different destinations. +// +// This first implementation of this bundle concept is somewhat primitive and +// only determines path usage by the number of connections using a path. +// Later on, we may want to consider bandwidth information from the path +// metadata, allow customized selectors like e.g. the pinging selector, allow +// defining priorities, etc. +type SelectorBundle struct { + mutex sync.Mutex + selectors []*bundledSelector +} + +// New creates a Selector for a dialed connection. The path chosen by this selector +// will attempt to minimize the usage overlap with other selectors from this bundle. +func (b *SelectorBundle) New() Selector { + b.mutex.Lock() + defer b.mutex.Unlock() + + s := &bundledSelector{ + bundle: b, + } + b.selectors = append(b.selectors, s) + return s +} + +func (b *SelectorBundle) remove(s *bundledSelector) { + b.mutex.Lock() + defer b.mutex.Unlock() + + if idx, ok := b.index(s); ok { + b.selectors = append(b.selectors[:idx], b.selectors[idx+1:]...) + } +} + +func (b *SelectorBundle) index(s *bundledSelector) (int, bool) { + for i := range b.selectors { + if b.selectors[i] == s { + return i, true + } + } + return -1, false +} + +func (b *SelectorBundle) firstMaxDisjoint(self *bundledSelector, paths []*Path) int { + if len(paths) <= 1 { + return 0 + } + + // build up path usage information + u := newBundlePathUsage() + for _, s := range b.selectors { + if s == self { + continue + } + if p := s.Path(); p != nil { + u.add(p) + } + } + + return u.firstMaxDisjoint(paths) +} + +func (b *SelectorBundle) firstMaxDisjointMoreAlive(self *bundledSelector, current *Path, paths []*Path) int { + moreAlive := make([]*Path, 0, len(paths)) + for _, p := range paths { + if stats.IsMoreAlive(p, current) { + moreAlive = append(moreAlive, p) + } + } + if len(moreAlive) == 0 { + return b.firstMaxDisjoint(self, paths) + } + best := moreAlive[b.firstMaxDisjoint(self, moreAlive)] + for i, p := range paths { + if p == best { + return i + } + } + panic("logic error, expected to find selected max disjoint in paths slice") +} + +// bundlePathUsage tracks the path usage by the selectors of a bundle. +// Currently, only per-interface usage counts are tracked. +type bundlePathUsage struct { + intfs map[PathInterface]int +} + +func newBundlePathUsage() bundlePathUsage { + return bundlePathUsage{ + intfs: make(map[PathInterface]int), + } +} + +func (u *bundlePathUsage) add(p *Path) { + intfs := p.Metadata.Interfaces + for _, intf := range intfs { + u.intfs[intf] = u.intfs[intf] + 1 + } +} + +// overlap returns how many (other) connections/selectors use paths that +// overlap with p (i.e. use the same path interfaces / hops). +func (u *bundlePathUsage) overlap(p *Path) (overlap int) { + intfs := p.Metadata.Interfaces + for _, intf := range intfs { + overlap = maxInt(overlap, u.intfs[intf]) + } + return +} + +func (u *bundlePathUsage) firstMaxDisjoint(paths []*Path) int { + best := 0 + bestOverlap := u.overlap(paths[0]) + for i := 1; i < len(paths); i++ { + overlap := u.overlap(paths[i]) + if overlap < bestOverlap { + best, bestOverlap = i, overlap + } + } + return best +} + +// bundledSelector is a Selector in a SelectorBundle. +type bundledSelector struct { + bundle *SelectorBundle + mutex sync.Mutex + paths []*Path + current int +} + +func (s *bundledSelector) Path() *Path { + s.mutex.Lock() + defer s.mutex.Unlock() + + if len(s.paths) == 0 { + return nil + } + return s.paths[s.current] +} + +func (s *bundledSelector) Initialize(local, remote UDPAddr, paths []*Path) { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.paths = paths + s.current = s.bundle.firstMaxDisjoint(s, paths) +} + +func (s *bundledSelector) Refresh(paths []*Path) { + s.mutex.Lock() + defer s.mutex.Unlock() + + newcurrent := -1 + if len(s.paths) > 0 { + currentFingerprint := s.paths[s.current].Fingerprint + for i, p := range paths { + if p.Fingerprint == currentFingerprint { + newcurrent = i + break + } + } + } + if newcurrent < 0 { + newcurrent = s.bundle.firstMaxDisjoint(s, paths) + } + s.paths = paths + s.current = newcurrent +} + +func (s *bundledSelector) PathDown(pf PathFingerprint, pi PathInterface) { + s.mutex.Lock() + defer s.mutex.Unlock() + + current := s.paths[s.current] + if isInterfaceOnPath(current, pi) || pf == current.Fingerprint { + fmt.Println("down:", s.current, len(s.paths)) + better := s.bundle.firstMaxDisjointMoreAlive(s, current, s.paths) + if better >= 0 { + s.current = better + fmt.Println("failover:", s.current, len(s.paths)) + } + } +} + +func (s *bundledSelector) Close() error { + s.bundle.remove(s) + return nil +} diff --git a/pkg/pan/bundle_test.go b/pkg/pan/bundle_test.go new file mode 100644 index 00000000..8ea87723 --- /dev/null +++ b/pkg/pan/bundle_test.go @@ -0,0 +1,188 @@ +// Copyright 2021 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pan + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestSelectorBundleSelectorsList checks that the management of the selectors +// list in the bundle works correctly. +func TestSelectorBundleSelectorsList(t *testing.T) { + bundledSelectors := func(selectors ...Selector) []*bundledSelector { + t.Helper() + r := make([]*bundledSelector, len(selectors)) + for i, s := range selectors { + r[i] = s.(*bundledSelector) + } + return r + } + + b := &SelectorBundle{} + s1 := b.New() + assert.Equal(t, bundledSelectors(s1), b.selectors) + s2 := b.New() + assert.Equal(t, bundledSelectors(s1, s2), b.selectors) + s2.Close() + assert.Equal(t, bundledSelectors(s1), b.selectors) + s3 := b.New() + assert.Equal(t, bundledSelectors(s1, s3), b.selectors) + s1.Close() + assert.Equal(t, bundledSelectors(s3), b.selectors) + s3.Close() + assert.Equal(t, bundledSelectors(), b.selectors) + s4 := b.New() + assert.Equal(t, bundledSelectors(s4), b.selectors) +} + +func TestBundlePathUsageOverlap(t *testing.T) { + mustParseIntf := func(intfShort string) PathInterface { + parts := strings.Split(intfShort, "#") + if len(parts) != 2 { + panic(fmt.Sprintf("bad interface %q", intfShort)) + } + ia := MustParseIA("1-ff00:0:" + parts[0]) + ifid, _ := strconv.Atoi(parts[1]) + return PathInterface{IA: ia, IfID: IfID(ifid)} + } + + makePath := func(intfStrs ...string) *Path { + intfs := make([]PathInterface, len(intfStrs)) + for i, intfStr := range intfStrs { + intfs[i] = mustParseIntf(intfStr) + } + return &Path{ + Metadata: &PathMetadata{ + Interfaces: intfs, + }, + } + } + + pAB1 := makePath("a#1", "b#1") + pAB2 := makePath("a#2", "b#2") + pACB11 := makePath("a#11", "c#11", "c#12", "b#11") + pACB22 := makePath("a#12", "c#13", "c#14", "b#12") + pACB12 := makePath("a#11", "c#11", "c#14", "b#12") + pACB21 := makePath("a#12", "c#13", "c#12", "b#11") + pADB := makePath("a#r21", "d#21", "d#22", "b#21") + pAD := makePath("a#r21", "d#21") + + overlapCases := []struct { + name string + usage []*Path + path *Path + overlap int + }{ + { + name: "empty", + usage: nil, + path: pAB1, + overlap: 0, + }, + { + name: "disjoint", + usage: []*Path{pAB1}, + path: pAB2, + overlap: 0, + }, + { + name: "same", + usage: []*Path{pAB1}, + path: pAB1, + overlap: 1, + }, + { + name: "same twice", + usage: []*Path{pAB1, pAB1}, + path: pAB1, + overlap: 2, + }, + { + name: "via D 1", + usage: []*Path{pAD}, + path: pADB, + overlap: 1, + }, + { + name: "via D 2", + usage: []*Path{pADB}, + path: pAD, + overlap: 1, + }, + { + name: "crisscross", + usage: []*Path{pACB11, pACB12, pACB21, pACB22}, + path: pACB12, + overlap: 2, + }, + } + for _, c := range overlapCases { + t.Run("overlap "+c.name, func(t *testing.T) { + u := newBundlePathUsage() + for _, p := range c.usage { + u.add(p) + } + actual := u.overlap(c.path) + assert.Equal(t, c.overlap, actual) + }) + } + + maxDisjointCases := []struct { + name string + usage []*Path + paths []*Path + expected int + }{ + { + name: "empty", + usage: nil, + paths: []*Path{pAB1, pAB2}, + expected: 0, + }, + { + name: "avoid existing", + usage: []*Path{pAB1}, + paths: []*Path{pAB1, pAB2}, + expected: 1, + }, + { + name: "balance 1", + usage: []*Path{pAB1, pAB1, pAB2}, + paths: []*Path{pAB1, pAB2}, + expected: 1, + }, + { + name: "balance 2", + usage: []*Path{pAB1, pAB2, pAB2}, + paths: []*Path{pAB1, pAB2}, + expected: 0, + }, + } + for _, c := range maxDisjointCases { + t.Run("firstMaxDisjoint "+c.name, func(t *testing.T) { + u := newBundlePathUsage() + for _, p := range c.usage { + u.add(p) + } + actual := u.firstMaxDisjoint(c.paths) + assert.Equal(t, c.expected, actual) + }) + } +} diff --git a/pkg/pan/def.go b/pkg/pan/def.go index 07a0b51b..78528e23 100644 --- a/pkg/pan/def.go +++ b/pkg/pan/def.go @@ -50,3 +50,10 @@ const ( // maxTime is the maximum usable time value (https://stackoverflow.com/a/32620397) var maxTime = time.Unix(1<<63-62135596801, 999999999) var maxDuration = time.Duration(1<<63 - 1) + +func maxInt(a, b int) int { + if a >= b { + return a + } + return b +}