diff --git a/pkg/pan/bundle.go b/pkg/pan/bundle.go new file mode 100644 index 00000000..36d25a4e --- /dev/null +++ b/pkg/pan/bundle.go @@ -0,0 +1,218 @@ +// Copyright 2021 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pan + +import ( + "fmt" + "sync" +) + +// SelectorBundle combines the selectors for multiple connections, attempting +// to use (and keep using, e.g under changing policies and paths expring or +// going down) disjoint paths. +// The idea is that the bandwidths of different paths can be accumulated and +// the load of the indivdual connections can be distributed over different +// network links. +// +// First create a SelectorBundle object and then use New() to create selectors +// for each individual Conn (i.e. for each Dial call). +// The bundle can be used for connections to different destinations. +// +// This first implementation of this bundle concept is somewhat primitive and +// only determines path usage by the number of connections using a path. +// Later on, we may want to consider bandwidth information from the path +// metadata, allow customized selectors like e.g. the pinging selector, allow +// defining priorities, etc. +type SelectorBundle struct { + mutex sync.Mutex + selectors []*bundledSelector +} + +// New creates a Selector for a dialed connection. The path chosen by this selector +// will attempt to minimize the usage overlap with other selectors from this bundle. +func (b *SelectorBundle) New() Selector { + b.mutex.Lock() + defer b.mutex.Unlock() + + s := &bundledSelector{ + bundle: b, + } + b.selectors = append(b.selectors, s) + return s +} + +func (b *SelectorBundle) remove(s *bundledSelector) { + b.mutex.Lock() + defer b.mutex.Unlock() + + if idx, ok := b.index(s); ok { + b.selectors = append(b.selectors[:idx], b.selectors[idx+1:]...) + } +} + +func (b *SelectorBundle) index(s *bundledSelector) (int, bool) { + for i := range b.selectors { + if b.selectors[i] == s { + return i, true + } + } + return -1, false +} + +func (b *SelectorBundle) firstMaxDisjoint(self *bundledSelector, paths []*Path) int { + if len(paths) <= 1 { + return 0 + } + + // build up path usage information + u := newBundlePathUsage() + for _, s := range b.selectors { + if s == self { + continue + } + if p := s.Path(); p != nil { + u.add(p) + } + } + + return u.firstMaxDisjoint(paths) +} + +func (b *SelectorBundle) firstMaxDisjointMoreAlive(self *bundledSelector, current *Path, paths []*Path) int { + moreAlive := make([]*Path, 0, len(paths)) + for _, p := range paths { + if stats.IsMoreAlive(p, current) { + moreAlive = append(moreAlive, p) + } + } + if len(moreAlive) == 0 { + return b.firstMaxDisjoint(self, paths) + } + best := moreAlive[b.firstMaxDisjoint(self, moreAlive)] + for i, p := range paths { + if p == best { + return i + } + } + panic("logic error, expected to find selected max disjoint in paths slice") +} + +// bundlePathUsage tracks the path usage by the selectors of a bundle. +// Currently, only per-interface usage counts are tracked. +type bundlePathUsage struct { + intfs map[PathInterface]int +} + +func newBundlePathUsage() bundlePathUsage { + return bundlePathUsage{ + intfs: make(map[PathInterface]int), + } +} + +func (u *bundlePathUsage) add(p *Path) { + intfs := p.Metadata.Interfaces + for _, intf := range intfs { + u.intfs[intf] = u.intfs[intf] + 1 + } +} + +// overlap returns how many (other) connections/selectors use paths that +// overlap with p (i.e. use the same path interfaces / hops). +func (u *bundlePathUsage) overlap(p *Path) (overlap int) { + intfs := p.Metadata.Interfaces + for _, intf := range intfs { + overlap = maxInt(overlap, u.intfs[intf]) + } + return +} + +func (u *bundlePathUsage) firstMaxDisjoint(paths []*Path) int { + best := 0 + bestOverlap := u.overlap(paths[0]) + for i := 1; i < len(paths); i++ { + overlap := u.overlap(paths[i]) + if overlap < bestOverlap { + best, bestOverlap = i, overlap + } + } + return best +} + +// bundledSelector is a Selector in a SelectorBundle. +type bundledSelector struct { + bundle *SelectorBundle + mutex sync.Mutex + paths []*Path + current int +} + +func (s *bundledSelector) Path() *Path { + s.mutex.Lock() + defer s.mutex.Unlock() + + if len(s.paths) == 0 { + return nil + } + return s.paths[s.current] +} + +func (s *bundledSelector) Initialize(local, remote UDPAddr, paths []*Path) { + s.mutex.Lock() + defer s.mutex.Unlock() + + s.paths = paths + s.current = s.bundle.firstMaxDisjoint(s, paths) +} + +func (s *bundledSelector) Refresh(paths []*Path) { + s.mutex.Lock() + defer s.mutex.Unlock() + + newcurrent := -1 + if len(s.paths) > 0 { + currentFingerprint := s.paths[s.current].Fingerprint + for i, p := range paths { + if p.Fingerprint == currentFingerprint { + newcurrent = i + break + } + } + } + if newcurrent < 0 { + newcurrent = s.bundle.firstMaxDisjoint(s, paths) + } + s.paths = paths + s.current = newcurrent +} + +func (s *bundledSelector) PathDown(pf PathFingerprint, pi PathInterface) { + s.mutex.Lock() + defer s.mutex.Unlock() + + current := s.paths[s.current] + if isInterfaceOnPath(current, pi) || pf == current.Fingerprint { + fmt.Println("down:", s.current, len(s.paths)) + better := s.bundle.firstMaxDisjointMoreAlive(s, current, s.paths) + if better >= 0 { + s.current = better + fmt.Println("failover:", s.current, len(s.paths)) + } + } +} + +func (s *bundledSelector) Close() error { + s.bundle.remove(s) + return nil +} diff --git a/pkg/pan/bundle_test.go b/pkg/pan/bundle_test.go new file mode 100644 index 00000000..8ea87723 --- /dev/null +++ b/pkg/pan/bundle_test.go @@ -0,0 +1,188 @@ +// Copyright 2021 ETH Zurich +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pan + +import ( + "fmt" + "strconv" + "strings" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestSelectorBundleSelectorsList checks that the management of the selectors +// list in the bundle works correctly. +func TestSelectorBundleSelectorsList(t *testing.T) { + bundledSelectors := func(selectors ...Selector) []*bundledSelector { + t.Helper() + r := make([]*bundledSelector, len(selectors)) + for i, s := range selectors { + r[i] = s.(*bundledSelector) + } + return r + } + + b := &SelectorBundle{} + s1 := b.New() + assert.Equal(t, bundledSelectors(s1), b.selectors) + s2 := b.New() + assert.Equal(t, bundledSelectors(s1, s2), b.selectors) + s2.Close() + assert.Equal(t, bundledSelectors(s1), b.selectors) + s3 := b.New() + assert.Equal(t, bundledSelectors(s1, s3), b.selectors) + s1.Close() + assert.Equal(t, bundledSelectors(s3), b.selectors) + s3.Close() + assert.Equal(t, bundledSelectors(), b.selectors) + s4 := b.New() + assert.Equal(t, bundledSelectors(s4), b.selectors) +} + +func TestBundlePathUsageOverlap(t *testing.T) { + mustParseIntf := func(intfShort string) PathInterface { + parts := strings.Split(intfShort, "#") + if len(parts) != 2 { + panic(fmt.Sprintf("bad interface %q", intfShort)) + } + ia := MustParseIA("1-ff00:0:" + parts[0]) + ifid, _ := strconv.Atoi(parts[1]) + return PathInterface{IA: ia, IfID: IfID(ifid)} + } + + makePath := func(intfStrs ...string) *Path { + intfs := make([]PathInterface, len(intfStrs)) + for i, intfStr := range intfStrs { + intfs[i] = mustParseIntf(intfStr) + } + return &Path{ + Metadata: &PathMetadata{ + Interfaces: intfs, + }, + } + } + + pAB1 := makePath("a#1", "b#1") + pAB2 := makePath("a#2", "b#2") + pACB11 := makePath("a#11", "c#11", "c#12", "b#11") + pACB22 := makePath("a#12", "c#13", "c#14", "b#12") + pACB12 := makePath("a#11", "c#11", "c#14", "b#12") + pACB21 := makePath("a#12", "c#13", "c#12", "b#11") + pADB := makePath("a#r21", "d#21", "d#22", "b#21") + pAD := makePath("a#r21", "d#21") + + overlapCases := []struct { + name string + usage []*Path + path *Path + overlap int + }{ + { + name: "empty", + usage: nil, + path: pAB1, + overlap: 0, + }, + { + name: "disjoint", + usage: []*Path{pAB1}, + path: pAB2, + overlap: 0, + }, + { + name: "same", + usage: []*Path{pAB1}, + path: pAB1, + overlap: 1, + }, + { + name: "same twice", + usage: []*Path{pAB1, pAB1}, + path: pAB1, + overlap: 2, + }, + { + name: "via D 1", + usage: []*Path{pAD}, + path: pADB, + overlap: 1, + }, + { + name: "via D 2", + usage: []*Path{pADB}, + path: pAD, + overlap: 1, + }, + { + name: "crisscross", + usage: []*Path{pACB11, pACB12, pACB21, pACB22}, + path: pACB12, + overlap: 2, + }, + } + for _, c := range overlapCases { + t.Run("overlap "+c.name, func(t *testing.T) { + u := newBundlePathUsage() + for _, p := range c.usage { + u.add(p) + } + actual := u.overlap(c.path) + assert.Equal(t, c.overlap, actual) + }) + } + + maxDisjointCases := []struct { + name string + usage []*Path + paths []*Path + expected int + }{ + { + name: "empty", + usage: nil, + paths: []*Path{pAB1, pAB2}, + expected: 0, + }, + { + name: "avoid existing", + usage: []*Path{pAB1}, + paths: []*Path{pAB1, pAB2}, + expected: 1, + }, + { + name: "balance 1", + usage: []*Path{pAB1, pAB1, pAB2}, + paths: []*Path{pAB1, pAB2}, + expected: 1, + }, + { + name: "balance 2", + usage: []*Path{pAB1, pAB2, pAB2}, + paths: []*Path{pAB1, pAB2}, + expected: 0, + }, + } + for _, c := range maxDisjointCases { + t.Run("firstMaxDisjoint "+c.name, func(t *testing.T) { + u := newBundlePathUsage() + for _, p := range c.usage { + u.add(p) + } + actual := u.firstMaxDisjoint(c.paths) + assert.Equal(t, c.expected, actual) + }) + } +} diff --git a/pkg/pan/def.go b/pkg/pan/def.go index 07a0b51b..78528e23 100644 --- a/pkg/pan/def.go +++ b/pkg/pan/def.go @@ -50,3 +50,10 @@ const ( // maxTime is the maximum usable time value (https://stackoverflow.com/a/32620397) var maxTime = time.Unix(1<<63-62135596801, 999999999) var maxDuration = time.Duration(1<<63 - 1) + +func maxInt(a, b int) int { + if a >= b { + return a + } + return b +}