From 2814b7b9b3a6599f1eb2ba8ce2ec8800a827131c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 14 Jun 2018 18:43:17 +0200 Subject: [PATCH 01/81] intel_mp: fix broken per queue stat accumulation --- src/apps/intel_mp/intel_mp.lua | 14 +++++++------- src/lib/hardware/register.lua | 15 ++++++++------- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/apps/intel_mp/intel_mp.lua b/src/apps/intel_mp/intel_mp.lua index 23f3d896d5..e5a7495e15 100644 --- a/src/apps/intel_mp/intel_mp.lua +++ b/src/apps/intel_mp/intel_mp.lua @@ -248,13 +248,13 @@ TXCTL 0xe014 +0x40*0..7 RW Tx DCA CTRL Register Queue } reg.i210 = { array = [[ -RQDPC 0x0C030 +0x40*0..4 RC Receive Queue Drop Packet Count -TQDPC 0x0E030 +0x40*0..4 RC Transmit Queue Drop Packet Count -PQGPRC 0x10010 +0x100*0..4 RC Per Queue Good Packets Received Count -PQGPTC 0x10014 +0x100*0..4 RC Per Queue Good Packets Transmitted Count -PQGORC 0x10018 +0x100*0..4 RC Per Queue Good Octets Received Count -PQGOTC 0x10034 +0x100*0..4 RC Per Queue Octets Transmitted Count -PQMPRC 0x10038 +0x100*0..4 RC Per Queue Multicast Packets Received +RQDPC 0x0C030 +0x40*0..4 RCR Receive Queue Drop Packet Count +TQDPC 0x0E030 +0x40*0..4 RCR Transmit Queue Drop Packet Count +PQGPRC 0x10010 +0x100*0..4 RCR Per Queue Good Packets Received Count +PQGPTC 0x10014 +0x100*0..4 RCR Per Queue Good Packets Transmitted Count +PQGORC 0x10018 +0x100*0..4 RCR Per Queue Good Octets Received Count +PQGOTC 0x10034 +0x100*0..4 RCR Per Queue Octets Transmitted Count +PQMPRC 0x10038 +0x100*0..4 RCR Per Queue Multicast Packets Received ]], inherit = "1000BaseX", singleton = [[ diff --git a/src/lib/hardware/register.lua b/src/lib/hardware/register.lua index 19a2b3b73c..cb6793343e 100644 --- a/src/lib/hardware/register.lua +++ b/src/lib/hardware/register.lua @@ -6,6 +6,7 @@ module(...,package.seeall) local ffi = require("ffi") local lib = require("core.lib") +local band = bit.band --- ### Register object --- There are eight types of register objects, set by the mode when created: @@ -30,12 +31,9 @@ function Register:readrc () end function Register:readrcr () - local val = self.ptr[0] - self.acc[0] = self.acc[0] + val - self.last - if val < self.last then - self.acc[0] = self.acc[0] + 2^32 - end - self.last = val + local val = self.ptr[0] + self.acc[0] = self.acc[0] + bit.band(val - self.prev[0], 0xFFFFFFFF) + self.prev[0] = val return self.acc[0] end @@ -134,7 +132,7 @@ local mt = { RCR = { __index = { read=Register.readrcr, reset = Register.reset, bits=ro_bits, byte=ro_byte, print=Register.printrc}, - __call = Register.readrc, __tostring = Register.__tostring } + __call = Register.readrcr, __tostring = Register.__tostring } } mt['RO64'] = mt.RO mt['RW64'] = mt.RW @@ -156,6 +154,9 @@ function new (name, longname, offset, base_ptr, mode) if string.find(mode, "64$") then o.ptr = ffi.cast("uint64_t*", o.ptr) end + if string.find(mode, "^RCR") then + o.prev = ffi.new("uint64_t[1]") + end return setmetatable(o, mt) end From 9c4e3ee603c9486fcb3ec9c2f7c35cb5f39e96de Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 13 Jun 2018 13:17:40 +0200 Subject: [PATCH 02/81] intel_mp: VMDq support for i350 NB: the behavior of the test_1g_2q_blast_vmdq_auto.sh is fishy and there is probably a bug here. Output looks like this: nic1: load: time: 5.00s fps: 1,483,058 fpGbps: 0.819 fpb: 2 bpp: 60 sleep: 1 us GPRC 1488238 RPTHC 1488238 nic2: load: time: 5.00s fps: 5,132 fpGbps: 0.003 fpb: 0 bpp: 60 sleep: 99 us I.e., both pools receive packets (and the right ones too) but the second pool receives way too little. FIXME? --- src/apps/intel_mp/README.md | 14 +- src/apps/intel_mp/intel_mp.lua | 860 +++++++++++------- src/apps/intel_mp/test_1g_1q_blast_vmdq.sh | 9 + .../intel_mp/test_1g_2q_blast_vmdq_auto.sh | 24 + src/apps/intel_mp/test_1g_vmdq_mirror.snabb | 75 ++ src/apps/intel_mp/test_1g_vmdq_pool_sel.snabb | 99 ++ src/apps/intel_mp/test_1g_vmdq_race.snabb | 68 ++ .../intel_mp/test_1g_vmdq_reconfig_mac.snabb | 80 ++ src/apps/intel_mp/test_1g_vmdq_tx.sh | 5 + 9 files changed, 913 insertions(+), 321 deletions(-) create mode 100755 src/apps/intel_mp/test_1g_1q_blast_vmdq.sh create mode 100755 src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh create mode 100755 src/apps/intel_mp/test_1g_vmdq_mirror.snabb create mode 100755 src/apps/intel_mp/test_1g_vmdq_pool_sel.snabb create mode 100755 src/apps/intel_mp/test_1g_vmdq_race.snabb create mode 100755 src/apps/intel_mp/test_1g_vmdq_reconfig_mac.snabb create mode 100755 src/apps/intel_mp/test_1g_vmdq_tx.sh diff --git a/src/apps/intel_mp/README.md b/src/apps/intel_mp/README.md index 0604c05a10..a9da128059 100644 --- a/src/apps/intel_mp/README.md +++ b/src/apps/intel_mp/README.md @@ -72,18 +72,19 @@ For a given NIC, all driver instances should have this parameter either enabled or disabled uniformly. If this is enabled, *macaddr* must be specified. -— Key **vmdq_queuing_mode** +— Key **vmdq_queueing_mode** -*Optional*. Sets the queuing mode to use in VMDq mode. Has no effect when -VMDq is disabled. The available queuing modes are `"rss-64-2"` +*Optional*. Sets the queueing mode to use in VMDq mode. Has no effect when +VMDq is disabled. The available queueing modes for the 82599 are `"rss-64-2"` (the default with 64 pools, 2 queues each) and `"rss-32-4"` -(32 pools, 4 queues each). +(32 pools, 4 queues each). The i350 provides only a single mode (8 pools, 2 +queues each) and hence ignores this option. — Key **poolnum** *Optional*. The VMDq pool to associate with, numbered from 0. The default is to select a pool number automatically. The maximum pool number depends -on the queuing mode. +on the queueing mode. — Key **macaddr** @@ -203,4 +204,5 @@ Each chipset supports a differing number of receive / transmit queues: The Intel82599 supports both VMDq and RSS with 32/64 pools and 4/2 RSS queues for each pool. -While the i350 supports VMDq, this driver does not currently support it. +Intel1g i350 supports both VMDq and RSS with 8 pools 2 queues for each pool. +Intel1g i210 does not support VMDq. diff --git a/src/apps/intel_mp/intel_mp.lua b/src/apps/intel_mp/intel_mp.lua index e5a7495e15..7ef8e96854 100644 --- a/src/apps/intel_mp/intel_mp.lua +++ b/src/apps/intel_mp/intel_mp.lua @@ -3,14 +3,16 @@ -- - Intel1G i210 and i350 based 1G network cards -- - Intel82599 82599 based 10G network cards -- The driver supports multiple processes connecting to the same physical nic. --- Per process RX / TX queues are available via RSS. Statistics collection --- processes can read counter registers +-- Per process RX / TX queues are available via RSS and VMDQ. Statistics +-- collection processes can read counter registers -- -- Data sheets (reference documentation): -- http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/ethernet-controller-i350-datasheet.pdf -- http://www.intel.com/content/dam/www/public/us/en/documents/datasheets/i210-ethernet-controller-datasheet.pdf -- http://www.intel.co.uk/content/dam/www/public/us/en/documents/datasheets/82599-10-gbe-controller-datasheet.pdf --- Note: section and page numbers in the comments below refer to the i210 data sheet +-- Note: section and page numbers in the comments below refer to the 82599 and +-- i210 data sheets, except where VMDQ behavior is being described, in which +-- case the 82599 and i350 data sheets are referenced. module(..., package.seeall) @@ -58,6 +60,7 @@ MPRC 0x0407C - RC Multicast Packets Received Count MPTC 0x040F0 - RC Multicast Packets Transmitted Count BPRC 0x04078 - RC Broadcast Packets Received Count BPTC 0x040F4 - RC Broadcast Packets Transmitted +RXCSUM 0x05000 - RW Receive Checksum Control ]] } reg['82599ES'] = { @@ -153,7 +156,6 @@ RTTUP2TC 0x0C800 - RW DCB Transmit User Priority to Traffic Class RTTDQSEL 0x04904 - RW DCB Transmit Descriptor Plane Queue Select RTTDT1C 0x04908 - RW DCB Transmit Descriptor Plane T1 Config RTTBCNRC 0x04984 - RW DCB Transmit Rate-Scheduler Config -RXCSUM 0x05000 - RW Receive Checksum Control RFCTL 0x05008 - RW Receive Filter Control Register RXCTRL 0x03000 - RW Receive Control RXDGPC 0x02F50 - RC DMA Good Rx Packet Counter @@ -194,6 +196,7 @@ ALLRXDCTL 0x0c028 +0x40*0..7 RW Re Descriptor Control Queue RAL64 0x05400 +0x08*0..15 RW64 Receive Address Low RAL 0x05400 +0x08*0..15 RW Receive Address Low RAH 0x05404 +0x08*0..15 RW Receive Address High +VFTA 0x05600 +0x04*0..127 RW VLAN Filter Table Array ]], inherit = "gbl", rxq = [[ @@ -264,6 +267,14 @@ EEC 0x12010 - RW EEPROM-Mode Control Register } reg.i350 = { array = [[ +VMVIR 0x03700 +0x04*0..7 RW VM VLAN insert register +PSRTYPE 0x05480 +0x04*0..7 RW Packet Split Receive Type +VMOLR 0x05AD0 +0x04*0..7 RW VM Offload register +VLVF 0x05d00 +0x04*0..31 RW VLAN VM Filter +DVMOLR 0x0C038 +0x04*0..7 RW DMA VM Offload register +VMRCTL 0x05D80 +0x04*0..7 RW Virtual Mirror rule control +VMRVLAN 0x05D90 +0x04*0..7 RW Virtual Mirror rule VLAN +VMRVM 0x05DA0 +0x04*0..7 RW Virtual Mirror rule VM RQDPC 0x0C030 +0x40*0..7 RCR Receive Queue Drop Packet Count TQDPC 0x0E030 +0x40*0..7 RCR Transmit Queue Drop Packet Count PQGPRC 0x10010 +0x100*0..7 RCR Per Queue Good Packets Received Count @@ -274,9 +285,16 @@ PQMPRC 0x10038 +0x100*0..7 RCR Per Queue Multicast Packets Received ]], inherit = "1000BaseX", singleton = [[ -EEMNGCTL 0x01010 - RW Manageability EEPROM-Mode Control Register -EEC 0x00010 - RW EEPROM-Mode Control Register -FACTPS 0x05B30 - Function Active and Power State to MNG +VFRE 0x00C8C - RW VF Receive Enable +VFTE 0x00C90 - RW VF Transmit Enable +EEMNGCTL 0x01010 - RW Manageability EEPROM-Mode Control Register +EEC 0x00010 - RW EEPROM-Mode Control Register +QDE 0x02408 - RW Queue Drop Enable Register +DTXCTL 0x03590 - RW DMA TX Control +RPLPSRTYPE 0x054C0 - RW Replicated Packet Split Receive Type +VT_CTL 0x0581C - RW VMDq Control Register +TXSWC 0x05ACC - RW TX Switch Control +FACTPS 0x05B30 - RW Function Active and Power State to MNG ]] } @@ -285,7 +303,8 @@ Intel = { pciaddr = {required=true}, ring_buffer_size = {default=2048}, vmdq = {default=false}, - vmdq_queuing_mode = {default="rss-64-2"}, + vmdq_queueing_mode = {default="rss-64-2"}, + vmdq_queuing_mode = {}, -- legacy misspelling macaddr = {}, poolnum = {}, vlan = {}, @@ -317,6 +336,14 @@ byPciID = { -- order to interchangeably use NIC drivers. driver = Intel +-- C type for VMDq enabled state +vmdq_enabled_t = ffi.typeof("struct { uint8_t enabled; }") +-- C type for shared memory indicating which pools are used +local vmdq_pools_t = ffi.typeof("struct { uint8_t pools[64]; }") +-- C type for VMDq queueing mode +-- mode = 0 for 32 pools/4 queues, 1 for 64 pools/2 queues +local vmdq_queueing_mode_t = ffi.typeof("struct { uint8_t mode; }") + function Intel:new (conf) local self = { r = {}, @@ -343,7 +370,7 @@ function Intel:new (conf) -- processes shm_root = "/intel-mp/" .. pci.canonical(conf.pciaddr) .. "/", -- only used for main process, affects max pool number - vmdq_queuing_mode = conf.vmdq_queuing_mode + vmdq_queueing_mode = conf.vmdq_queuing_mode or conf.vmdq_queueing_mode } local vendor = lib.firstline(self.path .. "/vendor") @@ -359,6 +386,33 @@ function Intel:new (conf) self.base, self.fd = pci.map_pci_memory_unlocked(self.pciaddress, 0) self.master = self.fd:flock("ex, nb") + if self.master then + -- set shm to indicate whether the NIC is in VMDq mode + local vmdq_shm = shm.create(self.shm_root .. "vmdq_enabled", + vmdq_enabled_t) + vmdq_shm.enabled = self.vmdq + shm.unmap(vmdq_shm) + if self.vmdq then + -- create shared memory for tracking VMDq pools + local vmdq_shm = shm.create(self.shm_root .. "vmdq_pools", + vmdq_pools_t) + -- explicitly initialize to 0 since we can't rely on cleanup + for i=0, 63 do vmdq_shm.pools[i] = 0 end + shm.unmap(vmdq_shm) + -- set VMDq pooling method for all instances on this NIC + local mode_shm = shm.create(self.shm_root .. "vmdq_queueing_mode", + vmdq_queueing_mode_t) + if self.vmdq_queueing_mode == "rss-32-4" then + mode_shm.mode = 0 + elseif self.vmdq_queueing_mode == "rss-64-2" then + mode_shm.mode = 1 + else + error("Invalid VMDq queueing mode") + end + shm.unmap(mode_shm) + end + end + self:load_registers(byid.registers) self:init() @@ -466,6 +520,108 @@ function Intel:wait_linkup (timeout) return false end +-- Implements various status checks related to VMDq configuration. +-- Also checks that the main process used the same VMDq setting if +-- this is a worker process +function Intel:check_vmdq () + local vmdq_shm = shm.open(self.shm_root .. "vmdq_enabled", vmdq_enabled_t) + + if not self.vmdq then + assert(not self.macaddr, "VMDq must be set to use MAC address") + assert(not self.mirror, "VMDq must be set to specify mirroring rules") + + if not self.master then + assert(vmdq_shm.enabled == 0, + "VMDq was set by the main process for this NIC") + end + else + assert(self.macaddr, "MAC address must be set in VMDq mode") + + if not self.master then + assert(vmdq_shm.enabled == 1, + "VMDq not set by the main process for this NIC") + end + end +end + +-- In VMDq mode, selects an available pool if one isn't provided by the user. +-- +-- This method runs before rxq/txq registers are loaded, because the rxq/txq registers +-- depend on the pool number prior to loading. +function Intel:select_pool() + if not self.vmdq then return end + + self:lock_sw_sem() + + if self.registers == "i350" then + self.max_pool = 8 + elseif self.registers == "82599ES" then + -- max queue number is different in VMDq mode + self.max_q = 128 + -- check the queueing mode in shm, adjust max pools based on that + local mode_shm = shm.open(self.shm_root .. "vmdq_queueing_mode", + vmdq_queueing_mode_t) + if mode_shm.mode == 0 then + self.max_pool = 32 + else + self.max_pool = 64 + end + shm.unmap(mode_shm) + else + error(self.registers .. " does not support VMDq") + end + + -- We use some shared memory to track which pool numbers are claimed + local pool_shm = shm.open(self.shm_root .. "vmdq_pools", vmdq_pools_t) + + -- if the poolnum was set manually in the config, just use that + if not self.poolnum then + local available_pool + + for poolnum = 0, self.max_pool-1 do + if pool_shm.pools[poolnum] == 0 then + available_pool = poolnum + break + end + end + + assert(available_pool, "No free VMDq pools are available") + self.poolnum = available_pool + else + assert(self.poolnum < self.max_pool, + string.format("Pool overflow: supports up to %d VMDq pools", + self.max_pool)) + end + + pool_shm.pools[self.poolnum] = 1 + shm.unmap(pool_shm) + + self:unlock_sw_sem() + + -- Once we know the pool number, figure out txq and rxq numbers. This + -- needs to be done prior to loading registers. + -- + -- for VMDq, make rxq/txq relative to the pool number + local max_rxq_or_txq = self.max_q / self.max_pool + assert(self.rxq >= 0 and self.rxq < max_rxq_or_txq, + "rxqueue must be in 0.." .. max_rxq_or_txq-1) + self.rxq = self.rxq + max_rxq_or_txq * self.poolnum + assert(self.txq >= 0 and self.txq < max_rxq_or_txq, + "txqueue must be in 0.." .. max_rxq_or_txq-1) + self.txq = self.txq + max_rxq_or_txq * self.poolnum +end + +-- used to disable the pool number for this instance on stop() +function Intel:unset_pool () + self:lock_sw_sem() + + local pool_shm = shm.open(self.shm_root .. "vmdq_pools", vmdq_pools_t) + pool_shm.pools[self.poolnum] = 0 + shm.unmap(pool_shm) + + self:unlock_sw_sem() +end + rxdesc_t = ffi.typeof([[ struct { uint64_t address; @@ -490,16 +646,8 @@ function Intel:init_rx_q () self.rxdesc = ffi.cast(ffi.typeof("$&", rxdesc_ring_t), memory.dma_alloc(ffi.sizeof(rxdesc_ring_t))) - -- VMDq pool state (4.6.10.1.4) if self.vmdq then - -- packet splitting none, enable 4 or 2 RSS queues per pool - if self.max_pool == 32 then - self.r.PSRTYPE[self.poolnum](bits { RQPL=30 }) - else - self.r.PSRTYPE[self.poolnum](bits { RQPL=29 }) - end - -- multicast promiscuous, broadcast accept, accept untagged pkts - self.r.PFVML2FLT[self.poolnum]:set(bits { MPE=28, BAM=27, AUPE=24 }) + self:set_vmdq_rx_pool() end -- Receive state @@ -524,7 +672,7 @@ function Intel:init_rx_q () -- enable VLAN tag stripping in VMDq mode if self.vmdq then - self.r.RXDCTL:set(bits { VME = 30 }) + self:enable_strip_vlan() end self.r.RXDCTL:set( bits { Enable = 25 }) @@ -533,15 +681,9 @@ function Intel:init_rx_q () self.r.RDT(self.ndesc - 1) self:rss_tab_build() - if self.driver == "Intel82599" then - self.r.RXCTRL:set(bits{ RXEN=0 }) - self.r.DCA_RXCTRL:clr(bits{RxCTRL=12}) - if self.vmdq then - -- enable packet reception for this pool/VF (4.6.10.1.4) - self.r.PFVFRE[math.floor(self.poolnum/32)]:set(bits{VFRE=self.poolnum%32}) - end - elseif self.driver == "Intel1g" then - self.r.RCTL:set(bits { RXEN = 1 }) + self:update_rx_filters() + if self.vmdq then + self:enable_vmdq_rx_pool() end self:unlock_sw_sem() end @@ -575,13 +717,7 @@ function Intel:init_tx_q () -- 4.5.10 -- for VMDq need some additional pool configs if self.vmdq then - self.r.RTTDQSEL(self.poolnum) - -- set baseline value for credit refill for tx bandwidth algorithm - self.r.RTTDT1C(0x80) - -- enables packet Tx for this VF's pool - self.r.PFVFTE[math.floor(self.poolnum/32)]:set(bits{VFTE=self.poolnum%32}) - -- enable TX loopback - self.r.PFVMTXSW[math.floor(self.poolnum/32)]:clr(bits{LLE=self.poolnum%32}) + self:set_vmdq_tx_pool() end if self.r.DMATXCTL then @@ -601,6 +737,7 @@ function Intel:load_registers(key) if v.inherit then self:load_registers(v.inherit) end if v.singleton then register.define(v.singleton, self.r, self.base) end if v.array then register.define_array(v.array, self.r, self.base) end + self.registers = key end function Intel:load_queue_registers(key) local v = reg[key] @@ -897,11 +1034,6 @@ function Intel:add_receive_MAC (mac) self:enable_MAC_for_pool(mac_index) end -function Intel:set_transmit_MAC (mac) - local poolnum = self.poolnum or 0 - self.r.PFVFSPOOF[math.floor(poolnum/8)]:set(bits{MACAS=poolnum%8}) -end - -- set VLAN for the driver instance function Intel:set_VLAN () local vlan = self.vlan @@ -911,148 +1043,6 @@ function Intel:set_VLAN () self:set_tag_VLAN(vlan) end -function Intel:add_receive_VLAN (vlan) - assert(vlan>=0 and vlan<4096, "bad VLAN number") - local vlan_index, first_empty - - -- works the same as add_receive_MAC - self:lock_sw_sem() - for idx=0, self.max_vlan-1 do - local valid = self.r.PFVLVF[idx]:bits(31, 1) - - if valid == 0 then - if not first_empty then - first_empty = idx - end - elseif self.r.PFVLVF[idx]:bits(0, 11) == vlan then - vlan_index = idx - break - end - end - self:unlock_sw_sem() - - if not vlan_index and first_empty then - vlan_index = first_empty - self.r.VFTA[math.floor(vlan/32)]:set(bits{Ena=vlan%32}) - self.r.PFVLVF[vlan_index](bits({Vl_En=31},vlan)) - end - - assert(vlan_index, "Max number of VLAN IDs reached") - - self.r.PFVLVFB[2*vlan_index + math.floor(self.poolnum/32)] - :set(bits{PoolEna=self.poolnum%32}) -end - -function Intel:set_tag_VLAN (vlan) - local poolnum = self.poolnum or 0 - self.r.PFVFSPOOF[math.floor(poolnum/8)]:set(bits{VLANAS=poolnum%8+8}) - -- set Port VLAN ID & VLANA to always add VLAN tag - -- TODO: on i350 it's the VMVIR register - self.r.PFVMVIR[poolnum](bits({VLANA=30}, vlan)) -end - -function Intel:unset_VLAN () - local r = self.r - local offs, mask = math.floor(self.poolnum/32), bits{PoolEna=self.poolnum%32} - - for vln_ndx = 0, 63 do - if band(r.PFVLVFB[2*vln_ndx+offs](), mask) ~= 0 then - -- found a vlan this pool belongs to - r.PFVLVFB[2*vln_ndx+offs]:clr(mask) - if r.PFVLVFB[2*vln_ndx+offs]() == 0 then - -- it was the last pool of the vlan - local vlan = tonumber(band(r.PFVLVF[vln_ndx](), 0xFFF)) - r.PFVLVF[vln_ndx](0x0) - r.VFTA[math.floor(vlan/32)]:clr(bits{Ena=vlan%32}) - end - end - end -end - -function Intel:set_mirror () - if not self.want_mirror then return end - want_mirror = self.want_mirror - - -- set MAC promiscuous - self.r.PFVML2FLT[self.poolnum]:set(bits{ - AUPE=24, ROMPE=25, ROPE=26, BAM=27, MPE=28}) - - -- pick one of a limited (4) number of mirroring rules - for idx=0, 3 do - -- check if no mirroring enable bits (3:0) are set - -- (i.e., this rule is unused and available) - if self.r.PFMRCTL[idx]:bits(0, 4) == 0 then - mirror_ndx = idx - break - -- there's already a rule for this pool, overwrite - elseif self.r.PFMRCTL[idx]:bits(8, 5) == self.poolnum then - mirror_ndx = idx - break - end - end - - assert(mirror_ndx, "Max number of mirroring rules reached") - - local mirror_rule = 0ULL - - -- mirror some or all pools - if want_mirror.pool then - mirror_rule = bor(bits{VPME=0}, mirror_rule) - if want_mirror.pool == true then -- mirror all pools - self.r.PFMRVM[mirror_ndx](0xFFFFFFFF) - self.r.PFMRVM[mirror_ndx+4](0xFFFFFFFF) - elseif type(want_mirror.pool) == 'table' then - local bm0 = self.r.PFMRVM[mirror_ndx]() - local bm1 = self.r.PFMRVM[mirror_ndx+4]() - for _, pool in ipairs(want_mirror.pool) do - if pool <= 64 then - bm0 = bor(lshift(1, pool), bm0) - else - bm1 = bor(lshift(1, pool-64), bm1) - end - end - self.r.PFMRVM[mirror_ndx](bm0) - self.r.PFMRVM[mirror_ndx+4](bm1) - end - end - - -- mirror hardware port - if want_mirror.port then - if want_mirror.port == true or - want_mirror.port == 'in' or - want_mirror.port == 'inout' then - mirror_rule = bor(bits{UPME=1}, mirror_rule) - end - if want_mirror.port == true or - want_mirror.port == 'out' or - want_mirror.port == 'inout' then - mirror_rule = bor(bits{DPME=2}, mirror_rule) - end - end - - -- TODO: implement VLAN mirroring - - if mirror_rule ~= 0 then - mirror_rule = bor(mirror_rule, lshift(self.poolnum, 8)) - self.r.PFMRCTL[mirror_ndx]:set(mirror_rule) - end -end - -function Intel:unset_mirror () - for rule_i = 0, 3 do - -- check if any mirror rule points here - local rule_dest = band(bit.rshift(self.r.PFMRCTL[rule_i](), 8), 63) - local bits = band(self.r.PFMRCTL[rule_i](), 0x07) - if bits ~= 0 and rule_dest == self.poolnum then - self.r.PFMRCTL[rule_i](0x0) -- clear rule - self.r.PFMRVLAN[rule_i](0x0) -- clear VLANs mirrored - self.r.PFMRVLAN[rule_i+4](0x0) - self.r.PFMRVM[rule_i](0x0) -- clear pools mirrored - self.r.PFMRVM[rule_i+4](0x0) - end - end -end - function Intel:rxpackets () return self.r.GPRC() end function Intel:txpackets () return self.r.GPTC() end function Intel:rxmcast () return self.r.MPRC() + self.r.BPRC() end @@ -1070,7 +1060,7 @@ Intel1g.offsets = { } } Intel1g.max_mac_addr = 15 -Intel1g.max_vlan = 8 +Intel1g.max_vlan = 32 function Intel1g:init_phy () -- 4.3.1.4 PHY Reset self.r.MANC:wait(bits { BLK_Phy_Rst_On_IDE = 18 }, 0) @@ -1138,6 +1128,11 @@ function Intel1g:init () self:rss_enable() + if self.vmdq then + self:vmdq_enable() + end + + self.r.RXCSUM(0) -- turn off all checksum offload self.r.RCTL:clr(bits { RXEN = 1 }) self.r.RCTL(bits { UPE = 3, -- Unicast Promiscuous @@ -1228,32 +1223,215 @@ end function Intel1g:set_rxstats () return end function Intel1g:set_txstats () return end -function Intel1g:check_vmdq () return end +-- enable VMDq mode, see 4.6.11.1 function Intel1g:vmdq_enable () - error("unimplemented") + -- enable legacy control flow, VLAN mode + self.r.CTRL:set(bits { RFCE=27, TFCE=28, VME=30 }) + + -- 4.6.11.1.1 Global Filtering and Offload Capabilities + assert(self.registers == "i350", "VMDq not supported by "..self.registers) + -- 011b = Multiple receive queues as defined by VMDq based on packet + -- destination MAC address (RAH.POOLSEL) and Ether-type queueing decision + -- filters. NB: ignore self.vmdq_queueing_mode, i350 only supports 8 pools + -- with one queue each. + self.r.MRQC:bits(0, 3, 0x3) + -- No packet splitting + self.r.RPLPSRTYPE(0) + -- VT_CTL.Dis_Def_pool: disable default pool, drop unmatched packets. + -- VT_CTL.Rpl_En: replicate broadcast/multicast packets to all queues. + self.r.VT_CTL:set(bits { Dis_Def_Pool=29, Rpl_En=30 }) + -- Enable loopback + self.r.TXSWC:set(band(bits { Loopback_en=31 }, bit.rshift(23, 0xFF))) -- LLE + -- clear VMVIR, VFTE for all pools, set them later + for pool = 0, 7 do + self.r.VFTE:clr(bits{VFTE=pool}) + self.r.VMVIR[pool](0) + end + -- enable vlan filter (8.10.1) + self.r.RCTL:set(bits { VFE=18 }) + + -- Set QDE bit for all queues + for queue = 0, 7 do + self.r.QDE:set(bits { QDE=queue }) + end end -function Intel1g:select_pool () return end -function Intel1g:enable_MAC_for_pool(mac_index) - self.r.RAH[mac_index]:set(bits { Ena = 18 + self.poolnum }) +-- VMDq pool state (4.6.11.1.3) +function Intel1g:set_vmdq_rx_pool () + -- long packets enabled, multicast promiscuous, broadcast accept, accept + -- untagged pkts + self.r.VMOLR[self.poolnum]:set(bits { LPE=16, MPE=28, BAM=26, AUPE=14 }) + -- packet splitting none + self.r.PSRTYPE[self.poolnum](0) end -function Intel1g:unset_MAC () - local msk = bits { Ena = 18 + self.poolnum } - for mac_index = 0, self.max_mac_addr do - pf.r.RAH[mac_index]:clr(msk) - end +-- enable packet reception for this pool/VF (4.6.9.2) +function Intel1g:enable_vmdq_rx_pool () + self.r.VFRE:set(bits { VFRE=self.poolnum }) end -function Intel1g:set_tx_rate () return end -function Intel1g:unset_tx_rate () return end +function Intel1g:update_rx_filters () + self.r.RCTL:set(bits { RXEN = 1 }) +end -Intel82599.driver = "Intel82599" -Intel82599.offsets = { - SRRCTL = { - Drop_En = 28 - }, - MRQC = { +function Intel1g:set_vmdq_tx_pool () + self.r.VFTE:set(bits{VFTE=self.poolnum}) +end + +function Intel1g:set_mirror () + if not self.want_mirror then return end + + -- pick one of a limited (4) number of mirroring rules + local mirror_ndx + for idx=0, 3 do + -- check if no mirroring enable bits (3:0) are set + -- (i.e., this rule is unused and available) + if self.r.VMRCTL[idx]:bits(0, 4) == 0 then + mirror_ndx = idx + break + -- there's already a rule for this pool, overwrite + elseif self.r.VMRCTL[idx]:bits(8, 3) == self.poolnum then + mirror_ndx = idx + break + end + end + + assert(mirror_ndx, "Max number of mirroring rules reached") + + local mirror_rule = 0 + + -- mirror some or all pools + if self.want_mirror.pool then + mirror_rule = bits { VPME=0 } + if self.want_mirror.pool == true then -- mirror all pools + self.r.VMRVM[mirror_ndx](0xFF) + elseif type(self.want_mirror.pool) == 'table' then + local vm = 0 + for _, pool in ipairs(self.want_mirror.pool) do + vm = bor(bits { VM=pool }, vm) + end + self.r.VMRVM[mirror_ndx](vm) + end + end + + -- mirror hardware port + if self.want_mirror.port then + if self.want_mirror.port == true or + self.want_mirror.port == 'in' or + self.want_mirror.port == 'inout' then + mirror_rule = bor(bits{UPME=1}, mirror_rule) + end + if self.want_mirror.port == true or + self.want_mirror.port == 'out' or + self.want_mirror.port == 'inout' then + mirror_rule = bor(bits{DPME=2}, mirror_rule) + end + end + + -- TODO: implement VLAN mirroring + + if mirror_rule ~= 0 then + mirror_rule = bor(mirror_rule, lshift(self.poolnum, 8)) + self.r.VMRCTL[mirror_ndx]:set(mirror_rule) + end +end + +function Intel1g:unset_mirror () + for rule_i = 0, 3 do + -- check if any mirror rule points here + local rule_dest = self.r.VMRCTL[rule_i]:bits(8, 3) + local bits = self.r.VMRCTL[rule_i]:bits(0, 4) + if bits ~= 0 and rule_dest == self.poolnum then + self.r.VMRCTL[rule_i](0x0) -- clear rule + self.r.VMRVLAN[rule_i](0x0) -- clear VLANs mirrored + self.r.VMRVM[rule_i](0x0) -- clear pools mirrored + end + end +end + +function Intel1g:enable_MAC_for_pool(mac_index) + self.r.RAH[mac_index]:set(bits { Ena = 18 + self.poolnum }) +end + +function Intel1g:set_transmit_MAC (mac) + local poolnum = self.poolnum or 0 + self.r.TXSWC:set(bits{MACAS=poolnum}) +end + +function Intel1g:unset_MAC () + local msk = bits { Ena = 18 + self.poolnum } + for mac_index = 0, self.max_mac_addr do + self.r.RAH[mac_index]:clr(msk) + end +end + +function Intel1g:add_receive_VLAN (vlan) + local vlan_index, first_empty + + -- works the same as add_receive_MAC + self:lock_sw_sem() + for idx=0, self.max_vlan-1 do + local valid = self.r.VLVF[idx]:bits(31, 1) + + if valid == 0 then + if not first_empty then + first_empty = idx + end + elseif self.r.VLVF[idx]:bits(0, 11) == vlan then + vlan_index = idx + break + end + end + self:unlock_sw_sem() + + if not vlan_index and first_empty then + vlan_index = first_empty + self.r.VLVF[vlan_index](bits({Vl_En=31},vlan)) + self.r.VFTA[math.floor(vlan/32)]:set(bits{Ena=vlan%32}) + end + + assert(vlan_index, "Max number of VLAN IDs reached") + + self.r.VLVF[vlan_index]:set(bits { POOLSEL=12+self.poolnum }) +end + +function Intel1g:set_tag_VLAN (vlan) + local poolnum = self.poolnum or 0 + self.r.TXSWC:set(bits{VLANAS=poolnum+8}) + self.r.VMVIR[poolnum](bits({VLANA=30}, vlan)) +end + +function Intel1g:unset_VLAN () + self.r.TXSWC:clr(bits{VLANAS=self.poolnum+8}) + + for vlan_index = 0, self.max_vlan-1 do + self.r.VMVIR[self.poolnum]:clr(bits( { VLANA=30 })) + if self.r.VLVF[vlan_index]:bits(12+self.poolnum, 1) ~= 0 then + -- found a vlan this pool belongs to + self.r.VLVF[vlan_index]:clr(bits { POOLSEL=12+self.poolnum }) + if self.r.VLVF[vlan_index]:bits(12,self.max_pool) == 0 then + -- it was the last pool of the vlan + local vlan = tonumber(band(self.r.VLVF[vlan_index](), 0xFFF)) + self.r.VLVF[vlan_index]:clr(bits { Vl_En=31 }) + self.r.VFTA[math.floor(vlan/32)]:clr(bits{ Ena=vlan%32 }) + end + end + end +end + +function Intel1g:enable_strip_vlan () + self.r.DVMOLR[self.poolnum]:set(bits { STRVLAN = 30 }) +end + +function Intel1g:set_tx_rate () return end +function Intel1g:unset_tx_rate () return end + +Intel82599.driver = "Intel82599" +Intel82599.offsets = { + SRRCTL = { + Drop_En = 28 + }, + MRQC = { RSS = 0 } } @@ -1306,9 +1484,6 @@ function Intel82599:init_queue_stats (frame) end end --- C type for VMDq enabled state -vmdq_enabled_t = ffi.typeof("struct { uint8_t enabled; }") - function Intel82599:init () if not self.master then return end pci.unbind_device_from_linux(self.pciaddress) @@ -1410,11 +1585,6 @@ function Intel82599:init () self:rss_enable() - -- set shm to indicate whether the NIC is in VMDq mode - local vmdq_shm = shm.create(self.shm_root .. "vmdq_enabled", vmdq_enabled_t) - vmdq_shm.enabled = self.vmdq - shm.unmap(vmdq_shm) - if self.vmdq then self:vmdq_enable() end @@ -1422,63 +1592,14 @@ function Intel82599:init () self:unlock_sw_sem() end --- Implements various status checks related to VMDq configuration. --- Also checks that the main process used the same VMDq setting if --- this is a worker process -function Intel82599:check_vmdq () - local vmdq_shm = shm.open(self.shm_root .. "vmdq_enabled", vmdq_enabled_t) - - if not self.vmdq then - assert(not self.macaddr, "VMDq must be set to use MAC address") - assert(not self.mirror, "VMDq must be set to specify mirroring rules") - - if not self.master then - assert(vmdq_shm.enabled == 0, - "VMDq was set by the main process for this NIC") - end - else - assert(self.driver == "Intel82599", "VMDq only supported on 82599") - assert(self.macaddr, "MAC address must be set in VMDq mode") - - if not self.master then - assert(vmdq_shm.enabled == 1, - "VMDq not set by the main process for this NIC") - end - end -end - --- C type for shared memory indicating which pools are used -local vmdq_pools_t = ffi.typeof("struct { uint8_t pools[64]; }") --- C type for VMDq queuing mode --- mode = 0 for 32 pools/4 queues, 1 for 64 pools/2 queues -local vmdq_queuing_mode_t = ffi.typeof("struct { uint8_t mode; }") - -- enable VMDq mode, see 4.6.10.1 -- follows the configuration flow in 4.6.11.3.3 -- (should only be called on the master instance) function Intel82599:vmdq_enable () - -- create shared memory for tracking VMDq pools - local vmdq_shm = shm.create(self.shm_root .. "vmdq_pools", vmdq_pools_t) - -- explicitly initialize to 0 since we can't rely on cleanup - for i=0, 63 do vmdq_shm.pools[i] = 0 end - shm.unmap(vmdq_shm) - - -- set VMDq queuing mode for all instances on this NIC - local mode_shm = shm.create(self.shm_root .. "vmdq_queuing_mode", - vmdq_queuing_mode_t) - if self.vmdq_queuing_mode == "rss-32-4" then - mode_shm.mode = 0 - elseif self.vmdq_queuing_mode == "rss-64-2" then - mode_shm.mode = 1 - else - error("Invalid VMDq queuing mode") - end - shm.unmap(mode_shm) - -- must be set prior to setting MTQC (7.2.1.2.1) self.r.RTTDCS:set(bits { ARBDIS=6 }) - if self.vmdq_queuing_mode == "rss-32-4" then + if self.vmdq_queueing_mode == "rss-32-4" then -- 1010 -> 32 pools, 4 RSS queues each self.r.MRQC:bits(0, 4, 0xA) -- Num_TC_OR_Q=10b -> 32 pools (4.6.11.3.3 and 8.2.3.9.15) @@ -1540,76 +1661,120 @@ function Intel82599:vmdq_enable () self.r.RTTDCS:clr(bits { ARBDIS=6 }) end --- In VMDq mode, selects an available pool if one isn't provided by the user. --- --- This method runs before rxq/txq registers are loaded, because the rxq/txq registers --- depend on the pool number prior to loading. -function Intel82599:select_pool() - if not self.vmdq then return end - - self:lock_sw_sem() - - -- check the queueing mode in shm, adjust max pools based on that - local mode_shm = shm.open(self.shm_root .. "vmdq_queuing_mode", vmdq_queuing_mode_t) - if mode_shm.mode == 0 then - self.max_pool = 32 +-- VMDq pool state (4.6.10.1.4) +function Intel82599:set_vmdq_rx_pool () + -- packet splitting none, enable 4 or 2 RSS queues per pool + if self.max_pool == 32 then + self.r.PSRTYPE[self.poolnum](bits { RQPL=30 }) else - self.max_pool = 64 + self.r.PSRTYPE[self.poolnum](bits { RQPL=29 }) end - shm.unmap(mode_shm) + -- multicast promiscuous, broadcast accept, accept untagged pkts + self.r.PFVML2FLT[self.poolnum]:set(bits { MPE=28, BAM=27, AUPE=24 }) +end - -- We use some shared memory to track which pool numbers are claimed - local pool_shm = shm.open(self.shm_root .. "vmdq_pools", vmdq_pools_t) +-- enable packet reception for this pool/VF (4.6.10.1.4) +function Intel82599:enable_vmdq_rx_pool () + self.r.PFVFRE[math.floor(self.poolnum/32)]:set(bits{VFRE=self.poolnum%32}) +end - -- if the poolnum was set manually in the config, just use that - if not self.poolnum then - local available_pool +function Intel82599:update_rx_filters () + self.r.RXCTRL:set(bits{ RXEN=0 }) + self.r.DCA_RXCTRL:clr(bits{RxCTRL=12}) +end - for poolnum = 0, self.max_pool-1 do - if pool_shm.pools[poolnum] == 0 then - available_pool = poolnum - break - end - end +function Intel82599:set_vmdq_tx_pool () + self.r.RTTDQSEL(self.poolnum) + -- set baseline value for credit refill for tx bandwidth algorithm + self.r.RTTDT1C(0x80) + -- enables packet Tx for this VF's pool + self.r.PFVFTE[math.floor(self.poolnum/32)]:set(bits{VFTE=self.poolnum%32}) + -- enable TX loopback + self.r.PFVMTXSW[math.floor(self.poolnum/32)]:clr(bits{LLE=self.poolnum%32}) +end - assert(available_pool, "No free VMDq pools are available") - self.poolnum = available_pool - else - assert(self.poolnum < self.max_pool, - string.format("Pool overflow: Intel 82599 supports up to %d VMDq pools", - self.max_pool)) +function Intel82599:set_mirror () + if not self.want_mirror then return end + + -- set MAC promiscuous + self.r.PFVML2FLT[self.poolnum]:set(bits{ + AUPE=24, ROMPE=25, ROPE=26, BAM=27, MPE=28}) + + -- pick one of a limited (4) number of mirroring rules + local mirror_ndx + for idx=0, 3 do + -- check if no mirroring enable bits (3:0) are set + -- (i.e., this rule is unused and available) + if self.r.PFMRCTL[idx]:bits(0, 4) == 0 then + mirror_ndx = idx + break + -- there's already a rule for this pool, overwrite + elseif self.r.PFMRCTL[idx]:bits(8, 5) == self.poolnum then + mirror_ndx = idx + break + end end - pool_shm.pools[self.poolnum] = 1 - shm.unmap(pool_shm) + assert(mirror_ndx, "Max number of mirroring rules reached") - self:unlock_sw_sem() + local mirror_rule = 0ULL - -- Once we know the pool number, figure out txq and rxq numbers. This - -- needs to be done prior to loading registers. - -- - -- for VMDq, make rxq/txq relative to the pool number - local max_rxq_or_txq = 128 / self.max_pool - assert(self.rxq >= 0 and self.rxq < max_rxq_or_txq, - "rxqueue must be in 0.." .. max_rxq_or_txq-1) - self.rxq = self.rxq + max_rxq_or_txq * self.poolnum - assert(self.txq >= 0 and self.txq < max_rxq_or_txq, - "txqueue must be in 0.." .. max_rxq_or_txq-1) - self.txq = self.txq + max_rxq_or_txq * self.poolnum + -- mirror some or all pools + if self.want_mirror.pool then + mirror_rule = bor(bits{VPME=0}, mirror_rule) + if self.want_mirror.pool == true then -- mirror all pools + self.r.PFMRVM[mirror_ndx](0xFFFFFFFF) + self.r.PFMRVM[mirror_ndx+4](0xFFFFFFFF) + elseif type(self.want_mirror.pool) == 'table' then + local bm0 = self.r.PFMRVM[mirror_ndx]() + local bm1 = self.r.PFMRVM[mirror_ndx+4]() + for _, pool in ipairs(self.want_mirror.pool) do + if pool <= 64 then + bm0 = bor(lshift(1, pool), bm0) + else + bm1 = bor(lshift(1, pool-64), bm1) + end + end + self.r.PFMRVM[mirror_ndx](bm0) + self.r.PFMRVM[mirror_ndx+4](bm1) + end + end - -- max queue number is different in VMDq mode - self.max_q = 128 -end + -- mirror hardware port + if self.want_mirror.port then + if self.want_mirror.port == true or + self.want_mirror.port == 'in' or + self.want_mirror.port == 'inout' then + mirror_rule = bor(bits{UPME=1}, mirror_rule) + end + if self.want_mirror.port == true or + self.want_mirror.port == 'out' or + self.want_mirror.port == 'inout' then + mirror_rule = bor(bits{DPME=2}, mirror_rule) + end + end --- used to disable the pool number for this instance on stop() -function Intel82599:unset_pool () - self:lock_sw_sem() + -- TODO: implement VLAN mirroring - local pool_shm = shm.open(self.shm_root .. "vmdq_pools", vmdq_pools_t) - pool_shm.pools[self.poolnum] = 0 - shm.unmap(pool_shm) + if mirror_rule ~= 0 then + mirror_rule = bor(mirror_rule, lshift(self.poolnum, 8)) + self.r.PFMRCTL[mirror_ndx]:set(mirror_rule) + end +end - self:unlock_sw_sem() +function Intel82599:unset_mirror () + for rule_i = 0, 3 do + -- check if any mirror rule points here + local rule_dest = band(bit.rshift(self.r.PFMRCTL[rule_i](), 8), 63) + local bits = band(self.r.PFMRCTL[rule_i](), 0x07) + if bits ~= 0 and rule_dest == self.poolnum then + self.r.PFMRCTL[rule_i](0x0) -- clear rule + self.r.PFMRVLAN[rule_i](0x0) -- clear VLANs mirrored + self.r.PFMRVLAN[rule_i+4](0x0) + self.r.PFMRVM[rule_i](0x0) -- clear pools mirrored + self.r.PFMRVM[rule_i+4](0x0) + end + end end function Intel82599:enable_MAC_for_pool (mac_index) @@ -1617,6 +1782,11 @@ function Intel82599:enable_MAC_for_pool (mac_index) :set(bits{Ena=self.poolnum%32}) end +function Intel82599:set_transmit_MAC (mac) + local poolnum = self.poolnum or 0 + self.r.PFVFSPOOF[math.floor(poolnum/8)]:set(bits{MACAS=poolnum%8}) +end + function Intel82599:unset_MAC () local msk = bits { Ena=self.poolnum%32 } for mac_index = 0, self.max_mac_addr do @@ -1624,6 +1794,66 @@ function Intel82599:unset_MAC () end end +function Intel82599:add_receive_VLAN (vlan) + local vlan_index, first_empty + + -- works the same as add_receive_MAC + self:lock_sw_sem() + for idx=0, self.max_vlan-1 do + local valid = self.r.PFVLVF[idx]:bits(31, 1) + + if valid == 0 then + if not first_empty then + first_empty = idx + end + elseif self.r.PFVLVF[idx]:bits(0, 11) == vlan then + vlan_index = idx + break + end + end + self:unlock_sw_sem() + + if not vlan_index and first_empty then + vlan_index = first_empty + self.r.VFTA[math.floor(vlan/32)]:set(bits{Ena=vlan%32}) + self.r.PFVLVF[vlan_index](bits({Vl_En=31},vlan)) + end + + assert(vlan_index, "Max number of VLAN IDs reached") + + self.r.PFVLVFB[2*vlan_index + math.floor(self.poolnum/32)] + :set(bits{PoolEna=self.poolnum%32}) +end + +function Intel82599:set_tag_VLAN (vlan) + local poolnum = self.poolnum or 0 + self.r.PFVFSPOOF[math.floor(poolnum/8)]:set(bits{VLANAS=poolnum%8+8}) + -- set Port VLAN ID & VLANA to always add VLAN tag + self.r.PFVMVIR[poolnum](bits({VLANA=30}, vlan)) +end + +function Intel82599:unset_VLAN () + local r = self.r + local offs, mask = math.floor(self.poolnum/32), bits{PoolEna=self.poolnum%32} + + for vln_ndx = 0, 63 do + if band(r.PFVLVFB[2*vln_ndx+offs](), mask) ~= 0 then + -- found a vlan this pool belongs to + r.PFVLVFB[2*vln_ndx+offs]:clr(mask) + if r.PFVLVFB[2*vln_ndx+offs]() == 0 then + -- it was the last pool of the vlan + local vlan = tonumber(band(r.PFVLVF[vln_ndx](), 0xFFF)) + r.PFVLVF[vln_ndx](0x0) + r.VFTA[math.floor(vlan/32)]:clr(bits{Ena=vlan%32}) + end + end + end +end + +function Intel82599:enable_strip_vlan () + self.r.RXDCTL:set(bits { VME = 30 }) +end + function Intel82599:set_tx_rate () if not self.txq then return end self.r.RTTDQSEL(self.poolnum or self.txq) diff --git a/src/apps/intel_mp/test_1g_1q_blast_vmdq.sh b/src/apps/intel_mp/test_1g_1q_blast_vmdq.sh new file mode 100755 index 0000000000..e966b94d50 --- /dev/null +++ b/src/apps/intel_mp/test_1g_1q_blast_vmdq.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +SNABB_SEND_BLAST=true ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source.pcap & +BLAST=$! + +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" nil 0 0 > results.0 + +kill -9 $BLAST +test `cat results.0 | grep "^GPRC" | awk '{print $2}'` -gt 10000 +exit $? diff --git a/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh b/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh new file mode 100755 index 0000000000..d1e5e9a558 --- /dev/null +++ b/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# +# Test VMDq with automatic pool selection + +SNABB_SEND_BLAST=true ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source2.pcap & +BLAST=$! + +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" nil nil nil > results.0 & +PID1=$! +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "12:34:56:78:9a:bc" nil nil nil > results.1 + +wait $PID1 +kill -9 $BLAST + +# FIXME: one pool receives a ton of packets, one only a tiny amount. Generally, +# the right packets seem to be received though??? Change bpp to fpb and $11 to +# $9 when this is figured out. + +# both queues should see packets +[[ `cat results.* | grep "^GPRC" | awk '{print $2}'` -gt 10000 ]] &&\ +[[ `cat results.0 | grep -m 1 bpp | awk '{print $11}'` -gt 0 ]] &&\ +[[ `cat results.1 | grep -m 1 bpp | awk '{print $11}'` -gt 0 ]] + +exit $? diff --git a/src/apps/intel_mp/test_1g_vmdq_mirror.snabb b/src/apps/intel_mp/test_1g_vmdq_mirror.snabb new file mode 100755 index 0000000000..cb3f785bc0 --- /dev/null +++ b/src/apps/intel_mp/test_1g_vmdq_mirror.snabb @@ -0,0 +1,75 @@ +#!../../snabb snsh + +-- Snabb test script for mirroring rules in VMDq mode +-- +-- Also tests rxcounters for consistency with link counts + +local basic_apps = require("apps.basic.basic_apps") +local intel = require("apps.intel_mp.intel_mp") +local pcap = require("apps.pcap.pcap") +local lib = require("core.lib") + +local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") +local pciaddr1 = lib.getenv("SNABB_PCI_INTEL1G1") + +local c = config.new() + +-- send packets on nic0 +config.app(c, "nic0", intel.Intel, + { pciaddr = pciaddr0, + txq = 0, + wait_for_link = true }) + +-- nic1 with three pools with several mirror configs +config.app(c, "nic1p0", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + poolnum = 0, + macaddr = "90:72:82:78:c9:7a", + rxq = 0, + rxcounter = 1, + wait_for_link = true }) + +config.app(c, "nic1p1", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + poolnum = 1, + mirror = { pool = { 0 } }, + macaddr = "12:34:56:78:9a:bc", + rxq = 0, + rxcounter = 2, + wait_for_link = true }) + +config.app(c, "nic1p2", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + poolnum = 2, + mirror = { pool = true }, + macaddr = "aa:aa:aa:aa:aa:aa", + rxq = 0, + rxcounter = 3, + wait_for_link = true }) + +config.app(c, "pcap", pcap.PcapReader, "source2.pcap") +config.app(c, 'sink', basic_apps.Sink) + +config.link(c, "pcap.output -> nic0.input") +config.link(c, "nic1p0.output -> sink.input0") +config.link(c, "nic1p1.output -> sink.input1") +config.link(c, "nic1p2.output -> sink.input2") + +engine.configure(c) +engine.main({ duration = 1 }) + +assert(link.stats(engine.app_table.sink.input.input0).rxpackets == 51, + "wrong number of packets received on pool 0") +assert(engine.app_table.nic1p0:get_rxstats().packets == 51, + "expected get_rxstats and link stats to agree") +assert(link.stats(engine.app_table.sink.input.input1).rxpackets == 102, + "wrong number of packets received on pool 1") +assert(engine.app_table.nic1p1:get_rxstats().packets == 102, + "expected get_rxstats and link stats to agree") +assert(link.stats(engine.app_table.sink.input.input2).rxpackets == 102, + "wrong number of packets received on pool 2") +assert(engine.app_table.nic1p2:get_rxstats().packets == 102, + "expected get_rxstats and link stats to agree") diff --git a/src/apps/intel_mp/test_1g_vmdq_pool_sel.snabb b/src/apps/intel_mp/test_1g_vmdq_pool_sel.snabb new file mode 100755 index 0000000000..7cbca311ab --- /dev/null +++ b/src/apps/intel_mp/test_1g_vmdq_pool_sel.snabb @@ -0,0 +1,99 @@ +#!../../snabb snsh + +-- Snabb test script for checking that pool selection is working +-- as expected across app shutdowns + +local basic_apps = require("apps.basic.basic_apps") +local intel = require("apps.intel_mp.intel_mp") +local pcap = require("apps.pcap.pcap") +local lib = require("core.lib") + +local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") +local pciaddr1 = lib.getenv("SNABB_PCI_INTEL1G1") + +local c = config.new() + +-- first add two apps in order & observe poolnums +config.app(c, "nicp0", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + rxq = 0, txq = 0, + macaddr = "90:72:82:78:c9:7a" }) +config.app(c, "source", basic_apps.Source) + +config.link(c, "source.out0 -> nicp0.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +assert(engine.app_table.nicp0.poolnum == 0, "wrong poolnum for nicp0") + +config.app(c, "nicp1", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + rxq = 0, txq = 0, + macaddr = "90:72:82:78:c9:7b" }) +config.link(c, "source.out1 -> nicp1.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +assert(engine.app_table.nicp1.poolnum == 1, "wrong poolnum for nicp1") + +-- now try removing the first app, then add a new one to use pool 0 +local c = config.new() + +config.app(c, "nicp1", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + rxq = 0, txq = 0, + macaddr = "90:72:82:78:c9:7b" }) +config.app(c, "source", basic_apps.Source) +config.link(c, "source.out1 -> nicp1.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +config.app(c, "nicp2", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + rxq = 0, txq = 0, + macaddr = "90:72:82:78:c9:7b" }) +config.link(c, "source.out2 -> nicp2.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +assert(engine.app_table.nicp1.poolnum == 1, "wrong poolnum for nicp1") +-- pool 0 should be freed by nicp0 being stopped +assert(engine.app_table.nicp2.poolnum == 0, "wrong poolnum for nicp2") + +-- next ensure that manually specified poolnums don't conflict with +-- auto pool selection +local c = config.new() + +config.app(c, "nicp1", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + -- this app will take pool 0, so p2 should take 1 + poolnum = 0, + rxq = 0, txq = 0, + macaddr = "90:72:82:78:c9:7b" }) +config.app(c, "source", basic_apps.Source) +config.link(c, "source.out1 -> nicp1.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +config.app(c, "nicp2", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + rxq = 0, txq = 0, + macaddr = "90:72:82:78:c9:7b" }) +config.link(c, "source.out2 -> nicp2.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +assert(engine.app_table.nicp1.poolnum == 0, "wrong poolnum for nicp1") +assert(engine.app_table.nicp2.poolnum == 1, "wrong poolnum for nicp2") diff --git a/src/apps/intel_mp/test_1g_vmdq_race.snabb b/src/apps/intel_mp/test_1g_vmdq_race.snabb new file mode 100755 index 0000000000..e49f90326c --- /dev/null +++ b/src/apps/intel_mp/test_1g_vmdq_race.snabb @@ -0,0 +1,68 @@ +#!../../snabb snsh + +-- Snabb test script that tests against race conditions in setting +-- VMDq parameters like MAC and VLAN + +local C = require("ffi").C +local intel = require("apps.intel_mp.intel_mp") +local lib = require("core.lib") +local worker = require("core.worker") + +local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") + +-- launch two worker processes each using VMDq with MAC addresses +-- and different VLAN tags and ensure that the chosen indices for +-- MAC & VLAN do not overlap. +-- +-- It's difficult for this test to fail unless delays are introduced +-- to deliberately trigger race conditions in the driver code (and +-- the locks disabled). +worker.start("worker0", [[ + local intel = require("apps.intel_mp.intel_mp") + local lib = require("core.lib") + local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") + local c = config.new() + config.app(c, "nic0", intel.Intel, + { pciaddr = pciaddr0, + vmdq = true, + poolnum = 0, + rxq = 0, txq = 0, + vlan = 0, + macaddr = "00:11:22:33:44:55" }) + engine.configure(c) + engine.main({ duration = 1 }) + assert(engine.app_table.nic0.r.RAH[1]:bits(31, 1) == 1) + assert(engine.app_table.nic0.r.VLVF[0]:bits(31, 1) == 1) +]]) + +worker.start("worker1", [[ + local intel = require("apps.intel_mp.intel_mp") + local lib = require("core.lib") + local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") + local c = config.new() + config.app(c, "nic1", intel.Intel, + { pciaddr = pciaddr0, + vmdq = true, + poolnum = 1, + rxq = 0, txq = 0, + vlan = 1, + macaddr = "55:44:33:22:11:00" }) + engine.configure(c) + engine.main({ duration = 1 }) + assert(engine.app_table.nic1.r.RAH[2]:bits(31, 1) == 1) + assert(engine.app_table.nic1.r.VLVF[1]:bits(31, 1) == 1) +]]) + +-- loop until all workers are done +while true do + local live = false + for w, s in pairs(worker.status()) do + live = live or s.alive + end + + if live then + C.sleep(0.1) + else + break + end +end diff --git a/src/apps/intel_mp/test_1g_vmdq_reconfig_mac.snabb b/src/apps/intel_mp/test_1g_vmdq_reconfig_mac.snabb new file mode 100755 index 0000000000..96aef1c3c5 --- /dev/null +++ b/src/apps/intel_mp/test_1g_vmdq_reconfig_mac.snabb @@ -0,0 +1,80 @@ +#!../../snabb snsh + +-- Snabb test script for checking that the MAC & VLAN indices +-- are selected correctly after an intel_mp instance is shut +-- down and restarted with a different config. +-- +-- This test exists to test the bug that was fixed in commit +-- 721f10bb0c3e076f79c54086fefd554851ac9679 + +local basic_apps = require("apps.basic.basic_apps") +local intel = require("apps.intel_mp.intel_mp") +local pcap = require("apps.pcap.pcap") +local lib = require("core.lib") + +local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") + +local c = config.new() + +-- first add two apps in order with different MACs and VLANs +config.app(c, "nicp0", intel.Intel, + { pciaddr = pciaddr0, + vmdq = true, + poolnum = 0, + rxq = 0, txq = 0, + vlan = 42, + macaddr = "00:11:22:33:44:55" }) +config.app(c, "source", basic_apps.Source) + +config.link(c, "source.out0 -> nicp0.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +config.app(c, "nicp1", intel.Intel, + { pciaddr = pciaddr0, + vmdq = true, + poolnum = 1, + rxq = 0, txq = 0, + vlan = 43, + macaddr = "55:44:33:22:11:00" }) +config.link(c, "source.out1 -> nicp1.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +-- then shut down the first one and start a new one on the same +-- pool as nicp1 +local c = config.new() + +config.app(c, "nicp1", intel.Intel, + { pciaddr = pciaddr0, + vmdq = true, + poolnum = 1, + rxq = 0, txq = 0, + vlan = 43, + macaddr = "55:44:33:22:11:00" }) +config.app(c, "source", basic_apps.Source) +config.link(c, "source.out1 -> nicp1.input") + +config.app(c, "nicp2", intel.Intel, + { pciaddr = pciaddr0, + vmdq = true, + poolnum = 1, + rxq = 0, txq = 0, + vlan = 43, + macaddr = "55:44:33:22:11:00" }) +config.link(c, "source.out2 -> nicp2.input") + +engine.configure(c) +engine.main({ duration = 0.1 }) + +-- MAC addresses are never disabled by intel_mp (or intel10g) +-- so both stay enabled. +-- TODO: this is issue #1205 (https://github.com/snabbco/snabb/issues/1205) +-- and these checks may need to change when that is resolved +assert(engine.app_table.nicp1.r.RAH[1]:bits(31, 1) == 1) +assert(engine.app_table.nicp2.r.RAH[2]:bits(31, 1) == 1) +-- only the second VLAN register should be enabled +assert(engine.app_table.nicp1.r.VLVF[0]:bits(31, 1) == 0) +assert(engine.app_table.nicp2.r.VLVF[1]:bits(31, 1) == 1) diff --git a/src/apps/intel_mp/test_1g_vmdq_tx.sh b/src/apps/intel_mp/test_1g_vmdq_tx.sh new file mode 100755 index 0000000000..dfca2c6062 --- /dev/null +++ b/src/apps/intel_mp/test_1g_vmdq_tx.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# +# Test packet transmit for VMDq mode + +./testvmdqtx.snabb $SNABB_PCI_INTEL1G0 $SNABB_PCI_INTEL1G1 "50:46:5d:74:1d:f9" source.pcap From 33c0912092e085cf9a7f68c3c89a7030dff86fa2 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Jun 2018 12:43:39 +0200 Subject: [PATCH 03/81] apps/intel_mp: RQDPC is actually an RC register on the i350 --- src/apps/intel_mp/intel_mp.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/intel_mp/intel_mp.lua b/src/apps/intel_mp/intel_mp.lua index 7ef8e96854..edbd461e2a 100644 --- a/src/apps/intel_mp/intel_mp.lua +++ b/src/apps/intel_mp/intel_mp.lua @@ -275,7 +275,7 @@ DVMOLR 0x0C038 +0x04*0..7 RW DMA VM Offload register VMRCTL 0x05D80 +0x04*0..7 RW Virtual Mirror rule control VMRVLAN 0x05D90 +0x04*0..7 RW Virtual Mirror rule VLAN VMRVM 0x05DA0 +0x04*0..7 RW Virtual Mirror rule VM -RQDPC 0x0C030 +0x40*0..7 RCR Receive Queue Drop Packet Count +RQDPC 0x0C030 +0x40*0..7 RC Receive Queue Drop Packet Count TQDPC 0x0E030 +0x40*0..7 RCR Transmit Queue Drop Packet Count PQGPRC 0x10010 +0x100*0..7 RCR Per Queue Good Packets Received Count PQGPTC 0x10014 +0x100*0..7 RCR Per Queue Good Packets Transmitted Count From 2bd39aee5d84cc74be4343438f20f4d257c2a68e Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 17 Sep 2018 13:54:38 +0200 Subject: [PATCH 04/81] intel_mp: fix 1G blast selftest via use of RateLimitedRepeater MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lack of backpressure in basic_apps.Repeater caused the testsend.snabb to predict unpredictable/unbalanced packet distributions, causing test cases on slower 1G NICs to fail. This changes testsend.snabb to use lwaftr’s RateLimitedRepeater, and sets an appropriate rate for the 1G tests. --- src/apps/intel_mp/test_1g_2q_blast.sh | 2 +- src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh | 6 +++--- src/apps/intel_mp/testsend.snabb | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/apps/intel_mp/test_1g_2q_blast.sh b/src/apps/intel_mp/test_1g_2q_blast.sh index 6f8271573e..1173315fa1 100755 --- a/src/apps/intel_mp/test_1g_2q_blast.sh +++ b/src/apps/intel_mp/test_1g_2q_blast.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -SNABB_SEND_BLAST=true ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source.pcap & +SNABB_SEND_BLAST=true SNABB_SEND_BLAST_RATE=1e9 ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source.pcap & BLAST=$! SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testrecv.snabb $SNABB_PCI_INTEL1G0 0 > results.0 & diff --git a/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh b/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh index d1e5e9a558..8ee7c48283 100755 --- a/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh +++ b/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh @@ -2,7 +2,7 @@ # # Test VMDq with automatic pool selection -SNABB_SEND_BLAST=true ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source2.pcap & +SNABB_SEND_BLAST=true SNABB_SEND_BLAST_RATE=1e9 ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source2.pcap & BLAST=$! SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" nil nil nil > results.0 & @@ -18,7 +18,7 @@ kill -9 $BLAST # both queues should see packets [[ `cat results.* | grep "^GPRC" | awk '{print $2}'` -gt 10000 ]] &&\ -[[ `cat results.0 | grep -m 1 bpp | awk '{print $11}'` -gt 0 ]] &&\ -[[ `cat results.1 | grep -m 1 bpp | awk '{print $11}'` -gt 0 ]] +[[ `cat results.0 | grep -m 1 fpb | awk '{print $9}'` -gt 0 ]] &&\ +[[ `cat results.1 | grep -m 1 fpb | awk '{print $9}'` -gt 0 ]] exit $? diff --git a/src/apps/intel_mp/testsend.snabb b/src/apps/intel_mp/testsend.snabb index baeb02b463..5f61cc978a 100755 --- a/src/apps/intel_mp/testsend.snabb +++ b/src/apps/intel_mp/testsend.snabb @@ -15,8 +15,9 @@ config.app(c, "nic", intel.Intel, {pciaddr=pciaddr, txq=qno, wait_for_link=true}) if os.getenv("SNABB_SEND_BLAST") then - local basic = require("apps.basic.basic_apps") - config.app(c, "repeat", basic.Repeater) + local loadgen = require("apps.lwaftr.loadgen") + local rate = tonumber(os.getenv("SNABB_SEND_BLAST_RATE")) or 10e9 + config.app(c, "repeat", loadgen.RateLimitedRepeater,{rate=rate}) config.link(c, "pcap.output -> repeat.input") config.link(c, "repeat.output -> nic.input") else From 9ea857e979e8701c64f6c6eb548474ea50ae5213 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 17 Sep 2018 13:58:10 +0200 Subject: [PATCH 05/81] intel_mp: port more vmdq tests to 1G cards. --- src/apps/intel_mp/test_1g_2q_blast_vlan.sh | 19 +++++++++++++++++++ src/apps/intel_mp/test_1g_2q_blast_vmdq.sh | 20 ++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100755 src/apps/intel_mp/test_1g_2q_blast_vlan.sh create mode 100755 src/apps/intel_mp/test_1g_2q_blast_vmdq.sh diff --git a/src/apps/intel_mp/test_1g_2q_blast_vlan.sh b/src/apps/intel_mp/test_1g_2q_blast_vlan.sh new file mode 100755 index 0000000000..c0893bf46f --- /dev/null +++ b/src/apps/intel_mp/test_1g_2q_blast_vlan.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# +# Test VMDq mode with two pools with different VLANs + +SNABB_SEND_BLAST=true SNABB_SEND_BLAST_RATE=1e9 ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source-vlan.pcap & +BLAST=$! + +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" 1 0 0 > results.0 & +PID1=$! +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" 2 1 0 > results.1 + +wait $PID1 +kill -9 $BLAST +[[ `cat results.* | grep "^GPRC" | awk '{print $2}'` -gt 10000 ]] &&\ +# both queues should see packets +[[ `cat results.0 | grep -m 1 fpb | awk '{print $9}'` -gt 0 ]] &&\ +[[ `cat results.1 | grep -m 1 fpb | awk '{print $9}'` -gt 0 ]] + +exit $? diff --git a/src/apps/intel_mp/test_1g_2q_blast_vmdq.sh b/src/apps/intel_mp/test_1g_2q_blast_vmdq.sh new file mode 100755 index 0000000000..f33ade96f9 --- /dev/null +++ b/src/apps/intel_mp/test_1g_2q_blast_vmdq.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Test VMDq mode with two pools with different MACs + +SNABB_SEND_BLAST=true SNABB_SEND_BLAST_RATE=1e9 ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source2.pcap & +BLAST=$! + +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" nil 0 0 > results.0 & +PID1=$! +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "12:34:56:78:9a:bc" nil 7 0 > results.1 + +wait $PID1 +kill -9 $BLAST + +# both queues should see packets +[[ `cat results.* | grep "^GPRC" | awk '{print $2}'` -gt 10000 ]] &&\ +[[ `cat results.0 | grep -m 1 fpb | awk '{print $9}'` -gt 0 ]] &&\ +[[ `cat results.1 | grep -m 1 fpb | awk '{print $9}'` -gt 0 ]] + +exit $? From afc971b71f84c614d0d597895cc598fa91e442cc Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 17 Sep 2018 14:05:45 +0200 Subject: [PATCH 06/81] intel_mp: revert queuing -> queueing spelling change from 9c4e3ee603 --- src/apps/intel_mp/intel_mp.lua | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/src/apps/intel_mp/intel_mp.lua b/src/apps/intel_mp/intel_mp.lua index edbd461e2a..93d64ec1eb 100644 --- a/src/apps/intel_mp/intel_mp.lua +++ b/src/apps/intel_mp/intel_mp.lua @@ -303,8 +303,7 @@ Intel = { pciaddr = {required=true}, ring_buffer_size = {default=2048}, vmdq = {default=false}, - vmdq_queueing_mode = {default="rss-64-2"}, - vmdq_queuing_mode = {}, -- legacy misspelling + vmdq_queuing_mode = {default="rss-64-2"}, macaddr = {}, poolnum = {}, vlan = {}, @@ -340,9 +339,9 @@ driver = Intel vmdq_enabled_t = ffi.typeof("struct { uint8_t enabled; }") -- C type for shared memory indicating which pools are used local vmdq_pools_t = ffi.typeof("struct { uint8_t pools[64]; }") --- C type for VMDq queueing mode +-- C type for VMDq queuing mode -- mode = 0 for 32 pools/4 queues, 1 for 64 pools/2 queues -local vmdq_queueing_mode_t = ffi.typeof("struct { uint8_t mode; }") +local vmdq_queuing_mode_t = ffi.typeof("struct { uint8_t mode; }") function Intel:new (conf) local self = { @@ -370,7 +369,7 @@ function Intel:new (conf) -- processes shm_root = "/intel-mp/" .. pci.canonical(conf.pciaddr) .. "/", -- only used for main process, affects max pool number - vmdq_queueing_mode = conf.vmdq_queuing_mode or conf.vmdq_queueing_mode + vmdq_queuing_mode = conf.vmdq_queuing_mode } local vendor = lib.firstline(self.path .. "/vendor") @@ -400,14 +399,14 @@ function Intel:new (conf) for i=0, 63 do vmdq_shm.pools[i] = 0 end shm.unmap(vmdq_shm) -- set VMDq pooling method for all instances on this NIC - local mode_shm = shm.create(self.shm_root .. "vmdq_queueing_mode", - vmdq_queueing_mode_t) - if self.vmdq_queueing_mode == "rss-32-4" then + local mode_shm = shm.create(self.shm_root .. "vmdq_queuing_mode", + vmdq_queuing_mode_t) + if self.vmdq_queuing_mode == "rss-32-4" then mode_shm.mode = 0 - elseif self.vmdq_queueing_mode == "rss-64-2" then + elseif self.vmdq_queuing_mode == "rss-64-2" then mode_shm.mode = 1 else - error("Invalid VMDq queueing mode") + error("Invalid VMDq queuing mode") end shm.unmap(mode_shm) end @@ -558,9 +557,9 @@ function Intel:select_pool() elseif self.registers == "82599ES" then -- max queue number is different in VMDq mode self.max_q = 128 - -- check the queueing mode in shm, adjust max pools based on that - local mode_shm = shm.open(self.shm_root .. "vmdq_queueing_mode", - vmdq_queueing_mode_t) + -- check the queuing mode in shm, adjust max pools based on that + local mode_shm = shm.open(self.shm_root .. "vmdq_queuing_mode", + vmdq_queuing_mode_t) if mode_shm.mode == 0 then self.max_pool = 32 else @@ -1231,8 +1230,8 @@ function Intel1g:vmdq_enable () -- 4.6.11.1.1 Global Filtering and Offload Capabilities assert(self.registers == "i350", "VMDq not supported by "..self.registers) -- 011b = Multiple receive queues as defined by VMDq based on packet - -- destination MAC address (RAH.POOLSEL) and Ether-type queueing decision - -- filters. NB: ignore self.vmdq_queueing_mode, i350 only supports 8 pools + -- destination MAC address (RAH.POOLSEL) and Ether-type queuing decision + -- filters. NB: ignore self.vmdq_queuing_mode, i350 only supports 8 pools -- with one queue each. self.r.MRQC:bits(0, 3, 0x3) -- No packet splitting @@ -1599,7 +1598,7 @@ function Intel82599:vmdq_enable () -- must be set prior to setting MTQC (7.2.1.2.1) self.r.RTTDCS:set(bits { ARBDIS=6 }) - if self.vmdq_queueing_mode == "rss-32-4" then + if self.vmdq_queuing_mode == "rss-32-4" then -- 1010 -> 32 pools, 4 RSS queues each self.r.MRQC:bits(0, 4, 0xA) -- Num_TC_OR_Q=10b -> 32 pools (4.6.11.3.3 and 8.2.3.9.15) From 8ef2d114a27b55bc9260a88110feba1be16710b6 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 22 Nov 2018 14:32:51 +0100 Subject: [PATCH 07/81] InboundDispatch: prepend/strip ethernet header and use pf.match --- src/program/vita/dispatch.lua | 48 +++++++++++++++++------------------ 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/src/program/vita/dispatch.lua b/src/program/vita/dispatch.lua index dd82428dda..9445981c53 100644 --- a/src/program/vita/dispatch.lua +++ b/src/program/vita/dispatch.lua @@ -180,50 +180,48 @@ InboundDispatch = { function InboundDispatch:new (conf) local o = { - node_ip4n = ipv4:pton(conf.node_ip4), - ip4 = ipv4:new({}) + eth4 = ethernet:new{type=0x0800}, + p_box = ffi.new("struct packet *[1]"), + dispatch = pf_match.compile(([[match { + ip dst host %s and icmp => icmp4 + ip dst host %s => protocol4_unreachable + ip => forward4 + otherwise => reject_protocol + }]]):format(conf.node_ip4, conf.node_ip4)) } return setmetatable(o, {__index=InboundDispatch}) end -function InboundDispatch:forward4 (p) +function InboundDispatch:forward4 () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) link.transmit(self.output.forward4, p) end -function InboundDispatch:icmp4 (p) +function InboundDispatch:icmp4 () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) link.transmit(self.output.icmp4, p) end -function InboundDispatch:protocol4_unreachable (p) +function InboundDispatch:protocol4_unreachable () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) link.transmit(self.output.protocol4_unreachable, p) end -function InboundDispatch:reject_protocol (p) - packet.free(p) +function InboundDispatch:reject_protocol () + packet.free(self.p_box[0]) counter.add(self.shm.protocol_errors) end -function InboundDispatch:dispatch (p) - local ip4 = self.ip4:new_from_mem(p.data, p.length) - if ip4 then - if ip4:dst_eq(self.node_ip4n) then - if ip4:protocol() == icmp.ICMP4.PROTOCOL then - self:icmp4(p) - else - self:protocol4_unreachable(p) - end - else - self:forward4(p) - end - else - self:reject_protocol(p) - end -end - function InboundDispatch:push () + local eth_hdr4 = self.eth4:header() for _, input in ipairs(self.input) do while not link.empty(input) do - self:dispatch(link.receive(input)) + local p = link.receive(input) + -- Prepend Ethernet pseudo header to please pf.match (we receive plain + -- IPv4 frames on the input port.) + p = packet.prepend(p, eth_hdr4, ethernet:sizeof()) + self.p_box[0] = p + self:dispatch(p.data, p.length) end end end From 2792df0bae63bc01a0f39d8d17507e5935a18748 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 29 Nov 2018 20:24:00 +0100 Subject: [PATCH 08/81] Makefile.vita: fixup INCLUDE_TEST pattern to work with POSIX shell (da)sh does not recognize {a,b} patterns. --- src/Makefile.vita | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Makefile.vita b/src/Makefile.vita index ee0fe319d1..86f492c0d4 100644 --- a/src/Makefile.vita +++ b/src/Makefile.vita @@ -16,7 +16,8 @@ INCLUDE_TEST = $(INCLUDE) \ lib/pcap apps/pcap \ program/snsh program/snabbmark \ lib/virtio apps/vhost apps/ipsec apps/ipv6 \ - apps/lwaftr/{loadgen,lwutil,constants}.* program/loadtest + apps/lwaftr/lwutil.* apps/lwaftr/constants.* \ + apps/lwaftr/loadgen.* program/loadtest all: INCLUDE='$(INCLUDE)' $(MAKE) From b8fffca3702484a4fe1233969caeb04e80dc2913 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 28 Nov 2018 14:13:51 +0100 Subject: [PATCH 09/81] vita-ske2a: design and implementation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the vita-ake protocol major version... The protocol fsm is split up to model a responder and an initiator, where the initiator begins by requesting a nonced challenge from the responder. Only an authentic challenges will be considered by the initiator, and enable it to craft an authentic nonce_key message to be accepted by the responder. See fsm-protocol.png These changes address issues with the previous protocol fsm which required a lot of quirky logic and jitter hacks to ensure it wouldn’t clog up (and there was a remaining defect where spoofing an initial nonce message in between two idle peers would cause them to DoS each other, OOPS.) Cryptographically, the new protocol version should be equivalent, with the exception that participants can no longer be tricked into accepting nonces from unauthenticated peers. On the KeyManager side, SAs are now maintained in a set of queues in an effort to prepare it for scaling Vita by negotiating multiple SA pairs for each route. Generally speaking, the new protocol allowed to simplify and streamline a few tricky parts of the implementation. Notably missing, as of yet, is a strategy to deal with compounded packet loss in a multiple SA per route setup. --- src/program/vita/README.exchange | 67 +- src/program/vita/exchange.lua | 995 ++++++++++++++++--------- src/program/vita/fsm-protocol.dot | 24 + src/program/vita/fsm-protocol.png | Bin 0 -> 59359 bytes src/program/vita/fsm-protocol.svg | 516 ------------- src/program/vita/vita-esp-gateway.yang | 37 +- 6 files changed, 738 insertions(+), 901 deletions(-) create mode 100644 src/program/vita/fsm-protocol.dot create mode 100644 src/program/vita/fsm-protocol.png delete mode 100644 src/program/vita/fsm-protocol.svg diff --git a/src/program/vita/README.exchange b/src/program/vita/README.exchange index c00f749b55..c2bf3efd5d 100644 --- a/src/program/vita/README.exchange +++ b/src/program/vita/README.exchange @@ -1,4 +1,4 @@ -VITA: SIMPLE KEY EXCHANGE (vita-ske, version 1i) +VITA: SIMPLE KEY EXCHANGE (vita-ske, version 2a) A simple key negotiation protocol based on pre-shared symmetric keys that provides authentication, perfect forward secrecy, and is immune to replay @@ -24,38 +24,57 @@ Notational Conventions: Description: Let k be a pre-shared symmetric key. Let r be a “Security Parameter Index” - (SPI). Let n1 and n2 be random 256‑bit nonces, where n1 is chosen by us. + (SPI). + + Two parties participate in the protocol: an initiator and a responder. + + ---- + + The initiator behaves as follows: Let n1 and n2 be random 256‑bit nonces, + where n1 is chosen by us. Let s1 be a random ephemeral key and p1 its + corresponding public key chosen by us, and p2 be a public key chosen by the + other party. Let spi1 and spi2 be child SPIs, where spi1 is choosen by us. ← n1 - → n2 + → n2 ‖ c - Let (p1, s1) and (p2, s2) be random, ephemeral, asymmetric key pairs, where - (p1, s1) is choosen by us. Let spi1 and spi2 be child SPIs, where spi1 is - choosen by us. Let h1 = HMAC(k, r ‖ n1 ‖ n2 ‖ spi1 ‖ p1). + Ensure c = HMAC(k, r ‖ n1 ‖ n2). Let h1 = HMAC(k, r ‖ n1 ‖ n2 ‖ spi1 ‖ p1). - ← spi1 ‖ p1 ‖ h1 + ← n1 ‖ spi1 ‖ p1 ‖ h1 → spi2 ‖ p2 ‖ h2 + Ensure that h2 = HMAC(k, r ‖ n2 ‖ n1 ‖ spi2 ‖ p2). + + Let q = DH(s1, p2), and ensure that p2 is a valid argument. Let + rx = HASH(q ‖ p1 ‖ p2) and tx = HASH(q ‖ p2 ‖ p1) be key material. Assign + (spi1, rx) to the inbound “Security Association” (SA), and (spi2, tx) to the + outbound SA. + + Discard s1, n1, n2, p1, and p2. + + ---- + + The responder behaves as follows: Let nc, n1 and n2 be random 256‑bit nonces, + where n1 is chosen by us. Let s1 be a random ephemeral key and p1 its + corresponding public key chosen by us, and p2 be a public key chosen by the + other party. Let spi1 and spi2 be child SPIs, where spi1 is choosen by us. + + ← nc + + Let c = HMAC(k, r ‖ nc ‖ n1). + + → n1 ‖ c + ← n2 ‖ spi2 ‖ p2 ‖ h2 + Ensure that h2 = HMAC(k, r ‖ n2 ‖ n1 ‖ spi2 ‖ p2). Let q = DH(s1, p2), and ensure that p2 is a valid argument. Let rx = HASH(q ‖ p1 ‖ p2) and tx = HASH(q ‖ p2 ‖ p1) be key material. Assign (spi1, rx) to the inbound “Security Association” (SA), and (spi2, tx) to the outbound SA. + Let h1 = HMAC(k, r ‖ n1 ‖ n2 ‖ spi1 ‖ p1). - The description above illustrates the perspective of an active party adhering - to the protocol, i.e. the exchange is initiated by us. An opposite, passive - party adhering to the protocol, i.e. one that is merely responding to a key - exchange initiated by an active party, must ensure that the tuple - (spi2, p2, h2) was received and authenticated before computing and sending its - response tuple (spi1, p1, h1). For a passive party the order of messages - during the exchange is reversed: - - → n2 - ← n1 - → spi2 ‖ p2 ‖ h2 (ensure h2 is verified before we reply) - ← spi1 ‖ p1 ‖ h1 + → spi1 ‖ p1 ‖ h1 - Note that there might be no passive party in an exchange if both parties have - initiated the exchange before receiving an initial nonce message. + Discard s1, n1, n2, p1, and p2. Security Proof: @@ -68,9 +87,9 @@ Security Proof: q = DH(s1, p2) or q = DH(s2, p1), and subsequently derive rx or tx, and thus perfect forward secrecy is provided. - A party passively adhering to the protocol will not produce a tuple - (spi, p1, h1) unless it has previously authenticated its counterpart tuple - (spi, p2, h2), and thus can not be used as an oracle. + A party that adheres to the protocol will not produce a tuple (spi1, p1, h1) + unless it has previously authenticated either the tuple (n2, spi2, p2, h2), or + the tuple (n2, c), and thus can not be used as an oracle. Notes: diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 97d22f502c..cc05822e7b 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -2,35 +2,34 @@ module(...,package.seeall) --- This module handles KEY NEGOTIATION with peers and SA CONFIGURATION, which --- includes dynamically reacting to changes to the routes defined in Vita’s --- root configuration. For each route defined in the gateway’s configuration a --- pair of SAs (inbound and outbound) is negotiated and maintained. On change, --- the set of SAs is written to configuration files picked up by the esp_worker --- and dsp_worker processes. --- --- (neg. proto.) +-- This module handles AUTHENTICATED KEY EXCHANGE with peers and +-- SECURITY ASSOCIATION (SA) CONFIGURATION, which includes dynamically reacting +-- to changes to the routes defined in Vita’s root configuration. For each +-- route defined in the gateway’s configuration pairs of SAs (inbound and +-- outbound) are negotiated and maintained. On change, the set of SAs is +-- written to the Security Association Database (SAD) which is used to +-- configure the data plane. +-- +-- (AKE protocol) -- || -- || --- --> KeyManager *--> esp_worker --- | --- \--> dsp_worker +-- --> KeyManager --> -- -- All things considered, this is the hairy part of Vita, as it covers touchy -- things such as key generation and expiration, and ultimately presents Vita’s -- main exploitation surface. On the upside, this module’s data plane doesn’t --- need worry as much about soft real-time requirements as others, as its +-- need to worry as much about soft real-time requirements as others, as its -- generally low-throughput. It can (and should) primarily focus on safety, -- and can afford more costly dynamic high-level language features to do so. --- At least to the extent where to doesn’t enable low-traffic DoS, that is. +-- At least to the extent where to doesn’t enable low-bandwidth DoS, that is. -- -- In order to insulate failure, this module is composed of three subsystems: -- --- 1. The KeyManager app handles the data plane traffic (key negotiation +-- 1. The KeyManager app handles AKE control plane traffic (key negotiation -- requests and responses) and configuration plane changes (react to --- configuration changes and generate configurations for negotiated SAs). +-- configuration changes and update the SAD.) -- --- It tries its best to avoid clobbering valid SA configurations too. I.e. +-- It tries its best to avoid clobbering valid SA configurations, too. I.e. -- SAs whose routes are not changed in a configuration transition are -- unaffected by the ensuing re-configuration, allowing for seamless -- addition of new routes and network address renumbering. @@ -40,12 +39,27 @@ module(...,package.seeall) -- lifetime of a SA pair has expired (sa_ttl), it is destroyed, and -- eventually re-negotiated if applicable. -- +-- Active SAs are maintained in three lists, inbound_sa, outbound_sa, and +-- outbound_sa_queue, which behave as FIFO queues. Inbound_sa and +-- outbound_sa reflect the active set of SAs as committed to the SAD. Newly +-- established inbound SAs are inserted to the front of inbound_sa, +-- displacing SAs at its end according to max_inbound_sa. Newly established +-- outbound SAs are inserted at the end of outbound_sa_queue. Outbound SAs +-- are removed from the front of outbound_sa_queue and inserted to the +-- front of outbound_sa once their activation_delay has elapsed. If there +-- are less than num_outbound_sa SAs in the outbound_sa list, outbound SAs +-- are removed from the front of outbound_sa_queue and inserted to the +-- front of outbound_sa. SAs inserted into outbound_sa displace SAs at the +-- end of outbound_sa according to num_outbound_sa. SAs that have expired +-- (sa_ttl) are removed from the inbound_sa and outbound_sa lists. +-- -- Note that the KeyManager app will attempt to re-negotiate SAs long --- before they expire (specifically, once half of sa_ttl has passed), in --- order to avoid loss of tunnel connectivity during re-negotiation. +-- before they expire (approximately once half of sa_ttl has passed, see +-- rekey_timeout), in order to avoid loss of tunnel connectivity during +-- re-negotiation. -- -- Negotiation requests are fed from the input port to the individual --- Protocol finite-state machine (described below in 2.) of a route, and +-- Protocol finite-state machines (described below in 2.) of a route, and -- associated to routes via the Transport wrapper (described below in 3.). -- Replies and outgoing requests (also obtained by mediating with the -- Protocol fsm) are sent via the output port. @@ -74,29 +88,32 @@ module(...,package.seeall) -- negotiations_expired count of negotiations expired -- (negotiation_ttl) -- --- nonces_negotiated count of nonce pairs that were exchanged --- (elevated count can indicate DoS attempts) +-- challenges_offered count of challenges that were offered +-- (responder) +-- +-- challenges_accepted count of challenges that were accepted +-- (initiator) +-- +-- keypairs_offered count of ephemeral key pairs that were +-- offered (responder) -- -- keypairs_negotiated count of ephemeral key pairs that were --- exchanged +-- negotiated (initiator) -- --- keypairs_expired count of ephemeral key pairs that have --- expired (sa_ttl) +-- inbound_sa_expired count of inbound SAs that have expired +-- (sa_ttl) -- --- 2. The Protocol subsysem implements vita-ske1 (the cryptographic key --- exchange protocol defined in README.exchange) as a finite-state machine --- with a timeout (negotiation_ttl) in a way that should be mostly DoS --- resistant, i.e. it can’t be put into a waiting state by inbound --- requests. +-- outbound_sa_expired count of outbound SAs that have expired +-- (sa_ttl) -- --- For a state transition diagram see: fsm-protocol.svg +-- outbound_sa_rejected count of outbound SAs that were rejected +-- because their SPI was not unique -- --- Alternatively it has been considered to implement the protocol on top of --- a connection based transport protocol (like TCP), i.e. allow multiple --- concurrent negotiations for each individual route. Such a protocol --- implementation wasn’t immediately available, and implementing one seemed --- daunting, and that’s why now each route has just its one own Protocol --- fsm. +-- 2. The Protocol subsystem implements vita-ske2 (the cryptographic key +-- exchange protocol defined in README.exchange) as two finite-state +-- machines (responder and initiator). +-- +-- For a state transition diagram see: fsm-protocol.png -- -- The Protocol fsm requires its user (the KeyManager app) to “know” about -- the state transitions of the exchange protocol, but it is written in a @@ -105,9 +122,12 @@ module(...,package.seeall) -- -- initiate_exchange -- receive_nonce --- exchange_key +-- offer_challenge +-- receive_challenge +-- offer_key -- receive_key --- derive_ephemeral_keys +-- offer_nonce_key +-- receive_nonce_key -- reset_if_expired -- -- which uphold invariants that should ensure any resulting key material is @@ -119,8 +139,9 @@ module(...,package.seeall) -- 3. The Transport header is a super-light transport header that encodes the -- target SPI and message type of the protocol requests it precedes. It is -- used by the KeyManager app to parse requests and associate them to the --- correct route by SPI. It uses the IP protocol type 99 for “any private --- encryption scheme”. +-- correct route and fsm by SPI and message type respectively. It uses the +-- IP protocol type 99 for “any private encryption scheme”. (Eventually, it +-- needs to be wrapped in UDP.) -- -- It exists explicitly separate from the KeyManager app and Protocol fsm, -- to clarify that it is interchangable, and logically unrelated to either @@ -147,6 +168,8 @@ KeyManager = { node_ip4 = {required=true}, routes = {required=true}, sa_db_path = {required=true}, + num_outbound_sa = {default=1}, + max_inbound_sa = {default=4}, negotiation_ttl = {default=5}, -- default: 5 seconds sa_ttl = {default=(10 * 60)} -- default: 10 minutes }, @@ -158,18 +181,16 @@ KeyManager = { public_key_errors = {counter}, negotiations_initiated = {counter}, negotiations_expired = {counter}, - nonces_negotiated = {counter}, + challenges_offered = {counter}, + challenges_accepted = {counter}, + keypairs_offered = {counter}, keypairs_negotiated = {counter}, - keypairs_expired = {counter} + inbound_sa_expired = {counter}, + outbound_sa_expired = {counter}, + outbound_sa_rejected = {counter} } } -local status = { expired = 0, rekey = 1, ready = 2 } - -local function jitter (s) -- compute random jitter of up to s seconds - return s * math.random(1000) / 1000 -end - function KeyManager:new (conf) local o = { routes = {}, @@ -177,6 +198,8 @@ function KeyManager:new (conf) transport = Transport.header:new({}), nonce_message = Protocol.nonce_message:new({}), key_message = Protocol.key_message:new({}), + challenge_message = Protocol.challenge_message:new({}), + nonce_key_message = Protocol.nonce_key_message:new({}), sa_db_updated = false, sa_db_commit_throttle = lib.throttle(1) } @@ -197,9 +220,15 @@ function KeyManager:reconfig (conf) and route.spi == spi end local function free_route (route) - if route.status ~= status.expired then - audit:log("Expiring keys for '"..route.id.."' (reconfig)") - self:expire_route(route) + for _, sa in ipairs(route.inbound_sa) do + audit:log(("Expiring inbound SA %d for '%s' (reconfig)") + :format(sa.spi, route.id)) + self.sa_db_updated = true + end + for _, sa in ipairs(route.outbound_sa) do + audit:log(("Expiring outbound SA %d for '%s' (reconfig)") + :format(sa.spi, route.id)) + self.sa_db_updated = true end end @@ -215,10 +244,15 @@ function KeyManager:reconfig (conf) -- if negotation_ttl has changed, swap out old protocol fsm for a new -- one with the adjusted timeout, effectively resetting the fsm if conf.negotiation_ttl ~= self.negotiation_ttl then - audit:log("Protocol reset for "..id.." (reconfig)") - old_route.protocol = Protocol:new(old_route.spi, - old_route.preshared_key, - conf.negotiation_ttl) + audit:log("Protocol reset for '"..id.."' (reconfig)") + old_route.initiator = Protocol:new('initiator', + old_route.spi, + old_route.preshared_key, + conf.negotiation_ttl) + old_route.responder = Protocol:new('responder', + old_route.spi, + old_route.preshared_key, + conf.negotiation_ttl) end else -- insert new new route @@ -227,12 +261,15 @@ function KeyManager:reconfig (conf) gw_ip4n = ipv4:pton(route.gw_ip4), preshared_key = new_key, spi = route.spi, - status = status.expired, - rx_sa = nil, prev_rx_sa = nil, tx_sa = nil, next_tx_sa = nil, - sa_timeout = nil, prev_sa_timeout = nil, rekey_timeout = nil, - next_tx_sa_activation_delay = nil, - protocol = Protocol:new(route.spi, new_key, conf.negotiation_ttl), - negotiation_delay = lib.timeout(0) + inbound_sa = {}, outbound_sa = {}, outbound_sa_queue = {}, + responder = Protocol:new('responder', + route.spi, + new_key, + conf.negotiation_ttl), + initiator = Protocol:new('initiator', + route.spi, + new_key, + conf.negotiation_ttl) } table.insert(new_routes, new_route) -- clean up after the old route if necessary @@ -249,6 +286,8 @@ function KeyManager:reconfig (conf) self.node_ip4n = ipv4:pton(conf.node_ip4) self.routes = new_routes self.sa_db_file = shm.root.."/"..shm.resolve(conf.sa_db_path) + self.num_outbound_sa = conf.num_outbound_sa + self.max_inbound_sa = conf.max_inbound_sa self.negotiation_ttl = conf.negotiation_ttl self.sa_ttl = conf.sa_ttl end @@ -263,32 +302,56 @@ function KeyManager:push () end for _, route in ipairs(self.routes) do - -- process protocol timeouts and initiate (re-)negotiation for SAs - if route.protocol:reset_if_expired() == Protocol.code.expired then + -- process protocol timeouts + if route.initiator:reset_if_expired() == Protocol.code.expired then counter.add(self.shm.negotiations_expired) - audit:log("Negotiation expired for '"..route.id.."' (negotiation_ttl)") - route.negotiation_delay = lib.timeout( - self.negotiation_ttl + jitter(.25) - ) + audit:log(("Negotiation expired for '%s' (negotiation_ttl)") + :format(route.id)) end - if route.status > status.expired and route.sa_timeout() then - counter.add(self.shm.keypairs_expired) - audit:log("Keys expired for '"..route.id.."' (sa_ttl)") - self:expire_route(route) - elseif route.prev_sa_timeout and route.prev_sa_timeout() then - self:expire_prev_sa(route) + for index, sa in ipairs(route.inbound_sa) do + if sa.ttl() then + table.remove(route.inbound_sa, index) + self.sa_db_updated = true + counter.add(self.shm.inbound_sa_expired) + audit:log(("Inbound SA %d expired for '%s' (sa_ttl)") + :format(sa.spi, route.id)) + end end - if route.status > status.rekey and route.rekey_timeout() then - route.status = status.rekey + for index, sa in ipairs(route.outbound_sa) do + if sa.ttl() then + table.remove(route.outbound_sa, index) + self.sa_db_updated = true + counter.add(self.shm.outbound_sa_expired) + audit:log(("Outbound SA %d expired for '%s' (sa_ttl)") + :format(sa.spi, route.id)) + end end - if route.status < status.ready and route.negotiation_delay() then - self:negotiate(route) + + -- activate new outbound SAs + for index, sa in ipairs(route.outbound_sa_queue) do + if sa.activation_delay() + or #route.outbound_sa < self.num_outbound_sa then + audit:log(("Activating outbound SA %d for '%s'") + :format(sa.spi, route.id)) + -- insert in front of SA list + table.insert(route.outbound_sa, 1, sa) + table.remove(route.outbound_sa_queue, index) + self.sa_db_updated = true + end + end + -- remove superfluous outbound SAs from the end of the list + while #route.outbound_sa > self.num_outbound_sa do + table.remove(route.outbound_sa) + self.sa_db_updated = true end - -- activate new tx SAs - if route.next_tx_sa and route.next_tx_sa_activation_delay() then - audit:log("Activating next outbound SA for '"..route.id.."'") - self:activate_next_tx_sa(route) + -- initiate (re-)negotiation of SAs + local num_sa = #route.outbound_sa_queue + for _, sa in ipairs(route.outbound_sa) do + if not sa.rekey_timeout() then num_sa = num_sa + 1 end + end + if num_sa < self.num_outbound_sa then + self:negotiate(route) end end @@ -300,44 +363,81 @@ function KeyManager:push () end function KeyManager:negotiate (route) + -- Inititate AKE if the protocol fsm permits (i.e., is in the initiator + -- state.) local ecode, nonce_message = - route.protocol:initiate_exchange(self.nonce_message) + route.initiator:initiate_exchange(self.nonce_message) if not ecode then counter.add(self.shm.negotiations_initiated) audit:log("Initiating negotiation for '"..route.id.."'") link.transmit(self.output.output, self:request(route, nonce_message)) - end + else assert(ecode == Protocol.code.protocol) end end function KeyManager:handle_negotiation (request) local route, message = self:parse_request(request) if not (self:handle_nonce_request(route, message) - or self:handle_key_request(route, message)) then + or self:handle_key_request(route, message) + or self:handle_challenge_request(route, message) + or self:handle_nonce_key_request(route, message)) then counter.add(self.shm.rxerrors) - audit:log("Rejected invalid negotiation request") + audit:log(("Rejected invalid negotiation request for '%s'") + :format(route or "")) end end function KeyManager:handle_nonce_request (route, message) if not route or message ~= self.nonce_message then return end - local ecode, response = route.protocol:receive_nonce(message) + -- Receive nonce message if the protocol fsm permits + -- (responder -> offer_challenge), otherwise reject the message and return. + local ecode = route.responder:receive_nonce(message) if ecode == Protocol.code.protocol then counter.add(self.shm.protocol_errors) return false else assert(not ecode) end - counter.add(self.shm.nonces_negotiated) - audit:log("Negotiated nonces for '"..route.id.."'") + -- If we got here we should respond with a challenge message + -- (offer_challenge -> responder). + local response = self.challenge_message + local ecode, response = route.responder:offer_challenge(response) + assert(not ecode) + link.transmit(self.output.output, self:request(route, response)) - if response then - link.transmit(self.output.output, self:request(route, response)) - else - audit:log("Offering keys for '"..route.id.."'") - local _, key_message = route.protocol:exchange_key(self.key_message) - link.transmit(self.output.output, self:request(route, key_message)) - end + counter.add(self.shm.challenges_offered) + audit:log("Offered challenge for '"..route.id.."'") + + return true +end + +function KeyManager:handle_challenge_request (route, message) + if not route or message ~= self.challenge_message then return end + + -- Receive challenge message if the protocol fsm permits + -- (accept_challenge -> offer_nonce_key), reject the message and return + -- otherwise or if... + local ecode = route.initiator:receive_challenge(message) + if ecode == Protocol.code.protocol then + counter.add(self.shm.protocol_errors) + return false + elseif ecode == Protocol.code.authentication then + -- ...the message failed to authenticate. + counter.add(self.shm.authentication_errors) + return false + else assert(not ecode) end + + counter.add(self.shm.challenges_accepted) + audit:log("Accepted challenge for '"..route.id.."'") + + -- If we got here we should offer our nonce and public key + -- (offer_nonce_key -> accept_key). + local response = self.nonce_key_message + local ecode, response = route.initiator:offer_nonce_key(response) + assert(not ecode) + + audit:log("Proposing key exchange for '"..route.id.."'") + link.transmit(self.output.output, self:request(route, response)) return true end @@ -345,98 +445,127 @@ end function KeyManager:handle_key_request (route, message) if not route or message ~= self.key_message then return end - local ecode, response = route.protocol:receive_key(message) + -- Receive an authenticated key message if the protocol fsm permits + -- (accept_key -> initiator), reject the message and return otherwise or + -- if... + local ecode, inbound_sa, outbound_sa = route.initiator:receive_key(message) if ecode == Protocol.code.protocol then counter.add(self.shm.protocol_errors) return false elseif ecode == Protocol.code.authentication then + -- ...the message failed to authenticate. counter.add(self.shm.authentication_errors) return false + elseif ecode == Protocol.code.parameter then + -- ...the message offered a bad public key. + counter.add(self.shm.public_key_errors) + return false else assert(not ecode) end - local ecode, rx, tx = route.protocol:derive_ephemeral_keys() - if ecode == Protocol.code.parameter then + counter.add(self.shm.keypairs_negotiated) + audit:log(("Completed AKE for '%s' (inbound SA %d, outbound SA %d)"): + format(route.id, inbound_sa.spi, outbound_sa.spi)) + + self:add_inbound_sa(route, inbound_sa) + self:add_outbound_sa(route, outbound_sa) + + return true +end + +function KeyManager:handle_nonce_key_request (route, message) + if not route or message ~= self.nonce_key_message then return end + + -- Receive an authenticated, combined nonce and key message if the protocol + -- fsm permits (responder -> offer_key), reject the message and return + -- otherwise or if... + local ecode, inbound_sa, outbound_sa = + route.responder:receive_nonce_key(message) + if ecode == Protocol.code.protocol then + counter.add(self.shm.protocol_errors) + return false + elseif ecode == Protocol.code.authentication then + -- ...the message failed to authenticate. + counter.add(self.shm.authentication_errors) + return false + elseif ecode == Protocol.code.parameter then + -- ...the message offered a bad public key. counter.add(self.shm.public_key_errors) return false else assert(not ecode) end - counter.add(self.shm.keypairs_negotiated) - audit:log(("Completed key exchange for '%s' (rx-spi %d, tx-spi %d)"): - format(route.id, rx.spi, tx.spi)) + -- If we got here we should respond with our own public key + -- (offer_key -> responder). + local ecode, response = route.responder:offer_key(self.key_message) + assert(not ecode) - if response then - link.transmit(self.output.output, self:request(route, response)) - end + link.transmit(self.output.output, self:request(route, response)) + + counter.add(self.shm.keypairs_offered) + audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %d)"): + format(route.id, inbound_sa.spi, outbound_sa.spi)) - self:configure_route(route, rx, tx) + self:add_inbound_sa(route, inbound_sa) + self:add_outbound_sa(route, outbound_sa) return true end -function KeyManager:configure_route (route, rx, tx) +function KeyManager:add_inbound_sa (route, sa) + -- invariant: inbound SA spi is unique for _, route in ipairs(self.routes) do - if (route.rx_sa and route.rx_sa.spi == rx.spi) - or (route.prev_rx_sa and route.prev_rx_sa.spi == rx.spi) then - error("PANIC: SPI collision detected.") + for _, inbound in ipairs(route.inbound_sa) do + assert(inbound.spi ~= sa.spi, "PANIC: SPI collision detected.") end end - route.status = status.ready - -- Cycle inbound SAs immediately (keep receiving packets on the previous - -- inbound SA for up to the duration of its sa_ttl timeout.) - route.prev_rx_sa = route.rx_sa - route.prev_sa_timeout = route.sa_timeout - route.rx_sa = { - route = route.id, - spi = rx.spi, + -- insert in front of SA list + table.insert(route.inbound_sa, 1, { + spi = sa.spi, aead = "aes-gcm-16-icv", - key = lib.hexdump(rx.key), - salt = lib.hexdump(rx.salt) - } - -- Activate the new SA immediately if there is no previous outbound SA or - -- there is a (stale) next outbound SA queued for activation. - -- Otherwise, queue the superseding SA for activation after a delay (in - -- order to give the other node time to install the matching inbound SA - -- before we send any packets on it.) - route.next_tx_sa = { - route = route.id, - spi = tx.spi, - aead = "aes-gcm-16-icv", - key = lib.hexdump(tx.key), - salt = lib.hexdump(tx.salt) - } - if not route.tx_sa or route.next_tx_sa_activation_delay then - self:activate_next_tx_sa(route) - else - route.next_tx_sa_activation_delay = lib.timeout(self.negotiation_ttl*1.5) + key = lib.hexdump(sa.key), + salt = lib.hexdump(sa.salt), + ttl = lib.timeout(self.sa_ttl) + }) + -- remove superfluous SAs from the end of the list + while #route.inbound_sa > self.max_inbound_sa do + table.remove(route.inbound_sa) end - route.sa_timeout = lib.timeout(self.sa_ttl) - route.rekey_timeout = lib.timeout(self.sa_ttl/2 + jitter(.25)) self.sa_db_updated = true end -function KeyManager:activate_next_tx_sa (route) - route.tx_sa = route.next_tx_sa - route.next_tx_sa = nil - route.next_tx_sa_activation_delay = nil - self.sa_db_updated = true -end - -function KeyManager:expire_route (route) - route.status = status.expired - route.tx_sa = nil - route.next_tx_sa = nil - route.next_tx_sa_activation_delay = nil - route.rx_sa = nil - route.prev_rx_sa = nil - route.prev_sa_timeout = nil - route.sa_timeout = nil - route.rekey_timeout = nil - self.sa_db_updated = true -end - -function KeyManager:expire_prev_sa (route) - route.prev_rx_sa = nil - route.prev_sa_timeout = nil +function KeyManager:add_outbound_sa (route, sa) + -- invariant: outbound SA spi is unique + for _, route in ipairs(self.routes) do + for _, outbound in ipairs(route.outbound_sa) do + if outbound.spi == sa.spi then + counter.add(self.shm.outbound_sa_rejected) + audit:log(("Rejecting outbound SA %d for '%s' (duplicate SPI)") + :format(sa.spi, route.id)) + return + end + end + end + -- enqueue new outbound SA at the end of the queue + table.insert(route.outbound_sa_queue, { + spi = sa.spi, + aead = "aes-gcm-16-icv", + key = lib.hexdump(sa.key), + salt = lib.hexdump(sa.salt), + ttl = lib.timeout(self.sa_ttl), + -- Rekey outbound SAs after approximately half of sa_ttl, with a second + -- of jitter to reduce the probability of two peers initating the key + -- exchange concurrently. + rekey_timeout = lib.timeout(self.sa_ttl/2 + math.random(1000)/1000), + -- Delay before activating redundant, newly established outbound SAs to + -- give the receiving end time to set up. Choosen so that when a + -- negotiation times out due to packet loss, the initiator can supersede + -- the bogus outbound SA before it gets activated. + activation_delay = lib.timeout(self.negotiation_ttl*1.5) + }) + -- remove superfluous outbound SAs from the front of the queue + assert(self.num_outbound_sa == 1, "FIXME: displacement of queued outbound SAs only works for num_outbound_sa = 1.") + if #route.outbound_sa_queue > self.num_outbound_sa then + table.remove(route.outbound_sa_queue, 1) + end self.sa_db_updated = true end @@ -460,7 +589,12 @@ function KeyManager:request (route, message) and Transport.message_type.nonce) or (message == self.key_message and Transport.message_type.key) + or (message == self.challenge_message + and Transport.message_type.challenge) + or (message == self.nonce_key_message + and Transport.message_type.nonce_key) }) + packet.append(request, self.transport:header(), Transport.header:sizeof()) packet.append(request, message:header(), message:sizeof()) @@ -489,10 +623,15 @@ function KeyManager:parse_request (request) local data = request.data + Transport.header:sizeof() local length = request.length - Transport.header:sizeof() - local message = (transport:message_type() == Transport.message_type.nonce - and self.nonce_message:new_from_mem(data, length)) - or (transport:message_type() == Transport.message_type.key - and self.key_message:new_from_mem(data, length)) + local message = + (transport:message_type() == Transport.message_type.nonce + and self.nonce_message:new_from_mem(data, length)) + or (transport:message_type() == Transport.message_type.key + and self.key_message:new_from_mem(data, length)) + or (transport:message_type() == Transport.message_type.challenge + and self.challenge_message:new_from_mem(data, length)) + or (transport:message_type() == Transport.message_type.nonce_key + and self.nonce_key_message:new_from_mem(data, length)) if not message then counter.add(self.shm.protocol_errors) return @@ -507,35 +646,42 @@ function KeyManager:commit_sa_db () -- Collect currently active SAs local esp_keys, dsp_keys = {}, {} for _, route in ipairs(self.routes) do - if route.status == status.ready then - esp_keys[route.tx_sa.spi] = route.tx_sa - dsp_keys[route.rx_sa.spi] = route.rx_sa - if route.prev_rx_sa then - dsp_keys[route.prev_rx_sa.spi] = route.prev_rx_sa - end + for _, sa in ipairs(route.outbound_sa) do + esp_keys[sa.spi] = { + route=route.id, spi=sa.spi, aead=sa.aead, key=sa.key, salt=sa.salt + } + end + for _, sa in ipairs(route.inbound_sa) do + dsp_keys[sa.spi] = { + route=route.id, spi=sa.spi, aead=sa.aead, key=sa.key, salt=sa.salt + } end end -- Commit active SAs to SA database - yang.compile_config_for_schema( + local success, err = pcall( + yang.compile_config_for_schema, schemata['ephemeral-keys'], {outbound_sa=esp_keys, inbound_sa=dsp_keys}, self.sa_db_file ) + if not success then + audit:log("Failed to commit SA database: "..err) + end end --- Vita: simple key exchange (vita-ske, version 1i). See README.exchange --- --- NB: this implementation introduces two pseudo states _send_key and _complete --- not present in fsm-protocol.svg. The _send_key state is inserted in between --- the transition from wait_nonce to wait_key. Its purpose is to codify and --- enforce that exchange_key is called exactly once by the active party. The --- _complete state is inserted in between the transition from wait_key back to --- idle, and ensures that exactly one ephemeral key pair is derived for each --- successful exchange. +-- Vita: simple key exchange (vita-ske, version 2a). See README.exchange + +Message = subClass(header) +function Message:new_from_mem (mem, size) + if size == self:sizeof() then + return Message:superClass().new_from_mem(self, mem, size) + end +end Protocol = { - status = { idle = 0, wait_nonce = 1, wait_key = 2, - _send_key = -1, _complete = -2 }, + status = { responder = 0, initiator = 1, + offer_challenge = 2, accept_challenge = 3, + offer_key = 4, offer_nonce_key = 5, accept_key = 7 }, code = { protocol = 0, authentication = 1, parameter = 2, expired = 3}, spi_counter = 0, preshared_key_bytes = C.crypto_auth_hmacsha512256_KEYBYTES, @@ -554,9 +700,12 @@ Protocol = { } __attribute__((packed)) slot; } ]], - nonce_message = subClass(header), - key_message = subClass(header) + nonce_message = subClass(Message), + challenge_message = subClass(Message), + key_message = subClass(Message), + nonce_key_message = subClass(Message) } + Protocol.nonce_message:init({ [1] = ffi.typeof([[ struct { @@ -564,6 +713,16 @@ Protocol.nonce_message:init({ } __attribute__((packed)) ]]) }) + +Protocol.challenge_message:init({ + [1] = ffi.typeof([[ + struct { + uint8_t nonce[]]..Protocol.nonce_bytes..[[]; + uint8_t auth_code[]]..Protocol.auth_code_bytes..[[]; + } __attribute__((packed)) + ]]) +}) + Protocol.key_message:init({ [1] = ffi.typeof([[ struct { @@ -574,6 +733,17 @@ Protocol.key_message:init({ ]]) }) +Protocol.nonce_key_message:init({ + [1] = ffi.typeof([[ + struct { + uint8_t nonce[]]..Protocol.nonce_bytes..[[]; + uint8_t spi[]]..ffi.sizeof(Protocol.spi_t)..[[]; + uint8_t public_key[]]..Protocol.public_key_bytes..[[]; + uint8_t auth_code[]]..Protocol.auth_code_bytes..[[]; + } __attribute__((packed)) + ]]) +}) + -- Public API function Protocol.nonce_message:new (config) @@ -582,12 +752,6 @@ function Protocol.nonce_message:new (config) return o end -function Protocol.nonce_message:new_from_mem (mem, size) - if size == self:sizeof() then - return self:superClass().new_from_mem(self, mem, size) - end -end - function Protocol.nonce_message:nonce (nonce) local h = self:header() if nonce ~= nil then @@ -596,6 +760,23 @@ function Protocol.nonce_message:nonce (nonce) return h.nonce end +function Protocol.challenge_message:new (config) + local o = Protocol.challenge_message:superClass().new(self) + o:nonce(config.nonce) + o:auth_code(config.auth_code) + return o +end + +Protocol.challenge_message.nonce = Protocol.nonce_message.nonce + +function Protocol.challenge_message:auth_code (auth_code) + local h = self:header() + if auth_code ~= nil then + ffi.copy(h.auth_code, auth_code, ffi.sizeof(h.auth_code)) + end + return h.auth_code +end + function Protocol.key_message:new (config) local o = Protocol.key_message:superClass().new(self) o:spi(config.spi) @@ -604,12 +785,6 @@ function Protocol.key_message:new (config) return o end -function Protocol.key_message:new_from_mem (mem, size) - if size == self:sizeof() then - return self:superClass().new_from_mem(self, mem, size) - end -end - function Protocol.key_message:spi (spi) local h = self:header() if spi ~= nil then @@ -626,19 +801,27 @@ function Protocol.key_message:public_key (public_key) return h.public_key end -function Protocol.key_message:auth_code (auth_code) - local h = self:header() - if auth_code ~= nil then - ffi.copy(h.auth_code, auth_code, ffi.sizeof(h.auth_code)) - end - return h.auth_code +Protocol.key_message.auth_code = Protocol.challenge_message.auth_code + +function Protocol.nonce_key_message:new (config) + local o = Protocol.nonce_key_message:superClass().new(self) + o:nonce(config.nonce) + o:spi(config.spi) + o:public_key(config.public_key) + o:auth_code(config.auth_code) + return o end -function Protocol:new (r, key, timeout) +Protocol.nonce_key_message.nonce = Protocol.nonce_message.nonce +Protocol.nonce_key_message.spi = Protocol.key_message.spi +Protocol.nonce_key_message.public_key = Protocol.key_message.public_key +Protocol.nonce_key_message.auth_code = Protocol.challenge_message.auth_code + +function Protocol:new (initial_status, r, key, timeout) local o = { - status = Protocol.status.idle, + status = assert(Protocol.status[initial_status]), timeout = timeout, - deadline = nil, + deadline = false, k = ffi.new(Protocol.buffer_t, Protocol.preshared_key_bytes), r = ffi.new(Protocol.spi_t), n1 = ffi.new(Protocol.buffer_t, Protocol.nonce_bytes), @@ -656,71 +839,115 @@ function Protocol:new (r, key, timeout) } ffi.copy(o.k, key, ffi.sizeof(o.k)) o.r.u32 = lib.htonl(r) - return setmetatable(o, {__index=Protocol}) + return setmetatable(o, {__index=Protocol}):reset() end function Protocol:initiate_exchange (nonce_message) - if self.status == Protocol.status.idle then - self.status = Protocol.status.wait_nonce + assert(nonce_message:class() == Protocol.nonce_message) + if self.status == Protocol.status.initiator then + -- initiator -> accept_challenge self:set_deadline() + self.status = Protocol.status.accept_challenge return nil, self:send_nonce(nonce_message) else return Protocol.code.protocol end end function Protocol:receive_nonce (nonce_message) - if self.status == Protocol.status.idle then - self:intern_nonce(nonce_message) - return nil, self:send_nonce(nonce_message) - elseif self.status == Protocol.status.wait_nonce then + assert(nonce_message:class() == Protocol.nonce_message) + if self.status == Protocol.status.responder then + -- responder -> offer_challenge self:intern_nonce(nonce_message) - self.status = Protocol.status._send_key - self:set_deadline() - return nil + self.status = Protocol.status.offer_challenge else return Protocol.code.protocol end end -function Protocol:exchange_key (key_message) - if self.status == Protocol.status._send_key then - self.status = Protocol.status.wait_key - return nil, self:send_key(key_message) +function Protocol:offer_challenge (challenge_message) + assert(challenge_message:class() == Protocol.challenge_message) + if self.status == Protocol.status.offer_challenge then + -- offer_challenge -> responder + self.status = Protocol.status.responder + return nil, self:send_challenge(challenge_message) + else return Protocol.code.protocol end +end + +function Protocol:receive_challenge (challenge_message) + assert(challenge_message:class() == Protocol.challenge_message) + if self.status == Protocol.status.accept_challenge then + -- accept_challenge -> offer_nonce_key + if self:intern_challenge_nonce(challenge_message) then + self.status = Protocol.status.offer_nonce_key + else return Protocol.code.authentication end + else return Protocol.code.protocol end +end + +function Protocol:offer_key (key_message) + assert(key_message:class() == Protocol.key_message) + if self.status == Protocol.status.offer_key then + -- offer_key -> responder + local response = self:send_key(key_message) + self:reset() + self.status = Protocol.status.responder + return nil, response else return Protocol.code.protocol end end function Protocol:receive_key (key_message) - if self.status == Protocol.status.idle - or self.status == Protocol.status.wait_key then + assert(key_message:class() == Protocol.key_message) + if self.status == Protocol.status.accept_key then + -- accept_key -> initiator if self:intern_key(key_message) then - local response = self.status == Protocol.status.idle - and self:send_key(key_message) - self.status = Protocol.status._complete - return nil, response + local ok, inbound_sa, outbound_sa = self:derive_ephemeral_keys() + if ok then + self:reset() + self.status = Protocol.status.initiator + return nil, inbound_sa, outbound_sa + else return Protocol.code.parameter end else return Protocol.code.authentication end else return Protocol.code.protocol end end -function Protocol:derive_ephemeral_keys () - if self.status == Protocol.status._complete then - self:reset() - if self:derive_shared_secret() then - local rx = self:derive_key_material(self.spi1, self.p1, self.p2) - local tx = self:derive_key_material(self.spi2, self.p2, self.p1) - return nil, rx, tx - else return Protocol.code.paramter end +function Protocol:offer_nonce_key (nonce_key_message) + assert(nonce_key_message:class() == Protocol.nonce_key_message) + if self.status == Protocol.status.offer_nonce_key then + -- offer_nonce_key -> accept_key + self.status = Protocol.status.accept_key + self:set_deadline() + return nil, self:send_key(self:send_nonce(nonce_key_message)) + else return Protocol.code.protocol end +end + +function Protocol:receive_nonce_key (nonce_key_message) + assert(nonce_key_message:class() == Protocol.nonce_key_message) + if self.status == Protocol.status.responder then + -- responder -> offer_key + self:intern_nonce(nonce_key_message) + if self:intern_key(nonce_key_message) then + local ok, inbound_sa, outbound_sa = self:derive_ephemeral_keys() + if ok then + self.status = Protocol.status.offer_key + return nil, inbound_sa, outbound_sa + else return Protocol.code.parameter end + else return Protocol.code.authentication end else return Protocol.code.protocol end end function Protocol:reset_if_expired () - if self.deadline and self.deadline() then - self:reset() - return Protocol.code.expired - end + if self.status == Protocol.status.accept_challenge + or self.status == Protocol.status.accept_key then + if self.deadline and self.deadline() then + -- accept_challenge, accept_key -> initiator + self:reset() + self.status = Protocol.status.initiator + return Protocol.code.expired + end + else return Protocol.code.protocol end end -- Internal methods function Protocol:send_nonce (nonce_message) - C.randombytes_buf(self.n1, ffi.sizeof(self.n1)) - return nonce_message:new({nonce=self.n1}) + nonce_message:nonce(self.n1) + return nonce_message end function Protocol:intern_nonce (nonce_message) @@ -728,12 +955,9 @@ function Protocol:intern_nonce (nonce_message) end function Protocol:send_key (key_message) - local r, k, n1, n2, spi1, s1, p1 = - self.r, self.k, self.n1, self.n2, self.spi1, self.s1, self.p1 + local r, k, n1, n2, spi1, p1 = + self.r, self.k, self.n1, self.n2, self.spi1, self.p1 local state, h1 = self.hmac_state, self.h - spi1.u32 = lib.htonl(Protocol:next_spi()) - C.randombytes_buf(s1, ffi.sizeof(s1)) - C.crypto_scalarmult_curve25519_base(p1, s1) C.crypto_auth_hmacsha512256_init(state, k, ffi.sizeof(k)) C.crypto_auth_hmacsha512256_update(state, r.bytes, ffi.sizeof(r)) C.crypto_auth_hmacsha512256_update(state, n1, ffi.sizeof(n1)) @@ -741,7 +965,10 @@ function Protocol:send_key (key_message) C.crypto_auth_hmacsha512256_update(state, spi1.bytes, ffi.sizeof(spi1)) C.crypto_auth_hmacsha512256_update(state, p1, ffi.sizeof(p1)) C.crypto_auth_hmacsha512256_final(state, h1) - return key_message:new({spi = spi1.bytes, public_key=p1, auth_code=h1}) + key_message:spi(spi1.bytes) + key_message:public_key(p1) + key_message:auth_code(h1) + return key_message end function Protocol:intern_key (m) @@ -762,6 +989,40 @@ function Protocol:intern_key (m) end end +function Protocol:intern_challenge_nonce (m) + local r, k, n1 = self.r, self.k, self.n1 + local state, c = self.hmac_state, self.h + C.crypto_auth_hmacsha512256_init(state, k, ffi.sizeof(k)) + C.crypto_auth_hmacsha512256_update(state, r.bytes, ffi.sizeof(r)) + C.crypto_auth_hmacsha512256_update(state, m:nonce(), ffi.sizeof(m:nonce())) + C.crypto_auth_hmacsha512256_update(state, n1, ffi.sizeof(n1)) + C.crypto_auth_hmacsha512256_final(state, c) + if C.sodium_memcmp(c, m:auth_code(), ffi.sizeof(c)) == 0 then + self:intern_nonce(m) + return true + end +end + +function Protocol:send_challenge (challenge_message) + local r, k, n1, n2 = self.r, self.k, self.n1, self.n2 + local state, c = self.hmac_state, self.h + C.crypto_auth_hmacsha512256_init(state, k, ffi.sizeof(k)) + C.crypto_auth_hmacsha512256_update(state, r.bytes, ffi.sizeof(r)) + C.crypto_auth_hmacsha512256_update(state, n1, ffi.sizeof(n1)) + C.crypto_auth_hmacsha512256_update(state, n2, ffi.sizeof(n2)) + C.crypto_auth_hmacsha512256_final(state, c) + challenge_message:auth_code(c) + return self:send_nonce(challenge_message) +end + +function Protocol:derive_ephemeral_keys () + if self:derive_shared_secret() then + local inbound = self:derive_key_material(self.spi1, self.p1, self.p2) + local outbound = self:derive_key_material(self.spi2, self.p2, self.p1) + return true, inbound, outbound + end +end + function Protocol:derive_shared_secret () return C.crypto_scalarmult_curve25519(self.q, self.s1, self.p2) == 0 end @@ -778,19 +1039,38 @@ function Protocol:derive_key_material (spi, salt_a, salt_b) salt = ffi.string(e.slot.salt, ffi.sizeof(e.slot.salt)) } end -function Protocol:reset () - self.deadline = nil - self.status = Protocol.status.idle +function Protocol:set_deadline (deadline) + if deadline == nil then self.deadline = lib.timeout(self.timeout) + else self.deadline = deadline end end -function Protocol:set_deadline () - self.deadline = lib.timeout(self.timeout) +function Protocol:reset () + self:set_deadline(false) + self:next_spi() + self:next_nonce() + self:next_dh_key() + self:clear_external_inputs() + return self end function Protocol:next_spi () - local current_spi = Protocol.spi_counter + 256 + self.spi1.u32 = lib.htonl(Protocol.spi_counter + 256) Protocol.spi_counter = (Protocol.spi_counter + 1) % (2^32 - 1 - 256) - return current_spi +end + +function Protocol:next_nonce () + C.randombytes_buf(self.n1, ffi.sizeof(self.n1)) +end + +function Protocol:next_dh_key () + C.randombytes_buf(self.s1, ffi.sizeof(self.s1)) + C.crypto_scalarmult_curve25519_base(self.p1, self.s1) +end + +function Protocol:clear_external_inputs () + ffi.fill(self.n2, ffi.sizeof(self.n2)) + ffi.fill(self.p2, ffi.sizeof(self.p2)) + ffi.fill(self.e.bytes, ffi.sizeof(self.e.bytes)) end -- Assertions about the world (-: @@ -808,7 +1088,7 @@ assert(ffi.sizeof(Protocol.key_t) <= C.crypto_generichash_blake2b_BYTES_MAX) -- requests through protocol filters. Transport = { - message_type = { nonce = 1, key = 3 }, + message_type = { nonce = 4, challenge = 5, key = 6, nonce_key = 7 }, header = subClass(header) } Transport.header:init({ @@ -851,137 +1131,144 @@ function selftest () local old_now = engine.now local now engine.now = function () return now end - local key1 = ffi.new("uint8_t[20]"); - local key2 = ffi.new("uint8_t[20]"); key2[0] = 1 - local A = Protocol:new(1234, key1, 2) - local B = Protocol:new(1234, key1, 2) - local C = Protocol:new(1234, key2, 2) - now = 0 + local function can_not_except (p, ops) + if not ops.initiate_exchange then + local e, m = p:initiate_exchange(Protocol.nonce_message:new{}) + assert(e == Protocol.code.protocol and not m) + end + if not ops.receive_nonce then + local e = p:receive_nonce(Protocol.nonce_message:new{}) + assert(e == Protocol.code.protocol) + end + if not ops.offer_challenge then + local e, m = p:offer_challenge(Protocol.challenge_message:new{}) + assert(e == Protocol.code.protocol and not m) + end + if not ops.receive_challenge then + local e = p:receive_challenge(Protocol.challenge_message:new{}) + assert(e == Protocol.code.protocol) + end + if not ops.offer_key then + local e, m = p:offer_key(Protocol.key_message:new{}) + assert(e == Protocol.code.protocol and not m) + end + if not ops.receive_key then + local e, rx, tx = p:receive_key(Protocol.key_message:new{}) + assert(e == Protocol.code.protocol and not (rx or tx)) + end + if not ops.offer_nonce_key then + local e, m = p:offer_nonce_key(Protocol.nonce_key_message:new{}) + assert(e == Protocol.code.protocol and not m) + end + if not ops.receive_nonce_key then + local e, rx, tx = p:receive_nonce_key(Protocol.nonce_key_message:new{}) + assert(e == Protocol.code.protocol and not (rx or tx)) + end + if not ops.reset_if_expired then + local e = p:reset_if_expired() + assert(e == Protocol.code.protocol) + end + end - -- Idle fsm can either receive_nonce, receive_key, or initiate_exchange + local key = ffi.new("uint8_t[20]"); + local A = Protocol:new('initiator', 1234, key, 2) + local B = Protocol:new('responder', 1234, key, 2) - local e, m = A:exchange_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_key(Protocol.key_message:new{}) - assert(e == Protocol.code.authentication and not m) - local e, rx, tx = A:derive_ephemeral_keys() - assert(e == Protocol.code.protocol and not (rx or tx)) + now = 0 + + can_not_except(A, {initiate_exchange=true}) - local e, m = A:receive_nonce(Protocol.nonce_message:new{}) - assert(not e and m) + can_not_except(B, {receive_nonce=true, receive_nonce_key=true}) + local e, rx, tx = B:receive_nonce_key(Protocol.nonce_key_message:new{}) + assert(e == Protocol.code.authentication and not (rx or tx)) + local C = Protocol:new('offer_nonce_key', 1234, key, 2) + ffi.copy(C.n2, B.n1, ffi.sizeof(C.n2)) + ffi.fill(C.p1, ffi.sizeof(C.p1)) + local _, nonce_key_c = C:offer_nonce_key(Protocol.nonce_key_message:new{}) + local e, rx, tx = B:receive_nonce_key(nonce_key_c) + assert(e == Protocol.code.parameter and not (rx or tx)) - -- idle -> wait_nonce + -- A: initiator -> accept_challenge local e, nonce_a = A:initiate_exchange(Protocol.nonce_message:new{}) assert(not e and nonce_a) - -- B receives nonce request - local e, nonce_b = B:receive_nonce(nonce_a) + can_not_except(A, {receive_challenge=true, reset_if_expired=true}) + + -- B: responder -> offer_challenge + local e = B:receive_nonce(nonce_a) assert(not e) - assert(nonce_b) - - -- Active fsm waiting for nonce can only receive nonce - - local e, m = A:initiate_exchange(Protocol.nonce_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:exchange_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, rx, tx = A:derive_ephemeral_keys() - assert(e == Protocol.code.protocol and not (rx or tx)) - - -- wait_nonce -> _send_key - local e, m = A:receive_nonce(nonce_b) - assert(not e and not m) - - -- Active fsm with exchanged nonces must offer key - - local e, m = A:initiate_exchange(Protocol.nonce_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_nonce(nonce_b) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, rx, tx = A:derive_ephemeral_keys() - assert(e == Protocol.code.protocol and not (rx or tx)) - - -- _send_key -> wait_key - local e, dh_a = A:exchange_key(Protocol.key_message:new{}) - assert(not e and dh_a) - - - -- B receives key request - local e, dh_b = B:receive_key(dh_a) - assert(not e and dh_b) - - -- Active fsm that offered its key must wait for matching offer - - local e, m = A:initiate_exchange(Protocol.nonce_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:exchange_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_nonce(nonce_b) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_key(Protocol.key_message:new{}) - assert(e == Protocol.code.authentication and not m) - local e, rx, tx = A:derive_ephemeral_keys() - assert(e == Protocol.code.protocol and not (rx or tx)) - - -- wait_key -> _complete - local e, m = A:receive_key(dh_b) - assert(not e and not m) - - -- Complete fsm must derive ephemeral keys - - local e, m = A:initiate_exchange(Protocol.nonce_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:exchange_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_nonce(nonce_b) - assert(e == Protocol.code.protocol and not m) - local e, m = A:receive_key(Protocol.key_message:new{}) - assert(e == Protocol.code.protocol and not m) - - -- _complete -> idle - local e, A_rx, A_tx = A:derive_ephemeral_keys() + + can_not_except(B, {offer_challenge=true}) + + -- B: offer_challenge -> responder + local e, challenge_b = B:offer_challenge(Protocol.challenge_message:new{}) + assert(not e and challenge_b) + + -- A: accept_challenge -> offer_nonce_key + local e = A:receive_challenge(challenge_b) assert(not e) + can_not_except(A, {offer_nonce_key=true}) + + -- A: offer_nonce_key -> accept_key + local e, nonce_key_a = A:offer_nonce_key(Protocol.nonce_key_message:new{}) + assert(not e and nonce_key_a) + + can_not_except(A, {receive_key=true, reset_if_expired=true}) + local e, rx, tx = A:receive_key(Protocol.key_message:new{}) + assert(e == Protocol.code.authentication and not (rx or tx)) + local C = Protocol:new('offer_key', 1234, key, 2) + ffi.copy(C.n1, B.n1, ffi.sizeof(C.n1)) + ffi.copy(C.n2, A.n1, ffi.sizeof(C.n2)) + ffi.fill(C.p1, ffi.sizeof(C.p1)) + ffi.copy(C.p2, A.p1, ffi.sizeof(C.p2)) + local _, key_c = C:offer_key(Protocol.key_message:new{}) + local e, rx, tx = A:receive_key(key_c) + assert(e == Protocol.code.parameter and not (rx or tx)) + + -- B: responder -> offer_key + local e, B_rx, B_tx = B:receive_nonce_key(nonce_key_a) + assert(not e and B_rx and B_tx) + + can_not_except(B, {offer_key=true}) + + -- B: offer_key -> responder + local e, key_b = B:offer_key(Protocol.key_message:new{}) + assert(not e and key_b) + + -- A: accept_key -> initator + local e, A_rx, A_tx = A:receive_key(key_b) + assert(not e and A_rx and A_tx) + -- Ephemeral keys should match - local e, B_rx, B_tx = B:derive_ephemeral_keys() - assert(not e) assert(A_rx.key == B_tx.key) assert(A_rx.salt == B_tx.salt) assert(A_tx.key == B_rx.key) assert(A_tx.salt == B_rx.salt) -- Test negotiation expiry - - -- Idle fsm should have its deadline reset now = 10 - assert(not A:reset_if_expired() and not B:reset_if_expired()) - -- idle -> wait_nonce + -- A: initiator -> accept_challenge A:initiate_exchange(Protocol.nonce_message:new{}) assert(not A:reset_if_expired()) - -- wait_nonce -> idle + -- accept_challenge -> initiator now = 12.0123 assert(A:reset_if_expired() == Protocol.code.expired) - -- idle -> wait_nonce + -- A: accept_challenge -> offer_nonce_key -> accept_key local _, nonce_a = A:initiate_exchange(Protocol.nonce_message:new{}) - - -- wait_nonce -> _send_key + B:receive_nonce(nonce_a) + local _, challenge_b = B:offer_challenge(Protocol.challenge_message:new{}) + A:receive_challenge(challenge_b) now = 20 - local _, nonce_b = B:receive_nonce(nonce_a) - A:receive_nonce(nonce_b) + A:offer_nonce_key(Protocol.nonce_key_message:new{}) - -- _send_key -> wait_key - local _, dh_a = A:exchange_key(Protocol.key_message:new{}) assert(not A:reset_if_expired()) - -- wait_key -> idle + -- A: accept_key -> initiator now = 30 assert(A:reset_if_expired() == Protocol.code.expired) diff --git a/src/program/vita/fsm-protocol.dot b/src/program/vita/fsm-protocol.dot new file mode 100644 index 0000000000..2082c11974 --- /dev/null +++ b/src/program/vita/fsm-protocol.dot @@ -0,0 +1,24 @@ +digraph VitaAKE { + subgraph responder { + responder [style=bold] + offer_challenge + offer_key + } + responder -> offer_challenge [label="→nc "] + offer_challenge -> responder [label="←n1‖c "] + responder -> offer_key [label="→n2‖spi2‖p2‖h2 "] + offer_key -> responder [label="←spi1‖p1‖h1 "] + + subgraph initiator { + initiator [style=bold] + accept_challenge [style=filled] + offer_nonce_key + accept_key [style=filled] + } + initiator -> accept_challenge [label="←n1 "] + accept_challenge -> offer_nonce_key [label="→n2‖c "] + accept_challenge-> initiator [label="ttl " style=dashed] + offer_nonce_key -> accept_key [label="←n1‖spi1‖p1‖h1 "] + accept_key -> initiator [label="→spi2‖p2‖h2 "] + accept_key -> initiator [label="ttl " style=dashed] +} diff --git a/src/program/vita/fsm-protocol.png b/src/program/vita/fsm-protocol.png new file mode 100644 index 0000000000000000000000000000000000000000..eab7494cd03c3c6062a61b324c32f996fff11a13 GIT binary patch literal 59359 zcmaI;c{rA98#avJRH!JFp-iC>6&lQBD9TU~$&?|5BxA^!S!75QC5evoeh-R0jO7iEF zkw@R>179tTy|9poe>d{aUAm$2n`}qHbcaWSUEjP@Ux$Ywbv64MUOz?!I{e=(lS0Xw z|NW~1UFy<{SrziD_^sin|Nnnia;M#qe4wtO;lzm(G@^$N{hS?9EplJH>-PBm{Xicd z>fwZ)^nFGUFu()8~cDu($ENNP0Gq*bai#b7f?}Db}|nS58KXJeNJ9~ z?%cWj(j$it?_dsol9))PuC9*nw%xpc|9*CM_G1?>^1Rf`@hK^h{q^hDj;)__xHX;W zUQ}{^)|GMlL%(m|J{46}MLoUk^U_C-M1Q?dba63#^VY3fn~Gc=*tc)5Xli1tuC5jq z74^Awiz@Wa9TRJ7`Za6TbTKh0&{?)Vmi4&3G(XM1Z{PZpCr_%VsVQk{GT)7gioAcn z##uT{Wv4>A&P!ucQ)(@(7SYubFX}i(*C)07mQAl;zqXz3mFX_^;+dcBQ#Li_n~xZ2 zNWAdoa(A>muAFkc@j%JNlUkaZK5=nztxFp=Zu~j=>7i&WmF@XM8#io-h>zd@CPO{$ z@U7I;ee=SH4*BAr^`Aa9p()1>dx3=LEUc54juY~tIs>)EL}#tr^W z9UXy@kxajS|4vLw^2OyRxz3Ghsw+A5oNQ8cT=!ub$GIH*Yqzw+93S(BO72#YtfM z;>KKlPq1Lm%`7iEqTZ)qHl4|)= z@6XQ8+F~V@PMqKq5ZG}4;Kj_=Hi=uuDE$2VQ)&8NbFUSQ#^Za0g@wJJK7GpB{^U<| zObnll3>VpN>gva_-?(LNzBhDLKY22>^-DvF`ooI$cGgXsHf81I1r`^}#J7=KyYjxK zhJVKn%B@?sg2KX1;Q9HRYo9x3Y+`cjfMH=-cQ@zm-MjJd74eOU85u$~t=jXGUk+uu z{`j{vZ}+`@e|M4VmeUEc|HhZP-de5^6BDE76lW}Un?KO9;dVqCc48`FVoOe*UbP<~q70mMM1m`0?Z2oA!!`Fs)y|J}4wa z5u3|-$jjAFfmiazBpod+ZGL`!Sz8;6q@<+1ng$(jvdET@s(jl4>M$;;0B>(fa!N{! zzQ)a@~?H>Q~W)IC~ z?$Ahn<)_C(SnV}bC#0~^B!fi)^7F;l($ezp-Ai}&>{%8ai>)jy%{f=kcXoD`b#=w1 z3v2GmzIi(&#BQvCEA!j*@9!OUKe~=!pTy`J$;(sa=H}WuIQTw%$RWGB><}yVjg-So zqvG4Xw3`P%MDn}*nY<9&5gi>pFg(2W%9ShP5)z4S^HX>J==S#(y9I@YD!I6bD<~>P z#>D8PgsSZHNlzD)mXR5lnhL{%oQ=5GDt69wr&E9upZvLVTf?|zZkyGFX?M~qDl608 zxN*bQ*47()%KF+hjvo#3MyF1lV&vcm86US$Qc~K1?ey*2w~?5(=H~LIL`4y+*1f-` zdL9bg#U-Zg$Bte4^TRhUPi$prx>WQP$F^-IW!_Zm;$pN?b;pK zHOjtX;QmV6ZAgv-o3i3n~mfk1QuQH#d1e9@lYU z0=rNEQDmyIJU{>bzM9&hmunr|oTOZdZ^mj%@5(4eM;Vx&4qrpNX$K!4HIL4^o4+Ug zQSVH~n-W`1xcB)Ag!{3fyo=lRQ*v#3ZsX%(T#L2Sb=V~x_jY%8_YV##Tw(s6|MV%J zUtl1=xHvlsgtEQ8eXM9oO3DLS56P01zgI1;UY(1$%c!7Yc>nHQp9Igp>#<%@+pk`@ zaA8~0i8~bwKZgeJK`-?#4gVaf5qBJCba!{hiQoA<&fkBX=gQ(0dM?R(8Y|43w7G)~ zcJ1Ch7%sO`R^qW#`{-nAM@P-fKqYx3;*ye|zhvH=oM=lIzxiwL^73*m{e7HtAKdfM z*XyPJIXOp8uV2sW*Sc%lwr#pORxCFrJ9*pm(-ISTFV{b~wNERhN%wt@3SZ<7VPQtH zas>`!$(bj*y1E8CE_xmBEAenuQ<>P1Ce2*nGSg2*E{s#$_E=hcF^X~k14jTp;mMW8 z$Bw(RZ<&Ph?%qxF?#7RbM8&`}XU>!r?`EXy$7YNcJpcJ0iyz}>6_M)x@83gEeR&TZ zih5{s`Eme$&vf`hBtxciYvI&X&pX}Xn@2WrN$xz7eE<3LJx8x~DMc@w2onfT)_Ina zVp?^Z#XK(ItxeB%t>+q5$tRm@0<_jUi%t=6Jpo0j+!c9Gfx;E zTX}c)qZ&VKj^S@uln0{b8*pL@EYVDLOWY-jii%!*{_y5&-gUDtug%4sCbl~7(!}xL z5VLuoZ#U$kp}e3Ft*j}ylX-ov^%ox=uVto!f`aZ}C?%$gvm@*h_B##Dt&`c!=;-Lm zQ2fdHwkdr5S~N!Jf=^Wlo8rln!T)``mA;A!-HCASJLg{+QMpJ>wxg8H428=*MS+oi z&OeQAR@s`W^-N<~BUW@>Z*TA0QX4N_0Wj`)WwWxg2PC$o zs+>Hz{-|}Q_ekA+hxz2LuU{+s`gr#1=MGD7Mef1Vj-8#K54?Y$&H4Jnv@`+o*z02t z1;!;r*o#zWU1{X5PmtrmUZ(VJGFTo<@G`9q-G;mHW8_m+jS(t;Rk~99PsVBDoQs>A zA_{HIx#ZW_mBOa)>7)U6C`~0EZa6$z>s~V@svc9?&2FZfXTv2eEuDWm>CKx%Hr@HR zut&~TMbhG7`XAfjU0N#FcUJb)4o2g__rVL(rK|aFdja--eiG|5Mn~z6wFGSY{S{RY zpMcf>Gx^2$$Je~Nj6oMX5W2P083qNKHggeGqTjc;xw#pw4kn@EYS2c*^)xHetqI? z{!|gP9JB&kvy+Pp3uPT0_s7nW;y?88dda{|Z9|-%6L+>(Iq>mq+`fIgxaYEX+>xs^ z6y0o7|8wCP$tF+H)pNzJ_Zt8G^$nm6tI%2MCD%byk_$*Ak4M>(qMq;gjrr!E$@M^D zon2k_=Imyj#cpDgKYjq0Z30>cz%!|f*>B#!e7nljy0>VX;kz6DF)^&b?Iu^RhA?m( zRbjARU9oRWklT#s`XVijbZBy1jEY>&?!Ibb!t43>+aAZsP9}m;fw*unzWx3E;hE>v z)aX(A*MTYk{+HF&g?wEsIVdH?fexjB?@WC9lv-b3e|fG^j>Yqa_M(BnoJ0q!gM-5( zY1g~&*e*|X7mQ@e@%lv+3d+iI@6mWfq2Ou<0(m@NF3f@-|}`l_g1a{#|*1kb?B|&;Pv;SqX|-`=ay!VjEvMAkv-z|KPYpuVx zzZ}UVM5*uW zd=M|;5HK=4oP|FB2JK2T@yv&~!G!eVdTr%&b|KYpAfXJ>TIb0j-0q3*ft71s}qjaPtE=_!YVg^ihm zl``x0plr*4&8q3^Kcq5#sh@Y*G&^7C{(N?=nfZMB$R~bj=|@z?;2~+PqAe{gvA}#% z4x{v{>gwiIRaKMVsgvW)NowlqcNo}(k6Cf^JoHiz;~y^Q^Sr;syVY)sr3<=B+AcxC zkoEowG~Nu`M3tuMyz=ZO(H#q-*R!#)r9633212Pfnj_^rB}}mfL)Ih&b&SO=fAZ)SNs1hgKMeiqlkzy{Y}_U(epN?)ZU-srSjVU?_IxsAuU?n znRB(H6R7wC?man^LTwCUqiHU8bajGT`GZ^OzlEP>HvjPb0(m3Mi%D+WCv$f0*`vr8 zNud-vPdDg?0X&`RVVglab>6`m0vfdz>;k2T&R+q4>+0%KDBNU0A+h~Hn|OF5y&EMhns#K?2cn*6U)BQx zqELVmH0Q*Gh5OsTdDz(4n5Ss}c5r?EwDE#b@9G`kx!D+Bpu>^kQn`KC<#wHp?s-+K z0FaK-H!wC9h+P%4%_c1?E9>)I+LWDN(fn7aAqF)P;BEH$2O|GFA0HbV#~!Drr_aRz z&_D0~HZ72An$3>CnS<9({)oPjsRj}~-(6d~!FjsZAAPGU_gcuzj4jZ0c|`^F&!0b! z)au6H$-8~~l>eN`k#XiCY)UB>kA~SrKPQIN_POTM`>zp9hP87RR^7FGqtk14WL0y0giOl&}s(? z1=#h*_8Ls^S}6xyIAdTCdhg!WM>1}#m4TaUa3l+6|NQyWnR|_cnVI>Fu5QqyN1WJt z0Kr>O6wF7xMrv8q;tmt-1_ePG>OsIJuG>J^q3j|)f@1c=DirU)Kw8}K3|LL_vu7r# zs-4+pwCEo{NA6$Lagzz$%(JPQonT(VpbsA0Ox6;Z?OFS)LaYZ9Gc$Gnk1yxZRvx!N zl}Qan&9!edGc|R=ngH%oys~Z>;2a zaPkU@*>oQ}dzOjvZ94w2G+)V@h`%TCpV;Oz^YhBq)Kf{tg{s+&Pv&a75U11)dT{E6 z!H|Qs^;ZBDAmY4gm5jauXeSP+H8?of^!zlBO(D=BmL`l-f=|07jXe1W4?e^j=Dc~c zG#p)n&H28nO8)fe@OKIRj0!=^Io4m8Q2EVOw-$a&`Cg*bTm4|Le!s`+{CjZ|WaOY>Ah2 z^0|F`gQJrZ7LLNh!(-i@pIyJq#mULY&K_jbQ)oA`M?KCDEo$nv5CzgB@8QE+1$8ox zp}#yyPF@QD1=WL)xR&RqUp3!6YWaEX>dMl=lEqPu@o(P}iH!02*+z{=;%99FCA9w; zEc~&l*~c0}bZwE$PVBSy*dYCH8fATP6NGc4mk%s2&YjWG$^87^QDP-Z z<4nIi<+>EFy`9~F1XHRy_3rQQ{7xo`0_Q%G_1I1k_W0{)4UjXYlp_d9Wep8|8CcEF z9_PNFT+hpw7RM5@0W2uTAP!|+?~{FBT|EfmOB5*s1J2$e*UI19^))psip4}kvNgJC z(M#@pNk9L}*uo+R8{zQPc6!u2;iE@KrC1|*4<0xbX6o^@scXz2?>b<(n@{!IfqdRE+)Gls1M|pXv2ob{p%y#;&^ZxyNTbae+ zD$~^4NoV#Z`)BPq0N2r z&Rg(-m6a6(w$9R(zk)#Z{B1H~yxP31AtG*ntbrLrrI_#^1-z0wnLqCFx`=vpMpHBG zcLpAd>^0o7K`9;h!~0*Vlw` z4Nd))g4AGOU|A3qrSe-v|rSSaDf^E2xO74`Kw(EQ&&Q@tPY__2|` z9jfR#pEG;kgP{|^N+D8&@RbJ6v}5gNV6;`O7HL2rvZ9=ti|N|#Em<+9Yu`WLc*Lml$9K(LX27Z#C&ko90_4%3VPdwW2 zqprY2!eU~|r%r8vKuA@@b+x{}o*qIA)S5Mtj^h+9EiI^b@`i?n2fi!gf%klRCE>E*^IZO^ ze97FL7wyD(s#_jvBoW>pJ*vMmww|J=r$>&wq|?M&aPzlUJHnQh3>~boHD7}n-a1&a zK>g}sDb37{AVI7EK*-FaQ1??Prk)D}w5dkplp+;Tt_XTSt1jYhm=qQh7N z3ZN3!W8Cu;_V}&$@0If3z2nDeC|m9Ov-+#$^vEW2`2{v`ORx{yzF1TY&&3h8)a;ih zhmPTBT?J6|+ji*kj=g(>9zq)u#=)3uO;Lc>LA%nV5V=azi6gNKziW)IX71sy6q6e zfsg7Awzg&9;V;&(a)Z+l^1J{98yOWDo($b=YW@?UFK#^h zsR*3A@cmWka98MVh{90h^PRr44h{|3E&aLHUFZ}d#rV0SLm8`IaQRh&$DeJI&QogY z7uWL+uO>f#{+~pdkdV;&3$V)!jC#B+olUNMdm^-VGdzF=knd-I2=wiIZN3p2e7%{Z zCL3hq4~9S)MB9R`VizA%lmuVr)*4!(B(e(|Q-CT@?>p(#gSGVZ^dzzax3tTS-*IR5 zDoytmi$Iv7=Qx@npRo5sZLNSqv?x3&lp}){9GZ8hWM24kE2x;UZrS2X2&qoCfpXaW!d`VBS8 zttvs-EijYGv1q0SpC@8JHWM)&x$oXR{SlTT*deSPCQi=K*49>0bG;lZe`q|W6RoLm z`v!TNrYg9JsumMFaOW5b?3d){9A8Z(^_VNnb0mKX(|OFzWWS1QZPsT@Y@ z;Yd*x6&Kr0c8Y9eV^b}zDJ!F(iUquTCp9@ax!qBf^};E2^(y#xcEcZ-{QUi+C0wt= zwjzRmcfOrt^!QlUF3rzW*;4HrAIl8cI4{oJprgP~JOREvQ^j`i37iv)-lBpcox}#g{ritY=W2K? z69T;mF5DZnZIIQ7-lX;1j(@)B|7HO+j^kjYj@izh1;r7aAN|DP=jT_}*ti*G3<@sO z&0piU;D4F^8gJH~ST~22fk-Hit*fA*01w#E!M?`tH49a$;r0! z--_KJ`DRqizj*PY3=4$=OI!sLGc!f0c)h!@^x>fa$??JTGchr7Sy`5Fc6xoiuY|Z$ z(1PLIFNCSm@v_v`*T2VkyoLVmD-wV%Vhm^wOSNrJpBwD8$**~vy;lE;K`N@--;!x4 zB`z*r*4)fQtjw&eh~EOO7r^!N(PGH6LF)|)3bGH6hsKM2BJ8>B63Q(Tr{@rP3chOO z`fS}$)a?GUTWiGa2G_!fZhq%{9VM1#YHCVU;sJ<%Ily3kx-lpU5KNVFUs7_iAAF0l z|Dh9jp+hU}ZEY{@QvU9R)WT6S1JMe;{Ci?#p{N%@-%grtYi;#|$%-rJA0H1w2Ww&Y zB@LFK0AH`7qJoi|I}B&^*iPnhn6l`Swl{7BBqi}P_$$(-Zh^vi3yM9FQ5l1k9vr#K z3OJ?p^4dN4?fCN`i;Rqn-av8ZblY(%jvhUl_4>6B4530;*E_05KYjdIj>TkRV+%wr z8-kvva_UrCmQzGZ%5Ib>5x^(+pH*y*{xRn^60BMzwPqaRGU!br%olb!PwDD#o`d&) z{=jwD9Qh8>OA)MU11R`!qiff$y|rwiA=dNJqX9Uh^b8Emg-%xygm_|J7xO#Q^cS`m z$yLDp`aecA@xIXj!ESkX?HY+sw3nTT89j{rtOjt%F=h z{9x<^xh^Xj3el}0>^^w#pbT9W;1X)PJXD;CbF?1NJ_sx#N(b=H&Rx6g8hLh-=9OgRzL^^k3<<4oS2@z5ug^*^b>HYGl)n)_C``7F)i&jN-O{` z7hfdZ?c29CrB1-`hdh_MhytLO_J+VN_|Wq3^bnIFL6yCGj{!BoqRbyVc(Fw3^5xwG z#c0*md?&!J`N2r%nyVT{9}MU$^>i{-Xc+ zVcBFjydEANBoPCSaqVla|KrEpZ{NN}V&)dPFFY=|31`lqKlr8;t!p^bs-!dl*i;^O zfRhms73B-ke(A~;en|BIP#;hh0JnZ3ZXoUPcPpvC=!DNYE}TBS83vWXm;yl^yVV|0 z&~*wPHSM{x2AlaA7H9w-EO8qS8033Hb}&Y1oB8$Y{aAumgLBbo0RaI8WN11J4k2g| z92^Y8tQ_BS7W~rrz1}mlH{1oO>;FqWL|N6Qr=`_8EAi37X)!oHi`GeM83=tdWD5H2 zhY6ka4h!4V2Sw_)fv2Y|sp>a==mjb95iDZUELEffl>v1v}z`mwk<@R7hkD6#*%)sH@vp8NeXu!1wUKY{lXW zC;|T!#~ahVq|4fPJL50(+F7pCyTQw|YvP!bva@B(yuadGo`8iSOTAw3Yb$MFJKUT*&?aX@|J)g8~F_E(}eS+;>b& zjC;*azgB)UaN^PGWa!>1Dv~Cu2fV`6miX*pI1=jeQFrgo%nT^v{11ToAt1DCEp@r_ z!+`bxl1jnO#Pf>H+_w$B9HvrlPfVD**QcaXUVws20Qa9Ey#N?JROVMBGrQ1ep@5U`mh=^=twq zfwm2_nwOW|5!v9we!~Kh!(?M~phM}n5bF@!3eD-phK87~^&XukkD(HiU{c~40Dp!T zsi`DrHZd{b@@t$48_W;-bwy?68l=qLHt!LQdiXE|l!};g;%+;5b*Qkv z$QG=G%oF5>iYM~E{Uamm#KiJw9L{>Tik|qeT3|o2VQzNT6gfJzlVC@RI^93+|%t&E6a=apOhyK7S48lKpJWg zWnckDJR=JWiFK*LeaHS_$W(X7Vi2%t{WX_?IXJnSV~ku4yp>aTj#LGr2!X9XK|GDr zkUl#V{>Zy`slnknWo~XoZzAQIh`48yP9)+1wC;?HI57^mM=}^BNdTrXNZ0}D>0+4n zei)cFxDX_uYMnCoFb50Y{AB@1^Ga_H)>|MxtHbdZQJ2{+y7RuNPvBz>0R+5%`o`Ae z|0Us+l$EtkcPHB~WEL}fqB_2Thn$TTcnAwd!?685VA}R>?Gx4PZVNNK4B;k5-jG)Ik)wj3DP#I%QtwBn#Q}0e&_%TYr z1GxO~=rz+%aYxK(XlN$k%4c@eY?7CkSA!j)rn(bfy@^{|_s1eJlO`sN5@bCtJt(wC z!0B&udN_arvAB@xsFmSJEgi2lqLh{ErTq<0z*9nsA`7mr4INu6Tss- zv&kq$C!E3^FkcAjKApdvq%dG=lb~5_Sk!S9Z5Tg8cu@c8#6v?SHa21Fx9m1vUib+S zc}$9%SE;%oK`tB^vLDn22a1VRNcYTiwW8;9L^~mY`5@S|mRdg6Y;EEh#H>QC`tlZG za+-qgb!TluB990?K*m=e6$=1#`9i!)@BZ(W#JyIeVQBB`P)qhqDlgt1JU7;u1VB#Y zD(W?BNWqtOon4E($@MH2WeVZD2v>}q*5SnF0!Mpr^Q_%k3Kju1(G>2jyE5+`T|kj6 zI3%Fc8}qLBRY&kDW_kX*i55yyZXmb}p6r|o+kH{l-C<|iIyw16!ky|Z4#9rw_?6p-aVU#}AZ`|jPl0!3_RkY2gyxoDnqA5Fst*2gJzK{!$n^neevek|$) zB2S8x#x8An)?lTDiS$0aW7C+zanw3uV7qoN1lU+) z$;c@LM9%ONrc?noY#X-XM*dZPogLakL#;SmyqK=jcRS`48<882TBjE~dF z$;ky(nU?S~E6A3ySEAEGa0`Y$fZzurA8c@j+?CU_!Jo;20xVk{e*gs}ZXESG=l&K> zPJ^jmh-NN>lNMPot^ECd5U@f4;1o!G@TN|@h#8HKkIxJws5+S~sR=J;JC7~HSSN8- z>?qtKAZY661uu3plAMDS=Z}CU`Lyn=(M=sJymRh zp_=d-!eAhs5Qu=@3akOO>cgJtle_&s)BvwF<8cTXgfqck=Q1)cCy? zF{xOK5ZK0;$B!GFw<-vXz~Qu>?95i%yHym_;>KwGwi14(&o8*-JRf}L4E35tuuw~I&usBUvycOp(aL)`>|B@rcI@J3Xsboog(xd0Iw2(-{y)<8zY zVZLis4Lqdx!4hP}R=srtmsaW*Wf2AK>IHp?NQaDX2sn7TEja8HZCDXJ%%Skchi7 zUvha!aIgt5y4v~k_v6Lwe5S?i=So-qg=6^;tFC4@6M}-Q9xLikWej=sWpmLx9Gn4d zWrzhS$;t02S?CfUOtr0g5RM!dk5XY0K5>8FLRn z^}f*QJ5pF1PoF+rGW6k7ns)k3-NDj+@N4kzL8(o=+Bd#^-sKKLM1pKtmp(EJ>Aj`H zZMxUsa=*aUC1EZ+iOHUKY^Xf$%6Ca97vkNPtL+)(AQsPzq&)r}CVUzBw{ur}L4!cM zj$<|N-MjY$$th%-poUdeSJU8l1m@-*g&jo#4|o_CGBjVt!Gy)1&JPQqOTLDOlP;Yv2T`Zaeyfec=&o}6-?oKOUVkSmg6<^8P*RTZgil5jRC zao+|_3VS3Sx(ctfG^cVH=PgJW-7)FUo~?VbqQgy92K5KYc0Ygul68iIJ)&nfHGrB;uf$ybKI7!)ChEx z_oX3WcI0Vu>^CcV{DE$T%)fDGmdV@5Jq*<(ya~YJzBqCpE@YW<%{l($Y=Hv@0uYCQ z1y2$W03)}jRqn(<8w2?Co-SDm!}nLRCz)lQxUKP}omcMPuPr(+_58V{Zk(ygX2wtg ziBbcRo1uZfWfBZTOykc~&qgfT^1t~$_ZnWM-J}!&yOZ3)`yeKC;z229UZ+!#PRRTM zun6syhDV5YHliMU_juE`xmo1=(DCRRNiuD)@a=g5SqcPEf~UoudI1;zlaqmu2yH)N zxDe&Ei6=rEtW@E=Q(?mI*DptLDXAf>J>BNbNp2$Dr~W(K(W=!B7i;CEn30$ouaFsX2A-cP3jd%*bW)s4_9 zAVCla8H=funpCalw%7+Knn3^pGY6p0AY4OoGmegjan5k=X8zQgc}!i4j+lqZbpL?yy8Keh*-PZ3zP2xiHKPf_Y2a$tA(P8L0 zbNKdNsDFu|cBIxKqJ!LVrSncB zPfF3?ktKm2_yA<`tl^Qw8kFgRD-f1$fo}YrpWckJf|7_}khZQTP8&oPk|5u~h&az_ zBfDqN0tqO_K2Ob)nSv!c1bup8w(ejvgafF|Nl3fT%*>Rb8P$4hPUtHKuin8(mxVoT zqi1*_u&b5{2;kN+Z*aply|B%$H*cCkPJ>#5vC?_&0tu%aM z*di2(IC}Vkh>-AN(gBtDzFR=yjLfb*d(5!ekwLzaGj_IlwJy#=?UPOiBv+_~yaEEZ zr}rkuoN<%mHiyKEL4qpeD4<_eI6snn3F%BRfL>dcu^#z&zg~u zK_bjd{3psje}3J2Bx(iN@h2Lnin`mM$!%na1v!O&;3}xDL;DOzTBDsTEF{`rT?_}d zH^s7vBR6)fDmtX_b$lY{Z!%H`F@?yB5fO|wXKuOcM)s6!M>LMS3 zSYZWn2hVCfCv&=S6;!dea%3|Cl_1q)#{zmjO8k0cFRePQ~kB zJPub(rgHG@oFFzXb)WH2NEHRZBzffP5>}>Iu>}N#g&juKuxsYHo}U7N61g#{i%19! zf?!~^{gatAvC;=(W*P)TDtkk zl=YRp%02VQ#$iZ@!^+AE3e*Tq^4OY$z9=QW@&_VjGlNwd5weN9A9IqYjp#3|Uam{R;3*u=Af=~P@u5s1 zRYYtg<|eKgkcN!gk(P;VQsDak8e``@(wEjPrPbo}!Ya3`_NT0DNef#20336;l?Uha zobp(l2IkLdz5eU*fw}K%&)-my006cj@Mtp6f@#y*{T+;d!V_NK{B1?VET|ZYkev!F zVO>yqZMLkXrkY>qyqa?y5j6lH=%r3C_a|i{zqWwloP}0OuoWcfVJYVSEi8PMik==W zVDcBAm!R-5!Yc>)=G(b*?ccwD&1<37vEVe0(li{{yVo1ZQxg2Z9OJDkI*SBw3&-kpZHnQm<9}Iw(0wFaycxEv620Vzj74S$s}D8GUGFrgF3SO{uUuca8&x0U!$_ zU6|TI#! zKg;IH^IG%#>z~CvFQp+sGn;d-tx{nrv<10SfZ zjh4e#;fG_}5eaEAOyQDWru6I%#D_2f@ZhkeW^~{8(b0gWrZYq-!ZUJ##vdz6%zD4| zTM%yyg26*-R%a(Wz;eFp-0t%>c%CGtL81tN;L@JU>=1$TZ~hh_YU$PKS0yDSx`mGW zh{z2r7p*^9y1L>E296YwFCI*JZ*PfSUE1WBw*JC0H;EvWl^rALEof#rqD_TDUT;Pa zHx2|Mhao$IvUgr^2%9W1B_)8|{QtCZ;L}R9!aA7;e~=o00zxhpQa@~Wk{-&(yeA6h z`H_)2Q?5&q>M;@v%NiJ*2qJn09KBtwgWiO>>UK(>s^ruIF7dRL@(FSVEQ~iJm@@pbTL? zX8<)dZS81iY$w$_nww21X}<`M=;dK&|3CpM_#1s81xT6;;q zd~8}#=WA@-itMHMq@#K77`~u!$6}64Kg(F&-+ZEOB|FzYFz^t?8n{;b=1bAlgAzWm zzy5C)AYA~X6!ZV2d0#7>=XSLPS?&DtMIHmf)pMpJkM@0s5*BL-C2!44uHbiP@PUme zWkwHEJR#7j5&(h$6**U)Gp^r_xT60RS*aQ>aFZb)Ak~bErQE%}y*Y&$am$zc?ju?a zaDf0Ni5NFDG_*CmMbsX0AFNn1*F};=vXa%tWRjqphq(I4`jN^PZImh0k+}Pd6+dl4 zUn=eloI3y{nR290XXf|P3kIiM9HbQP#m0mcrbrT_a2La4uI*i_2~g0|+MiN z=r(OS0eWVPtTO`NaE;DOg&=oL(z!tPR*i}}q*jB#5cL4cYUVk%B~TDDX$(6aWHI!` zi@g#%ql*iF$p9pt58h(8Fk=#1uN6~zfkH+jI5{~Zpv;1cWOfwAA;XETT8+V7 zyYVJoY4-(2WF8O9_YR{dL`&G7($J_zETXx@!>zYHG3|@hA!Y5>6Ppwj6-fc~w$5dl zUG`{ADSUM2+$5%8kJ;J^0|x4uG^R~o#p8@)+|b887}H_4SYwpFAA+UhNH$@?xpov5zB!qW%>e^Tz zcO~=qH1zBgE-tQxrKR(!HH}b|3ByL27qz6R4=kLODVhT^HAZj^afl+d>WRA}*Z%;X z**ZFs7gL~uqnTb=l*~$7yX3he`q#!@WOUUg zrlyj(`qXrswczmL(u-?O5*+*b(24(3kS>-Kom!RKUa|5s9uJK;2ij?q#ov>6<9cCm zMti=@#hWkAj{IDMscs^*mMqVzrkaGS^qf$1`tSuxD(}_}2*96tIgeJNe_2LH{edtQ zKncMZ&NJ<|Z%RZzBg1?jB6vU4)hQz51lKTKdNY(doPaik!qEQTt8Xjj-CZmz)4A<~Id0r*ah@0|Vd4 zJ=FG5=b=Q})6C(1Vx%K?wHIIx0$EMGziN_MH13B6F(xr3hbxU4&^@{Ylr6Ig}Wrykq zofnhO6l(Qer<}1k_+uouFE`zzpy#VS+y1++3X5=%YE=Y4tPB|C}v0`IdVl0#2s95g_^bW6<{i z5EPkq!Hpqd zY^0UHk5s>Jm0FLQ5{`qbd+&jw4ly=gGT&f6w5UE`^DI<}inT1G~) z)TW;-$)k~toz{QOE1&!>u(B|;nP^uSfmGuZEJLQ6ygvYIS`POOA(|8^X8E@tKz2x+ zxzLPJ>0xVoyD36pq~;;#H6B*!3_M7nDqJa0-I^asd+Sv`4Rzos4xrM0`1q0cNCb7* zR_?{nPnX4F&C=7?W12i>zdp1hDokrNa|JGJ24EG3>PuJ?Bh-@T{ z`|sbIg=S5B`}Z?o6-hVA#&k#F5aTrxls>c7A3lC!Vc~YNSEQIP-T@Ie-c!vwY`r`R zr2tX$84n7({0ax1{n5eI!$*%2Rv!5B<$j2(YR;T`LAP(;sC^g(zu8zc_Qk7LY(er{ z)<1p8zE$GkOf*2B=juP_sit3s$B~UE<46EAsA%$Ib6d74EN{vc+_7+{tRFk;u&%xs$mq|rMkET@CQuX(hm?HZ^ zsCv&?x~JOMuw?ce5@yU+PAT4oCvwSs{JfK7H4zXizx)OvB?afLCAilb)Z3pc|J(>i z#{xJi^K}IyREY?CL`)0~hQTo2)x;G%J_wyB6p$a5L!N3wurgg9bO;3uT@V$r=WNiX zvVs*}|C(H6oJgRqN2E|%(`ERO#@KwOePS`v@PI1(V1ICT9BX*HV(}6nDBuu+HV|pK z)GJ|1MTzbB*{z88(;>PZig?F=%LXlh2qs9EH_n$7I=aAe!^?v(a(5ipkK3T8h*~s= zJ&<-4#anvkw)bwZWM+pf$(4Lx5$~6u#D61k1iI#k~ zRoc@RAvInkn_waGt?_OSBFD7c7^3=<7B=qH#W5c1z7j4p-D+ra-@+6fg)J>D@#Yg_ zP}>Czi;(dpOtXhg=_;{B#>L%+_%;AHFaW8*9y>fuGTaCx2EXc^*6EQd)9ftsI#x0j zK|DD~iTOEphW@a*_DOrL!H7H5!2u{4Q}_61H*C~&NiIhkU69G zaZkg82@yjE=q7mG*8pT95;y3Hw7}@uu?^urj1fd3$z0->;AJ_Vxt`bY`YUe-Pa$Lb z*j4I}w~SJc;HmV(-oS7i9!SK-f&*&Uyyy*`Z!NcgiPGXlM56Ju%~M1@eDY)mnf1q{ zCpv(}NtAFMUEdwJ zCJ-6$`lz8^SI8C5q*$9zlKC96w;+0dcogPW=3r-6j;G-oH_L-h#P9{FuXjF5J_a4d zD|9f`cG`JsS7e3RQWs<+BqeDo&z!i0+|Zedc=A zw)b`gKDt25o=hfAsrXUskhrzC7p2+6Ne(S)!~M{^C)~Gy$60s1U61#T)nHl$+l+B_ zAB1hpEC-;|AQVjYGYM>vtP>Q8@i66?^_IC^D>z@{VND}J&0J5?_X!D+=`x}mK_MW4 z*geuLTee)ywG4T(gppQT)IfMrZ|cNO^2R;S$Or|RuYl!33KF-hdk{q6V;3&OPFeUx zEW!ZJvTWH+azohrNu<`2_nXY8T=?Jk0WTTYn= zsc;&QZK#2Rfj6enQo4QkXRhB#&T5mkPq_k$K(Y!rtW>+9Y?JK(h^$B!2Br$)3Wa7& z3YVZ95@8QvB|kJ_bkrSsejhK*su{kQn0})xlwnl79n=I_zO1JBRk`0dv3QM73zuSl zZA@ZaFr2PU z$b~NlwH|=$AmYAYON=^_yoE})V36Kp30r)?1EfhdnCZ<^tzAn6hfoSPO?IKvoZm2p^PsK+NNoV-|gww%E z6Sn-k6ZJ&y6BMB)9Wob^JHEbP#CFi5-!e$ZDhZ{i5f~La z!ojf-uXS67-?M=8fw-bS?6_twRSVk<;mLhwHNYMfkY<=zSp%R|HE}&}!I_tB!Sof# z{+q_LTX!}2D>vNZJpEX@oCJoRK2<0Ss|4=`mC-+qk_#h@yyy_9)KQt~6bA7anV5J#=`3EtZOs53K(V#OE7wrL zx1)kbVKL1{YNMt1FvuT}{=2j^l-K9!3kMgCiS&7U#I;l~(E$u?3t5BA)S%Eve=qV_ zVx`^0Ns0PM&_{2HhpLv|pnmojYogE~i89{)k^^rHAya5LEK2Wicqt)OgDVFBS;)JI zFw2l`wu6q3#+Qo5SKgUEaTj_BA|9G^ACwWXAQ>&1)y^TWI^ z{AMq_nu3$Z zY#(@yh~1zP84a{P`fy5IA$%*hjEGkT0w$>`7S^CoEuu9n8WKa!}J~XJQrVF&E+=(Fnk~m6?v_4hx5ph z)ZdN+2Q-`y`&gWR)fnU=$eIQ-no2+9hifNx>Vsrv*3?p{uJuSTVv>XrZ~ zR<-!cwo&vot>tbE=|FME48R&n*(Fg+jbC2kTK0I08*v$iH!3&P!Xbohya21+4sO)$ zPsPYCkf{*7$0RDTV4VTJMQHzi!kU)hI0S`<6NA1do9=0ko!$vb8DN?%#*{vM_|Wty zEfuwv{1nE7NV?_YvJ02Wf5gnPso2dK3U?D%k``lT^`H2AxP3&00}#}}d##u;(LkIJ z?jpH=i&|TcC*t*GUpm0JM_5(l_6Q2r70-+=cPA-to?68lo{l4iM+P5QL;QjNp1>IS z@yllY(;tL#LGt$a>CkAJO=bm=a2UiZy9ON!UkM&ONE`KWZPZ64C#NH_p35J!RJ)LD zxHM>i_s2cvU4fTQv;#bCMR0k9*3nql;$&4Igm&g#=fOIQSa%Aeos-u+IM(nLKI!Vx zgKmN$bt<*pTda?AK!fidgW6E)vISh|_Vdj{&&55uWIec8L)L+MEdzv+?Ekz(!(uKj zIU*v$8xvxE&r6@~-9XeJKsT&BVaDjGaos{w$CAGY@J_OCJ$}|PkWoj7b>nnj>DU42 z>n}1gl+4X{;akAK$~`jhZA%9Dh53q~a~oCUd7%p6;-KzA8r^_uKxRwu@=D~|$_>eD zFZS%I`Dcji1e)CNfwj$#w{vh1+K1P`c;(!P1O)&b(a*L17Qu-xh27$^^k+Xv7D>z- zqWt8`o!8Ov`FQCmE%OUoA*U<^G$L2G>1cw1*@5HeTfxT5=YPiy0af{hHZ*ib0ScBKxe+xSq* z!%rt%T_yJ3R={J-D|q(=y~H2)=Do@huVrNP*sW3p2^_cHXuRJP@N?e1USbmoZbCJK zOnmW?nJDZiwC?vP2}HfdC{x6vyhUTYe~v z03eVx&^jXoqMQ5W*1daMPY;j~3ML!9VQfQREAT(4!Qg{VF`sD$FcldQLBTBLW2p!< z=k6^Z$Hbiv$j#>{e0yFXu}7!DP^Y1kv{$Lun5&RpH-wu-JR0k|fjnhAKdfZbme&w` zPRqZ>d>Tyh6G&yjG&+q@2fKq7cvoqOiw{3t81#j~|A(wMfy+5<-~Vrfs1U{yB?^r# z`;shK%2HV;L_*mW(Mp?A)@l+$5`~y1*|L<1k{AzDicm>q%}61VrQiF!o9Fxe{r|st z&FgtRL*2{gx~}s&&*MCf<0P!tJuZoiHuw(hA7ND{%Gdl84!l8E@DlMv0EI;2AN0>Z zYh)WBeVPA}Q`)4^pVE5=_QT<~Zz>uP0n@$th3heli*KD>BZi+f+rCM6x* zbp1b9e90?KCr)g9>DfWpnBdA4?+>M48p5nd>#5O!Ek#> zNEVF)@#Ac))=oAWW&`h%G{#*9S;1&a0jvQ3-W14UT1n-{7az4eL`@E|9XY>~A%WmW ziV#)kY_q!3UjbX;mX@Lx%9ez17KP@<<-3!{{1`oD(j-r(u@MQQ=GfZ)J*ZOlJVhL~ zA|QfI4qHv&(Ht>o&Jka9b}svKJekehKJ93XUR!qlN@6sBeUk5KVW9ysf&a2vkFf1> z>iMLgHvH{folfvTc)c6vQC&Wxd<}%{C-5v>V>UCzlxNjO{}o+;-CDWY{P8k0b4gd! zF7UKo{#-D7_KhED>P;GPBn8ZZ(Y_NSi}i-y3HChk4w&bH~H6hHN-GHC0D~<%q*mCQo+yI}6R*YhY4&e4qnJ z3gsrc?|1aPc%dSp+67G#=%nV-((yP;UAeg?z;cfJpDaC?0@wQ8zWVF;@xj)wR^w>E zL%lca+i?yl8;6R%DeOYc9XrP@wqld@z1$u}72mU7QT1%}J`4&{cJ@ZoL7~RAyFqOa zdJ|z<)&2aD?0h|?H*0>aDA^6K+d%XPdc-IC8Y|YUQ32;BkZ>osA3VQf+ct545VrAv z0S$QgED!8_xkUIGk%iFd063E8eyNEi8bN?HwA1bWn0&RsFWNwKh2)ehEEn}yiYzkY z84DK%LlnObixRj1Y_t^AvySSI@dtgwv3>sn|H-uAvxt5)BoND=|*kRiW_bcxKn>eHK9L}lP)|Lco< zqYZd~qQAy*NBTuHV2_|^rTi2#XW9zGGEMMpOAM3^PQvjaZos zO4ONRpBoeuBv(IKtcvTDhNX9gXDzsQWeAqw1p0C8ze|sgv%<1~NYsN@uJYeT=@L?^L*A{{a@Guygp;lsSVf9_{J!DwVLrbjh3 zReHZJQu8e>U4mhRI(r2cQrE2bcCB`#+Yu$dq<4qhSMS=@wDNs+@*(SnmWySlQ0pJ- z@rRly?YQ(sNJ${BNaNs}0zuJ3z=^yOX)3ld(uATcu&CXq8EMAt;s*c>4j@0n;b8&9 zN*Bzgq`bVjo?c9kSzfLBW-L2jg-`f@ZQJR_5qc#tBRFBB^NbyuegA&+$X-=lBqrkt z!?jofS^x$or}fgjQzY8c;#>4Sj>p z%U!!syl-#NsEM@y`Xh!AM`*eo;t2in(|2-Bp#T%5P0#`Y?EYL^dy*0}Y{cF9W1lq- zArMO~c_sHpMbSt=C`XIUtSlP}th_1L;7tEXgRh?o-N<1*JjN#&owYQ6iS`O7X8xkaG>S!_tDCIt5!I}k~I-l9!% zeCg<79vzCyigq1Qwi$u>MY|rn8d+iSo8Tf2!d85h=_1$n07~mVR#ywp5N^l-z>}mh z)xl5HLik=h4HL>L))s~g=u9^50*)4;pyreix3}-ASw>(Hq12TT#JeWk+~SwcR}fPK z3G=aZ0_ZM=%mnJT1mxn_ z%^fWv0g7;%1OfD>Db6CIA>h9*`~U7_`Uw8b7-b{J+<7s%JM@U>LqlGwTU*VV)d-9m zgX*>t5Qhv2h>VQPF5dMQ1R=~(Fe+ZWbm>I5&ZVO7HNi832t4Wf&)zi)-N^{WG(xmD zl&MlFgO?CerJ{tNi?x2{pxgaSoR4uvjomB7I-K^+I1m$q9WpvWyc%dXiC4p}ik`ZZ zeofHIQ8G8(~3Lt}h&ASo~E-$7alB6I_ANh)*Ef?AeI z3mqI_Guauv%!Z13G3+fHy1SD$IE*tm^LUL2pm7ZVRT33AKqPsO6f|r-Tx-*DwGc=l z%V975J$|AD7*Y0Gk<+$mp5JRU!3j>r#|!&xW$5R|+6&!}88ip%6+$D$pC*J7xn9s% zCKF_KssAxnbr%-KC_2kzQMPT907<$b{u}AE9LXuUu}+L!7Cvr9Ta>!uss^1jgQz29 z40?8xqBOhCu(Aq-*d^>GsYT)biI zme%gnq6x9-_5)G0raNqm};D!mz3vVz{?X98KDOS}Pu zlBTp$H-I~9vCzUty7jw~t!Ff3XhaYdOU>7> z5_tJb(6>SCB{i_Bq_Sr5))Um$nQl(q@Tvu*8h`Rl>J2=2wAsosBP9MSbm)JT zOx7D(O_+b-O66Sm31oE{xE=s_!wS!!i-8l7nWf`|aH&qk-2`|eoho249tvDc8Dwey z8TO<(SxI_&fEu>-fptIL``Gv2+^+{dG=j2`QbR2E$0sO9d;3~qn$xG(GvwI=DoKRJ ztIxmd_&`;2hh`?KamXF$8B3S2%VmOM6eg0^QB66x?+|ik7y~u1L11ARa^p|%6RRLs)PlJU#(b}A#HrH- zv;?T^5!UQkvB_{+d%C30P5d|W9dZwSPNULPqudgJck5MeGV>1VP4J)l`XFmt0^7tN z1_4*9$hRFdA0eqGiu%?0un3_|$r6S+w6 zoZ@mcar@J?y^c-`vP*%m$-M+TVf$eut3PvH+i_A4;2e>Vu$E83WdiZP3K8J@xDZX- zo-F;P#`uREHSC((!(zH}_=Gga{1>d^{Rp;O3SI%TJ)q;^Hz!Y9#&} z_=@6NZuBBx-Fzxffy5;^dwsh&cE~LKtd<*ta311&iq{Do1Z8{wm@CcX=4>$;3W6s2 zvgg-?^g)-3K7^SUrvFXr5z84M3s zQUX9e9?+5ei_s9TgQq&r6m}I2gMJIIZUbH=+wC@D#DR*r4Hv_CC4`Ar7EGZx!5RLY z$Gjd!^dM#qwY};{L{=@#~-WRI3^R+%($GwOPFr$AdgmxI>?wBFKH~6|Y&IX5% zCHOtT-7kBy)@trtRRwI^u6O-aQ%mx!AQXtzPfH!+BO_bKbM$alQqZvm^WR@F+~W7L zt&8Xz-{6|7e4V3*h}XYjxWi03Ka}ux^zrAOvu}oWkmxgx1FJh4)k_eb0ctesGU1ks z{S+CS%dLx?K+4UhFVwoGPXODImvUr`XeEsUxR%uIsnN@!NEaPX!Y#0)E z@nSoX@?xX|9f#RiTe({F`t;egJY~r0hrZVrv3b=w@J;-$KbwalQ$s;_I{s5E3#i%W`yuB!-}sAvN!lIvVW(r^+402Pl4 z&IWnvmzJf_Cs`sa(9xd5>6!MYwIiB5i_}3;pL_3u6;B%v8_g!tMK8P(7TXe)Zenb&ZENI3sUt`e@tw!;vB_VIbovEWt6e;L7Vrh)!^_%+&w}0@G;Yc z5hv+kzYZR}r{P$L2s2pT!fHHq=2t&`v}WiKWK)KY!!WB~z6tXF3HN_8k z0Ih62Er9WQp$K38gOsDWA`z-|!sv$&f8^aSq`SH&G<3#!pXK+wYoW?KIb_y~X8A0b zD94~_o&w=ZDyu9aVjAF(^+#$d3;OmySuO9q#Y24Cw?`z^zk*_{q60CTa2FxQ=D_Nz|)*_lZaLkwO`I>rjZF z63y(Nb(Bi0INY}1$DmqRQN58&+;+iQc?r-ak%z z&PTQAJUsiQvEK!D-VE@IJhoB6kIUkc)?86dspIP75N$VP=+K!kjQ3C;{bplhL7*7C z>DvaPP6wtjeQA-bD|?K^5ec01frzfc8-ef4Wh@B*2$A9%w1#FoqUm{TC3p0XHaNVL zS5WwJXRYtj=x4k`3LIHBDn8o2&BNgR9$rUScNX*mFap7oXZAap%nUtiZd<6|kT;D)L(DDa>4 zVJi5EiO>MCnrB3#^MksK8U2c&K)J}UE-n1hx12b!EaJlt8gnprRt?-}4**CQ9Khv~ z8nPo97Jwvq^aBzK9O3{W%dTy1B$*d_KL}6&k=M3q>WQ{Vhydb529uH{Q~>uKVra)|PFoK6D232c2x7OWGsg;ya&`2A@S?r@jH72WU*}6bCtzLdHYfCYacjRjpvapv{bwnZJcmjAv;xD{N#aQ65u)5`Vcv2QG zx#o-Y=JG>r4v|Shm~BD_O)>#2DmbFPk&)Ve*P@^j5~m1kIgT<>21c>s+J;tEbOk|2 zL`x~m1TF{RM2Q^$AId6f!iXnHQPu}Xk_Qn}0XuI(jFa>h#LC^n1J(K2q7T{etg~G2 zSeFQA5sal#*zFr|SifGp%kA`x)4)WW#^9DBq!TL)B9X(rVI;R~vY`i>_?BSj9q4sD(|cYwRVC(fW4ok5T&z?p{?-}bF~JKAAo1*9*o z3kkuSY{c_UFlg?|6_#~Jih{sNBsKW?>rdhpf_a5FWld>G@2?7TQ|O!Mrg7483hA; zT3YPGDOf7=7D!O6FZ|@Dj4;MZq>;inGMU;;m8zAyWY{+&ZkhO&J1@SE-S-mocL=SZ zf`78BxwTB(-QAO)YpSsI#QK@!_*zPv$B$DyzqXd{?yOm51w%)Etnx3D0cils42f$# zZQ9>!@AZVfLvKrmWOxF3tq6DR2ZD*HQM@Cd_S*3BA-C+7)2I9Wo%yHkCS)>H?*%X3 z7ibd0&QO0z0Q~lyGwtE+LmZ$FpVDGe$fq22w_Uw{2E`R$I2FE#Z6 zOILkqFjFR}cI!6o<;J#4#BbK@h(*}JESjrw?5(X21{>Qx$mzLgXoSYI;sz~P7^uY_ z{mv)`&u@i$Sb6q>DBPD;edw^f*z=$ZUbMnDMkOsh1{j6wi*pH&1y_Gutl$y<{8aU~ zSL)MCJl8{iTh1|JtZH8Nde1liWQDM_xY22H(>Px2ICuUBFPu?YwQv9G;ZU@s(U2a9 z*Y9>;=_?w5W!`sz#!`p9a|j}23I>O(yg2rXV>{O~*Z0_|Q{yr&{r&YH7&jFkzApbT zHK4c2gZrBMN8;k;5_NIe^Vbg#MUBiTo=bu)WCXhvn*{%(q!Cu&%&>!I1QGU#9vw)w zsI&blD|%3xv|?EqQN0`g9k6A~y8N&0=wXaovZNckf_w9eMZ$w8??wY!5?3=xnhM^z zz?zf35D4Q3N~s~^Sz zTvGD#?5z)dBQucNaavbokS!VGQut(`qHu;o+EVZAeW~FOSN(U>&7{{T9U;Ckb#%z9 zD=%xE-JEs2MF5+Wv@tnkxB1JB`dfYrTPwCgVl@5g72SVBAjC$M?-zIg2MTwC1`Fev z=7{+T%M0j|KFq-mU8P`9oOSvnRKk9U2k16FBOBUpd&6_<1K6#7+Ndj7Gp zax4&=5HW%%g0sih ze(rqzP3%=zpdGdU5t)i628i1g0qgrDjm|_v$z8Yj!97{eP!({2Za&b z&o*@zlxCpp#M>W1hI*2hr>*S74`H=;LsA{@?={Qv=bf{^JXsvtqWQ5HUUUJa(0`$sjDEnl8BIlq^aWe!xip&l~61KLp* z3#mM%H_2(Qq^(}HY8)&GvP&82IQr?q>!sGU3$5c4zoj03k~D4pD)_ge4#({G_W6X4 z>P@hZTZb+G;g1{4UcFg?MA8I~5KwP!MFOrIaFDl5Uv;sCH}I!mk0{6c^!J*|{2Uoy zOWo55b>9{s2O7xusD*&%h>2D4RSimZy3dkY(;Xl^6T+6@&Y*8N0VHPnKe-@TjX>c7 z)Y9Lj028sC3=k!Y8#*%Ee5%0i(kq};CkF{2IgyE+bSNC-YKM8Aj`=c-UT3^EwMG`kV?6k-LGyJxW~N-Yt9YzSCKY`d4BIvzY4yPx7(E z+-wGq2;QD|a>j5P6zBEawK{ez0j&0;sX)mL)yEgG{uw#`Yo@17d-^XKfP=ZnxN%_8!exI(4ZZU2wXycEtPbrCxZ!SFoy)r2wHg_P+pUsB+Ce+m zWJrES$GU}uV*Bh<_mg|YaYNm3zZHgbPRTWeH^hxm+_cv!L%0#*-beDmVz~EnP1-@& zQSUfVv_DG7(n4**;6!MULU*OMv#1?Wbq5lqxSr$f+pNf~jwXnt%o4IV*&iIdbMN(_ z(vbl%^k@68eQFG8HEPlM$NQ1i!>*yf+&#O|#2@WRMF8&(4R>moR`+uxiPw{^1@pL( z*iKuPm3J*obAYo-*W^04k2to&KIj$%evvr;zuDk;KDiX#6govhWZ=RgaBa7)LIkbaVm=lPZsD z`;y1tlzZaaA`Wx`(WGU1zt; zY;)QJnMBS?{4edYkC>ok@H~$$bi$?)Wb~=uDn@>KUmR7?#hxo(%n_9ZeEyQ^>S|75 zf4YT2yd%D0Hu+CP?FHu^0?L6kVBFiVWy=uPNVAExi}V^c;{W{wWI`)l96nZjFVpcrC)+127kzkhC+ z$^2%~Q{(>plN6|@=&dLM7PD>Vo?oW%=Iy7(2;gPpM^n0%l3-C{M5ANHtELya!geI% zHxUkrB-ibj>zw)X)c{G6sCDAZi}C?sgUTxJrGBsf6j&bNpmfHFt8%QpnTM46d6tx& zTp_!_FLrnOD-!#SUwE)2S$^CBx!SQsiGNhy64DT{p()k1%vF&p8Kc`LrX_l+CURVP zS2^t@V{)sWS?qt!+(Ylyzuqm-iUp)lw%S8gwP=`2rw|YbSH|bV{!aV#HMN^BFiI(# zlr&HVn^E1jfT}X^_qHY%^matcI-}qL%NobQ6O&MSoG^%Pele!sLWwOpa+2$sRIg~e z#)$g3)y3Zcn#P;p+|y5HyFVAiNw{&;BQj+juvk=4iya&^G(+pwXijYvDR!tNt1{P- zP(Z@Dh$>blN4f1@6}=SFov15lZ7k-+r7wv45WMlxFcxouvB;=6M-AsWNk>+#Su=rB zNBn6^s8)qWj1)mkTp<#zx9q-cDBTjmGmdL!*fl_PQbXY>jQm+A%#SACd`~w5pm+kP z7Avvhj*0UjvZ%#az~Co;V&s(M+7sz|W@HrNO$G=`Gr@#iDs5}Pd_*MKIwP8mI|3mB zNc1|hL0C0YmKBMg1|Dq)l|x^iDt^r=V5J)I_B-*#C?RC%=Qehk313C-RZ&x;JitBg z{GDkTVq9@(q+5&nhW12?NHU9-)XV!LBEqyL`_(k3`;4d#enS~h4UIUl&=(R9Z&=}( z<9z4;$2sv^an=>3WG@f`Nha`~6z2hsX$H&*uvc<7{U3VGj)Ig1QK##Iis5Q70Xu#Y z`TjVE;ktEx*57Z&Z-q(4@mEj#{y~i+DN-W7P3Vup$wIxR&J`sHH;1S-p-2S z{fDdJOUsNY9%KZj2M<=7fK!MEKDVdD0dl^%$*z5s!u!F|8nj@mz}ZXLw-nx4xVVVo zY2c)pYQhzysm=W&G#tt(%69kWbY^xqXVFleGjHB@QdLq!p-~7If;vbv)bu6I_xCq3 zuukPZD&hF;ALBDx#LJmRe^%C9fr-w0HI*@vY*b_hs_?u(=B|UoGUfz&%S*Giel^kj z9H&~eynF&VXmW>~&$0z~iwkE9-Gd*zdmw!ivz?=TP(~O3vX^uht z5j7hj7sOz*m}j97@A<>8;3W4qAMi4C186W+&#pJ3?vEjfq(QbB{s&rhVT26at=Bb= zo(9Mrvgts;NSLI0efw%CaC1<>G#*^djs$t!uyuCKwh^r!TOzQ+KqY61XK!Qg9k=aE zNebl=vX0cUP%0JBHW5|`@=PzSfB*ivURm2(%wk*e2=Q;)|81UU-Oo{yYuArJLAkk@ zeB{`bM8#2q=Af-olhD2rODQOSN)wWiu6=kyZ-OU1EABB5WG{of!ikx*C?7@fBdSmK z4KtM^iIU=@Q+RMcxjF%EEa^0G3*li4nWrc|q1vCTYqoj9jJ7b2*;61kv*l}ZvN+1} z9N1y9)=;VVBC1{%$^53|bol7zXb0$8@7kAM_KG7SK^9D-;&wt(l9s^rya`k{4jlE? zl#1xq4#I@y%8`;3b(#qngp5N#=>Wk?JSu=XC~;&(e(lq|8T4|b2{N3D294Ajz-U4t zfCd=6uOvVD90v>3Sr11pk~2XIKpA99&@cPlB&23rv;oR|Sq$a3 zvr6Nc!6L~$0E#>Y4snuQ8yi(XT{5RDQkW23Tnwd4r=og0iP4YTvptGE@A;is){l-~URdp=-aoLMJoRA&Ey!0k}cwCv=19OHO^NG=l zYfxRO+m@XAM0+3VDAHeEuK^v~?Ohs7K9!jG3#uO}e|dQ!Rj}2Ch6ckH(tIPu$p@>5 zyA8Nb<*n$ba~`F?y}iq3XAye??vlmLFE<#h4ojw0+qPTgJ6>tr#p_dOo>T2z`@hPn zS0q|vCbsj1=q-TNMP#yR(@4d&d-p8+?1tn;%_OWHs%Em5#UH-?CuFdn?vblk#f(RO zn0yI?T*G~ttLr*Ge)LE{0X~h`+QkMnHD0(+UEnlq;lPi|s9eAv0~RJoR3ZQiW0x)h zXE%e%?pgi~(8QWhpb6`i0K^r?BIxXtguIuBrQ^6H+Sy2Z)#}wVxXS4MoYmb2g`BTL z1DwdA`D%_qTS+kKSluX!)`y3oQlcr{}shrRrdInT_K}$)6C3z z(>}~FBtItqE&ZxY%gnMT4VOH!XnNUtEc_f8W6oJ-6I>6zkWTI z8CA+Z$SHM+QOQ@XO#XiIR@;?>wYVv^ zXBtrM0W9Jai_Lb><@WxI{lzp8N7dEqmw!kOefe@3#^5oywG-D$Z(f3#xMHxD!Z$6= zC)RHiJVBjKW|&fzU+5HFg1C1V{W~rPnttOC*&d%M*4SX0rDi<5s+cLq{h%U)55n5* zUZU6O>yv_l9AXK#F@2090QV5n=y8!YB-VI9la5`Yx}YfWaaNXD=gl;%0}~PwYK9c| ztiHQCAtey51CwFLvNO$9Os+9%jY?t3IlX&&7mc58GmW`UCU@h*dc5e!UtD?(PW%39 zIb|Ze!m4Ekdjoy8kT!tzwXbWSm1@;%FA43w%A6Mc;;nW4@Dz^_MO6i!c z;};`ZskqP?<_k&db%=uip=91NqqC%68}SWU)WO4sjgNaUE+%JeOb(#Lj6PfQpOQ)P zGleTX0C3TwhT9S8*X*yQl}h>^g} zkL13&@pb9I12xujdhMNV(02N%+5E5AE-_5T=hC}>{xNdsJfqj%((P&LXFWu;SU`>YVy5sdWVD`Kf>?4zz5}>bK$|G-)n1XuF;@x9y9bKFTr=8DbLmz_oT$u1&w=Gn*w^Yw+*#<*zKeXbhuhs)~Vm>h>V~7HHA-;_okq zaVPkj_Xsm;p%KGK>k@bu7}#FE{p06Z;}++5cgq+joa&$B$yA6MnUr@X8-6z|2`ER}0Xw~i{k%N=;M5_52RlDL33sk#kf~m9UgE`z(Oi0W@7~RRTV48oG15a7 zld%vzW0yQEx_O@m^V&Jdv93#_IdxeJ6V@&K_6-*n^|9aUPH|kA&`AL-f+Ozm<^c^! zw5Rvl3-`RO&wGtdN4>vZ@-@E$fN_@Tty~r#*F1|crxK)AuX^uO*Qw>`nj>6`TkOh52xZQzu3kw$UHDil?QOvICvm8 zpOnIY4*gvAj4nH|0{A3)A=y&~sLb(T6TK}*jZt+>Ike^28G0o^{*$$1{+_Et`nJt+ zY;0}`RZ@?y%R+XAEuOP%pV`3feM0EuPZ%aH1s5(}JZF6Wh?BGPmIeLHY-}R=U>k#8 zgz~Gen*dwuhTW!Pros)Sx_J12u4X#7_na|qP10HCp+!8aszvP@49crWgM28ml11^1>f(x&rH@$M@&|Y@BdC*Q7rnj z{L!(JGJN2uB)b5j&NO?o?@C?+9{#pnzi}UOi_g&P5QI0f0)GTI%{(`bC=;pzcp{h3YCF}u7`)e)%~Bp z4Gep6%YLA&t34nS=QG%N_w7aXOVvV_O3N_cI}0q`hSCU~4$GYP{W;Xx(>o-S-XBo&+41*hne{ zVJ8qzAjh=j_|SrXH&qDsIPieFR?xEHcC*GD_{%X4LfP-P?cJHAkh${a2W#y=I+J(H zpS+E~6xb&#h%r3RPA6BSLp-Rh{!#R%V)^6w^{1rh_j3CYaR0L!wKVn>UPgVAJRX2c zW}1*BcI(=8@uNXG*)vYf?lJPiiarSsLJ!+POcu9k62{m!)NArq?-KKqJf}_jqM@QZ zCo_i8sP6u2$duzVJsVlWP9ic(oeq#j{LTuSHhRQA2Ufh^&!;EdYo&5Dcg5bbSFOx8 z2IEjmFJTcEm;a&5@BeeW#aPzcevS3~`7;(Q=rh^Fvm~H>lDQh-H7w7aG?Aj|=95Jc zdbB>T8C;XMY6vqIyQ(%sK|o}>F?V5{-l(k$`eoU;6$KckeQ%A3q|=|5f24;hcdYI3 z4KLN5jcgL`lmX_h zMH0=nlP*8;_uJJ4S{=A%6(~`d@;W{tH8wWCV5ec~Huyg&x!-AV!OPzQwtRSF`6HLzrVGuE6=xk!sK+i{w>H4t}+KyyiNyeX$Z)`Qo^r}w!L$F^ME~Bshz((%6Yi-=@ynN{q8W5aK13f>eET?DnPAn7`zd;F&eP$(C>-+OmYkt};VoZVhvcU| zed_tq0&GG2l-$2pWq}yp)JYi7p?&-O7juUryOAQEr~TF?@VaMw3;lxW&wsCV%imvr zf!mBv$ftyx-Y0}KBhvL-(o=YX8XP*JwHXxT@E2INI3HN%ZdYjY7;|Z{tUGu9yo_}~ zU7`L#ue+UY_#M3cbc2mse5!5-rI$KT=z4^7TD|Eqzn9*9m3?pjyM&nWd;Qar>$cWk zC|Y#$%!u_-raAzOJLj)W`GsPX-(qIs7vF&b=F>T$aKBDsm05PENjLbD5UPX}+T>~< ztp#2w56glUt|b>^V#H16Csak%eQzqSO4@66dcotR^Jr7Oo#_kuv<@ODs9r_jyui}L zCsAw0ABZsw9JFCtEG@MAIfsTsP%>Tdnb6bz;KfKqIrY8g9W6 zj!Df}N9oW&L(P2Sr~Aphc;IPub=fmxu;szzv5SRfQLTJ)#`n&x;a-J81K=O-$!|Uj_0R}xvSZ&;* zhuys%zhoRA4%j2w20&!#hf%)09c(LCubDC{K229}CrOsc{HN^F)5(cx$=`VYUC^3a zKB-OU9(Egp_TqP{Z6}+>Td`hl)A#+t{-)D;XYeqhJZU!S+(kP7RVWVm#Qw$Gc5UtGSN|EF)TGB>i}(pWTsWUH$Dtz+EaBnt>D zWNN$rjHo{uK2q2N#_u!J0)beS?K=N>=vtzu|InF5bLvx)I=e>)$M9UXhh?uAxIfGI z{;l@VnjouAet6eT`Tc(OWRHQdP_Ty@H4o~xDpAWizFmBLJSmgbK9|qV#!-2@Mnnvt z$&1pe=uGNDb^C@s17#N}e0v^t7hpAiRD;Gj)438RaPaX+Y2q0UZ*)#_e|5pxj8R#<{&FXVLMX z=IQLAA44s`C)g6V+&YmeJv=;|bhmBk3!__Y>jPa$`7JxFq$SzmFJ{I)D0JAnOGlZ5 zX}X{5ynCOrCi(YeRQ8$eZs#_?`>boqSB;rMNhagxP&W{16^erq5yBtmr_}H09cZZh za$3Jph6pZ7{JjoGSqSneU!JPLK9J``OTv^9wRr zE9h2nG^czUH;oOw zwC<`;^zC`~Xr}@HJpGUIc@I|a)2Gj~_wS3}d2LM5sr&wF%$2vg8okGBK646j7IQ=r z>H0&E(H+|YFsaBB=yF7{efh<{`TA2P#_XMPydjG>%$4dEfD}33vPb!M>sRyso?mAr z?J*2@cBOF%VGVuFCSY`JFl266Al5mZzS~X(?@vs z7#U63zr(b5Jth@wd|yLOFCLifJOPq%wp+~2`VVNuVko4W$0pf3TqDS}*WMj3LU(D^AB?bU z(o+)Q3zsg92c%2L-`Re4dzA(dPKvyvEMvz5Xk7Zz(Nu?Tu@I85Ytr9<-B)dUgvo5X zYY*B)ko!^(yNxJmku!GPrQZg+4m4?{a%w!p<(ykFxo^0O^q!JM9?e;S5OE8ch$!OV z0^AuKWGWtP*?(Bt+K#--I}6ww zWO|7zpf$Y!x*fj*G9=aK-{pJh#0xn5FZOR}Xa<{*cHea3(kTP0-+tq=I(layjOFCE zy#mHHfH;S2WP2_U7`8@@q-{#amt(>|0ClC?o+ZX?RtU`L`1Hs8P8mZ!c|&f|K&Le~ z)zc&V8(7#ngr164W+M``Oa4qq*s80F{|{K=R?`?0niPOoBlnhaB|Gl?V0in*2g`}~ z*KJ^$leEe~K4A<{d|gBOy>x>L)vf(M%0g8YU{?AJ;1*IIK=2Y|7LEQ@T*G9giesi5 z(Cg^BuQOOZ?Askt5J=p5nF!)6F!K!8Apqz26OP|qG;N5M2FsB7!REK$9-Q(;a3tDG zDvf$Cx}~w2b9veVI%DaD)|S)n1k9aNlS~I+lklN zzP6%K!-hgcLolCEzIZcPd! zm&H&PtEFxR29tR232Zp%Vt$6HH4VQMZV0g-q1@NIc9G8V0J-;n$Nz_B6-Cyz7!y#@ z(!MaKn+J?5zcn}=0=JBFsIOut;7~;YL9lAhX+i8JDHybcbk_CCouy+Ko**8~z#{zN zZle|*Ly6y$EK2 zSyI)8RJ}Qu{@P>su5AlLPEvj!RVucB&M2IKE zl)ZVg{ng=3Env0pGiu&Z)1=jn(3QU`0a6W3uol?A!r>EK67kbkAgt1RBi~QJGl4#D z`qU-aC#C8TqK3Y{7<>pt1X}wU4Ej+xx8*$UHHw}A{q$y}+tclH7J5u9H<)0fOtv)6 zktkjL@^q67+6I_`*U%%?gUZLtqkwu(s?|fR<_C0Dr4#|E9{^up?LdtW3!A!Had-3tc??r=4YcnmhSTwAQVakiR#wu=a|E@uRi=b}L(K+SE0@Vj1by%lh3r=B`B9^Vj`521%eR|Pn&gYf z2Bh<7z95dI3nFD@6Xq}6qyga2N~HmwIH5-Ug4Sd6bRFh&)upr_Ab@KCHC2e9$AX90 z3yv@6gcBZaWZF~thAE$zfmw<@ym);;salt%?KVmKlr5DCwhF!^oVU2Ua~nLnSm5YS zH8HztTM~5nYp5g{4AGd?ee+f$JOV$#<4G6@8d8_-HjFHKBFoBzLk=lXPwv_ zTr>lgEXl?3VEv@^FPbm(c<~GE79pIWS-4#Na6W^gpIUyDE&`8_)e!Ft`+@Uf3*a+0 zP{8|W6WdYM&li6q(cDit-dX|`p?TB$l>8SK6^U6dwMVR1LC-;pi2(LKw=BtZ#QzJl z7UUdwI=cN2n^axZn&)BD(3oh@@FndMod`q>P4L>Z!wH-JB4`pZY2e-nyYtEQ2SPD{ zqRUUkCfR)EDDEnMGzXxlYRC|{&7_P|v{z;WbkV7W>@`I@K>ySArd_T;z~oi@y*6)p7^C7>Q@ z6~-Lf$bs)3pEpzJ7H;@^l%FJn#oS5XK0Q*ua|IEMO7Grr{oV!z2UmRAkQ4R7&DmKM z6-jc%$1^^4KfOoUdU=f&$jj*o8dU?nteESK)Z_t}xQZej^33tzajjQ6I=0JOl_*nvPsKPZyC=?K ze$a{*b^xEAKdegd`7u?Y%lsa-VxPJcxK6t8WlW@>(D=%Ne)I;RP%e#3&ANWwWdDc_ z%fgqYRHmynRB-m=azCL?Xs+18ZM&FrJI=x)=t$0rE%2>WQK<1VLx%izsCfE%g(_0I zR_u6{NXz_JS+4Kaqd@PDp-6M?wsGs%pJNPUUqvehKj*xE6`gRWn6U*=$p#UUm$VSIu(3dwONqK9P zn=D&)LNOK@S(wq3OH`DnBi^6Q(rKem?c!N!68pxhZ^*D>&-07G#7K!&9mZ8S9p97%VWYsqNc@NZAOzS!_FK=^j#*48fgg3 z{bBdgX8cgztZ6R4zVx#3S-maxeNYQNn0I`HJgRE+e9ZT=9H4|}jYhg5KEn?uWW zg`JhMuqBimVz$V_AX{$As&{+$st!vn)Q(fnX875_*$DsBli!K2bZL_QT+% z3y~RS_PYUVeWDVcZUn$S%b3eTbD{dM1D!&+Z$u4pr z`)FjHwWve+tN#lCb{t+jeY7nUc{3=UJUyBIfq1h)f)C}B0*i>NMP@o%seq<-+8onhSsT40;5U9cB5;OWfS&in^2@>@Av#SvB zsQ}$;-^6b9oW1GD=$|!qTs=Z!n`zcjI`GNy!AroyXPRB^S7u>K&7uMdcXml;bJ=J$ z4-&i^Q+b}!bNRsGDTzx+6+cAp>Ns1MD|PXgp87>Z6!<Y9HH<_4A(aySH9iQi9bQV`D z(6Td*MMg~D5mpSBCIMv-HHAN1dJT7NZdxAo01>6K2;*Rw@l%{ad6CqwFC#h~p0I4V zdi92`;dew)=gIVJuwSut5XvutXQ{W0tM08EZA&qIorH%lu(_#@y8NC|XTsL7*H8;Tf1l(-8rl?@Gzuko2(r>pZ?(GAD;`vG7sVDxTDi2asQ6``aFynmPxbqSn#;K zW=hCgz}Ra>2nWJ$NuhZ}GEo>lhiYAhW5=);Ou0-e*>E(a4Cjetc{$^pkw$fvUX* zvsmVhGMY=wSJ<#62bLeE_2HkFv#rSuO4GGhs{&`(92TO4d*yYeuV{)PCd}$G#*;t} zKaiIb*fzwYvo-`v$K-cI$z@H;4N`6@f$rK$BJx+UyX{7E*G8s<5z*mm{fy^-z|5cGHkJ$qZ# zqP-HWuMUIr?foUW_$XY*v~NZ4D{uFV5+avW$nJN?bSsbKXiBe8Sow$$g1d|nB_2Zl z<+Fb=dsglkr*JPUQ?)|do%a2Op*z5z#fjk!l^JZK6IO9m_oo^jWiW|o*3cyj#~swS zEHQuK_{EEl=4o!dnwJ1K{7rAXB%IsHV_=3 z{fYt18G;8httV+uTMxJlvNQpIOOUZ+!Q=h^VarHw9`;4$Gyj%Tv}30!mpt(Ig$Y?4 zp|iZ2=a1@q(uqvL9oTQ=TY4nD+#oZv^tEOp4Uq2v6=*A4xD*UZD6;$d+3XNiw!Z#^ z@@=NPZDTUz8pu(Eqi(0lvUPU`1qHz58! z$Lq?QQ65EFX$mWeLhOgcSe>4{(a+Bps*9?i-xQeIw`uPzHjIuDmt92mw)`UjG+FF*CCx zi=LS^WFD;J%bbF$zj`N3(hit(Ll9~EWAFw1L48C=fM$b~U0A`CbTqzY=m&Vf8> z31ipYO>Ahs%D&`_Zo35GA9k4tFSrK8C5@dUNDR2;WNnWdP_M58$BRwkshC#Vm7B+W z3m7cnKYZ8+l3r0aV>k=~saapqNk5x^SXhuYy*a2{=r~gkm$53OKxW`hz;rOlNarsf zO+S+ZwzXpRLQ}iTede80p|&Oz(FEEyLAiWA%oFnEH)R5xd}tbIG9?bO2DNol1<>X8 zi35qc^j|n70q}MAOgVld#5Zs%S3DrE*hYYMAPHxDx3=0E<>74kYD~t_~p> zGR(f=KVF~bC`n2}1u)aBYilJK6Ev2^L}p^Fy*a)k*#)?}n)`O2&_1TV@TWMfp^40BK znsp>(HsR_}X6VgBZ!IWS&@UTY1Eh}LWs9TX4@qCpI=l61$Jv+GJw+}fExO-Z4fhe2 zjbJX^jBh^Vz=6lmMmUU-kwxo9-m?M4}~F zz>GlnV(`t@ho4(gv!cW8fq>?LfSB$;^X+yXNBqh!9i80^BnhO43OzF-CZ=p#=aMyR z*9x%DVdEX>YN&7yNzP@M46-e9&-$Sg7U2<=v2QOJ!*3B>or?)4i6U^^o^THex6wH;So^q(zY!Xd{r7i1xu4yYp6}(U+D<3 z2sf6fE$F-XTHP;m%*?VZWBv#XePC;;SAP3{sj_Yx`|hraW9!iMdU-E$C+z*n+RLZc zKhZ!Md9Bs*7ip?AKyfLh_E~HaOl*dRwgs(+**t%)js?vdchWh;Z6mrR(F5aX4>D?Q4BhMghsdsbz(FRl2j*G#e^%>!4F%5iE5V}!jG z(>s{Hnx|ut#bs3mAxTs6edU!mNFNyLzdtY1)xtw-9ah*=Vg3(u_GCCPDZX!Gp22P1 zYxLLeCCYZG@*OJ9P0`y?a?GN$$Kbw@x!!(0W}}yR%gbJlVlNtC(lj6|g&>)8gdTOD z5c}E6ka+9Z(&ft=DF7Y2TaFuinXfMG!gOW3ut#4ock#h?aoMZG(^&XNRb)PLOzP~s?;=aqMu;2-=V$6c^0C82acp!UB%K_43WLFnazKb&g$I#r;< zV^7VVqP_HyGH!+5{v$q~I~WN=0@1Wd6*Q*q>)&#baD?l7W4G){ zvKlCmW$W`5YUE`Zp_LBXaOURY+y2W12p}C@jMMMTtk8ObjDuA}QL zBT2-h8B#%S7ya8x&^q)sG6etR5+J!9*X#XKAFq1+Nh$(7CeOWfpOBmWnN_`%G=YKhkf;8B z=c{X=X%E_#uv(-AtMz^LRQ5m*>ULA3g>OT5HLH69zdH2x{kPRHAfid^AdBQYvz-sL zGH_mJk7J>h@m3q2JMHH>DI*rK5TQmpa!Qpw>s$78!(&oXY*^j#rpISGJ$LFrsFOAX z$*#$U4TI>|nGhpuJg7R#{QW?@d;MSGtr1S@8nhji~?4oxIdx6gmGOE|r8{z#< z);1fAODbn%lD6I0T`vag`rP?u(q?;oq;QvMHAoUf^#njLy?l~sh?4zie~yzVlnS;G z*h49n^3r|=P^aS%Ze$q(b*HQ9xbRRjI}i1kQB4gW3>Zx7v+`P_mXgUmVh}4i;jv)* z*8eUl3h|Z-JOowbPR;!uMZ<>~opJ>HB_zQ)#)nc9B3T+$i^8_kR;Mu4D=(WxtY zT*0@W)rW#UkV0Ubi=qCoLtpUcrfkDZ-Zbi=)4M9C{|D$@@yIlQkSiUCGiUmroZU;P zMWlLEO--PpB;WnUiR(&I4egd~InmS8n68Ua-!B`pj9UbxF6-1{=1=oY=DVbm1BqQGFX|oN8{i){0w=GxZ`|?~ zjJx|U0oJ?kc|(Hf#H&aaB2O$`whU2u8?t+WmyL?T(Y9e6Oz*pqLIlP@he9(&!nk|k zj%#zzP5v?Hod)FJXXVHBL5%b*$H!1`(pRzAcAU&(mTlWNJ<4WAhMTd9A8GK0`?>xi_p(>*I zdlTM)MvRb~xO!;*RfYI`x4ZT6^NoK$moRlrhiFZSQC7~H2QJX3rB~{@>3(NG8)@1q zyZh?!U^bigNG@?80k+xqOFi3E#%CZj79WShK0iGa9ri`bbaQ9};?pj}H_&2{KUXrb zw2UrB#lSk0YSLubMS)KGD+PAykI(a|?WF%U{_v%qH+Nf881gO$ zsS&+U>CWn^4!g>w94#33h}2eiqI>uLf}K9@ z<-JVOuP=6V6jr?O{<(a7#KuL$>X@9^yewk$LU==>#}?eB7NPgg_~pv{n}_apAhPbu z8okQFAycdh(5n5PqRs@Y$MpT;jb$t$8Ecdjh7l5jLYazGL@}1k6qS*TY?T&div~rJ z%C1zlY^kKOM)sy;X+k*--3 zWdIhq`637N%6^~9D?(zRv-a>A;*Rr;&;-Lxwcaz|13%N<4`Ab~QS(|cW3DDl-)!8lT>&#+(BiE8AH z$NMuz5UF za6^=MY8^+H25oJw5N<`NUZ-#ZO&E$-g`R>sstN3yoEH zWpnjSW_0}606i@hs}4uy<{)uii&d?4+@>V!-3jKBskTUJ6Q_2 z5{w!!frq6RCz#NDs8#sPHG6Dhe_Uhjo7Dr&9vy0t%)vqg8QedjsPs*>gE1eFAlDor zeZEKDB1H$ZgY~7Zw~L((E3KIWObXHBi*qC&1_WIvzNSkVV;k0dFDB1Rm9X-v-`N~MEqxSbXuhAi=eG43OE*EL7zzWFJa=) zUWn@*23q62cPeb@g{}eG+niQ5yy&!K1RjWsHl=I$v=J>MruQ2|&jx39yH#=qpLKc5 zY|Dn5Q{KO0>v{lhI<>cS%JzrU5Yn>CKuvW`P;y>576Tb7r2toc-VNLBD_?x-($A+b zVoN66wFwmqabFv=Go5#KUl3YO*Iq19lq@n{16XoY+j?+=+V;y;Noiw}vE)3_uo7uE z5%24W-#HqY*>B$YY1C|VEOe>ExxYm?>4tv&^!QQ{d8BNtQQ2pXbWwaq|C5e1HKl(p zMR2rtlxVngHAwnLu)RApirv2~Y~2d8Md_bzX$z}*oV$R1^L zL&zw}E~v;V5l9M|B%%^b!-oSWM6Q{eXf?Iay@qv^hE55S^d`xd0kPVcMHn`^{Gx8& zoFKSD;sFqjzm)=!!%q=sdeX$ig3@7)`GH~S#4jKi8Ap4Hbe#e2J<3~z>vr(i+m{DA zSTa1t#I_Q)TM;{EVp8?QahvqW#S)uc<=l)x5$kPZZxACQ!W2l)evM|O)ytS&wAlCa zY)J~Fc_0^;^Fg1*{-eZBacPk5404~S_5+X#sn{g%7eNzyDppsfW=mz|Yp9Pz%krF# z56$o~>_Xh+kUz<#vTKr39gLEO5-u=M}`$7ODAY22Pb*27-Hxi5XGSbT;=MfO}r zWM)#DLz`>6Pt^O(B|i7n9Y;$`Wr^Pi{LQ4nKj`dFuuieuX_3SM*U>Z0Wl2eb9R#1a zviL$aT@)e80G3P;B2|AXWR3jZOL|GBgSE!yKT*$M@e7SG=*d zeq7fx{eW)MqJwSLX=)y9vcB)uBhH$37QLt0#x8kq&u!7edlOg04LmyL=K*oH1@B(# zpIEric0k!l@8PB!=9+aY~l{oF-`JcNa>Fa7d`9*FLW_e46p|pg<;L;M&Zf>%hQ^&98g*U!A??ho;_tSA}igK9+E;)_&`- zqrBV|sC~izM8Yo{+8ei=+l#;%q*Dy)%HYP{BKw5X9v(rYY@D}NlpO=rr@Arp)d0lr zow}djpr^pde$By>r_Jo>G_p)PRO_RFMf{oWt}+!{=eaCIaEqiTfKPHU~gpd4%OylHqi;#Ey+YaT)xaZu!a($X>6Y{sWD_SZPV%Y!WT zLZ%BY;B8Q#YHMk&Gt=eQ(s zx<_qAUKhkLFeO|Rep3x6C^Nrg;~zDO0CyCUt8j!wGG`U$oIdh_cG{k7y-_wfm3ysX z7AB4jAZm$@i#W0}v$Blgy|^EQAJT*3H_(TOCGX*uRtL*)CEUZ*{(JUp4I2w}=wA$s zKq%tK5IR&Z8SvZSron0f(Q%iSWi~ah>L;rq^IErn@jR^>b44Z~QFs~1k6fM4Xs9b|$vB3^`2prU36xf(Kq4}1=6W&m;z zot_u^-|u}rlF!@AbTCOI`bB!x{xXL)D}sV`20`u1e$AbQYIk54~#;=~KKendpU zAt53e#qRqIV-^bxn|Rf`cUSM+IbPtSqM}k~_st%5j%Z(QU6?)XzWzEohcO!{af^}R zg76VY>oL%aa|p7(v(@0r4UNk;HJ==ldwwtIPLg?}PsEb+9wercB2SMYVeOQU=MSb~ znS}P0@P0`_qkSRCh_{?%?j5>2QfI`KnVycZc#YkdWlw%0X@ow>Wcmlt_+sCq)?-}c zR-7TEr%NIf>K$ST?{YVF)Mw>4>964@SEX_Yu$T&TIjQ&vbC_G%obru&tMHGB8~yx- z%(=W~P|~GL=d|41Wuvaf6SKa1_idVE_q5|sNXtNbuJoaYF0I15|HOK+`P+4F}F9U5PO|`u!I8{+uhlD)*^H2NtIuPFgRjJhk(rq2D4Fe$T8WY1p z+16(p6s;Q9u%oriyrL3w0qMgoq(7P1DTuj&HFBLX#SogQG~zTy3bt~-+ug?(?EYop zi72SA)n3opv*2)K@B$JWF$?nXWqYQ*3`@4}@q*0AGB*PR+0>M!PrrQ~VG$L~fqI6+ z0mQGP{^p`)!!x_#eE;=V#(H=R=R6(K6Y&U>T7GQP^)jmaZgg5@uL6*dNZE+`mQhj^ z&@co9a+$1tx1Z}wYUo__S zwX-&^T^jF#aemOZf+of62!Et{hd{%vE}uM6f+S;OXA$3hD)oaJV0vM#7oALjVI;{q z5lknlUC}U(8kKyqGWeArRpQ5&M&HbVQXC8~=;W88;=K;>o0VCq-?;kyt(Y}o!akI| z;J8ugr|Rv3nyHFkz*VLo#6y(2l|tjJTbT_4o8B2iHqW-1HR~3gXPJ|X3E23 z?6w(rsM@2)T~t8bhB!2XB$9!b0plOIHoprbp-&?!nwg2N>bOS3noq4=4W%d31tB}m z@_1XOmo0@9>c$vHyaZZ!Naw?0;iG+!b;i3-lPTjQZE}jpE!+*qMEQyaw|0v4_Q+KD}KK>4<`t*8caMSgm4s zH9jO!h9W>fb27_d`YDG+Wl9PFRP^;N3o0BDd{vCmm!o;g=`xmyrgwXW260=%F?~Vj zynq88@%LiAaH;p?ejv1h@P50+zJA?gqc7PhPckYJEGJA57|Nf^AR5|DlB92NV&b0Q zU{uueSQdK?+&80(lD`&lJ&c?=4bP+sfVb3wN{Cl*`|jR$pJEP2ac>o)dR;^>i4Q~h zYLO3Jyr}#adsgtQMm%)Mxwy(F#woQExj@v}GO3sPAb)nQ&G`Dztqli(|DYLDr(PrQ z;0DE6@~U$&psM-%y`l?}AoXPe9IHX(XEgguGk})>MIlrxR?{QgM8}?Ru~u!{rp9_m zQE6^r;qq9L6jHGMB7Xa4M4@M-NhLaRk(_e*l~8aK!5|~RDI^KNSWz9HpUo*SE7ki~ z8mKaHm89dma!X)7x;`jNCQi#3b0mK}a>_OC*kM;m&`m0EHCL@8l1N2I+`I;L3 zzwcCw%E)oiqI>e4z7+EHDc;uN>qm8V7x9E7XA^ar_UO?rT8Qxr50GQt#eU^ls7y3J z2KKA#`T+9&qW8{5mYz~pADb`=^h@d!0R)C0pEYGOm3($`8|E`NVI>;7y+*zlO)6O= z0?)Bp%3p(L`FY8G7V(`Dw{SfAOeGde?*RkutjrIPYsLi1V4C%(m%p^4;Z=ELFpFgv zMJl9SLh@=*D~ld=R-V@J*B6qLt6B0O^o4`&U%z+jrUGegc(n|AhuUgIopVmW$Wf!> z=&%@mdZkVGWFWL?maD}A%t^@~@SwMN1~JO%n!WPtr|H^>c}zrJ1*DCGN=8z9_a#I` z4hmBte<4uHbhVB@beYAI%N10jslSV9#qi>yG^2zqQTP4XD4K%+0)5XIpARCkts=j` zVbPbTRnd7BcPNW5F z>W!+mn>jNqX-Ufc2M>1eIc;Xn{J5hwW$NsNzj2zLVMm9S^J{nDsjEHvza;S!Zd=N! zL;T-^>}*rc4b&%ynB2^bT>TCl=m|cfac^1Wxy32>m|!@rH@ByAH}__eO5(g|(L9_; zLIoq4VWhD+NS0+#@$R$z;A30IN7gY-qoQ+Y-I=Qi$&=s;<83F=+#(12_1n6A@8_}U z(AYaob^OdI@?V2}fjsI|K1IC3TgZ|x7pQtEL2G*ld$M_(!isTMKJ^|zYE1eo=X>X^+}N3955x!ZD4vx7C;RY&ps)FwUp;8 zc7%jnC-mKl3jiqPs|)GM+$^Z}QV+_xHT)5Ybd{ezvshSY>h#j8C}x*2o1uFCiri4q zpz3y&cydg0jFP)_ZD)R^2LTXha%py~8@K#@-|5lc3TBR1oOb?JELua^xB+j;^Nzhz zSp~gQ{~_^G#geZ`Kv7wQ@O~c6gEO6-$lGQv)vJw<^Ptobz9i<~chWn<0bbo?)QEQ& zCBn>PFtJ#Z_`H+_EBYLXGs$KejrML{vPWWEQF)EejkGjT8%t~KDrzq? zDm#HevybxS&k2g4-7rMrS%p znF-Q+PEWYvk#!8_o*{0prpWA`GB>$QtnPQT9^MgyvYfBtG20(3p-QDm+#9f#8VF<} za{yRU&50ahBY8SKnIui>!UfLG(%&G94KWX<0Y{>iD7tQq{&Md8`7n4PxrBp#>nvG> zN0Gr@xw00D7&P;*>#P>P;Li5a5FD28aT!6%YG?OPVGEqbCeD#U6Mx*GozK1 zT8uR}{cT#D=nyI0&)HTiswvsTlZ~H27j)&wY~mH~7w61EXnCJuJvrP7Kyl}LS$l4N z-#$AdR(C@9F#214HOM|YYpM5%?g32-mA{fObC!LX?L2o(TuuQclNYT@(agaJpJUIO zu-{8j(6lemqGSo^K7zg3a^=dDsA7Lt+?CY7&DivMjU4$T!p{*rCL^BNmvHmU)G=G! z=Xk8581r|nrpG8r^vp%Pg9JE(q2j*R`Ue(O;u={?>*QT@1u8uYzE^0;6EZDuO02DW z$p~e)oK)AoBSwUBdIhl}m&2oOF0?t}y#uj=Ow*?ppR+9Aq!@3JEQW*CT zyI;bS4?Q^5Mpxe*%1H$%?VGnezn533qAoI?nTFL*WWuLx)l^mImQ5(qm1&vOWd|YS1Q@=|6SOdh+BZe#{(tX9Vs@sxgFfn3yn8 zhgha3$9#wJ-YG<};P1SG82+ZjbA^U>p@gJ`FQ@^R-n=sZ2xa06rMv{M30nZz2Qlmc zy?0=QiSl!|pT48$Q*#+J149lxre83{)^@YJT#Qu-Rs&%!o&fp`qF*(GY7vPFGbKgN zX?#eN!j^JR=8~dR6MfsRCNbq@PH)hUfk;i5F;VV0Z{%f>Mbf$-sG-%@mJAkSc72(5_U}p)mUt zme60OQ8rTR5N<0nvhdMfXX|IbpYy$T`+OAyqR(g$8iGX}OMRJeOEilT%*Iyapc2dh zuZ=7lktR8qovjLdx(1AATY=B&J$Udobi;5rC{tew57x0YR(y^Pss)s@7M#u&QCV4; z*9ZQuVopRmNh7_lph?*&AW@V!t67XU5LDKe(*K{kz%Lcp_(hpOFZT0qgjcZUQq}c*N;Wh0q;<19oJKXS17$| zb7+c8K_yAsX|+4-EqcLtTf9b*bq|?SncjIf{yO+hLO#tP7TJ3^!>Obf81e{87@&7* zVZ9H6w2zG3%XEfh8@esuWtiHmRy7)Z8aavfi8tryPMJPk!UkA0rK;!Yi|&VMhL03- z61Mr;n_~}MAS%|tm~dHH#1S?}$-e@Bwtz92_DhJ@+At>H#Sa*XJ~B5zM*U`jx+*pJ zc%_+_xJ8s<=$y@B9)v#~F?={w-rs{dpXze$F;RtF(4{L@q+>v*l}<3WFq3E)_z&im zIh8#4y#4$2aXSm;%lYhJ2pJ7fil&z0wHruacH3oS;#d3Zvohl{BEq6?=<)b(SJ!AD zPk}y~M-QXCoSbmwi7i^RU@qhqz76e>`a-mwAr^jdaYM<)0N(G}hP?$jX7YzSs;Rra zzjw=CuE*wErF3q%Q!O(izpr`tt?2K&n{msfQ!Sny*ws#PHt?^A`JautJhG`tGX7IV PiC^a9tV~ZD&sq0>tR;Cr literal 0 HcmV?d00001 diff --git a/src/program/vita/fsm-protocol.svg b/src/program/vita/fsm-protocol.svg deleted file mode 100644 index 6ec7fb1bc1..0000000000 --- a/src/program/vita/fsm-protocol.svg +++ /dev/null @@ -1,516 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index ed7fc7e8a4..a994c04693 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -494,23 +494,46 @@ module vita-esp-gateway { type yang:zero-based-counter64; description "Count of key exchange negotiations expired due to exceeding - negotiation-ttl."; + negotiation-ttl."; } - leaf nonces-negotiated { + leaf challenges-offered { type yang:zero-based-counter64; description - "Count of nonce pairs that were exchanged (elevated count can - indicate a denial-of-service attack.)"; + "Count of challenges offered as the protocol responder (elevated + count can indicate a denial-of-service attempt.)"; + } + leaf challenges-accepted { + type yang:zero-based-counter64; + description + "Count of challenges accepted as the protocol initator."; + } + leaf keypairs-offered { + type yang:zero-based-counter64; + description + "Count of ephemeral key pairs that were offered as the protocol + responder."; } leaf keypairs-negotiated { type yang:zero-based-counter64; description - "Count of ephemeral key pairs that were exchanged."; + "Count of ephemeral key pairs that were negotiated as the protocol + initiator."; + } + leaf inbound-sa-expired { + type yang:zero-based-counter64; + description + "Count of inbound SAs that have expired due to sa-ttl."; + } + leaf outbound-sa-expired { + type yang:zero-based-counter64; + description + "Count of outbound SAs that have expired due to sa-ttl."; } - leaf keypairs-expired { + leaf outbound-sa-rejected { type yang:zero-based-counter64; description - "Count of ephemeral key pairs that have expired due to sa-ttl."; + "Count of outbound SAs that were rejected because they had a + non-unique SPI (indicates a misbehaving peer.)"; } } } From b08c8b62bc957ff285223669e0fb3035c6e79790 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 00:08:53 +0100 Subject: [PATCH 10/81] Revert "lib.yang.binary: caller of data_compiler_from_grammar ensures atomicity" This reverts commit d8cbcdfb9425c764e2b7336ccd88137b3f33208c. --- src/lib/ptree/action_codec.lua | 6 ++---- src/lib/yang/binary.lua | 2 ++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/ptree/action_codec.lua b/src/lib/ptree/action_codec.lua index 0cc4ed80eb..61bacb2a1a 100644 --- a/src/lib/ptree/action_codec.lua +++ b/src/lib/ptree/action_codec.lua @@ -129,15 +129,13 @@ local function encoder() end function encoder:config(class, arg) local file_name = random_file_name() - local tmp_name = file_name..".tmp" if class.yang_schema then yang.compile_config_for_schema_by_name(class.yang_schema, arg, - tmp_name) + file_name) else if arg == nil then arg = {} end - binary.compile_ad_hoc_lua_data_to_file(tmp_name, arg) + binary.compile_ad_hoc_lua_data_to_file(file_name, arg) end - assert(S.rename(tmp_name, file_name)) -- ensure atomic update self:string(file_name) end function encoder:finish() diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index 359a4ebb3f..c906fbde13 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -323,6 +323,7 @@ end function data_compiler_from_grammar(emit_data, schema_name, schema_revision) return function(data, filename, source_mtime) source_mtime = source_mtime or {sec=0, nsec=0} +-- local stream = file.tmpfile("rusr,wusr,rgrp,roth", lib.dirname(filename)) local stream = assert(file.open(filename, 'w')) local strtab = string_table_builder() local header = header_t( @@ -340,6 +341,7 @@ function data_compiler_from_grammar(emit_data, schema_name, schema_revision) assert(stream:seek('set', 0)) -- Fix up header. stream:write_struct(header_t, header) +-- stream:rename(filename) stream:close() end end From 5722fe389ac364183e92d6fcd28a84d1e9d4c557 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 00:09:07 +0100 Subject: [PATCH 11/81] Revert "lib.yang.binary: [hack/fixme] write to filename instead of renaming" This reverts commit 70be7bb96d6a03edc32782cde16bc2c8aa6b560f. --- src/lib/yang/binary.lua | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/lib/yang/binary.lua b/src/lib/yang/binary.lua index c906fbde13..4327417cd9 100644 --- a/src/lib/yang/binary.lua +++ b/src/lib/yang/binary.lua @@ -323,8 +323,7 @@ end function data_compiler_from_grammar(emit_data, schema_name, schema_revision) return function(data, filename, source_mtime) source_mtime = source_mtime or {sec=0, nsec=0} --- local stream = file.tmpfile("rusr,wusr,rgrp,roth", lib.dirname(filename)) - local stream = assert(file.open(filename, 'w')) + local stream = file.tmpfile("rusr,wusr,rgrp,roth", lib.dirname(filename)) local strtab = string_table_builder() local header = header_t( MAGIC, VERSION, source_mtime.sec, source_mtime.nsec, @@ -341,7 +340,7 @@ function data_compiler_from_grammar(emit_data, schema_name, schema_revision) assert(stream:seek('set', 0)) -- Fix up header. stream:write_struct(header_t, header) --- stream:rename(filename) + stream:rename(filename) stream:close() end end From eb020bfd3f40197680b9296acf562dec27584206 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 00:19:47 +0100 Subject: [PATCH 12/81] vita: replace broken inotify usage for SAD change detection... ...in favor of built in atomicity of configuration compilation. --- src/program/vita/vita.lua | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/program/vita/vita.lua b/src/program/vita/vita.lua index 5e4e40e21a..f25dbd07d0 100644 --- a/src/program/vita/vita.lua +++ b/src/program/vita/vita.lua @@ -157,18 +157,12 @@ function run_vita (opt) } -- Listen for SA database changes. - local notify_fd, sa_db_wd = assert(S.inotify_init("cloexec, nonblock")) + local sa_db_last_modified local function sa_db_needs_reload () - if not sa_db_wd then - sa_db_wd = notify_fd:inotify_add_watch(sa_db_path, "close_write") - -- sa_db_wd ~= nil means the SA database was newly created and we - -- should load it. - return (sa_db_wd ~= nil) - else - local events, err = notify_fd:inotify_read() - -- Any event indicates the SA database was written to and we should - -- reload it. - return not (err and assert(err.again, err)) and #events > 0 + local stat = S.stat(sa_db_path) + if stat and stat.st_mtime ~= sa_db_last_modified then + sa_db_last_modified = stat.st_mtime + return true end end From afc72af742dd4113cab3f2b9e9d80178a58b76cf Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 00:55:47 +0100 Subject: [PATCH 13/81] KeyExchange: set sa_db_commit_throttle = negotation_ttl --- src/program/vita/exchange.lua | 4 ++-- src/program/vita/test.lua | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index cc05822e7b..ee0da2d7ac 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -200,8 +200,7 @@ function KeyManager:new (conf) key_message = Protocol.key_message:new({}), challenge_message = Protocol.challenge_message:new({}), nonce_key_message = Protocol.nonce_key_message:new({}), - sa_db_updated = false, - sa_db_commit_throttle = lib.throttle(1) + sa_db_updated = false } local self = setmetatable(o, { __index = KeyManager }) self:reconfig(conf) @@ -290,6 +289,7 @@ function KeyManager:reconfig (conf) self.max_inbound_sa = conf.max_inbound_sa self.negotiation_ttl = conf.negotiation_ttl self.sa_ttl = conf.sa_ttl + self.sa_db_commit_throttle = lib.throttle(conf.negotiation_ttl) end function KeyManager:push () diff --git a/src/program/vita/test.lua b/src/program/vita/test.lua index 7bffdef583..909827a173 100644 --- a/src/program/vita/test.lua +++ b/src/program/vita/test.lua @@ -114,7 +114,6 @@ function run_softbench (pktsize, npackets, nroutes, cpuspec) }, packet_size = pktsize, nroutes = nroutes, - negotiation_ttl = nroutes, sa_ttl = 16 } @@ -197,7 +196,7 @@ defaults = { nroutes = {default=1}, packet_size = {default="IMIX"}, sa_ttl = {}, - negotiation_ttl = {default=1} + negotiation_ttl = {} } private_interface_defaults = { pci = {default="00:00.0"}, From d4d0c598476bfd412d54027e5b843f0a0b37cef4 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 15:12:37 +0100 Subject: [PATCH 14/81] KeyManager: optimize SA configuration for loopback testing scenario --- src/program/vita/exchange.lua | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index ee0da2d7ac..5fadb908a0 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -462,11 +462,18 @@ function KeyManager:handle_key_request (route, message) return false else assert(not ecode) end + -- This is an optimization for loopback testing: if we are negotiating with + -- ourselves, configure an outbound SA only (inbound SA has been configured + -- by the responder.) + local is_loopback = self.ip:src_eq(route.gw_ip4n) + counter.add(self.shm.keypairs_negotiated) - audit:log(("Completed AKE for '%s' (inbound SA %d, outbound SA %d)"): - format(route.id, inbound_sa.spi, outbound_sa.spi)) + audit:log(("Completed AKE for '%s' (inbound SA %s, outbound SA %d)"): + format(route.id, + (is_loopback and "-") or inbound_sa.spi, + outbound_sa.spi)) - self:add_inbound_sa(route, inbound_sa) + if not is_loopback then self:add_inbound_sa(route, inbound_sa) end self:add_outbound_sa(route, outbound_sa) return true @@ -500,12 +507,19 @@ function KeyManager:handle_nonce_key_request (route, message) link.transmit(self.output.output, self:request(route, response)) + -- This is an optimization for loopback testing: if we are negotiating with + -- ourselves, configure an inbound SA only (outbound SA will be configured + -- by the initiator.) + local is_loopback = self.ip:src_eq(route.gw_ip4n) + counter.add(self.shm.keypairs_offered) - audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %d)"): - format(route.id, inbound_sa.spi, outbound_sa.spi)) + audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %s)"): + format(route.id, + inbound_sa.spi, + (is_loopback and "-") or outbound_sa.spi)) self:add_inbound_sa(route, inbound_sa) - self:add_outbound_sa(route, outbound_sa) + if not is_loopback then self:add_outbound_sa(route, outbound_sa) end return true end From 2fa257042a5a31c1c42b859eb2adf3b9a3793946 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 15:13:14 +0100 Subject: [PATCH 15/81] Revert "KeyExchange: set sa_db_commit_throttle = negotation_ttl" This reverts commit afc72af742dd4113cab3f2b9e9d80178a58b76cf. --- src/program/vita/exchange.lua | 4 ++-- src/program/vita/test.lua | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 5fadb908a0..73e7231be2 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -200,7 +200,8 @@ function KeyManager:new (conf) key_message = Protocol.key_message:new({}), challenge_message = Protocol.challenge_message:new({}), nonce_key_message = Protocol.nonce_key_message:new({}), - sa_db_updated = false + sa_db_updated = false, + sa_db_commit_throttle = lib.throttle(1) } local self = setmetatable(o, { __index = KeyManager }) self:reconfig(conf) @@ -289,7 +290,6 @@ function KeyManager:reconfig (conf) self.max_inbound_sa = conf.max_inbound_sa self.negotiation_ttl = conf.negotiation_ttl self.sa_ttl = conf.sa_ttl - self.sa_db_commit_throttle = lib.throttle(conf.negotiation_ttl) end function KeyManager:push () diff --git a/src/program/vita/test.lua b/src/program/vita/test.lua index 909827a173..7bffdef583 100644 --- a/src/program/vita/test.lua +++ b/src/program/vita/test.lua @@ -114,6 +114,7 @@ function run_softbench (pktsize, npackets, nroutes, cpuspec) }, packet_size = pktsize, nroutes = nroutes, + negotiation_ttl = nroutes, sa_ttl = 16 } @@ -196,7 +197,7 @@ defaults = { nroutes = {default=1}, packet_size = {default="IMIX"}, sa_ttl = {}, - negotiation_ttl = {} + negotiation_ttl = {default=1} } private_interface_defaults = { pci = {default="00:00.0"}, From 9dfdda483201c09c3e6ab5ab1e0fdc0005e62903 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 15:14:32 +0100 Subject: [PATCH 16/81] KeyManager: minor code reorganization to reflect temporal ordering --- src/program/vita/exchange.lua | 72 +++++++++++++++++------------------ 1 file changed, 36 insertions(+), 36 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 73e7231be2..af10dce926 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -442,13 +442,14 @@ function KeyManager:handle_challenge_request (route, message) return true end -function KeyManager:handle_key_request (route, message) - if not route or message ~= self.key_message then return end +function KeyManager:handle_nonce_key_request (route, message) + if not route or message ~= self.nonce_key_message then return end - -- Receive an authenticated key message if the protocol fsm permits - -- (accept_key -> initiator), reject the message and return otherwise or - -- if... - local ecode, inbound_sa, outbound_sa = route.initiator:receive_key(message) + -- Receive an authenticated, combined nonce and key message if the protocol + -- fsm permits (responder -> offer_key), reject the message and return + -- otherwise or if... + local ecode, inbound_sa, outbound_sa = + route.responder:receive_nonce_key(message) if ecode == Protocol.code.protocol then counter.add(self.shm.protocol_errors) return false @@ -462,31 +463,37 @@ function KeyManager:handle_key_request (route, message) return false else assert(not ecode) end + -- If we got here we should respond with our own public key + -- (offer_key -> responder). + local ecode, response = route.responder:offer_key(self.key_message) + assert(not ecode) + + link.transmit(self.output.output, self:request(route, response)) + -- This is an optimization for loopback testing: if we are negotiating with - -- ourselves, configure an outbound SA only (inbound SA has been configured - -- by the responder.) + -- ourselves, configure an inbound SA only (outbound SA will be configured + -- by the initiator.) local is_loopback = self.ip:src_eq(route.gw_ip4n) - counter.add(self.shm.keypairs_negotiated) - audit:log(("Completed AKE for '%s' (inbound SA %s, outbound SA %d)"): + counter.add(self.shm.keypairs_offered) + audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %s)"): format(route.id, - (is_loopback and "-") or inbound_sa.spi, - outbound_sa.spi)) + inbound_sa.spi, + (is_loopback and "-") or outbound_sa.spi)) - if not is_loopback then self:add_inbound_sa(route, inbound_sa) end - self:add_outbound_sa(route, outbound_sa) + self:add_inbound_sa(route, inbound_sa) + if not is_loopback then self:add_outbound_sa(route, outbound_sa) end return true end -function KeyManager:handle_nonce_key_request (route, message) - if not route or message ~= self.nonce_key_message then return end +function KeyManager:handle_key_request (route, message) + if not route or message ~= self.key_message then return end - -- Receive an authenticated, combined nonce and key message if the protocol - -- fsm permits (responder -> offer_key), reject the message and return - -- otherwise or if... - local ecode, inbound_sa, outbound_sa = - route.responder:receive_nonce_key(message) + -- Receive an authenticated key message if the protocol fsm permits + -- (accept_key -> initiator), reject the message and return otherwise or + -- if... + local ecode, inbound_sa, outbound_sa = route.initiator:receive_key(message) if ecode == Protocol.code.protocol then counter.add(self.shm.protocol_errors) return false @@ -500,26 +507,19 @@ function KeyManager:handle_nonce_key_request (route, message) return false else assert(not ecode) end - -- If we got here we should respond with our own public key - -- (offer_key -> responder). - local ecode, response = route.responder:offer_key(self.key_message) - assert(not ecode) - - link.transmit(self.output.output, self:request(route, response)) - -- This is an optimization for loopback testing: if we are negotiating with - -- ourselves, configure an inbound SA only (outbound SA will be configured - -- by the initiator.) + -- ourselves, configure an outbound SA only (inbound SA has been configured + -- by the responder.) local is_loopback = self.ip:src_eq(route.gw_ip4n) - counter.add(self.shm.keypairs_offered) - audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %s)"): + counter.add(self.shm.keypairs_negotiated) + audit:log(("Completed AKE for '%s' (inbound SA %s, outbound SA %d)"): format(route.id, - inbound_sa.spi, - (is_loopback and "-") or outbound_sa.spi)) + (is_loopback and "-") or inbound_sa.spi, + outbound_sa.spi)) - self:add_inbound_sa(route, inbound_sa) - if not is_loopback then self:add_outbound_sa(route, outbound_sa) end + if not is_loopback then self:add_inbound_sa(route, inbound_sa) end + self:add_outbound_sa(route, outbound_sa) return true end From 39ce848989fe7a1aa87c9e2e691e5792c094b7dc Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 1 Dec 2018 15:24:06 +0100 Subject: [PATCH 17/81] KeyManager:add_outbound_sa does not update SAD --- src/program/vita/exchange.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index af10dce926..57371a0e22 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -580,7 +580,6 @@ function KeyManager:add_outbound_sa (route, sa) if #route.outbound_sa_queue > self.num_outbound_sa then table.remove(route.outbound_sa_queue, 1) end - self.sa_db_updated = true end function KeyManager:request (route, message) From 099cf5363d8b179988d07741082947da68146267 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 7 Dec 2018 13:18:06 +0100 Subject: [PATCH 18/81] core.worker: set SNABB_WORKER_NAME in worker environment --- src/core/worker.lua | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 2450abf7ab..4c9f13ed1d 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -41,10 +41,13 @@ function start (name, luacode) shm.alias("group", "/"..S.getppid().."/group") -- Save the code we want to run in the environment. S.setenv("SNABB_PROGRAM_LUACODE", luacode, true) + -- Save this worker's qualified name in the environment. + local parent = engine.program_name or S.getppid() + S.setenv("SNABB_WORKER_NAME", parent.."/"..name, true) -- Restart the process with execve(). -- /proc/$$/exe is a link to the same Snabb executable that we are running local filename = ("/proc/%d/exe"):format(S.getpid()) - local argv = { ("[snabb worker '%s' for %d]"):format(name, S.getppid()) } + local argv = { ("[snabb worker '%s' for %s]"):format(name, parent) } lib.execv(filename, argv) else -- Parent process From 44e81bfc2d0bff578b15994a460d49f53393894b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 7 Dec 2018 13:19:56 +0100 Subject: [PATCH 19/81] lib.ptree.worker: claim SNABB_WORKER_NAME --- src/lib/ptree/worker.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/ptree/worker.lua b/src/lib/ptree/worker.lua index b779cae0b0..c38cbc7727 100644 --- a/src/lib/ptree/worker.lua +++ b/src/lib/ptree/worker.lua @@ -108,6 +108,7 @@ function Worker:main () end function main (opts) + engine.claim_name(os.getenv("SNABB_WORKER_NAME")) return new_worker(opts):main() end From 482c3a510ba3d650cb32fa7482418f92ac1da9df Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 5 Sep 2018 12:39:51 +0200 Subject: [PATCH 20/81] Work-in-progress implementation of Poptrie --- src/lib/poptrie.lua | 415 ++++++++++++++++++++++++++++++++++++ src/lib/poptrie_lookup.dasl | 127 +++++++++++ 2 files changed, 542 insertions(+) create mode 100644 src/lib/poptrie.lua create mode 100644 src/lib/poptrie_lookup.dasl diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua new file mode 100644 index 0000000000..4c5083caa4 --- /dev/null +++ b/src/lib/poptrie.lua @@ -0,0 +1,415 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(...,package.seeall) + +local debug = false + +-- Poptrie, see +-- http://conferences.sigcomm.org/sigcomm/2015/pdf/papers/p57.pdf + +local ffi = require("ffi") +local bit = require("bit") +local band, bor, lshift, rshift, bnot = + bit.band, bit.bor, bit.lshift, bit.rshift, bit.bnot + +local poptrie_lookup = require("lib.poptrie_lookup") + +local Poptrie = { + leaf_compression = true, + k = 6, + leaf_t = ffi.typeof("uint16_t"), + vector_t = ffi.typeof("uint64_t"), + base_t = ffi.typeof("uint32_t") +} +Poptrie.node_t = ffi.typeof([[struct { + $ leafvec, vector; + $ base0, base1; +} __attribute__((packed))]], Poptrie.vector_t, Poptrie.base_t) + +local function array (t, n) + return ffi.new(ffi.typeof("$[?]", t), n) +end + +function new (init) + local num_default = 4 + local pt = { + nodes = init.nodes or array(Poptrie.node_t, num_default), + num_nodes = (init.nodes and assert(init.num_nodes)) or num_default, + leaves = init.leaves or array(Poptrie.leaf_t, num_default), + num_leaves = (init.leaves and assert(init.num_leaves)) or num_default + } + return setmetatable(pt, {__index=Poptrie}) +end + +function Poptrie:grow_nodes () + self.num_nodes = self.num_nodes * 2 + local new_nodes = array(Poptrie.node_t, self.num_nodes) + ffi.copy(new_nodes, self.nodes, ffi.sizeof(self.nodes)) + self.nodes = new_nodes +end + +function Poptrie:grow_leaves () + self.num_leaves = self.num_leaves * 2 + local new_leaves = array(Poptrie.leaf_t, self.num_leaves) + ffi.copy(new_leaves, self.leaves, ffi.sizeof(self.leaves)) + self.leaves = new_leaves +end + +-- XXX - Generalize for key=uint8_t[?] +local function extract (key, offset, length) + return band(rshift(key+0ULL, offset), lshift(1ULL, length) - 1) +end + +-- Add key/value pair to RIB (intermediary binary trie) +-- key=uint64_t, length=uint16_t, value=uint16_t +function Poptrie:add (key, length, value) + assert(value) + local function add (node, offset) + if offset == length then + node.value = value + elseif extract(key, offset, 1) == 0 then + node.left = add(node.left or {}, offset + 1) + elseif extract(key, offset, 1) == 1 then + node.right = add(node.right or {}, offset + 1) + else error("invalid state") end + return node + end + self.rib = add(self.rib or {}, 0) +end + +-- Longest prefix match on RIB +function Poptrie:rib_lookup (key, length, root) + local function lookup (node, offset, value) + value = node.value or value + if offset == length then + return {value=value, left=node.left, right=node.right} + elseif extract(key, offset, 1) == 0 and node.left then + return lookup(node.left, offset + 1, value) + elseif extract(key, offset, 1) == 1 and node.right then + return lookup(node.right, offset + 1, value) + else + -- No match: return longest prefix key value, but no child nodes. + return {value=value} + end + end + return lookup(root or self.rib, 0) +end + +-- Compress RIB into Poptrie +function Poptrie:build (rib, node_index, leaf_base, node_base) + local function allocate_leaf () + while leaf_base >= self.num_leaves do + self:grow_leaves() + end + leaf_base = leaf_base + 1 + return leaf_base - 1 + end + local function allocate_node () + while node_base >= self.num_nodes do + self:grow_nodes() + end + node_base = node_base + 1 + return node_base - 1 + end + local function node () + return self.nodes[node_index] + end + -- When called without arguments, create the root node. + rib = rib or self.rib + leaf_base = leaf_base or 0 + node_base = node_base or 0 + node_index = node_index or allocate_node() + -- Initialize node base pointers. + node().base0 = leaf_base + node().base1 = node_base + -- Compute children + local children = {} + for index = 0, 2^Poptrie.k - 1 do + children[index] = self:rib_lookup(index, Poptrie.k, rib) + end + -- Allocate and initialize node.leafvec and leaves. + local last_leaf_value = nil + for index = 0, 2^Poptrie.k - 1 do + local child = children[index] + if not (child.left or child.right) then + local value = child.value or 0 + if value ~= last_leaf_value then -- always true when leaf_compression=false + if Poptrie.leaf_compression then + node().leafvec = bor(node().leafvec, lshift(1ULL, index)) + last_leaf_value = value + end + local leaf_index = allocate_leaf() + self.leaves[leaf_index] = value + end + end + end + -- Allocate child nodes (this has to be done before recursing into build() + -- because their indices into the nodes array need to be node.base1 + index, + -- and build() will advance the node_base.) + local child_nodes = {} + for index = 0, 2^Poptrie.k - 1 do + local child = children[index] + if child.left or child.right then + child_nodes[index] = allocate_node() + end + end + -- Initialize node.vector and child nodes. + for index = 0, 2^Poptrie.k - 1 do + local child = children[index] + if child.left or child.right then + node().vector = bor(node().vector, lshift(1ULL, index)) + leaf_base, node_base = + self:build(child, child_nodes[index], leaf_base, node_base) + end + end + -- Return new leaf_base and node_base indices. + return leaf_base, node_base +end + +-- http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetNaive +local function popcnt (v) -- XXX - popcaan is 64-bit only + local c = 0 + while v > 0 do + c = c + band(v, 1ULL) + v = rshift(v, 1ULL) + end + return c +end + +-- [Algorithm 1] lookup(t = (N , L), key); the lookup procedure for the address +-- key in the tree t (when k = 6). The function extract(key, off, len) extracts +-- bits of length len, starting with the offset off, from the address key. +-- N and L represent arrays of internal nodes and leaves, respectively. +-- << denotes the shift instruction of bits. Numerical literals with the UL and +-- ULL suffixes denote 32-bit and 64-bit unsigned integers, respectively. +-- Vector and base are the variables to hold the contents of the node’s fields. +-- +-- if [direct_pointing] then +-- index = extract(key, 0, t.s); +-- dindex = t.D[index].direct index; +-- if (dindex & (1UL << 31)) then +-- return dindex & ((1UL << 31) - 1); +-- end if +-- index = dindex; +-- offset = t.s; +-- else +-- index = 0; +-- offset = 0; +-- end if +-- vector = t.N [index].vector; +-- v = extract(key, offset, 6); +-- while (vector & (1ULL << v)) do +-- base = t.N [index].base1; +-- bc = popcnt(vector & ((2ULL << v) - 1)); +-- index = base + bc - 1; +-- vector = t.N [index].vector; +-- offset += 6; +-- v = extract(key, offset, 6); +-- end while +-- base = t.N [index].base0; +-- if [leaf_compression] then +-- bc = popcnt(t.N [index].leafvec & ((2ULL << v) - 1)); +-- else +-- bc = popcnt((∼t.N [index].vector) & ((2ULL << v) - 1)); +-- end if +-- return t.L[base + bc - 1]; +-- +function Poptrie:lookup (key) + local N, L = self.nodes, self.leaves + local index = 0 + local node = N[index] + local offset = 0 + local v = extract(key, offset, Poptrie.k) + if debug then print(index, bin(node.vector), bin(v)) end + while band(node.vector, lshift(1ULL, v)) ~= 0 do + local base = N[index].base1 + local bc = popcnt(band(node.vector, lshift(2ULL, v) - 1)) + index = base + bc - 1 + node = N[index] + offset = offset + Poptrie.k + v = extract(key, offset, Poptrie.k) + if debug then print(index, bin(node.vector), bin(v)) end + end + if debug then print(node.base0, bin(node.leafvec), bin(v)) end + local base = node.base0 + local bc + if Poptrie.leaf_compression then + bc = popcnt(band(node.leafvec, lshift(2ULL, v) - 1)) + else + bc = popcnt(band(bnot(node.vector), lshift(2ULL, v) - 1)) + end + if debug then print(base + bc - 1) end + return L[base + bc - 1] +end + +Poptrie.asm_lookup64 = poptrie_lookup.generate(Poptrie, 64) +function Poptrie:lookup64 (key) + return Poptrie.asm_lookup64(self.leaves, self.nodes, key) +end + +function selftest () + local t = new{} + -- Test RIB + t:add(0x00, 8, 1) -- 00000000 + t:add(0x0F, 8, 2) -- 00001111 + t:add(0x07, 4, 3) -- 0111 + t:add(0xFF, 8, 4) -- 11111111 + t:add(0xFF, 5, 5) -- 11111 + local n = t:rib_lookup(0x0, 1) + assert(not n.value and n.left and not n.right) + local n = t:rib_lookup(0x00, 8) + assert(n.value == 1 and not (n.left or n.right)) + local n = t:rib_lookup(0x07, 3) + assert(not n.value and (n.left and n.right)) + local n = t:rib_lookup(0x0, 1, n) + assert(n.value == 3 and not (n.left or n.right)) + local n = t:rib_lookup(0xFF, 5) + assert(n.value == 5 and (not n.left) and n.right) + local n = t:rib_lookup(0x0F, 3, n) + assert(n.value == 4 and not (n.left or n.right)) + local n = t:rib_lookup(0x3F, 8) + assert(n.value == 5 and not (n.left or n.right)) + -- Test FIB + local leaf_base, node_base = t:build() + if debug then + for i=0, node_base-1 do + print("node:", i) + print(t.nodes[i].base0, bin(t.nodes[i].leafvec)) + print(t.nodes[i].base1, bin(t.nodes[i].vector)) + end + for i=0, leaf_base-1 do + print("leaf:", i, t.leaves[i]) + end + end + assert(t:lookup(0x00) == 1) -- 00000000 + assert(t:lookup(0x03) == 0) -- 00000011 + assert(t:lookup(0x07) == 3) -- 00000111 + assert(t:lookup(0x0F) == 2) -- 00001111 + assert(t:lookup(0x1F) == 5) -- 00011111 + assert(t:lookup(0x3F) == 5) -- 00111111 + assert(t:lookup(0xFF) == 4) -- 11111111 + assert(t:lookup64(0x00) == 1) + assert(t:lookup64(0x03) == 0) + assert(t:lookup64(0x07) == 3) + assert(t:lookup64(0x0F) == 2) + assert(t:lookup64(0x1F) == 5) + assert(t:lookup64(0x3F) == 5) + assert(t:lookup64(0xFF) == 4) + + -- Random testing + local function reproduce (cases) + debug = true + print("repoducing...") + local t = new{} + for entry, case in ipairs(cases) do + print("key:", entry, bin(case[1])) + print("prefix:", entry, bin(case[1], case[2])) + t:add(case[1], case[2], entry) + end + local leaf_base, node_base = t:build() + for i=0, node_base-1 do + print("node:", i) + print(t.nodes[i].base0, bin(t.nodes[i].leafvec)) + print(t.nodes[i].base1, bin(t.nodes[i].vector)) + end + for i=0, leaf_base-1 do + if t.leaves[i] > 0 then print("leaf:", i, t.leaves[i]) end + end + for _, case in ipairs(cases) do + print("rib:", t:rib_lookup(case[1]).value) + print("fib:", t:lookup(case[1])) + print("64:", t:lookup64(case[1])) + end + end + local function r_assert (condition, cases) + if condition then return end + reproduce(cases) + print("selftest failed") + main.exit(1) + end + local lib = require("core.lib") + local seed = lib.getenv("SNABB_RANDOM_SEED") or 0 + for keysize = 1, 64 do + print("keysize:", keysize) + -- ramp up the geometry below to crank up test coverage + for entries = 1, 8 do + for i = 1, 4 do + math.randomseed(seed+i) + cases = {} + local t = new{} + local k = {} + for entry= 1, entries do + local a, l = math.random(2^keysize - 1), math.random(keysize) + cases[entry] = {a, l} + t:add(a, l, entry) + k[entry] = a + end + local v = {} + for entry, a in ipairs(k) do + v[entry] = t:rib_lookup(a, keysize).value + r_assert(v[entry] > 0, cases) + end + t:build() + for entry, a in ipairs(k) do + r_assert(t:lookup(a) == v[entry], cases) + r_assert(t:lookup64(a) == v[entry], cases) + end + end + end + end + + -- PMU analysis + local pmu = require("lib.pmu") + local function measure (description, f, iterations) + local set = pmu.new_counter_set() + pmu.switch_to(set) + f(iterations) + pmu.switch_to(nil) + local tab = pmu.to_table(set) + print(("%s: %.2f cycles/lookup %.2f instructions/lookup") + :format(description, + tab.cycles / iterations, + tab.instructions / iterations)) + end + if pmu.is_available() then + local t = new{} + local k = {} + local numentries = 10000 + local keysize = 64 + for entry = 1, numentries do + local a, l = math.random(2^keysize - 1), math.random(keysize) + t:add(a, l, entry) + k[entry] = a + end + t:build() + print("PMU analysis (numentries="..numentries..", keysize="..keysize..")") + pmu.setup() + measure("lookup", + function (iter) + for i=1,iter do t:lookup(k[i%#k+1]) end + end, + 1e5) + measure("lookup64", + function (iter) + for i=1,iter do t:lookup64(k[i%#k+1]) end + end, + 1e7) + else + print("No PMU available.") + end +end + +-- debugging utils +function bin (number, length) + local digits = {"0", "1"} + local s = "" + local i = 0 + repeat + local remainder = number % 2 + s = digits[tonumber(remainder+1)]..s + number = (number - remainder) / 2 + i = i + 1 + if i % Poptrie.k == 0 then s = " "..s end + until number == 0 or (i == length) + return s +end diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl new file mode 100644 index 0000000000..27ae3288d3 --- /dev/null +++ b/src/lib/poptrie_lookup.dasl @@ -0,0 +1,127 @@ +-- Use of this source code is governed by the Apache 2.0 license; see COPYING. + +module(..., package.seeall) + +local debug = false + +local ffi = require("ffi") +local dasm = require("dasm") + +|.arch x64 +|.actionlist actions +|.globalnames globalnames + +-- Table keeping machine code alive to the GC. +local anchor = {} + +-- Assemble a lookup routine +function generate (Poptrie, keysize) + local offsets = + -- Assert assumptions about lib.poptrie + assert(Poptrie.k == 6) + assert(ffi.sizeof(Poptrie.leaf_t) == 2) + assert(ffi.sizeof(Poptrie.vector_t) == 8) + assert(ffi.sizeof(Poptrie.base_t) == 4) + assert(ffi.offsetof(Poptrie.node_t, 'leafvec') == 0) + assert(ffi.offsetof(Poptrie.node_t, 'vector') == 8) + assert(ffi.offsetof(Poptrie.node_t, 'base0') == 16) + assert(ffi.offsetof(Poptrie.node_t, 'base1') == 20) + + local name = "poptrie_lookup(k="..Poptrie.k..", keysize="..keysize..")" + + local Dst = dasm.new(actions) + lookup(Dst, Poptrie, keysize) + local mcode, size = Dst:build() + table.insert(anchor, mcode) + + if debug then + print("mcode dump: "..name) + dasm.dump(mcode, size) + end + + local prototype + if keysize <= 64 then + prototype = ffi.typeof("$ (*) ($ *, $ *, uint64_t)", + Poptrie.leaf_t, Poptrie.leaf_t, Poptrie.node_t) + else error("NYI") end + + return ffi.cast(prototype, mcode) +end + +|.define leaves, rdi -- pointer to leaves array +|.define nodes, rsi -- pointer to nodes array +|.define key, rdx -- key to look up +|.define index, r8d -- index into node array +|.define node, r8 -- pointer into node array +|.define offset, r9 -- offset into key +|.define v, r10 -- k bits extracted from key +|.define vec, r11 -- 64-bit vector or leafvec + +-- lookup(leaf_t *leaves, node_t *nodes, key) -> leaf_t +function lookup (Dst, Poptrie, keysize) + -- index, node, offset = 0, nodes[index], 0 + | mov index, 0 + | lea node, [nodes+0] -- nodes[0] + | mov offset, 0 + -- while band(vec, lshift(1ULL, v)) ~= 0 + |1: + -- v = extract(key, offset, k=6) + if keysize <= 64 then + -- v = rshift(key, offset) + | mov v, key + | mov rcx, offset + | shr v, cl + -- v = band(v, lshift(1, k=6) - 1) + | and v, 63 + else error("NYI") end + -- vec = nodes[index].vector + | mov vec, qword [node+8] + -- rax = lshift(1ULL, v) + | mov rax, 1 + | mov rcx, v + | shl rax, cl + -- rax = band(vec, rax) + | and rax, vec + -- is bit v set in vec? + | cmp rax, 0 + | je >2 -- reached leaf, exit loop + -- rax = lshift(2ULL, v) - 1 + | mov rax, 2 + | mov rcx, v + | shl rax, cl + | sub rax, 1 + -- rax = popcnt(band(vec, rax)) + | and rax, vec + | popcnt rax, rax + -- index = base + bc - 1 + | mov index, dword [node+20] -- nodes[index].base1 + | add index, eax + | sub index, 1 + -- node = nodes[index] + | imul index, 24 -- multiply by node size + | lea node, [nodes+index] + -- offset = offset + k + | add offset, 6 + | jmp <1 -- loop + -- end while + |2: + -- rax = lshift(2ULL, v) - 1 + | mov rax, 2 + | mov rcx, v + | shl rax, cl + | sub rax, 1 + if Poptrie.leaf_compression then + -- vec = nodes[index].leafvec + | mov vec, qword [node+0] + else error("NYI") end + -- rax = popcnt(band(vec, rax)) - 1 + | and rax, vec + | popcnt rax, rax + -- return leaves[base + bc - 1] + | mov index, dword [node+16] -- nodes[index].base0 + | add index, eax + | sub index, 1 + -- assumption: at most ax is set at this point because popcnt(qword) < byte + | mov ax, word [leaves+index*2] -- leaves[index] + | ret +end From 3234b1fef37838d7482f70639281bf14c47de7fb Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 14 Dec 2018 14:10:44 +0100 Subject: [PATCH 21/81] Add basic documentation for lib.poptrie --- src/doc/genbook.sh | 2 ++ src/lib/README.poptrie.md | 70 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 src/lib/README.poptrie.md diff --git a/src/doc/genbook.sh b/src/doc/genbook.sh index c8182f6a84..79274460df 100755 --- a/src/doc/genbook.sh +++ b/src/doc/genbook.sh @@ -84,6 +84,8 @@ $(cat $mdroot/lib/README.checksum.md) $(cat $mdroot/lib/README.ctable.md) +$(cat $mdroot/lib/README.poptrie.md) + $(cat $mdroot/lib/README.pmu.md) $(cat $mdroot/lib/yang/README.md) diff --git a/src/lib/README.poptrie.md b/src/lib/README.poptrie.md new file mode 100644 index 0000000000..42df1272b6 --- /dev/null +++ b/src/lib/README.poptrie.md @@ -0,0 +1,70 @@ +### Poptrie (lib.poptrie) + +An implementation of +[Poptrie](http://conferences.sigcomm.org/sigcomm/2015/pdf/papers/p57.pdf). +Includes high-level functions for building the Poptrie data structure, as well +as a hand-written, optimized assembler lookup routine. + +#### Example usage + +```lua +local pt = poptrie.new{} +-- Associate prefixes of length to values (uint16_t) +pt:add(0x00FF, 8, 1) +pt:add(0x000F, 4, 2) +pt:build() +pt:lookup64(0x001F) ⇒ 2 +pt:lookup64(0x10FF) ⇒ 1 +-- The value zero denotes "no match" +pt:lookup64(0x0000) ⇒ 0 +-- You can create a pre-built poptrie from nodes and leaves arrays. +local pt2 = poptrie.new{nodes=pt.nodes, leaves=pt.leaves} +``` + +#### Known bugs and limitations + + - Only supports keys up to 64 bits wide + - *Direct pointing* is not yet implemented + +#### Performance + +``` +PMU analysis (numentries=10000, keysize=64) +lookup: 30001.41 cycles/lookup 59357.23 instructions/lookup +lookup64: 179.39 cycles/lookup 205.72 instructions/lookup +``` + +#### Interface + +— Function **new** *init* + +Creates and returns a new `Poptrie` object. + +*Init* is a table with the following keys: + +* `leaves` - *Optional*. An array of leaves. When *leaves* is supplied *nodes* + must be supplied as well. +* `nodes` - *Optional*. An array of nodes. When *nodes* is supplied *leaves* + must be supplied as well. + +— Method **Poptrie:add** *prefix* *length* *value* + +Associates *value* to *prefix* of *length*. *Prefix* must be an unsigned +integer (little-endian) of up to 64 bits. *Length* must be an an unsigned +integer between 1 and 64. *Value* must be a 16‑bit unsigned integer, and should +be greater than zero (see `lookup64` as to why.) + +— Method **Poptrie:build** + +Compiles the optimized poptrie data structure used by `lookup64`. After calling +this method, the *leaves* and *nodes* fields of the `Poptrie` object will +contain the leaves and nodes arrays respectively. These arrays can be used to +construct a `Poptrie` object. + +— Method **Poptrie:lookup64** *key* + +Looks up *key* in the `Poptrie` object and returns the associated value or +zero. *Key* must be an unsigned, little-endian integer of up to 64 bits. + +Unless the `Poptrie` object was initialized with leaves and nodes arrays, the +user must call `Poptrie:build` before calling `Poptrie:lookup64`. From 498b38891dc05bce4db70676afce4c135ca13d86 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 14 Dec 2018 15:32:34 +0100 Subject: [PATCH 22/81] lib.protocol.ipv4: add ipv4:pton_cidr --- src/lib/protocol/README.md | 6 ++++++ src/lib/protocol/ipv4.lua | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/src/lib/protocol/README.md b/src/lib/protocol/README.md index 9e008af461..a9d7f3e786 100644 --- a/src/lib/protocol/README.md +++ b/src/lib/protocol/README.md @@ -213,6 +213,12 @@ Returns the binary representation of IPv4 address denoted by *string*. Returns the string representation of *ip* address. +— Function **ipv4:pton** *string* + +Returns the binary representation of the IPv4 address prefix and prefix length +encoded denoted by *string* of the form `/`. +See [CIDR notation](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing). + ### IPv6 (lib.protocol.ipv6) diff --git a/src/lib/protocol/ipv4.lua b/src/lib/protocol/ipv4.lua index 0f050ff540..9bb19515ee 100644 --- a/src/lib/protocol/ipv4.lua +++ b/src/lib/protocol/ipv4.lua @@ -91,6 +91,13 @@ function ipv4:ntop (n) return ffi.string(c_str) end +function ipv4:pton_cidr (p) + local prefix, length = p:match("([^/]*)/([0-9]*)") + return + ipv4:pton(prefix), + assert(tonumber(length), "Invalid length "..length) +end + function ipv4:set(addr) return ipv4:pton(addr) end From a9ba0d49bf2979cd287a0526b65b4b9b24892926 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 14 Dec 2018 15:33:07 +0100 Subject: [PATCH 23/81] vita/route: use lib.poptrie instead of lib.lpm and lib.ctable --- src/Makefile.vita | 5 +-- src/program/vita/route.lua | 63 +++++++++++--------------------------- 2 files changed, 21 insertions(+), 47 deletions(-) diff --git a/src/Makefile.vita b/src/Makefile.vita index ee0fe319d1..ef92c96835 100644 --- a/src/Makefile.vita +++ b/src/Makefile.vita @@ -2,9 +2,10 @@ INCLUDE = *.* core arch jit syscall pf \ lib/token_bucket.* lib/tsc.* \ lib/lua lib/protocol lib/checksum.* lib/ipsec \ lib/yang lib/stream.* lib/stream lib/buffer.* \ - lib/xsd_regexp.* lib/maxpc.* lib/ctable.* lib/binary_search.* \ + lib/xsd_regexp.* lib/maxpc.* lib/ctable.* lib/cltable.* \ + lib/binary_search.* lib/multi_copy.* lib/hash \ lib/ptree lib/rrd.* lib/fibers \ - lib/multi_copy.* lib/hash lib/cltable.* lib/lpm lib/interlink.* \ + lib/poptrie* lib/interlink.* \ lib/hardware lib/macaddress.* lib/numa.* lib/cpuset.* \ lib/scheduling.* lib/timers \ apps/interlink apps/intel_mp \ diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index 7a6909fb27..dd05bdc496 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -6,8 +6,7 @@ local counter = require("core.counter") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") local esp = require("lib.protocol.esp") -local lpm = require("lib.lpm.lpm4_248").LPM4_248 -local ctable = require("lib.ctable") +local poptrie = require("lib.poptrie") local ffi = require("ffi") @@ -27,29 +26,21 @@ PrivateRouter = { } function PrivateRouter:new (conf) - local keybits = 15 -- see lib/lpm/README.md local o = { ports = {}, routes = {}, mtu = conf.mtu, ip4 = ipv4:new({}), - routing_table4 = lpm:new({keybits=keybits}) + routing_table4 = poptrie.new{} } for id, route in pairs(conf.routes) do local index = #o.ports+1 - assert(index < 2^keybits, "index overflow") - o.routing_table4:add_string( - assert(route.net_cidr4, "Missing net_cidr4"), - index - ) + assert(ffi.cast("uint16_t", index) == index, "index overflow") + assert(route.net_cidr4, "Missing net_cidr4") + local prefix, length = ipv4:pton_cidr(route.net_cidr4) + o.routing_table4:add(ffi.cast("uint32_t *", prefix)[0], length, index) o.ports[index] = id end - -- NB: need to add default LPM entry until #1238 is fixed, see - -- https://github.com/snabbco/snabb/issues/1238#issuecomment-345362030 - -- Zero maps to nil in o.routes (which is indexed starting at one), hence - -- packets that match the default entry will be dropped (and route_errors - -- incremented.) - o.routing_table4:add_string("0.0.0.0/0", 0) o.routing_table4:build() return setmetatable(o, {__index = PrivateRouter}) end @@ -61,7 +52,8 @@ function PrivateRouter:link () end function PrivateRouter:find_route4 (dst) - return self.routes[self.routing_table4:search_bytes(dst)] + local address = ffi.cast("uint32_t *", dst)[0] + return self.routes[self.routing_table4:lookup64(address)] end function PrivateRouter:route (p) @@ -110,37 +102,19 @@ PublicRouter = { function PublicRouter:new (conf) local o = { - routing_table4 = ctable.new{ - key_type = ffi.typeof("uint32_t"), - value_type = ffi.typeof("uint32_t") - }, + ports = {}, + routes = {}, + routing_table4 = poptrie.new{}, esp = esp:new({}) } - self.build_fib(o, conf) - return setmetatable(o, {__index = PublicRouter}) -end - -function PublicRouter:reconfig (conf) - self:build_fib(conf) - self:link() -- links might have changed before reconfig -end - -function PublicRouter:build_fib (conf) - self.ports = {} - self.routes = {} - -- Update FIB entries for SAs for spi, sa in pairs(conf.sa) do - local index = #self.ports+1 - assert(ffi.cast("uint32_t", index) == index, "index overflow") - self.routing_table4:add(spi, index, 'update') - self.ports[index] = sa.route.."_"..spi - end - -- Remove obsolete FIB entries - for entry in self.routing_table4:iterate() do - if not conf.sa[tonumber(entry.key)] then - self.routing_table4:remove_ptr(entry) - end + local index = #o.ports+1 + assert(ffi.cast("uint16_t", index) == index, "index overflow") + o.routing_table4:add(spi, 32, index) + o.ports[index] = sa.route.."_"..spi end + o.routing_table4:build() + return setmetatable(o, {__index = PublicRouter}) end function PublicRouter:link () @@ -150,8 +124,7 @@ function PublicRouter:link () end function PublicRouter:find_route4 (spi) - local entry = self.routing_table4:lookup_ptr(spi) - return entry and self.routes[entry.value] + return self.routes[self.routing_table4:lookup64(spi)] end function PublicRouter:push () From 9343b36ab576c1c20cc13e364b61300387f4750b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 14 Dec 2018 15:34:24 +0100 Subject: [PATCH 24/81] lib.poptrie: fix building empty RIB and repeated calls to build() --- src/lib/poptrie.lua | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua index 4c5083caa4..8e5abbf933 100644 --- a/src/lib/poptrie.lua +++ b/src/lib/poptrie.lua @@ -92,7 +92,7 @@ function Poptrie:rib_lookup (key, length, root) return {value=value} end end - return lookup(root or self.rib, 0) + return lookup(root or self.rib or {}, 0) end -- Compress RIB into Poptrie @@ -115,6 +115,11 @@ function Poptrie:build (rib, node_index, leaf_base, node_base) return self.nodes[node_index] end -- When called without arguments, create the root node. + if not rib then + -- Clear previous FIB + ffi.fill(self.leaves, ffi.sizeof(self.leaves)) + ffi.fill(self.nodes, ffi.sizeof(self.nodes)) + end rib = rib or self.rib leaf_base = leaf_base or 0 node_base = node_base or 0 @@ -249,6 +254,8 @@ end function selftest () local t = new{} + -- Tets building empty RIB + t:build() -- Test RIB t:add(0x00, 8, 1) -- 00000000 t:add(0x0F, 8, 2) -- 00001111 From 4e48a25f61c3e4c1db967828d4064316b5ba1e4b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 14 Dec 2018 18:49:14 +0100 Subject: [PATCH 25/81] lib.poptrie: refactor :build() into smaller methods... ...to make implementing direct pointing for straight forward. --- src/lib/poptrie.lua | 114 ++++++++++++++++++++++---------------------- 1 file changed, 57 insertions(+), 57 deletions(-) diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua index 8e5abbf933..86d9812cb5 100644 --- a/src/lib/poptrie.lua +++ b/src/lib/poptrie.lua @@ -95,38 +95,36 @@ function Poptrie:rib_lookup (key, length, root) return lookup(root or self.rib or {}, 0) end --- Compress RIB into Poptrie -function Poptrie:build (rib, node_index, leaf_base, node_base) - local function allocate_leaf () - while leaf_base >= self.num_leaves do - self:grow_leaves() - end - leaf_base = leaf_base + 1 - return leaf_base - 1 - end - local function allocate_node () - while node_base >= self.num_nodes do - self:grow_nodes() - end - node_base = node_base + 1 - return node_base - 1 - end - local function node () - return self.nodes[node_index] +function Poptrie:clear_fib () + self.leaf_base, self.node_base = 0, 0 + ffi.fill(self.leaves, ffi.sizeof(self.leaves), 0) + ffi.fill(self.nodes, ffi.sizeof(self.nodes), 0) +end + +function Poptrie:allocate_leaf () + while self.leaf_base >= self.num_leaves do + self:grow_leaves() end - -- When called without arguments, create the root node. - if not rib then - -- Clear previous FIB - ffi.fill(self.leaves, ffi.sizeof(self.leaves)) - ffi.fill(self.nodes, ffi.sizeof(self.nodes)) + self.leaf_base = self.leaf_base + 1 + return self.leaf_base - 1 +end + +function Poptrie:allocate_node () + while self.node_base >= self.num_nodes do + self:grow_nodes() end - rib = rib or self.rib - leaf_base = leaf_base or 0 - node_base = node_base or 0 - node_index = node_index or allocate_node() + self.node_base = self.node_base + 1 + return self.node_base - 1 +end + +function Poptrie:build_node (rib, node_index) -- Initialize node base pointers. - node().base0 = leaf_base - node().base1 = node_base + do local node = self.nodes[node_index] + -- Note: have to be careful about keeping direct references of nodes + -- around as they can get invalidated when the backing array is grown. + node.base0 = self.leaf_base + node.base1 = self.node_base + end -- Compute children local children = {} for index = 0, 2^Poptrie.k - 1 do @@ -140,10 +138,11 @@ function Poptrie:build (rib, node_index, leaf_base, node_base) local value = child.value or 0 if value ~= last_leaf_value then -- always true when leaf_compression=false if Poptrie.leaf_compression then - node().leafvec = bor(node().leafvec, lshift(1ULL, index)) + local node = self.nodes[node_index] + node.leafvec = bor(node.leafvec, lshift(1ULL, index)) last_leaf_value = value end - local leaf_index = allocate_leaf() + local leaf_index = self:allocate_leaf() self.leaves[leaf_index] = value end end @@ -155,20 +154,26 @@ function Poptrie:build (rib, node_index, leaf_base, node_base) for index = 0, 2^Poptrie.k - 1 do local child = children[index] if child.left or child.right then - child_nodes[index] = allocate_node() + child_nodes[index] = self:allocate_node() end end -- Initialize node.vector and child nodes. for index = 0, 2^Poptrie.k - 1 do local child = children[index] if child.left or child.right then - node().vector = bor(node().vector, lshift(1ULL, index)) - leaf_base, node_base = - self:build(child, child_nodes[index], leaf_base, node_base) + local node = self.nodes[node_index] + node.vector = bor(node.vector, lshift(1ULL, index)) + self:build_node(child, child_nodes[index]) end end - -- Return new leaf_base and node_base indices. - return leaf_base, node_base +end + +-- Compress RIB into Poptrie +function Poptrie:build () + -- Clear previous FIB + self:clear_fib() + -- Build root node (and children) + self:build_node(self.rib, self:allocate_node()) end -- http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetNaive @@ -252,6 +257,17 @@ function Poptrie:lookup64 (key) return Poptrie.asm_lookup64(self.leaves, self.nodes, key) end +function Poptrie:fib_info () + for i=0, self.node_base-1 do + print("node:", i) + print(self.nodes[i].base0, bin(self.nodes[i].leafvec)) + print(self.nodes[i].base1, bin(self.nodes[i].vector)) + end + for i=0, self.leaf_base-1 do + print("leaf:", i, self.leaves[i]) + end +end + function selftest () local t = new{} -- Tets building empty RIB @@ -277,17 +293,8 @@ function selftest () local n = t:rib_lookup(0x3F, 8) assert(n.value == 5 and not (n.left or n.right)) -- Test FIB - local leaf_base, node_base = t:build() - if debug then - for i=0, node_base-1 do - print("node:", i) - print(t.nodes[i].base0, bin(t.nodes[i].leafvec)) - print(t.nodes[i].base1, bin(t.nodes[i].vector)) - end - for i=0, leaf_base-1 do - print("leaf:", i, t.leaves[i]) - end - end + t:build() + if debug then t:fib_info() end assert(t:lookup(0x00) == 1) -- 00000000 assert(t:lookup(0x03) == 0) -- 00000011 assert(t:lookup(0x07) == 3) -- 00000111 @@ -313,15 +320,8 @@ function selftest () print("prefix:", entry, bin(case[1], case[2])) t:add(case[1], case[2], entry) end - local leaf_base, node_base = t:build() - for i=0, node_base-1 do - print("node:", i) - print(t.nodes[i].base0, bin(t.nodes[i].leafvec)) - print(t.nodes[i].base1, bin(t.nodes[i].vector)) - end - for i=0, leaf_base-1 do - if t.leaves[i] > 0 then print("leaf:", i, t.leaves[i]) end - end + t:build() + t:fib_info() for _, case in ipairs(cases) do print("rib:", t:rib_lookup(case[1]).value) print("fib:", t:lookup(case[1])) From 77cd0a737e8c232c03660fe3cc0ebfa6440c0601 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 14 Dec 2018 21:59:33 +0100 Subject: [PATCH 26/81] lib.poptrie: implement direct pointing The slowness of rib_lookup (and interestingly, extract as well) is becoming a serious problem with regard to :build() performance. --- src/lib/poptrie.lua | 54 +++++++++++++++++++++++++-------- src/lib/poptrie_lookup.dasl | 59 ++++++++++++++++++++++++++++--------- 2 files changed, 87 insertions(+), 26 deletions(-) diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua index 86d9812cb5..12d29c5fc7 100644 --- a/src/lib/poptrie.lua +++ b/src/lib/poptrie.lua @@ -16,10 +16,13 @@ local poptrie_lookup = require("lib.poptrie_lookup") local Poptrie = { leaf_compression = true, + direct_pointing = true, k = 6, + s = 18, + leaf_tag = lshift(1, 31), leaf_t = ffi.typeof("uint16_t"), vector_t = ffi.typeof("uint64_t"), - base_t = ffi.typeof("uint32_t") + base_t = ffi.typeof("uint32_t"), } Poptrie.node_t = ffi.typeof([[struct { $ leafvec, vector; @@ -36,7 +39,8 @@ function new (init) nodes = init.nodes or array(Poptrie.node_t, num_default), num_nodes = (init.nodes and assert(init.num_nodes)) or num_default, leaves = init.leaves or array(Poptrie.leaf_t, num_default), - num_leaves = (init.leaves and assert(init.num_leaves)) or num_default + num_leaves = (init.leaves and assert(init.num_leaves)) or num_default, + directmap = init.directmap or array(Poptrie.base_t, 2^Poptrie.s) } return setmetatable(pt, {__index=Poptrie}) end @@ -110,6 +114,11 @@ function Poptrie:allocate_leaf () end function Poptrie:allocate_node () + if Poptrie.direct_pointing then + -- When using direct_pointing, the node index space is split into half in + -- favor of a bit used for disambiguation in Poptrie:build_directmap. + assert(band(self.node_base, Poptrie.leaf_tag) == 0, "Node overflow") + end while self.node_base >= self.num_nodes do self:grow_nodes() end @@ -168,12 +177,26 @@ function Poptrie:build_node (rib, node_index) end end +function Poptrie:build_directmap (rib) + for index = 0, 2^Poptrie.s - 1 do + local child = self:rib_lookup(index, Poptrie.s) + if child.left or child.right then + self.directmap[index] = self:allocate_node() + self:build_node(child, self.directmap[index]) + else + self.directmap[index] = bor(child.value or 0, Poptrie.leaf_tag) + end + end +end + -- Compress RIB into Poptrie function Poptrie:build () - -- Clear previous FIB self:clear_fib() - -- Build root node (and children) - self:build_node(self.rib, self:allocate_node()) + if Poptrie.direct_pointing then + self:build_directmap(self.rib) + else + self:build_node(self.rib, self:allocate_node()) + end end -- http://graphics.stanford.edu/~seander/bithacks.html#CountBitsSetNaive @@ -225,10 +248,17 @@ end -- return t.L[base + bc - 1]; -- function Poptrie:lookup (key) - local N, L = self.nodes, self.leaves - local index = 0 + local N, L, D = self.nodes, self.leaves, self.directmap + local index, offset = 0, 0 + if Poptrie.direct_pointing then + offset = Poptrie.s + index = D[extract(key, 0, offset)] + if debug then print(bin(index), band(index, Poptrie.leaf_tag - 1)) end + if band(index, Poptrie.leaf_tag) ~= 0 then + return band(index, Poptrie.leaf_tag - 1) -- direct leaf, strip tag + end + end local node = N[index] - local offset = 0 local v = extract(key, offset, Poptrie.k) if debug then print(index, bin(node.vector), bin(v)) end while band(node.vector, lshift(1ULL, v)) ~= 0 do @@ -254,7 +284,7 @@ end Poptrie.asm_lookup64 = poptrie_lookup.generate(Poptrie, 64) function Poptrie:lookup64 (key) - return Poptrie.asm_lookup64(self.leaves, self.nodes, key) + return Poptrie.asm_lookup64(self.leaves, self.nodes, key, self.directmap) end function Poptrie:fib_info () @@ -336,11 +366,11 @@ function selftest () end local lib = require("core.lib") local seed = lib.getenv("SNABB_RANDOM_SEED") or 0 - for keysize = 1, 64 do + for _, keysize in ipairs{33} do print("keysize:", keysize) -- ramp up the geometry below to crank up test coverage - for entries = 1, 8 do - for i = 1, 4 do + for entries = 1, 2 do + for i = 1, 2 do math.randomseed(seed+i) cases = {} local t = new{} diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl index 27ae3288d3..f69258c12d 100644 --- a/src/lib/poptrie_lookup.dasl +++ b/src/lib/poptrie_lookup.dasl @@ -16,9 +16,12 @@ local anchor = {} -- Assemble a lookup routine function generate (Poptrie, keysize) - local offsets = -- Assert assumptions about lib.poptrie assert(Poptrie.k == 6) + if Poptrie.direct_pointing then + assert(Poptrie.s == 18) + assert(Poptrie.leaf_tag == bit.lshift(1, 31)) + end assert(ffi.sizeof(Poptrie.leaf_t) == 2) assert(ffi.sizeof(Poptrie.vector_t) == 8) assert(ffi.sizeof(Poptrie.base_t) == 4) @@ -40,9 +43,11 @@ function generate (Poptrie, keysize) end local prototype - if keysize <= 64 then - prototype = ffi.typeof("$ (*) ($ *, $ *, uint64_t)", - Poptrie.leaf_t, Poptrie.leaf_t, Poptrie.node_t) + if keysize <= 64 and Poptrie.direct_pointing then + prototype = ffi.typeof( + "$ (*) ($ *, $ *, uint64_t, $ *)", + Poptrie.leaf_t, Poptrie.leaf_t, Poptrie.node_t, Poptrie.base_t + ) else error("NYI") end return ffi.cast(prototype, mcode) @@ -51,20 +56,46 @@ end |.define leaves, rdi -- pointer to leaves array |.define nodes, rsi -- pointer to nodes array |.define key, rdx -- key to look up +|.define dmap, rcx -- pointer to directmap |.define index, r8d -- index into node array |.define node, r8 -- pointer into node array |.define offset, r9 -- offset into key -|.define v, r10 -- k bits extracted from key +|.define v, r10 -- k or s bits extracted from key |.define vec, r11 -- 64-bit vector or leafvec -- lookup(leaf_t *leaves, node_t *nodes, key) -> leaf_t function lookup (Dst, Poptrie, keysize) - -- index, node, offset = 0, nodes[index], 0 - | mov index, 0 - | lea node, [nodes+0] -- nodes[0] - | mov offset, 0 + if Poptrie.direct_pointing then + -- v = extract(key, 0, s=18) + -- v = band(key, lshift(1, s=18) - 1) + | mov v, 0x3FFFF + | and v, key + -- index = dmap[v] + | mov index, dword [dmap+v*4] + -- eax = band(index, leaf_tag) + | mov eax, 0x80000000 + | and eax, index + -- is leaf_tag set? + | cmp eax, 0 + | je >1 -- leaf_tag not set, index is a node + -- eax = leaf_tag - 1 (tag inverted) + | mov eax, 0x7FFFFFFF + | and eax, index + | ret + -- node, offset = nodes[index], s + |1: + | imul index, 24 -- multiply by node size + | lea node, [nodes+index] + -- offset = s + | mov offset, 18 + else + -- index, node, offset = 0, nodes[index], 0 + | mov index, 0 + | lea node, [nodes+0] -- nodes[0] + | mov offset, 0 + end -- while band(vec, lshift(1ULL, v)) ~= 0 - |1: + |2: -- v = extract(key, offset, k=6) if keysize <= 64 then -- v = rshift(key, offset) @@ -72,7 +103,7 @@ function lookup (Dst, Poptrie, keysize) | mov rcx, offset | shr v, cl -- v = band(v, lshift(1, k=6) - 1) - | and v, 63 + | and v, 0x3F else error("NYI") end -- vec = nodes[index].vector | mov vec, qword [node+8] @@ -84,7 +115,7 @@ function lookup (Dst, Poptrie, keysize) | and rax, vec -- is bit v set in vec? | cmp rax, 0 - | je >2 -- reached leaf, exit loop + | je >3 -- reached leaf, exit loop -- rax = lshift(2ULL, v) - 1 | mov rax, 2 | mov rcx, v @@ -102,9 +133,9 @@ function lookup (Dst, Poptrie, keysize) | lea node, [nodes+index] -- offset = offset + k | add offset, 6 - | jmp <1 -- loop + | jmp <2 -- loop -- end while - |2: + |3: -- rax = lshift(2ULL, v) - 1 | mov rax, 2 | mov rcx, v From b5fcbfb3c2c7a40b54509957b95db30b484f0df7 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 15 Dec 2018 13:12:26 +0100 Subject: [PATCH 27/81] lib.poptrie: do not print zero value leaves in fib_info --- src/lib/poptrie.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua index 12d29c5fc7..c470d7f44e 100644 --- a/src/lib/poptrie.lua +++ b/src/lib/poptrie.lua @@ -294,7 +294,9 @@ function Poptrie:fib_info () print(self.nodes[i].base1, bin(self.nodes[i].vector)) end for i=0, self.leaf_base-1 do - print("leaf:", i, self.leaves[i]) + if self.leaves[i] > 0 then + print("leaf:", i, self.leaves[i]) + end end end From 92bc6c6373c5fd3cbca4d4391236ff583af19546 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 15 Dec 2018 13:12:56 +0100 Subject: [PATCH 28/81] lib.poptrie: optimize build(), cover direct pointing in PMU analysis - avoid allocation in rib_lookup - introduce rib_map to squash build complexity --- src/lib/README.poptrie.md | 28 ++++-- src/lib/poptrie.lua | 188 ++++++++++++++++++++++-------------- src/lib/poptrie_lookup.dasl | 2 +- 3 files changed, 138 insertions(+), 80 deletions(-) diff --git a/src/lib/README.poptrie.md b/src/lib/README.poptrie.md index 42df1272b6..8b56d7dc96 100644 --- a/src/lib/README.poptrie.md +++ b/src/lib/README.poptrie.md @@ -8,7 +8,7 @@ as a hand-written, optimized assembler lookup routine. #### Example usage ```lua -local pt = poptrie.new{} +local pt = poptrie.new{direct_pointing=true} -- Associate prefixes of length to values (uint16_t) pt:add(0x00FF, 8, 1) pt:add(0x000F, 4, 2) @@ -17,21 +17,30 @@ pt:lookup64(0x001F) ⇒ 2 pt:lookup64(0x10FF) ⇒ 1 -- The value zero denotes "no match" pt:lookup64(0x0000) ⇒ 0 --- You can create a pre-built poptrie from nodes and leaves arrays. -local pt2 = poptrie.new{nodes=pt.nodes, leaves=pt.leaves} +-- You can create a pre-built poptrie from its backing memory. +local pt2 = poptrie.new{ + nodes = pt.nodes, + leaves = pt.leaves, + directmap = pt.directmap +} ``` #### Known bugs and limitations - Only supports keys up to 64 bits wide - - *Direct pointing* is not yet implemented #### Performance +- Intel(R) Xeon(R) CPU E3-1246 v3 @ 3.50GHz (Haswell, Turbo off) + ``` -PMU analysis (numentries=10000, keysize=64) -lookup: 30001.41 cycles/lookup 59357.23 instructions/lookup -lookup64: 179.39 cycles/lookup 205.72 instructions/lookup +PMU analysis (numentries=10000, keysize=32) +build: 0.1290 seconds +lookup: 13217.09 cycles/lookup 28014.35 instructions/lookup +lookup64: 122.94 cycles/lookup 133.22 instructions/lookup +build(direct_pointing): 0.1056 seconds +lookup(direct_pointing): 5519.01 cycles/lookup 11412.01 instructions/lookup +lookup64(direct_pointing): 89.82 cycles/lookup 70.72 instructions/lookup ``` #### Interface @@ -42,10 +51,15 @@ Creates and returns a new `Poptrie` object. *Init* is a table with the following keys: +* `direct_pointing` - *Optional*. Boolean that governs whether to use the + *direct pointing* optimization. Default is `false`. * `leaves` - *Optional*. An array of leaves. When *leaves* is supplied *nodes* must be supplied as well. * `nodes` - *Optional*. An array of nodes. When *nodes* is supplied *leaves* must be supplied as well. +* `directmap` - *Optional*. A direct map array. When *directmap* is supplied, + *nodes* and *leaves* must be supplied as well and *direct_pointing* is + implicit. — Method **Poptrie:add** *prefix* *length* *value* diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua index c470d7f44e..515bf8da42 100644 --- a/src/lib/poptrie.lua +++ b/src/lib/poptrie.lua @@ -16,13 +16,15 @@ local poptrie_lookup = require("lib.poptrie_lookup") local Poptrie = { leaf_compression = true, - direct_pointing = true, + direct_pointing = false, k = 6, s = 18, leaf_tag = lshift(1, 31), leaf_t = ffi.typeof("uint16_t"), vector_t = ffi.typeof("uint64_t"), base_t = ffi.typeof("uint32_t"), + num_leaves = 100, + num_nodes = 10 } Poptrie.node_t = ffi.typeof([[struct { $ leafvec, vector; @@ -34,15 +36,29 @@ local function array (t, n) end function new (init) - local num_default = 4 - local pt = { - nodes = init.nodes or array(Poptrie.node_t, num_default), - num_nodes = (init.nodes and assert(init.num_nodes)) or num_default, - leaves = init.leaves or array(Poptrie.leaf_t, num_default), - num_leaves = (init.leaves and assert(init.num_leaves)) or num_default, - directmap = init.directmap or array(Poptrie.base_t, 2^Poptrie.s) - } - return setmetatable(pt, {__index=Poptrie}) + local self = setmetatable({}, {__index=Poptrie}) + if init.leaves and init.nodes then + self.leaves, self.num_leaves = init.leaves, assert(init.num_leaves) + self.nodes, self.num_nodes = init.nodes, assert(init.num_nodes) + elseif init.nodes or init.leaves or init.directmap then + error("partial init") + else + self.leaves = array(Poptrie.leaf_t, Poptrie.num_leaves) + self.nodes = array(Poptrie.node_t, Poptrie.num_nodes) + end + if init.directmap then + self.directmap = init.directmap + self.direct_pointing = true + else + if init.direct_pointing ~= nil then + self.direct_pointing = init.direct_pointing + end + if self.direct_pointing then + self.directmap = array(Poptrie.base_t, 2^Poptrie.s) + end + end + self.asm_lookup64 = poptrie_lookup.generate(self, 64) + return self end function Poptrie:grow_nodes () @@ -61,7 +77,7 @@ end -- XXX - Generalize for key=uint8_t[?] local function extract (key, offset, length) - return band(rshift(key+0ULL, offset), lshift(1ULL, length) - 1) + return band(rshift(key+0ULL, offset), lshift(1, length) - 1) end -- Add key/value pair to RIB (intermediary binary trie) @@ -86,17 +102,32 @@ function Poptrie:rib_lookup (key, length, root) local function lookup (node, offset, value) value = node.value or value if offset == length then - return {value=value, left=node.left, right=node.right} - elseif extract(key, offset, 1) == 0 and node.left then + return value, (node.left or node.right) and node + elseif node.left and extract(key, offset, 1) == 0 then return lookup(node.left, offset + 1, value) - elseif extract(key, offset, 1) == 1 and node.right then + elseif node.right and extract(key, offset, 1) == 1 then return lookup(node.right, offset + 1, value) else - -- No match: return longest prefix key value, but no child nodes. - return {value=value} + -- No match: return longest prefix key value, but no node. + return value + end + end + return lookup(root or self.rib, 0) +end + +-- Map f over keys of length in RIB +function Poptrie:rib_map (f, length, root) + local function map (node, offset, key, value) + value = (node and node.value) or value + local left, right = node and node.left, node and node.right + if offset == length then + f(key, value, (left or right) and node) + else + map(left, offset + 1, key, value) + map(right, offset + 1, bor(key, lshift(1, offset)), value) end end - return lookup(root or self.rib or {}, 0) + return map(root or self.rib, 0, 0) end function Poptrie:clear_fib () @@ -114,7 +145,7 @@ function Poptrie:allocate_leaf () end function Poptrie:allocate_node () - if Poptrie.direct_pointing then + if self.direct_pointing then -- When using direct_pointing, the node index space is split into half in -- favor of a bit used for disambiguation in Poptrie:build_directmap. assert(band(self.node_base, Poptrie.leaf_tag) == 0, "Node overflow") @@ -126,7 +157,7 @@ function Poptrie:allocate_node () return self.node_base - 1 end -function Poptrie:build_node (rib, node_index) +function Poptrie:build_node (rib, node_index, default) -- Initialize node base pointers. do local node = self.nodes[node_index] -- Note: have to be careful about keeping direct references of nodes @@ -134,17 +165,17 @@ function Poptrie:build_node (rib, node_index) node.base0 = self.leaf_base node.base1 = self.node_base end - -- Compute children - local children = {} - for index = 0, 2^Poptrie.k - 1 do - children[index] = self:rib_lookup(index, Poptrie.k, rib) + -- Compute leaves and children + local leaves, children = {}, {} + local function collect (key, value, node) + leaves[key], children[key] = value, node end + self:rib_map(collect, Poptrie.k, rib) -- Allocate and initialize node.leafvec and leaves. local last_leaf_value = nil for index = 0, 2^Poptrie.k - 1 do - local child = children[index] - if not (child.left or child.right) then - local value = child.value or 0 + if not children[index] then + local value = leaves[index] or default or 0 if value ~= last_leaf_value then -- always true when leaf_compression=false if Poptrie.leaf_compression then local node = self.nodes[node_index] @@ -161,38 +192,39 @@ function Poptrie:build_node (rib, node_index) -- and build() will advance the node_base.) local child_nodes = {} for index = 0, 2^Poptrie.k - 1 do - local child = children[index] - if child.left or child.right then + if children[index] then child_nodes[index] = self:allocate_node() end end -- Initialize node.vector and child nodes. for index = 0, 2^Poptrie.k - 1 do - local child = children[index] - if child.left or child.right then + if children[index] then local node = self.nodes[node_index] node.vector = bor(node.vector, lshift(1ULL, index)) - self:build_node(child, child_nodes[index]) + self:build_node(children[index], + child_nodes[index], + leaves[index] or default) end end end +-- Build direct index array for RIB function Poptrie:build_directmap (rib) - for index = 0, 2^Poptrie.s - 1 do - local child = self:rib_lookup(index, Poptrie.s) - if child.left or child.right then + local function build (index, value, node) + if node then self.directmap[index] = self:allocate_node() - self:build_node(child, self.directmap[index]) + self:build_node(node, self.directmap[index], value) else - self.directmap[index] = bor(child.value or 0, Poptrie.leaf_tag) + self.directmap[index] = bor(value or 0, Poptrie.leaf_tag) end end + self:rib_map(build, Poptrie.s, rib) end -- Compress RIB into Poptrie function Poptrie:build () self:clear_fib() - if Poptrie.direct_pointing then + if self.direct_pointing then self:build_directmap(self.rib) else self:build_node(self.rib, self:allocate_node()) @@ -250,7 +282,7 @@ end function Poptrie:lookup (key) local N, L, D = self.nodes, self.leaves, self.directmap local index, offset = 0, 0 - if Poptrie.direct_pointing then + if self.direct_pointing then offset = Poptrie.s index = D[extract(key, 0, offset)] if debug then print(bin(index), band(index, Poptrie.leaf_tag - 1)) end @@ -282,9 +314,8 @@ function Poptrie:lookup (key) return L[base + bc - 1] end -Poptrie.asm_lookup64 = poptrie_lookup.generate(Poptrie, 64) function Poptrie:lookup64 (key) - return Poptrie.asm_lookup64(self.leaves, self.nodes, key, self.directmap) + return self.asm_lookup64(self.leaves, self.nodes, key, self.directmap) end function Poptrie:fib_info () @@ -301,6 +332,7 @@ function Poptrie:fib_info () end function selftest () + -- To test direct pointing: Poptrie.direct_pointing = true local t = new{} -- Tets building empty RIB t:build() @@ -310,20 +342,20 @@ function selftest () t:add(0x07, 4, 3) -- 0111 t:add(0xFF, 8, 4) -- 11111111 t:add(0xFF, 5, 5) -- 11111 - local n = t:rib_lookup(0x0, 1) - assert(not n.value and n.left and not n.right) - local n = t:rib_lookup(0x00, 8) - assert(n.value == 1 and not (n.left or n.right)) - local n = t:rib_lookup(0x07, 3) - assert(not n.value and (n.left and n.right)) - local n = t:rib_lookup(0x0, 1, n) - assert(n.value == 3 and not (n.left or n.right)) - local n = t:rib_lookup(0xFF, 5) - assert(n.value == 5 and (not n.left) and n.right) - local n = t:rib_lookup(0x0F, 3, n) - assert(n.value == 4 and not (n.left or n.right)) - local n = t:rib_lookup(0x3F, 8) - assert(n.value == 5 and not (n.left or n.right)) + local v, n = t:rib_lookup(0x0, 1) + assert(not v and n.left and not n.right) + local v, n = t:rib_lookup(0x00, 8) + assert(v == 1 and not n) + local v, n = t:rib_lookup(0x07, 3) + assert(not v and (n.left and n.right)) + local v, n = t:rib_lookup(0x0, 1, n) + assert(v == 3 and not n) + local v, n = t:rib_lookup(0xFF, 5) + assert(v == 5 and (not n.left) and n.right) + local v, n = t:rib_lookup(0x0F, 3, n) + assert(v == 4 and not n) + local v, n = t:rib_lookup(0x3F, 8) + assert(v == 5 and not n) -- Test FIB t:build() if debug then t:fib_info() end @@ -355,7 +387,7 @@ function selftest () t:build() t:fib_info() for _, case in ipairs(cases) do - print("rib:", t:rib_lookup(case[1]).value) + print("rib:", t:rib_lookup(case[1])) print("fib:", t:lookup(case[1])) print("64:", t:lookup64(case[1])) end @@ -368,16 +400,16 @@ function selftest () end local lib = require("core.lib") local seed = lib.getenv("SNABB_RANDOM_SEED") or 0 - for _, keysize in ipairs{33} do + for keysize = 1, 64 do print("keysize:", keysize) -- ramp up the geometry below to crank up test coverage - for entries = 1, 2 do - for i = 1, 2 do + for entries = 1, 3 do + for i = 1, 10 do math.randomseed(seed+i) cases = {} local t = new{} local k = {} - for entry= 1, entries do + for entry = 1, entries do local a, l = math.random(2^keysize - 1), math.random(keysize) cases[entry] = {a, l} t:add(a, l, entry) @@ -385,7 +417,7 @@ function selftest () end local v = {} for entry, a in ipairs(k) do - v[entry] = t:rib_lookup(a, keysize).value + v[entry] = t:rib_lookup(a, keysize) r_assert(v[entry] > 0, cases) end t:build() @@ -410,8 +442,12 @@ function selftest () tab.cycles / iterations, tab.instructions / iterations)) end + local function time (description, f) + local start = os.clock(); f() + print(("%s: %.4f seconds"):format(description, os.clock() - start)) + end if pmu.is_available() then - local t = new{} + local t = new{direct_pointing=false} local k = {} local numentries = 10000 local keysize = 64 @@ -420,19 +456,27 @@ function selftest () t:add(a, l, entry) k[entry] = a end - t:build() + local function build () + t:build() + end + local function lookup (iter) + for i=1,iter do t:lookup(k[i%#k+1]) end + end + local function lookup64 (iter) + for i=1,iter do t:lookup64(k[i%#k+1]) end + end print("PMU analysis (numentries="..numentries..", keysize="..keysize..")") pmu.setup() - measure("lookup", - function (iter) - for i=1,iter do t:lookup(k[i%#k+1]) end - end, - 1e5) - measure("lookup64", - function (iter) - for i=1,iter do t:lookup64(k[i%#k+1]) end - end, - 1e7) + time("build", build) + measure("lookup", lookup, 1e5) + measure("lookup64", lookup64, 1e7) + do local rib = t.rib + t = new{direct_pointing=true} + t.rib = rib + end + time("build(direct_pointing)", build) + measure("lookup(direct_pointing)", lookup, 1e5) + measure("lookup64(direct_pointing)", lookup64, 1e7) else print("No PMU available.") end diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl index f69258c12d..0295d1bd82 100644 --- a/src/lib/poptrie_lookup.dasl +++ b/src/lib/poptrie_lookup.dasl @@ -43,7 +43,7 @@ function generate (Poptrie, keysize) end local prototype - if keysize <= 64 and Poptrie.direct_pointing then + if keysize <= 64 then prototype = ffi.typeof( "$ (*) ($ *, $ *, uint64_t, $ *)", Poptrie.leaf_t, Poptrie.leaf_t, Poptrie.node_t, Poptrie.base_t From 55931c963786047ce66f77852cc77b70a7b66333 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sun, 16 Dec 2018 16:21:45 +0100 Subject: [PATCH 29/81] PublicRouter: revert "vita/route: use lib.poptrie instead of lib.ctable" This reverts parts of commit a9ba0d49bf2979cd287a0526b65b4b9b24892926. --- src/program/vita/route.lua | 40 ++++++++++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index dd05bdc496..c88fdc754f 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -7,6 +7,7 @@ local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") local esp = require("lib.protocol.esp") local poptrie = require("lib.poptrie") +local ctable = require("lib.ctable") local ffi = require("ffi") @@ -102,19 +103,37 @@ PublicRouter = { function PublicRouter:new (conf) local o = { - ports = {}, - routes = {}, - routing_table4 = poptrie.new{}, + routing_table4 = ctable.new{ + key_type = ffi.typeof("uint32_t"), + value_type = ffi.typeof("uint32_t") + }, esp = esp:new({}) } + self.build_fib(o, conf) + return setmetatable(o, {__index = PublicRouter}) +end + +function PublicRouter:reconfig (conf) + self:build_fib(conf) + self:link() -- links might have changed before reconfig +end + +function PublicRouter:build_fib (conf) + self.ports = {} + self.routes = {} + -- Update FIB entries for SAs for spi, sa in pairs(conf.sa) do - local index = #o.ports+1 - assert(ffi.cast("uint16_t", index) == index, "index overflow") - o.routing_table4:add(spi, 32, index) - o.ports[index] = sa.route.."_"..spi + local index = #self.ports+1 + assert(ffi.cast("uint32_t", index) == index, "index overflow") + self.routing_table4:add(spi, index, 'update') + self.ports[index] = sa.route.."_"..spi + end + -- Remove obsolete FIB entries + for entry in self.routing_table4:iterate() do + if not conf.sa[tonumber(entry.key)] then + self.routing_table4:remove_ptr(entry) + end end - o.routing_table4:build() - return setmetatable(o, {__index = PublicRouter}) end function PublicRouter:link () @@ -124,7 +143,8 @@ function PublicRouter:link () end function PublicRouter:find_route4 (spi) - return self.routes[self.routing_table4:lookup64(spi)] + local entry = self.routing_table4:lookup_ptr(spi) + return entry and self.routes[entry.value] end function PublicRouter:push () From 0f248728d9a2c351e1d5e8f04bc5a68e2c86f2b6 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sun, 16 Dec 2018 16:58:06 +0100 Subject: [PATCH 30/81] PublicRouter: simplify SPI lookup (simple array) Exploit our ability to restrict inbound SPIs (to be within uint16_t) in KeyManager to replace the ctable used for lookups in PublicRouter with a dead simple array of size 2^16. --- src/program/vita/exchange.lua | 2 +- src/program/vita/route.lua | 20 +++++--------------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 97d22f502c..305a122be8 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -789,7 +789,7 @@ end function Protocol:next_spi () local current_spi = Protocol.spi_counter + 256 - Protocol.spi_counter = (Protocol.spi_counter + 1) % (2^32 - 1 - 256) + Protocol.spi_counter = (Protocol.spi_counter + 1) % (2^16 - 1 - 256) return current_spi end diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index 7a6909fb27..bba99dd999 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -7,7 +7,6 @@ local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") local esp = require("lib.protocol.esp") local lpm = require("lib.lpm.lpm4_248").LPM4_248 -local ctable = require("lib.ctable") local ffi = require("ffi") @@ -110,10 +109,7 @@ PublicRouter = { function PublicRouter:new (conf) local o = { - routing_table4 = ctable.new{ - key_type = ffi.typeof("uint32_t"), - value_type = ffi.typeof("uint32_t") - }, + routing_table4 = ffi.new("uint32_t[?]", 2^16), esp = esp:new({}) } self.build_fib(o, conf) @@ -128,19 +124,14 @@ end function PublicRouter:build_fib (conf) self.ports = {} self.routes = {} - -- Update FIB entries for SAs + ffi.fill(self.routing_table4, ffi.sizeof(self.routing_table4), 0) for spi, sa in pairs(conf.sa) do local index = #self.ports+1 + assert(spi < 2^16, "SPI overflow") assert(ffi.cast("uint32_t", index) == index, "index overflow") - self.routing_table4:add(spi, index, 'update') + self.routing_table4[spi] = index self.ports[index] = sa.route.."_"..spi end - -- Remove obsolete FIB entries - for entry in self.routing_table4:iterate() do - if not conf.sa[tonumber(entry.key)] then - self.routing_table4:remove_ptr(entry) - end - end end function PublicRouter:link () @@ -150,8 +141,7 @@ function PublicRouter:link () end function PublicRouter:find_route4 (spi) - local entry = self.routing_table4:lookup_ptr(spi) - return entry and self.routes[entry.value] + return self.routes[self.routing_table4[spi]] end function PublicRouter:push () From 659c655225a59c37fb25e37964873b85b0f4dc6c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sun, 16 Dec 2018 17:01:39 +0100 Subject: [PATCH 31/81] PrivateRouter: use direct pointing optimization for Poptrie. --- src/program/vita/route.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index c88fdc754f..08c58bc2a6 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -32,7 +32,7 @@ function PrivateRouter:new (conf) routes = {}, mtu = conf.mtu, ip4 = ipv4:new({}), - routing_table4 = poptrie.new{} + routing_table4 = poptrie.new{direct_pointing=true} } for id, route in pairs(conf.routes) do local index = #o.ports+1 From 3fcc794be3a1c3410ad8b4882a48561d4c34423c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 17 Dec 2018 17:11:16 +0100 Subject: [PATCH 32/81] lib.poptrie: make s (bits used for direct pointing) configurable --- src/lib/README.poptrie.md | 2 ++ src/lib/poptrie.lua | 9 ++++++--- src/lib/poptrie_lookup.dasl | 8 ++++---- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/src/lib/README.poptrie.md b/src/lib/README.poptrie.md index 8b56d7dc96..9756b6274f 100644 --- a/src/lib/README.poptrie.md +++ b/src/lib/README.poptrie.md @@ -53,6 +53,8 @@ Creates and returns a new `Poptrie` object. * `direct_pointing` - *Optional*. Boolean that governs whether to use the *direct pointing* optimization. Default is `false`. +* `s` - *Optional*. Bits to use for the *direct pointing* optimization. + Default is 18. Note that the direct map array will be 2×2ˢ bytes in size. * `leaves` - *Optional*. An array of leaves. When *leaves* is supplied *nodes* must be supplied as well. * `nodes` - *Optional*. An array of nodes. When *nodes* is supplied *leaves* diff --git a/src/lib/poptrie.lua b/src/lib/poptrie.lua index 515bf8da42..ab768eb651 100644 --- a/src/lib/poptrie.lua +++ b/src/lib/poptrie.lua @@ -53,8 +53,11 @@ function new (init) if init.direct_pointing ~= nil then self.direct_pointing = init.direct_pointing end + if init.s ~= nil then + self.s = init.s + end if self.direct_pointing then - self.directmap = array(Poptrie.base_t, 2^Poptrie.s) + self.directmap = array(Poptrie.base_t, 2^self.s) end end self.asm_lookup64 = poptrie_lookup.generate(self, 64) @@ -218,7 +221,7 @@ function Poptrie:build_directmap (rib) self.directmap[index] = bor(value or 0, Poptrie.leaf_tag) end end - self:rib_map(build, Poptrie.s, rib) + self:rib_map(build, self.s, rib) end -- Compress RIB into Poptrie @@ -283,7 +286,7 @@ function Poptrie:lookup (key) local N, L, D = self.nodes, self.leaves, self.directmap local index, offset = 0, 0 if self.direct_pointing then - offset = Poptrie.s + offset = self.s index = D[extract(key, 0, offset)] if debug then print(bin(index), band(index, Poptrie.leaf_tag - 1)) end if band(index, Poptrie.leaf_tag) ~= 0 then diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl index 0295d1bd82..3f80bd435c 100644 --- a/src/lib/poptrie_lookup.dasl +++ b/src/lib/poptrie_lookup.dasl @@ -19,7 +19,6 @@ function generate (Poptrie, keysize) -- Assert assumptions about lib.poptrie assert(Poptrie.k == 6) if Poptrie.direct_pointing then - assert(Poptrie.s == 18) assert(Poptrie.leaf_tag == bit.lshift(1, 31)) end assert(ffi.sizeof(Poptrie.leaf_t) == 2) @@ -66,9 +65,10 @@ end -- lookup(leaf_t *leaves, node_t *nodes, key) -> leaf_t function lookup (Dst, Poptrie, keysize) if Poptrie.direct_pointing then - -- v = extract(key, 0, s=18) - -- v = band(key, lshift(1, s=18) - 1) - | mov v, 0x3FFFF + -- v = extract(key, 0, Poptrie.s) + local direct_mask = bit.lshift(1ULL, Poptrie.s) - 1 + -- v = band(key, direct_mask) + | mov v, direct_mask | and v, key -- index = dmap[v] | mov index, dword [dmap+v*4] From b629d5d37bbf7168e7728c2026afd1b5752d53f0 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 17 Dec 2018 18:25:20 +0100 Subject: [PATCH 33/81] KeyManager: allow initiator to update outbound SAs This should resolve the remaining issue from b8fffc. Initiators can now update unrequited outbound SAs that were added by the responder during a negotiation that ultimately failed (e.g., due to packet loss.) --- src/program/vita/exchange.lua | 52 +++++++++++++------------- src/program/vita/vita-esp-gateway.yang | 5 +-- 2 files changed, 28 insertions(+), 29 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 57371a0e22..beb3053a69 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -106,8 +106,7 @@ module(...,package.seeall) -- outbound_sa_expired count of outbound SAs that have expired -- (sa_ttl) -- --- outbound_sa_rejected count of outbound SAs that were rejected --- because their SPI was not unique +-- outbound_sa_updated count of outbound SAs that were updated -- -- 2. The Protocol subsystem implements vita-ske2 (the cryptographic key -- exchange protocol defined in README.exchange) as two finite-state @@ -187,7 +186,7 @@ KeyManager = { keypairs_negotiated = {counter}, inbound_sa_expired = {counter}, outbound_sa_expired = {counter}, - outbound_sa_rejected = {counter} + outbound_sa_updated = {counter} } } @@ -481,8 +480,8 @@ function KeyManager:handle_nonce_key_request (route, message) inbound_sa.spi, (is_loopback and "-") or outbound_sa.spi)) - self:add_inbound_sa(route, inbound_sa) - if not is_loopback then self:add_outbound_sa(route, outbound_sa) end + self:insert_inbound_sa(route, inbound_sa) + if not is_loopback then self:upsert_outbound_sa(route, outbound_sa) end return true end @@ -518,13 +517,13 @@ function KeyManager:handle_key_request (route, message) (is_loopback and "-") or inbound_sa.spi, outbound_sa.spi)) - if not is_loopback then self:add_inbound_sa(route, inbound_sa) end - self:add_outbound_sa(route, outbound_sa) + if not is_loopback then self:insert_inbound_sa(route, inbound_sa) end + self:upsert_outbound_sa(route, outbound_sa) return true end -function KeyManager:add_inbound_sa (route, sa) +function KeyManager:insert_inbound_sa (route, sa) -- invariant: inbound SA spi is unique for _, route in ipairs(self.routes) do for _, inbound in ipairs(route.inbound_sa) do @@ -546,17 +545,23 @@ function KeyManager:add_inbound_sa (route, sa) self.sa_db_updated = true end -function KeyManager:add_outbound_sa (route, sa) - -- invariant: outbound SA spi is unique - for _, route in ipairs(self.routes) do - for _, outbound in ipairs(route.outbound_sa) do +function KeyManager:upsert_outbound_sa (route, sa) + -- possibly replace existing or queued outbound SA + local function remove_existing_sa_for_update () + for index, outbound in ipairs(route.outbound_sa_queue) do if outbound.spi == sa.spi then - counter.add(self.shm.outbound_sa_rejected) - audit:log(("Rejecting outbound SA %d for '%s' (duplicate SPI)") - :format(sa.spi, route.id)) - return + return table.remove(route.outbound_sa_queue, index) end end + for index, outbound in ipairs(route.outbound_sa) do + if outbound.spi == sa.spi then + return table.remove(route.outbound_sa, index) + end + end + end + if remove_existing_sa_for_update() then + counter.add(self.shm.outbound_sa_updated) + audit:log("Updating outbound SA "..sa.spi.." for '"..route.id.."'") end -- enqueue new outbound SA at the end of the queue table.insert(route.outbound_sa_queue, { @@ -571,15 +576,10 @@ function KeyManager:add_outbound_sa (route, sa) rekey_timeout = lib.timeout(self.sa_ttl/2 + math.random(1000)/1000), -- Delay before activating redundant, newly established outbound SAs to -- give the receiving end time to set up. Choosen so that when a - -- negotiation times out due to packet loss, the initiator can supersede - -- the bogus outbound SA before it gets activated. + -- negotiation times out due to packet loss, the initiator can update + -- unrequited outbound SAs before they get activated. activation_delay = lib.timeout(self.negotiation_ttl*1.5) }) - -- remove superfluous outbound SAs from the front of the queue - assert(self.num_outbound_sa == 1, "FIXME: displacement of queued outbound SAs only works for num_outbound_sa = 1.") - if #route.outbound_sa_queue > self.num_outbound_sa then - table.remove(route.outbound_sa_queue, 1) - end end function KeyManager:request (route, message) @@ -949,7 +949,7 @@ function Protocol:reset_if_expired () or self.status == Protocol.status.accept_key then if self.deadline and self.deadline() then -- accept_challenge, accept_key -> initiator - self:reset() + self:reset('reuse_spi') -- renegotiate this SA and possibly update it self.status = Protocol.status.initiator return Protocol.code.expired end @@ -1057,9 +1057,9 @@ function Protocol:set_deadline (deadline) else self.deadline = deadline end end -function Protocol:reset () +function Protocol:reset (reuse_spi) self:set_deadline(false) - self:next_spi() + if not reuse_spi then self:next_spi() end self:next_nonce() self:next_dh_key() self:clear_external_inputs() diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index a994c04693..64b2f07bfa 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -529,11 +529,10 @@ module vita-esp-gateway { description "Count of outbound SAs that have expired due to sa-ttl."; } - leaf outbound-sa-rejected { + leaf outbound-sa-updated { type yang:zero-based-counter64; description - "Count of outbound SAs that were rejected because they had a - non-unique SPI (indicates a misbehaving peer.)"; + "Count of outbound SAs that were updated."; } } } From 497a28a89994a902f8bef4329e0c68d79fd3da64 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 20 Dec 2018 13:43:50 +0100 Subject: [PATCH 34/81] PrivateRouter: use 24 bits for poptrie direct pointing --- src/program/vita/route.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index 08c58bc2a6..3161adfaef 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -32,7 +32,7 @@ function PrivateRouter:new (conf) routes = {}, mtu = conf.mtu, ip4 = ipv4:new({}), - routing_table4 = poptrie.new{direct_pointing=true} + routing_table4 = poptrie.new{direct_pointing=true, s=24} } for id, route in pairs(conf.routes) do local index = #o.ports+1 From 703eb9a49a130b15531975bf181fa420fc5905dd Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 8 Jan 2019 17:35:42 +0100 Subject: [PATCH 35/81] intel_mp: work around flaky mirror test on i350 --- src/apps/intel_mp/test_1g_vmdq_mirror.snabb | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/apps/intel_mp/test_1g_vmdq_mirror.snabb b/src/apps/intel_mp/test_1g_vmdq_mirror.snabb index cb3f785bc0..3bf1383be5 100755 --- a/src/apps/intel_mp/test_1g_vmdq_mirror.snabb +++ b/src/apps/intel_mp/test_1g_vmdq_mirror.snabb @@ -58,8 +58,16 @@ config.link(c, "nic1p0.output -> sink.input0") config.link(c, "nic1p1.output -> sink.input1") config.link(c, "nic1p2.output -> sink.input2") -engine.configure(c) -engine.main({ duration = 1 }) +local retries = 10 +repeat + -- This test is flaky on my hardware, apparently because sometimes the first + -- so-many packets are not delivered. This works around that. - Max + engine.configure(config.new()) + engine.configure(c) + engine.main({ duration = 1 }) + retries = retries - 1 +until link.stats(engine.app_table.sink.input.input0).rxpackets > 0 + or retries == 0 assert(link.stats(engine.app_table.sink.input.input0).rxpackets == 51, "wrong number of packets received on pool 0") From 65fc22ed0837eed70c84d583ce88bac46080e347 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 9 Jan 2019 19:49:41 +0100 Subject: [PATCH 36/81] vita/tunnel: add support for tunneling over IPv6 --- src/program/vita/tunnel.lua | 64 ++++++++++++++++++++++++++++++++----- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/src/program/vita/tunnel.lua b/src/program/vita/tunnel.lua index e317d3713b..3e0138c9ab 100644 --- a/src/program/vita/tunnel.lua +++ b/src/program/vita/tunnel.lua @@ -5,6 +5,7 @@ module(...,package.seeall) local counter = require("core.counter") local esp = require("lib.ipsec.esp") local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") -- sa := { spi=(SPI), aead=(STRING), key=(KEY), salt=(SALT), -- [ window_size=(INT), @@ -13,6 +14,7 @@ local ipv4 = require("lib.protocol.ipv4") -- https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xhtml local NextHeaderIPv4 = 4 +local NextHeaderIPv6 = 41 Encapsulate = { name = "Encapsulate", @@ -30,12 +32,22 @@ end function Encapsulate:push () local output, sa = self.output.output, self.sa - local input4 = self.input.input4 - while not link.empty(input4) do - link.transmit( - output, - sa:encapsulate_tunnel(link.receive(input4), NextHeaderIPv4) - ) + local input4, input6 = self.input.input4, self.input.input6 + if input4 then + while not link.empty(input4) do + link.transmit( + output, + sa:encapsulate_tunnel(link.receive(input4), NextHeaderIPv4) + ) + end + end + if input6 then + while not link.empty(input6) do + link.transmit( + output, + sa:encapsulate_tunnel(link.receive(input6), NextHeaderIPv6) + ) + end end end @@ -65,12 +77,14 @@ end function Decapsulate:push () local input, sa = self.input.input, self.sa - local output4 = self.output.output4 + local output4, output6 = self.output.output4, self.output.output6 while not link.empty(input) do local p_enc = link.receive(input) local p, next_header = sa:decapsulate_tunnel(p_enc) - if p and next_header == NextHeaderIPv4 then + if p and next_header == NextHeaderIPv4 and output4 then link.transmit(output4, p) + elseif p and next_header == NextHeaderIPv6 and output6 then + link.transmit(output6, p) elseif p then counter.add(self.shm.rxerrors) counter.add(self.shm.protocol_errors) @@ -120,3 +134,37 @@ function Tunnel4:encapsulate (p) self.ip:checksum() return p end + + +Tunnel6 = { + name = "Tunnel6", + config = { + src = {required=true}, + dst = {required=true} + } +} + +function Tunnel6:new (conf) + local o = { + ip_template = ipv6:new{ + src = ipv6:pton(conf.src), + dst = ipv6:pton(conf.dst), + next_header = esp.PROTOCOL, + hop_limit = 64 + } + } + return setmetatable(o, {__index = Tunnel6}) +end + +function Tunnel6:push () + local input, output = self.input.input, self.output.output + while not link.empty(input) do + link.transmit(output, self:encapsulate(link.receive(input))) + end +end + +function Tunnel6:encapsulate (p) + self.ip_template:payload_length(p.length) + p = packet.prepend(p, self.ip_template:header(), ipv6:sizeof()) + return p +end From ff7467aff4cd223a4d5847f12647f59ea014fe18 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 16 Jan 2019 13:21:51 +0100 Subject: [PATCH 37/81] vita: exclude Ethernet header from MTU calculations --- src/program/vita/README.config | 12 ++++++------ src/program/vita/route.lua | 2 +- src/program/vita/vita.lua | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/program/vita/README.config b/src/program/vita/README.config index b6b06d8525..6d67017f6f 100644 --- a/src/program/vita/README.config +++ b/src/program/vita/README.config @@ -75,12 +75,12 @@ NOTES head -c 32 /dev/urandom | hexdump -e '/1 "%02X"' - Optionally, the gateway’s MTU can be specified in bytes. The default and - maximum permitted value is 8937. Since Vita performs neither fragmentation - nor reassembly it may be necessary to adjust the next-hop MTU accordingly. - Note that packets leaving the public interface will have an added packet size - overhead due to encapsulation (up to 57 bytes for IPv4 and up to 77 bytes for - IPv6.) + Optionally, the gateway’s MTU (excluding Ethernet overhead) can be specified + in bytes. The default and maximum permitted value is 8923. Since Vita + performs neither fragmentation nor reassembly it may be necessary to adjust + the next-hop MTU accordingly. Note that packets leaving the public interface + will have an added packet size overhead due to encapsulation (up to 57 bytes + for IPv4 and up to 77 bytes for IPv6.) While the default configuration should be generally applicable, the negotiation timeout and lifetime of Security Associations (SA) can be diff --git a/src/program/vita/route.lua b/src/program/vita/route.lua index 7a6909fb27..a74526e0a8 100644 --- a/src/program/vita/route.lua +++ b/src/program/vita/route.lua @@ -68,7 +68,7 @@ function PrivateRouter:route (p) assert(self.ip4:new_from_mem(p.data, p.length)) local route = self:find_route4(self.ip4:dst()) if route then - if p.length + ethernet:sizeof() <= self.mtu then + if p.length <= self.mtu then link.transmit(route, p) else counter.add(self.shm.rxerrors) diff --git a/src/program/vita/vita.lua b/src/program/vita/vita.lua index 5e4e40e21a..610bb9f610 100644 --- a/src/program/vita/vita.lua +++ b/src/program/vita/vita.lua @@ -31,7 +31,7 @@ local confighelp = require("program.vita.README_config_inc") local confspec = { private_interface = {}, public_interface = {}, - mtu = {default=8937}, + mtu = {default=8923}, route = {default={}}, negotiation_ttl = {}, sa_ttl = {}, From 0bebcc91c2b8102daa48fa0657116c21f62a4c94 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 16 Jan 2019 14:57:02 +0100 Subject: [PATCH 38/81] vita/icmp: use self.icmp_in for parsing Be a bit more deliberate in distinguishing parsed headers from emitted headers (previously used self.icmp for both) to avoid nasty bugs. --- src/program/vita/icmp.lua | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/program/vita/icmp.lua b/src/program/vita/icmp.lua index a4282085dc..57e1130fcb 100644 --- a/src/program/vita/icmp.lua +++ b/src/program/vita/icmp.lua @@ -71,6 +71,7 @@ function ICMP4:new (conf) src = ipv4:pton(conf.node_ip4) }), ip4_in = ipv4:new({}), + icmp_in = icmp:new(), icmp = icmp:new(), nexthop_mtu = conf.nexthop_mtu or 1024, nexthop_mtu_configured = conf.nexthop_mtu ~= nil, @@ -106,8 +107,8 @@ function ICMP4:log (p) ) self.logger:log(("received message from %s [type %d code %d packet %s ...]") :format(ipv4:ntop(self.ip4_in:src()), - self.icmp:type(), - self.icmp:code(), + self.icmp_in:type(), + self.icmp_in:code(), lib.hexdump(excerpt))) end @@ -123,7 +124,7 @@ ICMP4.handlers[3] = function (self, p) [3] = self.shm.port_unreachable, [4] = self.shm.fragmentation_needed, [5] = self.shm.source_route_failed }) - [self.icmp:code()] + [self.icmp_in:code()] or self.shm.code_not_implemented_errors ) end @@ -136,7 +137,7 @@ ICMP4.handlers[11] = function (self, p) counter.add( ({ [0] = self.shm.transit_ttl_exceeded, [1] = self.shm.fragment_reassembly_time_exceeded }) - [self.icmp:code()] + [self.icmp_in:code()] or self.shm.code_not_implemented_errors ) end @@ -158,7 +159,7 @@ ICMP4.handlers[5] = function (self, p) [1] = self.shm.redirect_host, [2] = self.shm.redirect_tos_net, [3] = self.shm.redirect_tos_host }) - [self.icmp:code()] + [self.icmp_in:code()] or self.shm.code_not_implemented_errors ) end @@ -196,9 +197,9 @@ function ICMP4:handle_msg (p) if not self.ip4_in:new_from_mem(p.data, p.length) or self.ip4_in:protocol() ~= ICMP4.PROTOCOL or self.ip4_in:is_fragment() - or not self.icmp:new_from_mem(p.data + ipv4:sizeof(), - p.length - ipv4:sizeof()) - or not self.icmp:checksum_check(self:msg_payload(p)) + or not self.icmp_in:new_from_mem(p.data + ipv4:sizeof(), + p.length - ipv4:sizeof()) + or not self.icmp_in:checksum_check(self:msg_payload(p)) then packet.free(p) counter.add(self.shm.rxerrors) @@ -206,7 +207,7 @@ function ICMP4:handle_msg (p) return end -- Ensure we have a handler for ICMP type of packet. - local handler = self.handlers[self.icmp:type()] + local handler = self.handlers[self.icmp_in:type()] if not handler then packet.free(p) counter.add(self.shm.rxerrors) From 1eec1f9b80408d842626bc058954308370108c6c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 10 Jan 2019 15:44:23 +0100 Subject: [PATCH 39/81] vita/imcp: added ICMP6 app --- src/program/vita/icmp.lua | 299 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 299 insertions(+) diff --git a/src/program/vita/icmp.lua b/src/program/vita/icmp.lua index 57e1130fcb..8cbcbdb19b 100644 --- a/src/program/vita/icmp.lua +++ b/src/program/vita/icmp.lua @@ -3,6 +3,7 @@ module(..., package.seeall) local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") local icmp = require("lib.protocol.icmp.header") local counter = require("core.counter") local lib = require("core.lib") @@ -285,3 +286,301 @@ function ICMP4:push () end -- ...remainder is NYI. end + + +ICMP6 = { + name = "ICMP6", + config = { + node_ip6 = ICMP4.config.node_ip4, + nexthop_mtu = ICMP4.config.nexthop_mtu, + max_pps = ICMP4.config.max_pps + }, + shm = { + rxerrors = {counter}, + protocol_errors = {counter}, + type_not_implemented_errors = {counter}, + code_not_implemented_errors = {counter}, + destination_unreachable = {counter}, + net_unreachable = {counter}, + destination_denied = {counter}, + scope_denied = {counter}, + host_unreachable = {counter}, + port_unreachable = {counter}, + source_denied = {counter}, + net_denied = {counter}, + packet_too_big = {counter}, + time_exceeded = {counter}, + transit_hop_limit_exceeded = {counter}, + fragment_reassembly_time_exceeded = {counter}, + parameter_problem = {counter}, + header_field_problem = {counter}, + next_header_problem = {counter}, + option_problem = {counter}, + echo_request = {counter} + }, + PROTOCOL = 58, -- ICMPv6 = 58 + MIN_MTU = 1280, -- IPv6 minimum MTU + payload_offset = ipv6:sizeof() + icmp:sizeof(), + handlers = {} +} + +ICMP6.payload_base_t = ffi.typeof([[struct { + uint8_t unused[4]; + uint8_t excerpt[40]; + } __attribute__((packed))]]) + +ICMP6.packet_too_big_t = ffi.typeof([[struct { + uint32_t nexthop_mtu; + } __attribute__((packed))]]) + +ICMP6.parameter_problem_t = ffi.typeof([[struct { + uint32_t pointer; + } __attribute__((packed))]]) + +ICMP6.payload_t = ffi.typeof([[union { + $ base; + $ packet_too_big; + $ parameter_problem; + } __attribute__((packed))]], + ICMP6.payload_base_t, + ICMP6.packet_too_big_t, + ICMP6.parameter_problem_t) + +ICMP6.payload_ptr_t = ffi.typeof("$ *", ICMP6.payload_t) + +function ICMP6:new (conf) + local o = { + ip6 = ipv6:new({ + hop_limit = 64, + next_header = ICMP6.PROTOCOL, + src = ipv6:pton(conf.node_ip6) + }), + ip6_in = ipv6:new({}), + icmp_in = icmp:new(), + icmp = icmp:new(), + nexthop_mtu = conf.nexthop_mtu or 1280, + nexthop_mtu_configured = conf.nexthop_mtu ~= nil, + throttle = lib.throttle(1), + max_pps = conf.max_pps, + buckets = {rx = 0, tx = 0}, + num_buckets = 2, + logger = nil + } + for bucket, _ in pairs(o.buckets) do + o.buckets[bucket] = floor(o.max_pps / o.num_buckets) + end + return setmetatable(o, {__index=ICMP6}) +end + +function ICMP6:link () + if not self.logger then + self.logger = lib.logger_new({module = self.appname}) + end + -- input link aliases + self.input.packet_too_big = + self.input.packet_too_big or self.input.fragmentation_needed + self.input.transit_hop_limit_exceeded = + self.input.transit_hop_limit_exceeded or self.input.transit_ttl_exceeded + -- a helpful warning + if self.input.packet_too_big and not self.nexthop_mtu_configured then + self.logger:log(("WARNING, 'packet_too_big' or 'fragmentation_needed' link attached but nexthop_mtu not configured, defaulting to %d.") + :format(self.nexthop_mtu)) + end +end + +function ICMP6:log (p) + local payload = ffi.cast(ICMP6.payload_ptr_t, p.data + ICMP6.payload_offset) + local payload_length = p.length - ICMP6.payload_offset + local excerpt = ffi.string( + payload.base.excerpt, + min(payload_length - ffi.sizeof(payload.base.unused), + ffi.sizeof(payload.base.excerpt)) + ) + self.logger:log(("received message from %s [type %d code %d packet %s ...]") + :format(ipv6:ntop(self.ip6_in:src()), + self.icmp_in:type(), + self.icmp_in:code(), + lib.hexdump(excerpt))) +end + +-- Destination Unreachable +ICMP6.handlers[1] = function (self, p) + self:log(p) + packet.free(p) + counter.add(self.shm.destination_unreachable) + counter.add( + ({ [0] = self.shm.net_unreachable, + [1] = self.shm.destination_denied, + [2] = self.shm.scope_denied, + [3] = self.shm.host_unreachable, + [4] = self.shm.port_unreachable, + [5] = self.shm.source_denied, + [6] = self.shm.net_denied }) + [self.icmp_in:code()] + or self.shm.code_not_implemented_errors + ) +end + +-- Packet Too Big +ICMP6.handlers[2] = function (self, p) + self:log(p) + packet.free(p) + counter.add(self.shm.packet_too_big) +end + +-- Time Exceeded +ICMP6.handlers[3] = function (self, p) + self:log(p) + packet.free(p) + counter.add(self.shm.time_exceeded) + counter.add( + ({ [0] = self.shm.transit_hop_limit_exceeded, + [1] = self.shm.fragment_reassembly_time_exceeded }) + [self.icmp_in:code()] + or self.shm.code_not_implemented_errors + ) +end + +-- Parameter Problem +ICMP6.handlers[4] = function (self, p) + self:log(p) + packet.free(p) + counter.add(self.shm.parameter_problem) + counter.add( + ({ [0] = self.shm.header_field_problem, + [1] = self.shm.next_header_problem, + [2] = self.shm.option_problem }) + [self.icmp_in:code()] + or self.shm.code_not_implemented_errors + ) +end + +-- Echo +ICMP6.handlers[128] = function (self, p) + -- Copy payload. + local reply = packet.from_pointer(self:msg_payload(p)) + -- Prepend ICMP and IP headers. + self.ip6:dst(self.ip6_in:src()) + self.ip6:payload_length(reply.length + icmp:sizeof()) + self.icmp:type(129) + self.icmp:code(0) + self.icmp:checksum(reply.data, reply.length, self.ip6) + reply = packet.prepend(reply, self.icmp:header(), icmp:sizeof()) + reply = packet.prepend(reply, self.ip6:header(), ipv6:sizeof()) + -- Send reply. + link.transmit(self.output.output, reply) + packet.free(p) + counter.add(self.shm.echo_request) +end + +function ICMP6:msg_payload (p) + local payload = p.data + ICMP6.payload_offset + local declared_length = self.ip6_in:payload_length() - icmp:sizeof() + local actual_length = p.length - ICMP6.payload_offset + local length = max(0, min(actual_length, declared_length)) + return payload, length +end + +function ICMP6:handle_msg (p) + -- Ensure packet is a valid ICMP message. + local function parse_and_check (p) + if not self.ip6_in:new_from_mem(p.data, p.length) + or self.ip6_in:next_header() ~= ICMP6.PROTOCOL + or not self.icmp_in:new_from_mem(p.data + ipv6:sizeof(), + p.length - ipv6:sizeof()) + then return false end + local payload, length = self:msg_payload(p) + return self.icmp_in:checksum_check(payload, length, self.ip6_in) + end + if not parse_and_check(p) then + packet.free(p) + counter.add(self.shm.rxerrors) + counter.add(self.shm.protocol_errors) + return + end + -- Ensure we have a handler for ICMP type of packet. + local handler = self.handlers[self.icmp_in:type()] + if not handler then + packet.free(p) + counter.add(self.shm.rxerrors) + counter.add(self.shm.type_not_implemented_errors) + return + end + -- Handle incoming message. + handler(self, p) +end + +function ICMP6:send_msg (msgtype, code, p, opt) + opt = opt or {} + local msg = packet.resize(packet.allocate(), ffi.sizeof(ICMP6.payload_t)) + local payload = ffi.cast(ICMP6.payload_ptr_t, msg.data) + -- Set fields, copy packet excerpt. + if opt.nexthop_mtu then + payload.fragmentation_needed.nexthop_mtu = lib.htonl(opt.nexthop_mtu) + end + if opt.pointer then + payload.parameter_problem.pointer = lib.htonl(opt.pointer) + end + local excerpt_max = -- As much of invoking packet as possible without the + -- ICMPv6 packet exceeding the minimum IPv6 MTU + ICMP6.MIN_MTU - ICMP6.payload_offset - ffi.sizeof(payload.base.unused) + local excerpt_length = min(p.length, excerpt_max) + msg = packet.resize(msg, ffi.sizeof(payload.base.unused) + excerpt_length) + ffi.copy(payload.base.excerpt, p.data, excerpt_length) + -- Prepend ICMP and IP headers + assert(self.ip6_in:new_from_mem(p.data, p.length)) + self.ip6:dst(self.ip6_in:src()) + self.ip6:payload_length(msg.length + icmp:sizeof()) + self.icmp:type(msgtype) + self.icmp:code(code) + self.icmp:checksum(msg.data, msg.length, self.ip6) + msg = packet.prepend(msg, self.icmp:header(), icmp:sizeof()) + msg = packet.prepend(msg, self.ip6:header(), ipv6:sizeof()) + -- Send message, free packet. + link.transmit(self.output.output, msg) + packet.free(p) +end + +function ICMP6:rate_limit (bucket) + if self.throttle() then + self.buckets[bucket] = + min(floor(self.buckets[bucket] + self.max_pps / self.num_buckets), + self.max_pps) + end + if self.buckets[bucket] > 0 then + self.buckets[bucket] = self.buckets[bucket] - 1 + return true + end +end + +function ICMP6:push () + -- Process ingoing messages. + while not link.empty(self.input.input) and self:rate_limit('rx') do + self:handle_msg(link.receive(self.input.input)) + end + + -- Process outgoing messages. + if self.input.protocol_unreachable then + while not link.empty(self.input.protocol_unreachable) + and self:rate_limit('tx') do + self:send_msg(4, 1, link.receive(self.input.protocol_unreachable), { + pointer = 6 -- Next Header + }) + end + end + if self.input.packet_too_big then + while not link.empty(self.input.packet_too_big) + and self:rate_limit('tx') do + self:send_msg(2, 0, link.receive(self.input.packet_too_big), { + nexthop_mtu = self.nexthop_mtu + }) + end + end + if self.input.transit_hop_limit_exceeded then + while not link.empty(self.input.transit_hop_limit_exceeded) + and self:rate_limit('tx') do + self:send_msg(3, 0, link.receive(self.input.transit_hop_limit_exceeded)) + end + end + -- ...remainder is NYI. +end From 37a9de5beb10defe92fd33169c454ac81d05ef7e Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 14 Jan 2019 14:30:31 +0100 Subject: [PATCH 40/81] vita/dispatch: support IPv6 in PublicDispatch --- src/program/vita/dispatch.lua | 48 ++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/src/program/vita/dispatch.lua b/src/program/vita/dispatch.lua index dd82428dda..1d2036a462 100644 --- a/src/program/vita/dispatch.lua +++ b/src/program/vita/dispatch.lua @@ -7,6 +7,7 @@ local icmp = require("program.vita.icmp") local counter = require("core.counter") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") local pf_match = require("pf.match") local ffi = require("ffi") @@ -86,7 +87,8 @@ end PublicDispatch = { name = "PublicDispatch", config = { - node_ip4 = {required=true} + node_ip4 = {}, + node_ip6 = {} }, shm = { rxerrors = {counter}, @@ -98,8 +100,10 @@ PublicDispatch = { function PublicDispatch:new (conf) local o = { - p_box = ffi.new("struct packet *[1]"), - dispatch = pf_match.compile(([[match { + p_box = ffi.new("struct packet *[1]") + } + if conf.node_ip4 then + o.dispatch = pf_match.compile(([[match { ip[6:2] & 0x3FFF != 0 => reject_fragment ip proto esp => forward4 ip proto %d => protocol @@ -109,7 +113,17 @@ function PublicDispatch:new (conf) arp => arp otherwise => reject_ethertype }]]):format(exchange.PROTOCOL, conf.node_ip4, conf.node_ip4)) - } + elseif conf.node_ip6 then + o.dispatch = pf_match.compile(([[match { + ip6 proto esp => forward6 + ip6 proto %d => protocol6 + ip6 and icmp6 and (ip6[40] = 135 or ip6[40] = 136) => nd + ip6 dst host %s and icmp6 => icmp6 + ip6 dst host %s => protocol6_unreachable + ip6 => reject_protocol + otherwise => reject_ethertype + }]]):format(exchange.PROTOCOL, conf.node_ip6, conf.node_ip6, conf.node_ip6)) + else error("Need either node_ip4 or node_ip6.") end return setmetatable(o, {__index=PublicDispatch}) end @@ -120,26 +134,52 @@ function PublicDispatch:forward4 () link.transmit(self.output.forward4, p) end +function PublicDispatch:forward6 () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof() + ipv6:sizeof()) + -- NB: Ignore potential differences between IP datagram and Ethernet size + -- since the minimum ESP packet exceeds 60 bytes in payload. + link.transmit(self.output.forward6, p) +end + function PublicDispatch:protocol () local p = packet.shiftleft(self.p_box[0], ethernet:sizeof() + ipv4:sizeof()) link.transmit(self.output.protocol, p) end +function PublicDispatch:protocol6 () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof() + ipv6:sizeof()) + link.transmit(self.output.protocol, p) +end + function PublicDispatch:icmp4 () local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) link.transmit(self.output.icmp4, p) end +function PublicDispatch:icmp6 () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) + link.transmit(self.output.icmp6, p) +end + function PublicDispatch:arp () local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) link.transmit(self.output.arp, p) end +function PublicDispatch:nd () + link.transmit(self.output.nd, self.p_box[0]) +end + function PublicDispatch:protocol4_unreachable () local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) link.transmit(self.output.protocol4_unreachable, p) end +function PublicDispatch:protocol6_unreachable () + local p = packet.shiftleft(self.p_box[0], ethernet:sizeof()) + link.transmit(self.output.protocol6_unreachable, p) +end + function PublicDispatch:reject_fragment () packet.free(self.p_box[0]) counter.add(self.shm.rxerrors) From f81816b8398ef4f4aea1b231d973717c2bed1d97 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 14 Jan 2019 15:12:36 +0100 Subject: [PATCH 41/81] vita/exchange: support IPv6 --- src/program/vita/exchange.lua | 46 ++++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 97d22f502c..764a8878d0 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -133,6 +133,7 @@ local counter = require("core.counter") local header = require("lib.protocol.header") local lib = require("core.lib") local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") local yang = require("lib.yang.yang") local schemata = require("program.vita.schemata") local audit = lib.logger_new({rate=32, module='KeyManager'}) @@ -144,7 +145,8 @@ PROTOCOL = 99 -- “Any private encryption scheme” KeyManager = { name = "KeyManager", config = { - node_ip4 = {required=true}, + node_ip4 = {}, + node_ip6 = {}, routes = {required=true}, sa_db_path = {required=true}, negotiation_ttl = {default=5}, -- default: 5 seconds @@ -173,7 +175,8 @@ end function KeyManager:new (conf) local o = { routes = {}, - ip = ipv4:new({}), + ip4 = ipv4:new({}), + ip6 = ipv6:new({}), transport = Transport.header:new({}), nonce_message = Protocol.nonce_message:new({}), key_message = Protocol.key_message:new({}), @@ -222,9 +225,11 @@ function KeyManager:reconfig (conf) end else -- insert new new route + assert(route.gw_ip4 or route.gw_ip6, "Need either gw_ip4 or gw_ip6") local new_route = { id = id, - gw_ip4n = ipv4:pton(route.gw_ip4), + gw_ip4n = route.gw_ip4 and ipv4:pton(route.gw_ip4), + gw_ip6n = route.gw_ip6 and ipv6:pton(route.gw_ip6), preshared_key = new_key, spi = route.spi, status = status.expired, @@ -246,7 +251,9 @@ function KeyManager:reconfig (conf) end -- switch to new configuration - self.node_ip4n = ipv4:pton(conf.node_ip4) + assert(conf.node_ip4 or conf.node_ip6, "Need either node_ip4 or node_ip6") + self.node_ip4n = conf.node_ip4 and ipv4:pton(conf.node_ip4) + self.node_ip6n = conf.node_ip6 and ipv6:pton(conf.node_ip6) self.routes = new_routes self.sa_db_file = shm.root.."/"..shm.resolve(conf.sa_db_path) self.negotiation_ttl = conf.negotiation_ttl @@ -443,16 +450,27 @@ end function KeyManager:request (route, message) local request = packet.allocate() - self.ip:new({ - total_length = ipv4:sizeof() - + Transport.header:sizeof() - + message:sizeof(), - ttl = 64, - protocol = PROTOCOL, - src = self.node_ip4n, - dst = route.gw_ip4n - }) - packet.append(request, self.ip:header(), ipv4:sizeof()) + if self.node_ip4n then + self.ip4:new({ + total_length = ipv4:sizeof() + + Transport.header:sizeof() + + message:sizeof(), + ttl = 64, + protocol = PROTOCOL, + src = self.node_ip4n, + dst = route.gw_ip4n + }) + packet.append(request, self.ip4:header(), ipv4:sizeof()) + elseif self.node_ip6n then + self.ip6:new({ + payload_length = Transport.header:sizeof() + message:sizeof(), + hop_limit = 64, + next_header = PROTOCOL, + src = self.node_ip6n, + dst = route.gw_ip6n + }) + packet.append(request, self.ip6:header(), ipv6:sizeof()) + else error("BUG") end self.transport:new({ spi = route.spi, From a1b8a8386ee0bfd678f02dda3fb5aa3da9422e8d Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 14 Jan 2019 16:21:58 +0100 Subject: [PATCH 42/81] apps.ipv6.nd_light: patch to prepend eth header to north packets --- src/apps/ipv6/README.md | 19 +++++++++---------- src/apps/ipv6/nd_light.lua | 10 ++-------- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/src/apps/ipv6/README.md b/src/apps/ipv6/README.md index df1636fc90..b4bf5db12b 100644 --- a/src/apps/ipv6/README.md +++ b/src/apps/ipv6/README.md @@ -6,19 +6,18 @@ The `nd_light` app implements a small subset of IPv6 neighbor discovery (RFC4861). It has two duplex ports, `north` and `south`. The `south` port attaches to a network on which neighbor discovery (ND) must be performed. The `north` port attaches to an app that processes IPv6 -packets (including full ethernet frames). Packets transmitted to the +packets (without ethernet headers). Packets transmitted to the `north` port must be wrapped in full Ethernet frames (which may be empty). -The `nd_light` app replies to neighbor solicitations for which it is -configured as a target and performs rudimentary address resolution for -its configured *next-hop* address. If address resolution succeeds, the -Ethernet headers of packets from the `north` port will be overwritten -with headers containing the discovered destination address and the -configured source address before they are transmitted over the `south` -port. All packets from the `north` port are discarded as long as ND has -not yet succeeded. Packets received from the `south` port are transmitted -to the `north` port unaltered. +The `nd_light` app replies to neighbor solicitations for which it is configured +as a target and performs rudimentary address resolution for its configured +*next-hop* address. If address resolution succeeds, the Ethernet headers +containing the discovered destination address and the configured source address +are prepended to packets received from the `north` port before they are +transmitted over the `south` port. All packets from the `north` port are +discarded as long as ND has not yet succeeded. Packets received from the +`south` port are transmitted to the `north` port unaltered. DIAGRAM: nd_light +----------+ diff --git a/src/apps/ipv6/nd_light.lua b/src/apps/ipv6/nd_light.lua index fc1febb6a8..57800d6677 100644 --- a/src/apps/ipv6/nd_light.lua +++ b/src/apps/ipv6/nd_light.lua @@ -62,7 +62,6 @@ nd_light.config = { nd_light.shm = { status = {counter, 2}, -- Link down rxerrors = {counter}, - txerrors = {counter}, txdrop = {counter}, ns_checksum_errors = {counter}, ns_target_address_errors = {counter}, @@ -371,13 +370,8 @@ function nd_light:push () else local p = cache.p p[0] = link.receive(l_in) - if p[0].length >= self._eth_header:sizeof() then - self._eth_header:copy(p[0].data) - link.transmit(l_out, p[0]) - else - packet.free(p[0]) - counter.add(self.shm.txerrors) - end + link.transmit(l_out, packet.prepend( + p[0], self._eth_header:header(), ethernet:sizeof())) end end end From 8c414d18cbc5bbc930f941807fad52bd8dc1a140 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 14 Jan 2019 16:22:32 +0100 Subject: [PATCH 43/81] vita: support tunneling over IPv6 (caveat: weak schema) --- src/program/vita/vita-esp-gateway.yang | 9 ++- src/program/vita/vita.lua | 81 ++++++++++++++++++-------- 2 files changed, 64 insertions(+), 26 deletions(-) diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index ed7fc7e8a4..f04e525903 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -10,9 +10,11 @@ module vita-esp-gateway { // APPLICATION MODEL grouping interface { leaf pci { type pci-address; mandatory true; } - leaf ip4 { type inet:ipv4-address-no-zone; mandatory true; } + leaf ip4 { type inet:ipv4-address-no-zone; } + leaf ip6 { type inet:ipv6-address-no-zone; } leaf mac { type yang:mac-address; } - leaf nexthop-ip4 { type inet:ipv4-address-no-zone; mandatory true; } + leaf nexthop-ip4 { type inet:ipv4-address-no-zone; } + leaf nexthop-ip6 { type inet:ipv6-address-no-zone; } leaf nexthop-mac { type yang:mac-address; } } @@ -25,7 +27,8 @@ module vita-esp-gateway { key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; leaf id { type id; mandatory true; } leaf net-cidr4 { type inet:ipv4-prefix; mandatory true; } - leaf gw-ip4 { type inet:ipv4-address-no-zone; mandatory true; } + leaf gw-ip4 { type inet:ipv4-address-no-zone; } + leaf gw-ip6 { type inet:ipv6-address-no-zone; } leaf spi { type spi; mandatory true; } leaf preshared-key { type key32; mandatory true; } } diff --git a/src/program/vita/vita.lua b/src/program/vita/vita.lua index 5e4e40e21a..6337fa131a 100644 --- a/src/program/vita/vita.lua +++ b/src/program/vita/vita.lua @@ -16,8 +16,11 @@ local icmp = require("program.vita.icmp") local Receiver = require("apps.interlink.receiver") local Transmitter = require("apps.interlink.transmitter") local intel_mp = require("apps.intel_mp.intel_mp") +local nd_light = require("apps.ipv6.nd_light").nd_light +local Join = require("apps.basic.basic_apps").Join local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") local numa = require("lib.numa") local yang = require("lib.yang.yang") local ptree = require("lib.ptree.ptree") @@ -42,17 +45,21 @@ local confspec = { local ifspec = { pci = {required=true}, - ip4 = {required=true}, - nexthop_ip4 = {required=true}, + ip4 = {}, + ip6 = {}, + nexthop_ip4 = {}, + nexthop_ip6 = {}, mac = {}, nexthop_mac = {} } -local function derive_local_unicast_mac (prefix, ip4) +local function derive_local_unicast_mac (prefix, ip) local mac = ffi.new("uint8_t[?]", 6) mac[0] = prefix[1] mac[1] = prefix[2] - ffi.copy(mac+2, ipv4:pton(ip4), 4) + local n, offset = ipv4:pton(ip), 0 + if not n then n, offset = ipv6:pton(ip), 12 end + ffi.copy(mac+2, ffi.cast("uint8_t *", n) + offset, 4) -- First bit = 0 indicates unicast, second bit = 1 means locally -- administered. assert(bit.band(bit.bor(prefix[1], 0x02), 0xFE) == prefix[1], @@ -63,7 +70,8 @@ end local function parse_ifconf (conf, mac_prefix) if not conf then return end conf = lib.parse(conf, ifspec) - conf.mac = conf.mac or derive_local_unicast_mac(mac_prefix, conf.ip4) + conf.mac = conf.mac or + derive_local_unicast_mac(mac_prefix, conf.ip4 or conf.ip6) return conf end @@ -290,25 +298,45 @@ function configure_public_router (conf, append) if not conf.public_interface then return c end config.app(c, "PublicDispatch", dispatch.PublicDispatch, { - node_ip4 = conf.public_interface.ip4 + node_ip4 = conf.public_interface.ip4, + node_ip6 = conf.public_interface.ip6 }) config.app(c, "PublicRouter", route.PublicRouter, { sa = conf.inbound_sa }) - config.app(c, "PublicICMP4", icmp.ICMP4, { - node_ip4 = conf.public_interface.ip4 - }) - config.app(c, "PublicNextHop", nexthop.NextHop4, { - node_mac = conf.public_interface.mac, - node_ip4 = conf.public_interface.ip4, - nexthop_ip4 = conf.public_interface.nexthop_ip4, - nexthop_mac = conf.public_interface.nexthop_mac - }) - config.link(c, "PublicDispatch.forward4 -> PublicRouter.input") - config.link(c, "PublicDispatch.icmp4 -> PublicICMP4.input") - config.link(c, "PublicDispatch.arp -> PublicNextHop.arp") - config.link(c, "PublicDispatch.protocol4_unreachable -> PublicICMP4.protocol_unreachable") - config.link(c, "PublicICMP4.output -> PublicNextHop.icmp4") + if conf.public_interface.ip4 then + config.app(c, "PublicICMP4", icmp.ICMP4, { + node_ip4 = conf.public_interface.ip4 + }) + config.app(c, "PublicNextHop", nexthop.NextHop4, { + node_mac = conf.public_interface.mac, + node_ip4 = conf.public_interface.ip4, + nexthop_ip4 = conf.public_interface.nexthop_ip4, + nexthop_mac = conf.public_interface.nexthop_mac + }) + config.link(c, "PublicDispatch.forward4 -> PublicRouter.input") + config.link(c, "PublicDispatch.icmp4 -> PublicICMP4.input") + config.link(c, "PublicDispatch.arp -> PublicNextHop.arp") + config.link(c, "PublicDispatch.protocol4_unreachable -> PublicICMP4.protocol_unreachable") + config.link(c, "PublicICMP4.output -> PublicNextHop.icmp4") + elseif conf.public_interface.ip6 then + config.app(c, "PublicICMP6", icmp.ICMP6, { + node_ip6 = conf.public_interface.ip6 + }) + config.app(c, "PublicNextHop", Join) + config.app(c, "PublicND", nd_light, { + local_mac = conf.public_interface.mac, + local_ip = conf.public_interface.ip6, + next_hop = conf.public_interface.nexthop_ip6, + remote_mac = conf.public_interface.nexthop_mac + }) + config.link(c, "PublicDispatch.forward6 -> PublicRouter.input") + config.link(c, "PublicDispatch.icmp6 -> PublicICMP6.input") + config.link(c, "PublicDispatch.nd -> PublicND.south") + config.link(c, "PublicDispatch.protocol6_unreachable -> PublicICMP6.protocol_unreachable") + config.link(c, "PublicICMP6.output -> PublicNextHop.icmp6") + config.link(c, "PublicNextHop.output -> PublicND.north") + else error("Need either public_interface.ip4 or public_interface.ip6") end if not conf.data_plane then config.app(c, "Protocol_in_Tx", Transmitter, "Protocol_in") @@ -322,8 +350,13 @@ function configure_public_router (conf, append) local ESP_out = "ESP_"..id.."_out" local Tunnel = "Tunnel_"..id config.app(c, ESP_out.."_Rx", Receiver, ESP_out) - config.app(c, Tunnel, tunnel.Tunnel4, - {src=conf.public_interface.ip4, dst=route.gw_ip4}) + if route.gw_ip4 then + config.app(c, Tunnel, tunnel.Tunnel4, + {src=conf.public_interface.ip4, dst=route.gw_ip4}) + elseif route.gw_ip6 then + config.app(c, Tunnel, tunnel.Tunnel6, + {src=conf.public_interface.ip6, dst=route.gw_ip6}) + else error("Need either route.gw_ip4 or route.gw_ip6") end config.link(c, ESP_out.."_Rx.output -> "..Tunnel..".input") config.link(c, Tunnel..".output -> "..public_out) end @@ -337,7 +370,8 @@ function configure_public_router (conf, append) local public_links = { input = "PublicDispatch.input", - output = "PublicNextHop.output" + output = (conf.public_interface.ip4 and "PublicNextHop.output") + or "PublicND.south" } return c, public_links @@ -390,6 +424,7 @@ function configure_exchange (conf, append) config.app(c, "KeyExchange", exchange.KeyManager, { node_ip4 = conf.public_interface.ip4, + node_ip6 = conf.public_interface.ip6, routes = conf.route, sa_db_path = sa_db_path, negotiation_ttl = conf.negotiation_ttl, From c6652e82ede6c8b1262fa533f9bbf663587d4f31 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 14 Jan 2019 16:22:40 +0100 Subject: [PATCH 44/81] Add program/vita/test6.snabb to benchmark tunneling over IPv6 --- .travis.yml | 1 + src/program/vita/test.lua | 22 +++++++++++++++++++--- src/program/vita/test6.snabb | 14 ++++++++++++++ 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100755 src/program/vita/test6.snabb diff --git a/.travis.yml b/.travis.yml index ae6ccc4223..fc706beccb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,3 +13,4 @@ script: - (cd src && sudo ./snabb snsh -t program.vita.exchange) - (cd src && sudo program/vita/conftest.snabb) - (cd src && sudo program/vita/test.snabb IMIX 1e6 3) + - (cd src && sudo program/vita/test6.snabb IMIX 1e6 3) diff --git a/src/program/vita/test.lua b/src/program/vita/test.lua index 7bffdef583..77be094251 100644 --- a/src/program/vita/test.lua +++ b/src/program/vita/test.lua @@ -107,11 +107,14 @@ end -- Testing setups for Vita -- Run Vita in software benchmark mode. -function run_softbench (pktsize, npackets, nroutes, cpuspec) +function run_softbench (pktsize, npackets, nroutes, cpuspec, use_v6) local testconf = { private_interface = { nexthop_ip4 = private_interface_defaults.ip4.default }, + public_interface = { + ip6 = use_v6 and public_interface_defaults6.ip6.default + }, packet_size = pktsize, nroutes = nroutes, negotiation_ttl = nroutes, @@ -213,6 +216,13 @@ public_interface_defaults = { nexthop_ip4 = {default="172.16.0.10"}, nexthop_mac = {} } +public_interface_defaults6 = { + pci = {default="00:00.0"}, + mac = {}, + ip6 = {default="172:16:0::10"}, + nexthop_ip6 = {default="172:16:0::10"}, + nexthop_mac = {} +} traffic_templates = { -- Internet Mix, see https://en.wikipedia.org/wiki/Internet_Mix @@ -223,8 +233,13 @@ local function parse_gentestconf (conf) conf = lib.parse(conf, defaults) conf.private_interface = lib.parse(conf.private_interface, private_interface_defaults) - conf.public_interface = lib.parse(conf.public_interface, - public_interface_defaults) + if conf.public_interface and conf.public_interface.ip6 then + conf.public_interface = lib.parse(conf.public_interface, + public_interface_defaults6) + else + conf.public_interface = lib.parse(conf.public_interface, + public_interface_defaults) + end assert(conf.nroutes >= 0 and conf.nroutes <= 255, "Invalid number of routes: "..conf.nroutes) return conf @@ -273,6 +288,7 @@ function gen_configuration (conf) cfg.route["test"..route] = { net_cidr4 = conf.route_prefix.."."..route..".0/24", gw_ip4 = conf.public_interface.nexthop_ip4, + gw_ip6 = conf.public_interface.nexthop_ip6, preshared_key = ("%064x"):format(route), spi = 1000+route } diff --git a/src/program/vita/test6.snabb b/src/program/vita/test6.snabb new file mode 100755 index 0000000000..d35dd3143e --- /dev/null +++ b/src/program/vita/test6.snabb @@ -0,0 +1,14 @@ +#!snabb snsh + +-- Use of this source code is governed by the GNU AGPL license; see COPYING. + +local vita_test = require("program.vita.test") + +-- IPv6 version of test.snabb + +local pktsize = tonumber(main.parameters[1]) or main.parameters[1] +local npackets = tonumber(main.parameters[2]) or 10e6 +local nroutes = tonumber(main.parameters[3]) +local cpuspec = main.parameters[4] + +vita_test.run_softbench(pktsize, npackets, nroutes, cpuspec, 'IPv6') From e996b68164d33d2bb252d2e20ba8c12e7aa1f894 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 15 Jan 2019 18:31:52 +0100 Subject: [PATCH 45/81] apps.ipv6.nd_light: patch to hold packets while link is down --- src/apps/ipv6/nd_light.lua | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/src/apps/ipv6/nd_light.lua b/src/apps/ipv6/nd_light.lua index 57800d6677..45006addf6 100644 --- a/src/apps/ipv6/nd_light.lua +++ b/src/apps/ipv6/nd_light.lua @@ -62,7 +62,6 @@ nd_light.config = { nd_light.shm = { status = {counter, 2}, -- Link down rxerrors = {counter}, - txdrop = {counter}, ns_checksum_errors = {counter}, ns_target_address_errors = {counter}, na_duplicate_errors = {counter}, @@ -361,18 +360,16 @@ function nd_light:push () l_in = self.input.north l_out = self.output.south + -- Do not forward packets from north until ND for the next-hop has + -- completed. + if not self._eth_header then + return + end while not link.empty(l_in) do - if not self._eth_header then - -- Drop packets until ND for the next-hop - -- has completed. - packet.free(link.receive(l_in)) - counter.add(self.shm.txdrop) - else - local p = cache.p - p[0] = link.receive(l_in) - link.transmit(l_out, packet.prepend( - p[0], self._eth_header:header(), ethernet:sizeof())) - end + local p = packet.prepend( + link.receive(l_in), self._eth_header:header(), ethernet:sizeof() + ) + link.transmit(l_out, p) end end From 6448af04c967454793f5abcc248928486440671f Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 15 Jan 2019 18:32:30 +0100 Subject: [PATCH 46/81] lib.protocol.ipv6: initialize payload_length in :new() --- src/lib/protocol/ipv6.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib/protocol/ipv6.lua b/src/lib/protocol/ipv6.lua index f1edcdde09..91ec1dde5d 100644 --- a/src/lib/protocol/ipv6.lua +++ b/src/lib/protocol/ipv6.lua @@ -66,6 +66,7 @@ function ipv6:new (config) o:version(6) o:traffic_class(config.traffic_class or defaults.traffic_class) o:flow_label(config.flow_label or defaults.flow_label) + o:payload_length(config.payload_length or 0) o:next_header(config.next_header or defaults.next_header) o:hop_limit(config.hop_limit or defaults.hop_limit) o:src(config.src) From 54bd2576f23008e2f2af892b3d4134ac02901b3b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 16 Jan 2019 10:44:11 +0100 Subject: [PATCH 47/81] vita: add selftest6.snabb to exercise IPv6 paths --- .travis.yml | 1 + src/program/vita/selftest6-pcaps.snabb | 256 ++++++++++++++++++++ src/program/vita/selftest6-private-out.pcap | Bin 0 -> 24 bytes src/program/vita/selftest6-public-in.pcap | Bin 0 -> 5481 bytes src/program/vita/selftest6-public-out.pcap | Bin 0 -> 1973 bytes src/program/vita/selftest6.snabb | 209 ++++++++++++++++ 6 files changed, 466 insertions(+) create mode 100755 src/program/vita/selftest6-pcaps.snabb create mode 100644 src/program/vita/selftest6-private-out.pcap create mode 100644 src/program/vita/selftest6-public-in.pcap create mode 100644 src/program/vita/selftest6-public-out.pcap create mode 100755 src/program/vita/selftest6.snabb diff --git a/.travis.yml b/.travis.yml index fc706beccb..2a088d9496 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ before_script: script: - make -j - (cd src && sudo program/vita/selftest.snabb) + - (cd src && sudo program/vita/selftest6.snabb) - (cd src && sudo ./snabb snsh -t program.vita.exchange) - (cd src && sudo program/vita/conftest.snabb) - (cd src && sudo program/vita/test.snabb IMIX 1e6 3) diff --git a/src/program/vita/selftest6-pcaps.snabb b/src/program/vita/selftest6-pcaps.snabb new file mode 100755 index 0000000000..1f528ff4a6 --- /dev/null +++ b/src/program/vita/selftest6-pcaps.snabb @@ -0,0 +1,256 @@ +#!snabb snsh + +-- Use of this source code is governed by the GNU AGPL license; see COPYING. + +local pcap = require("lib.pcap.pcap") +local ethernet = require("lib.protocol.ethernet") +local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") +local icmp = require("lib.protocol.icmp.header") +local esp = require("lib.ipsec.esp") +local datagram = require("lib.protocol.datagram") +local ffi = require("ffi") + +-- Synopsis: +-- +-- sudo selftest6-pcaps.snabb +-- +-- Source selftest-*-in.pcap with packets that exercise various corner cases in +-- Vita. Anything that’s not the happy path. IPv6 version. + +PcapLog = {} + +function PcapLog:new (filename) + local o = {} + o.file = io.open(filename, "w") + pcap.write_file_header(o.file) + return setmetatable(o, {__index=PcapLog}) +end + +function PcapLog:write (p) + pcap.write_record(self.file, p.data, p.length) +end + +local public = PcapLog:new("program/vita/selftest6-public-in.pcap") + +local private_src = ipv4:pton("192.168.0.1") +local private_dst = ipv4:pton("192.168.10.1") +local public_src = ipv6:pton("203:0:0::1") +local public_dst = ipv6:pton("203:0:113::1") +local remote_dst = ipv4:pton("192.168.10.2") + +function icmp4 (conf) + local payload = conf.payload or packet.from_string("0000Hello, World!") + local length = conf.payload_length or payload.length + local msg = datagram:new(payload) + local icm = icmp:new(conf.type, conf.code) + icm:checksum(msg:payload(), conf.payload_length or payload.length) + icm:header().checksum = conf.icmp_checksum or icm:header().checksum + msg:push(icm) + local ip4 = ipv4:new{ + flags = conf.flags, + frag_off = conf.frag_off, + total_length = ipv4:sizeof() + icmp:sizeof() + length, + ttl = conf.ttl or 64, + protocol = conf.protocol or 1, + src = conf.src, + dst = conf.dst + } + ip4:header().checksum = conf.ipv4_checksum or ip4:header().checksum + msg:push(ip4) + msg:push(ethernet:new{type=0x0800}) + return msg:packet() +end + +function icmp6 (conf) + local payload = conf.payload or packet.from_string("0000Hello, World!") + local length = conf.payload_length or payload.length + local msg = datagram:new(payload) + local icm = icmp:new(conf.type, conf.code) + local ip6 = ipv6:new{ + payload_length = icmp:sizeof() + length, + hop_limit = conf.ttl or 64, + next_header = conf.protocol or 58, + src = conf.src, + dst = conf.dst + } + icm:checksum(msg:payload(), conf.payload_length or payload.length, ip6) + icm:header().checksum = conf.icmp_checksum or icm:header().checksum + msg:push(icm) + msg:push(ip6) + msg:push(ethernet:new{type=0x86dd}) + return msg:packet() +end + +local sa = esp.encrypt:new{ + aead = "aes-gcm-16-icv", + spi = 1001, + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00" +} + +local sa_bad_spi = esp.encrypt:new{ + aead = "aes-gcm-16-icv", + spi = 0, + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00" +} + +local sa_replay = esp.encrypt:new{ + aead = "aes-gcm-16-icv", + spi = 1001, + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00" +} + +function encap6 (payload, conf) + payload = (conf.sa or sa):encapsulate_tunnel( + packet.shiftleft(payload, ethernet:sizeof()), + conf.nh or 4 + ) + local d = datagram:new(payload) + d:push(ipv6:new{ + payload_length = (conf.length or payload.length), + hop_limit = conf.ttl or 64, + next_header = esp.PROTOCOL, + src = conf.src, + dst = conf.dst + }) + d:push(ethernet:new{type=0x86dd}) + return d:packet() +end + +-- Echo request +public:write(icmp6{ + type = 128, + src = public_src, + dst = public_dst +}) +-- Broken echo request (too short) +public:write(icmp6{ + type = 128, + src = public_src, + dst = public_dst, + payload_length = 10000 +}) +-- Broken echo request (too long) +public:write(icmp6{ + type = 128, + src = public_src, + dst = public_dst, + payload_length = 4 +}) +-- Echo reply +public:write(icmp6{ + type = 129, + src = public_src, + dst = public_dst +}) +-- Encapsulated echo request +public:write(encap6( + icmp4{ + type = 8, + src = remote_dst, + dst = private_dst + }, + { + src = public_src, + dst = public_dst + } +)) +-- Unreachable protocol (private/public/inbound) +public:write(icmp6{ + protocol = 42, + src = public_src, + dst = public_dst, + payload = packet.from_string(("x"):rep(2000)) +}) +public:write(encap6( + icmp4{ + protocol = 42, + src = remote_dst, + dst = private_dst + }, + { + src = public_src, + dst = public_dst + } +)) +-- TTL expired (private/inbound) +public:write(encap6( + icmp4{ + type = 8, + src = remote_dst, + dst = remote_dst, + ttl = 0 + }, + { + src = public_src, + dst = public_dst + } +)) +-- Bogus SPI +public:write(encap6( + icmp4{ + type = 8, + src = remote_dst, + dst = private_dst + }, + { + sa = sa_bad_spi, + src = public_src, + dst = public_dst + } +)) +-- Bogus SeqNo +public:write(encap6( + icmp4{ + type = 8, + src = remote_dst, + dst = private_dst + }, + { + sa = sa_replay, + src = public_src, + dst = public_dst + } +)) +-- Bogus NextHeader +public:write(encap6( + icmp4{ + type = 8, + src = remote_dst, + dst = private_dst + }, + { + src = public_src, + dst = public_dst, + nh = 42 + } +)) +-- Bogus checksums +public:write(icmp6{ + type = 128, + src = public_src, + dst = public_dst, + icmp_checksum = 42 +}) +-- Various ICMPv6 messages +local payload = packet.from_string("....012345678901234567890123456789012345") +for _, msgtype in ipairs({ + {type=1, codes={0,1,2,3,4,5,6,100}}, + {type=2, codes={0,100}}, + {type=3, codes={0,1,100}}, + {type=4, codes={0,1,2,100}}, + {type=100, codes={0}} +}) do + for _, code in ipairs(msgtype.codes) do + public:write(icmp6{ + payload = packet.clone(payload), + type = msgtype.type, + code = code, + src = public_src, + dst = public_dst + }) + end +end diff --git a/src/program/vita/selftest6-private-out.pcap b/src/program/vita/selftest6-private-out.pcap new file mode 100644 index 0000000000000000000000000000000000000000..a324304509bdd12aef4a69458a409cefefafe614 GIT binary patch literal 24 Ycmca|c+)~A1{MYw`2U}Qff2|708wKE@Bjb+ literal 0 HcmV?d00001 diff --git a/src/program/vita/selftest6-public-in.pcap b/src/program/vita/selftest6-public-in.pcap new file mode 100644 index 0000000000000000000000000000000000000000..cd1ed1648147607b13457306e0b486e6e194082a GIT binary patch literal 5481 zcmca|c+)~A1{MYw`2U}Qff2?5(%w*vB*f5mHvz;IwQ^u$Miysa1o0V#;bM#p3^zFp zfWRX)CnsM=Aw0h*Cq)q^joWB-5n_zq#7TzH3>?H5E#*Xx(L@ADBf|~c0n!8v84yMW z(6II~A{f>T49qWqIvHWHH$y$7dDE*2lV`oTtZ*dP+E#q}q5Yn^eYd7hd&|9)>F)y; z$LY4mX4HPpOq?ZZbSGz7>gKmSl3P|Ea%=rR5#|Ce4h9AgMg|ZUv0u?5!bLF-6{BD@ z1V%$(Gz3ONU^E0qLtw~<0IVfIm9`2K6Vy26fIX*flqny{-Ch0v(u&i&%3pmJUZcUk ztzwtf>IM;(Jt%fg`r*C}M{U$_J7cs*cw84?p?1x)e zC;bs)30TBiee>PEOWpRhKBDelWrYthsM&Z_-^M`fa46V#*30Ho- z>;68OS}p>cLsolkY3RF-H{L29I_sMMdeq&&KrI(hzlBMabi@MdzibM)z}?MvHYzj3 zbm0QWM)&mGq-FIECsu#klDFvFGZTBW4bC&fe$1V@Q|VFs6o-ahpNU4o&wiZT@rC#N zTv!<4?v@bO^|++5y7I0NTLUA>1utWcZ$?2LuL&M#d(lX66=_xLGh8D72CB zdnTQ2Wcr>#XB(Nnr_I#F*~wC-YmT! zZms#W-sIbPpO4+NlP>w__u^xUhnD@!%pXuA7$yN3AdJODF+{s4Ee&eim$K4bld_b| zO|5MFg+b=+=c5b>vZm)w4T+wcINjc`M22#x0X+0 z^@)Eewt(kb(rmrW-PxC>|5?=bsa`oS&;GEM(7)4lJS`t;xaP_`ynLjgl#{lBCxs!~ J=j6I~Cjjw2)JXsU literal 0 HcmV?d00001 diff --git a/src/program/vita/selftest6.snabb b/src/program/vita/selftest6.snabb new file mode 100755 index 0000000000..1ff0ea4879 --- /dev/null +++ b/src/program/vita/selftest6.snabb @@ -0,0 +1,209 @@ +#!snabb snsh + +-- Use of this source code is governed by the GNU AGPL license; see COPYING. + +local vita = require("program.vita.vita") +local ARP = require("apps.ipv4.arp").ARP +local ethernet = require("lib.protocol.ethernet") +local ipv4 = require("lib.protocol.ipv4") +local ipv6 = require("lib.protocol.ipv6") +local basic_apps = require("apps.basic.basic_apps") +local pcap = require("apps.pcap.pcap") +local filter = require("apps.packet_filter.pcap_filter") +local match = require("apps.test.match") +local counter = require("core.counter") +local shm = require("core.shm") +local cltable = require("lib.cltable") +local ffi = require("ffi") + +-- Synopsis: +-- +-- sudo selftest6.snabb [regenerate] +-- +-- Basic event-sourced (selftest-*-in.pcap) test that exercises various +-- non-happy paths of Vita. Regenerates reference outputs (selftest-*-out.pcap) +-- when called with an argument. IPv6 version. +-- +-- TODO: doesn’t exercise KeyManager yet. + +local regenerate_pcaps = main.parameters[1] + +local cfg = { + private_interface = { + pci = "00:00.0", + ip4 = "192.168.10.1", + nexthop_ip4 = "192.168.0.1", + mac = "52:54:00:00:00:00" + }, + public_interface = { + pci = "00:00.0", + ip6 = "203:0:113::1", + nexthop_ip6 = "203:0:0::1", + mac = "52:54:00:00:00:FF", + nexthop_mac = "52:54:00:00:00:FE" + }, + mtu = 500, + route = { + loopback = { + net_cidr4 = "192.168.10.0/24", + gw_ip6 = "203:0:113::1", + preshared_key = string.rep("00", 32), + spi = 1001 + } + }, + outbound_sa = { + [1001] = { + route = "loopback", + aead = "aes-gcm-16-icv", + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00" + } + }, + inbound_sa = { + [1001] = { + route = "loopback", + aead = "aes-gcm-16-icv", + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00", + auditing = true + } + } +} + +local c = config.new() + +-- Configure Vita app network (all-in-one process.) +local _, private = vita.configure_private_router(cfg, c) +local _, public = vita.configure_public_router(cfg, c) +vita.configure_esp(cfg, c) +vita.configure_dsp(cfg, c) + +-- Add ARP resolvers. +config.app(c, "private_arp", ARP, { + self_mac = ethernet:pton("52:54:00:00:00:00"), + self_ip = ipv4:pton("192.168.0.1"), + next_ip = ipv4:pton("192.168.10.1") +}) +config.link(c, "private_arp.south -> "..private.input) +config.link(c, private.output.." -> private_arp.south") + +-- Loopback ESP traffic. +config.app(c, "public_out", basic_apps.Tee) +config.app(c, "join", basic_apps.Join) +config.app(c, "filter", filter.PcapFilter, {filter="ip proto esp"}) +config.link(c, public.output.." -> public_out.input") +config.link(c, "public_out.loopback -> filter.input") +config.link(c, "filter.output -> join.loopback") +config.link(c, "join.output -> "..public.input) + +-- Add PCAP sources and sinks. +--config.app(c, "private_pcap_in", pcap.PcapReader, +-- "program/vita/selftest-private-in.pcap") +config.app(c, "private_pcap_in", basic_apps.Sink) -- null +config.app(c, "public_pcap_in", pcap.PcapReader, + "program/vita/selftest6-public-in.pcap") +config.link(c, "private_pcap_in.output -> private_arp.north") +config.link(c, "public_pcap_in.output -> join.input") +if regenerate_pcaps then + -- Regenerate reference outputs. + config.app(c, "private_pcap_out", pcap.PcapWriter, + "program/vita/selftest6-private-out.pcap") + config.app(c, "public_pcap_out", pcap.PcapWriter, + "program/vita/selftest6-public-out.pcap") + config.link(c, "private_arp.north -> private_pcap_out.input") + config.link(c, "public_out.output -> public_pcap_out.input") +else + -- Match reference outputs. + config.app(c, "private_pcap_out", pcap.PcapReader, + "program/vita/selftest6-private-out.pcap") + config.app(c, "public_pcap_out", pcap.PcapReader, + "program/vita/selftest6-public-out.pcap") + config.app(c, "match_private", match.Match, {}) + config.link(c, "private_pcap_out.output -> match_private.comparator") + config.link(c, "private_arp.north -> match_private.rx") + config.app(c, "match_public", match.Match, {}) + config.link(c, "public_pcap_out.output -> match_public.comparator") + config.link(c, "public_out.output -> match_public.rx") +end + +engine.configure(c) + +-- Hack to avoid ESP seq# reuse because of packets from public_in.pcap +engine.app_table.ESP_loopback.sa.seq.no = 100 + +-- Run engine until its idle (all packets have been processed). +local last_frees = counter.read(engine.frees) +local function is_idle () + if counter.read(engine.frees) == last_frees then return true + else last_frees = counter.read(engine.frees) end +end +engine.main({done=is_idle}) + +if regenerate_pcaps then + -- Print final statistics. + engine.report_links() + for appname, app in pairs(engine.app_table) do + if app.shm then + print() + print(appname) + for name, _ in pairs(app.shm.specs) do + local value = counter.read(app.shm[name]) + if value > 0 then + print(("%00d %s"):format(tonumber(value), name)) + end + end + end + end +else + -- Assert application state is as expected. + if #engine.app_table.match_private:errors() > 0 then + engine.app_table.match_private:report() + main.exit(1) + end + if #engine.app_table.match_public:errors() > 0 then + engine.app_table.match_public:report() + main.exit(1) + end + for app, counters in pairs{ + PublicRouter = { + route_errors = 1, -- Bogus SPI + }, + DSP_loopback_1001 = { -- Bogus SeqNo, Bogus NextHeader + rxerrors = 2, + protocol_errors = 1, + decrypt_errors = 1 + }, + PublicICMP6 = { + echo_request = 2, -- Echo request, Broken echo request (too long) + destination_unreachable = 8,-- Standard codes, Code 100 + net_unreachable = 1, + destination_denied = 1, + scope_denied = 1, + host_unreachable = 1, + port_unreachable = 1, + source_denied = 1, + net_denied = 1, + packet_too_big = 2, -- Code 0, Code 100 + time_exceeded = 3, -- Standard codes, Code 100 + fragment_reassembly_time_exceeded = 1, + transit_hop_limit_exceeded = 1, + parameter_problem = 4, -- Standard codes, Code 100 + next_header_problem = 1, + header_field_problem = 1, + option_problem = 1, + rxerrors = 4, + protocol_errors = 2, -- Broken echo request (too short), Bogus checksum + type_not_implemented_errors = 2, -- Echo reply, Type 100 + code_not_implemented_errors = 3 -- Code 100 + }, + InboundICMP4 = { -- Encapsulated echo request + echo_request = 1 + } + } do + for name, should in pairs(counters) do + local actual = tonumber(counter.read(engine.app_table[app].shm[name])) + assert(should == actual, + name.." should be "..should.." but is "..actual) + end + end +end From 56d4cb0d619ae1c554cf47d5ae0461207125321b Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 17 Jan 2019 11:15:06 +0100 Subject: [PATCH 48/81] vita/schemata: extend application state to cover public-imcp6 stats --- src/program/vita/schemata.lua | 1 + src/program/vita/vita-esp-gateway.yang | 130 +++++++++++++++++++++++++ 2 files changed, 131 insertions(+) diff --git a/src/program/vita/schemata.lua b/src/program/vita/schemata.lua index 541f43d818..431ff542d1 100644 --- a/src/program/vita/schemata.lua +++ b/src/program/vita/schemata.lua @@ -75,6 +75,7 @@ function compute_state_reader (schema_name) PublicRouter = "public-router", PrivateICMP4 = "private-icmp4", PublicICMP4 = "public-icmp4", + PublicICMP6 = "public-icmp6", InboundICMP4 = "inbound-icmp4", KeyManager = "key-manager" } diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index f04e525903..9221267374 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -414,6 +414,131 @@ module vita-esp-gateway { } } + grouping icmp6-state { + leaf rxerrors { + type yang:zero-based-counter64; + description + "Total ICMPv6 messages that were dropped because they were invalid."; + } + leaf protocol-errors { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages that were dropped because their checksum + was invalid, or they were IP fragments."; + } + leaf type-not-implemented-errors { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages received that had an unrecognized type."; + } + leaf code-not-implemented-errors { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages received that had an unrecognized code."; + } + leaf destination-unreachable { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages received of type + 'destination unreachable'."; + } + leaf net-unreachable { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'no route to destination'."; + } + leaf destination-denied { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'communication with destination administratively prohibited'."; + } + leaf scope-denied { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'beyond scope of source address'."; + } + leaf host-unreachable { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'address unreachable'."; + } + leaf port-unreachable { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'port unreachable'."; + } + leaf source-denied { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'source address failed ingress/egress policy'."; + } + leaf net-denied { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'destination unreachable' messages received with + code 'reject route to destination'."; + } + leaf packet-too-big { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages received of type + 'packet too big'."; + } + leaf time-exceeded { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages received of type + 'time exceeded'."; + } + leaf transit-hop-limit-exceeded { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'time exceeded' messages received with code + 'hop limit exceeded in transit'."; + } + leaf fragment-reassembly-time-exceeded { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'time exceeded' messages received with code + 'fragment reassembly time exceeded'."; + } + leaf parameter-problem { + type yang:zero-based-counter64; + description + "Count of ICMPv6 messages received of type + 'parameter problem'."; + } + leaf header-field-problem { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'parameter exceeded' messages received with code + 'erroneous header field encountered'."; + } + leaf next-header-problem { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'parameter exceeded' messages received with code + 'unrecognized next header type encountered'."; + } + leaf option-problem { + type yang:zero-based-counter64; + description + "Count of ICMPv6 'parameter exceeded' messages received with code + 'unrecognized IPv6 option encountered'."; + } + leaf echo-request { + type yang:zero-based-counter64; + description + "Count of ICMPv6 echo requests handled."; + } + } + container private-icmp4 { description "ICMPv4 events on the private interface."; uses icmp4-state; @@ -424,6 +549,11 @@ module vita-esp-gateway { uses icmp4-state; } + container public-icmp6 { + description "ICMPv6 events on the public interface."; + uses icmp6-state; + } + container inbound-icmp4 { description "ICMPv4 events triggered by encapsulated messages on the public interface."; From a58e6cfb91f828b43c23cc1b47204e1c66073b0c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Thu, 17 Jan 2019 16:22:22 +0100 Subject: [PATCH 49/81] vita/schemata: encode valid configurations in YANG schema This fixes the caveat in 8c414d18c. --- src/program/vita/README.config | 52 ++++++----- src/program/vita/conftest.snabb | 16 ++-- src/program/vita/selftest.snabb | 6 +- src/program/vita/selftest6.snabb | 6 +- src/program/vita/test.lua | 73 +++++++++------ src/program/vita/vita-esp-gateway.yang | 47 +++++++--- src/program/vita/vita-gentest.yang | 12 ++- src/program/vita/vita-loadtest.md | 14 +-- src/program/vita/vita.lua | 123 ++++++++++++++----------- 9 files changed, 207 insertions(+), 142 deletions(-) diff --git a/src/program/vita/README.config b/src/program/vita/README.config index b6b06d8525..3f689c884d 100644 --- a/src/program/vita/README.config +++ b/src/program/vita/README.config @@ -1,9 +1,10 @@ CONFIGURATION SYNTAX configuration:= - private-interface { } - public-interface { } - [ route { } ]* + private-interface4 { } + public-interface4|public-interface6 { } + [ route4|route46 { } ]* + [ mtu ; ] [ negotiation-ttl ; ] [ sa-ttl ; ] [ data-pane ; ] @@ -12,18 +13,17 @@ CONFIGURATION SYNTAX interface:= pci ; - ip4 ; - nexthop-ip4 ; + ip4|ip6 ; + nexthop-ip4 ; [ mac ; ] [ nexthop-mac ; ] route:= id ; - net-cidr4 ; - gw-ip4 ; + net-cidr4 ; + gw-ip4|gw-ip6 ; preshared-key ; spi ; - [ private-mtu ; ] sa:= route ; @@ -34,19 +34,19 @@ CONFIGURATION SYNTAX NOTES - A Vita configuration defines the a private and a public network interface for - use by the gateway, a set of routes to other Vita nodes, and the gateway’s - MTU among other miscellaneous operational parameters. + A Vita configuration defines a private and a public network interface for use + by the gateway, a set of routes to other Vita nodes, and the gateway’s MTU + among other miscellaneous operational parameters. Each interface is identified by a Linux PCI bus address, and assigned an - IPv4 address as well as the IPv4 address of the next hop through which all + IP address as well as the IP address of the next hop through which all packets leaving the interface will be routed. The IP addresses assigned to both interfaces may be the same. If the Ethernet (MAC) address of the interface is not explicitly specified, - it is automatically derived from its assigned IPv4 address by prepending the - bytes 2a:bb: or 3a:bb: depending on whether it is the private or public - interface respectively. + it is automatically derived from its assigned IP address by prepending the + bytes 2a:bb: or 3a:bb: to its least significant 32 bits depending on whether + it is the private or public interface respectively. Given that the NIC driver in use supports VMDq, it is possible to pass the same PCI bus address for both interfaces to have them share a single physical @@ -58,11 +58,10 @@ NOTES Each route is given a unique, human readable identifier that must satisfy the pattern [a-zA-Z0-9_]+ (i.e., one or more alphanumeric ASCII and underscore - characters.) The route’s destination IPv4 subnet and gateway are specified - with an IPv4 prefix in CIDR notation, and an IPv4 address respectively. - Packets that arrive on the private interface and which are destined to a - route’s specified IPv4 subnetwork are tunneled to the destination gateway of - that route. + characters.) The route’s destination IP subnet and gateway are specified with + an IP prefix in CIDR notation, and an IP address respectively. Packets that + arrive on the private interface and which are destined to a route’s specified + IP subnetwork are tunneled to the destination gateway of that route. For authentication, each route is assigned a unique, pre-shared 256-bit key, encoded as a hexadecimal string (two digits for each octet, most significant @@ -94,21 +93,26 @@ NOTES 32-bit salt respectively, encoded as hexadecimal strings (two digits for each octet, most significant digit first). +REFERENCES + + Refer to vita-esp-gateway.yang for a formal YANG model that defines Vita’s + configuration as well as the statistics counters queryable at runtime. + EXAMPLE - private-interface { + private-interface4 { pci 0c:00.0; ip4 172.16.1.10; nexthop-ip4 172.16.1.1; } - public-interface { + public-interface4 { pci 0c:00.1; ip4 203.0.113.10; nexthop-ip4 172.16.1.1; } - route { + route4 { id site2; net-cidr4 172.16.2.0/24; gw-ip4 203.0.113.2; @@ -116,7 +120,7 @@ EXAMPLE spi 1001; } - route { + route4 { id site3; net-cidr4 172.16.3.0/24; gw-ip4 203.0.113.3; diff --git a/src/program/vita/conftest.snabb b/src/program/vita/conftest.snabb index fe1b97ab3d..ed43642419 100755 --- a/src/program/vita/conftest.snabb +++ b/src/program/vita/conftest.snabb @@ -26,22 +26,22 @@ test_conf(vita_test.gen_configuration({nroutes=1})) print("Change route key...") local conf = vita_test.gen_configuration({nroutes=1}) -conf.route.test1.preshared_key = string.rep("FF", 32) +conf.route4.test1.preshared_key = string.rep("FF", 32) test_conf(conf) print("Change route gw_ip4...") -conf.route.test1.gw_ip4 = "172.17.1.10" +conf.route4.test1.gw_ip4 = "172.17.1.10" test_conf(conf) print("Change public_ip4 and route net_cidr4...") -conf.public_interface.ip4 = "172.16.0.11" -conf.public_interface.nexthop_ip4 = conf.public_interface.ip4 -conf.route.test1.net_cidr4 = "172.16.0.0/16" +conf.public_interface4.ip4 = "172.16.0.11" +conf.public_interface4.nexthop_ip4 = conf.public_interface4.ip4 +conf.route4.test1.net_cidr4 = "172.16.0.0/16" test_conf(conf) print("Change route id...") -conf.route.test2 = conf.route.test1 -conf.route.test1 = nil +conf.route4.test2 = conf.route4.test1 +conf.route4.test1 = nil test_conf(conf) print("Change negotiation_ttl...") @@ -53,5 +53,5 @@ conf.sa_ttl = 42 test_conf(conf) print("Remove route...") -conf.route.test2 = nil +conf.route4.test2 = nil test_conf(conf) diff --git a/src/program/vita/selftest.snabb b/src/program/vita/selftest.snabb index f0c62f7881..7502befcd9 100755 --- a/src/program/vita/selftest.snabb +++ b/src/program/vita/selftest.snabb @@ -28,20 +28,20 @@ local ffi = require("ffi") local regenerate_pcaps = main.parameters[1] local cfg = { - private_interface = { + private_interface4 = { pci = "00:00.0", ip4 = "192.168.10.1", nexthop_ip4 = "192.168.0.1", mac = "52:54:00:00:00:00" }, - public_interface = { + public_interface4 = { pci = "00:00.0", ip4 = "203.0.113.1", nexthop_ip4 = "203.0.0.1", mac = "52:54:00:00:00:FF" }, mtu = 500, - route = { + route4 = { loopback = { net_cidr4 = "192.168.10.0/24", gw_ip4 = "203.0.113.1", diff --git a/src/program/vita/selftest6.snabb b/src/program/vita/selftest6.snabb index 1ff0ea4879..d5bb414492 100755 --- a/src/program/vita/selftest6.snabb +++ b/src/program/vita/selftest6.snabb @@ -29,13 +29,13 @@ local ffi = require("ffi") local regenerate_pcaps = main.parameters[1] local cfg = { - private_interface = { + private_interface4 = { pci = "00:00.0", ip4 = "192.168.10.1", nexthop_ip4 = "192.168.0.1", mac = "52:54:00:00:00:00" }, - public_interface = { + public_interface6 = { pci = "00:00.0", ip6 = "203:0:113::1", nexthop_ip6 = "203:0:0::1", @@ -43,7 +43,7 @@ local cfg = { nexthop_mac = "52:54:00:00:00:FE" }, mtu = 500, - route = { + route46 = { loopback = { net_cidr4 = "192.168.10.0/24", gw_ip6 = "203:0:113::1", diff --git a/src/program/vita/test.lua b/src/program/vita/test.lua index 77be094251..a8e6d34b72 100644 --- a/src/program/vita/test.lua +++ b/src/program/vita/test.lua @@ -109,22 +109,26 @@ end -- Run Vita in software benchmark mode. function run_softbench (pktsize, npackets, nroutes, cpuspec, use_v6) local testconf = { - private_interface = { - nexthop_ip4 = private_interface_defaults.ip4.default - }, - public_interface = { - ip6 = use_v6 and public_interface_defaults6.ip6.default + private_interface4 = { + nexthop_ip4 = private_interface4_defaults.ip4.default }, packet_size = pktsize, nroutes = nroutes, negotiation_ttl = nroutes, sa_ttl = 16 } + if use_v6 then + testconf.public_interface6 = { + ip6 = public_interface6_defaults.ip6.default + } + end local function configure_private_router_softbench (conf) local c, private = vita.configure_private_router(conf) - if not conf.private_interface then return c end + if not (conf.private_interface4 or conf.private_interface6) then + return c + end config.app(c, "bridge", basic_apps.Join) config.link(c, "bridge.output -> "..private.input) @@ -178,7 +182,9 @@ end function configure_public_router_loopback (conf, append) local c, public = vita.configure_public_router(conf, append) - if not conf.public_interface then return c end + if not (conf.public_interface4 or conf.public_interface6) then + return c + end config.link(c, public.output.." -> "..public.input) @@ -194,29 +200,30 @@ end -- destination. defaults = { - private_interface = {}, - public_interface = {}, + private_interface4 = {}, + public_interface4 = {}, + public_interface6 = {}, route_prefix = {default="172.16"}, nroutes = {default=1}, packet_size = {default="IMIX"}, sa_ttl = {}, negotiation_ttl = {default=1} } -private_interface_defaults = { +private_interface4_defaults = { pci = {default="00:00.0"}, mac = {default="02:00:00:00:00:01"}, -- needed because used in sim. packets ip4 = {default="172.16.0.10"}, nexthop_ip4 = {default="172.16.1.1"}, nexthop_mac = {} } -public_interface_defaults = { +public_interface4_defaults = { pci = {default="00:00.0"}, mac = {}, ip4 = {default="172.16.0.10"}, nexthop_ip4 = {default="172.16.0.10"}, nexthop_mac = {} } -public_interface_defaults6 = { +public_interface6_defaults = { pci = {default="00:00.0"}, mac = {}, ip6 = {default="172:16:0::10"}, @@ -230,16 +237,19 @@ traffic_templates = { } local function parse_gentestconf (conf) + -- default to v4 + conf.private_interface4 = (not conf.private_interface6) and + (conf.private_interface4 or {}) + conf.public_interface4 = (not conf.public_interface6) and + (conf.public_interface4 or {}) + -- populate defaults conf = lib.parse(conf, defaults) - conf.private_interface = lib.parse(conf.private_interface, - private_interface_defaults) - if conf.public_interface and conf.public_interface.ip6 then - conf.public_interface = lib.parse(conf.public_interface, - public_interface_defaults6) - else - conf.public_interface = lib.parse(conf.public_interface, - public_interface_defaults) - end + conf.private_interface4 = conf.private_interface4 and + lib.parse(conf.private_interface4, private_interface4_defaults) + conf.public_interface4 = conf.public_interface4 and + lib.parse(conf.public_interface4, public_interface4_defaults) + conf.public_interface6 = conf.public_interface6 and + lib.parse(conf.public_interface6, public_interface6_defaults) assert(conf.nroutes >= 0 and conf.nroutes <= 255, "Invalid number of routes: "..conf.nroutes) return conf @@ -249,11 +259,11 @@ function gen_packet (conf, route, size) local payload_size = size - ethernet:sizeof() - ipv4:sizeof() assert(payload_size >= 0, "Negative payload_size :-(") local d = datagram:new(packet.resize(packet.allocate(), payload_size)) - d:push(ipv4:new{ src = ipv4:pton(conf.private_interface.nexthop_ip4), + d:push(ipv4:new{ src = ipv4:pton(conf.private_interface4.nexthop_ip4), dst = ipv4:pton(conf.route_prefix.."."..route..".1"), total_length = ipv4:sizeof() + payload_size, ttl = 64 }) - d:push(ethernet:new{ dst = ethernet:pton(conf.private_interface.mac), + d:push(ethernet:new{ dst = ethernet:pton(conf.private_interface4.mac), type = 0x0800 }) local p = d:packet() -- Pad to minimum Ethernet frame size (excluding four octet CRC) @@ -278,17 +288,22 @@ end function gen_configuration (conf) conf = parse_gentestconf(conf) local cfg = { - private_interface = conf.private_interface, - public_interface = conf.public_interface, - route = {}, + private_interface4 = conf.private_interface4, + public_interface4 = conf.public_interface4, + public_interface6 = conf.public_interface6, + route4 = {}, + route46 = {}, negotiation_ttl = conf.negotiation_ttl, sa_ttl = conf.sa_ttl } + local routes = + (cfg.private_interface4 and cfg.public_interface4 and cfg.route4) or + (cfg.private_interface4 and cfg.public_interface6 and cfg.route46) for route = 1, conf.nroutes do - cfg.route["test"..route] = { + routes["test"..route] = { net_cidr4 = conf.route_prefix.."."..route..".0/24", - gw_ip4 = conf.public_interface.nexthop_ip4, - gw_ip6 = conf.public_interface.nexthop_ip6, + gw_ip4 = cfg.public_interface4 and cfg.public_interface4.nexthop_ip4, + gw_ip6 = cfg.public_interface6 and cfg.public_interface6.nexthop_ip6, preshared_key = ("%064x"):format(route), spi = 1000+route } diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index 9221267374..4921ddbbfb 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -8,31 +8,50 @@ module vita-esp-gateway { import ietf-inet-types { prefix inet; } // APPLICATION MODEL - grouping interface { + grouping interface4 { leaf pci { type pci-address; mandatory true; } leaf ip4 { type inet:ipv4-address-no-zone; } - leaf ip6 { type inet:ipv6-address-no-zone; } leaf mac { type yang:mac-address; } leaf nexthop-ip4 { type inet:ipv4-address-no-zone; } + leaf nexthop-mac { type yang:mac-address; } + } + grouping interface6 { + leaf pci { type pci-address; mandatory true; } + leaf ip6 { type inet:ipv6-address-no-zone; } + leaf mac { type yang:mac-address; } leaf nexthop-ip6 { type inet:ipv6-address-no-zone; } leaf nexthop-mac { type yang:mac-address; } } - container private-interface { uses interface; } - container public-interface { uses interface; } - - leaf mtu { type uint16 { range "0..8937"; } } + grouping route { + leaf id { type id; mandatory true; } + leaf spi { type spi; mandatory true; } + leaf preshared-key { type key32; mandatory true; } + } - list route { - key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; - leaf id { type id; mandatory true; } - leaf net-cidr4 { type inet:ipv4-prefix; mandatory true; } - leaf gw-ip4 { type inet:ipv4-address-no-zone; } - leaf gw-ip6 { type inet:ipv6-address-no-zone; } - leaf spi { type spi; mandatory true; } - leaf preshared-key { type key32; mandatory true; } + choice router { + case v4-over-v4 { + container private-interface4 { uses interface4; } + container public-interface4 { uses interface4; } + list route4 { + key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; uses route; + leaf net-cidr4 { type inet:ipv4-prefix; mandatory true; } + leaf gw-ip4 { type inet:ipv4-address-no-zone; } + } + } + case v4-over-v6 { + container private-interface4 { uses interface4; } + container public-interface6 { uses interface6; } + list route46 { + key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; uses route; + leaf net-cidr4 { type inet:ipv4-prefix; mandatory true; } + leaf gw-ip6 { type inet:ipv6-address-no-zone; } + } + } } + leaf mtu { type uint16 { range "0..8937"; } } + leaf negotiation-ttl { type time-to-live; } leaf sa-ttl { type time-to-live; } diff --git a/src/program/vita/vita-gentest.yang b/src/program/vita/vita-gentest.yang index 249705b10f..df276884b8 100644 --- a/src/program/vita/vita-gentest.yang +++ b/src/program/vita/vita-gentest.yang @@ -4,8 +4,16 @@ module vita-gentest { import vita-esp-gateway { prefix vita; } - container private-interface { uses vita:interface; } - container public-interface { uses vita:interface; } + choice router { + case v4-over-v4 { + container private-interface4 { uses vita:interface4; } + container public-interface4 { uses vita:interface4; } + } + case v4-over-v6 { + container private-interface4 { uses vita:interface4; } + container public-interface6 { uses vita:interface6; } + } + } leaf negotiation-ttl { type vita:time-to-live; } leaf sa-ttl { type vita:time-to-live; } diff --git a/src/program/vita/vita-loadtest.md b/src/program/vita/vita-loadtest.md index d7af6bc8af..aa63fa2430 100644 --- a/src/program/vita/vita-loadtest.md +++ b/src/program/vita/vita-loadtest.md @@ -74,18 +74,18 @@ First of all, we start two Vita nodes, and assign them the names `node1` and `no Most of it should be fairly self explanatory: we assign the desired ports via their PCI bus addresses, and set the interface’s own addresses as well as the addresses of the next hops. In this case, the private and public interface addresses are the same, but they need not be. The next hop of the private interface (this would normally be your local router) will be `snabb loadtest`. Since `loadtest` does not speak ARP, we configure a fixed MAC destination address for this next hop. This will prevent Vita from attempting to look up the next hop’s MAC addresses via ARP, and instead use the preconfigured address. The next hop of the public interface (this would normally be your gateway to the Internet) is configured to be the other Vita node in the test setup. Finally, we define a single route to the subnet `172.17.1.0/24` via the second Vita node with a dummy key. For the other Vita node, `node2.conf` is symmetric: - private-interface { + private-interface4 { pci 22:00.2; ip4 172.17.0.10; nexthop-ip4 172.17.0.1; nexthop-mac 02:00:00:00:00:00; } - public-interface { + public-interface4 { pci 23:00.1; ip4 172.17.0.10; nexthop-ip4 172.16.0.10; } - route { + route4 { id test1; gw-ip4 172.16.0.10; net-cidr4 "172.16.1.0/24"; @@ -95,13 +95,13 @@ Most of it should be fairly self explanatory: we assign the desired ports via th Because typing out configuration files for testing gets old fast, and we still need matching Pcap records, Vita comes with a utility that generates both of these from a meta-configuration file. For the first node we have `gentest-node1.conf`: - private-interface { + private-interface4 { pci 23:00.0; ip4 172.16.0.10; nexthop-ip4 172.16.0.1; nexthop-mac 02:00:00:00:00:00; } - public-interface { + public-interface4 { pci 22:00.1; ip4 172.16.0.10; nexthop-ip4 172.17.0.10; @@ -112,13 +112,13 @@ Because typing out configuration files for testing gets old fast, and we still n …and for the second node `gentest-node2.conf`: - private-interface { + private-interface4 { pci 22:00.2; ip4 172.17.0.10; nexthop-ip4 172.17.0.1; nexthop-mac 02:00:00:00:00:00; } - public-interface { + public-interface4 { pci 23:00.1; ip4 172.17.0.10; nexthop-ip4 172.16.0.10; diff --git a/src/program/vita/vita.lua b/src/program/vita/vita.lua index 6337fa131a..78df2d9483 100644 --- a/src/program/vita/vita.lua +++ b/src/program/vita/vita.lua @@ -32,10 +32,12 @@ local usage = require("program.vita.README_inc") local confighelp = require("program.vita.README_config_inc") local confspec = { - private_interface = {}, - public_interface = {}, + private_interface4 = {}, + public_interface4 = {}, + public_interface6 = {}, mtu = {default=8937}, - route = {default={}}, + route4 = {default={}}, + route46 = {default={}}, negotiation_ttl = {}, sa_ttl = {}, data_plane = {}, @@ -77,8 +79,9 @@ end local function parse_conf (conf) conf = lib.parse(conf, confspec) - conf.private_interface = parse_ifconf(conf.private_interface, {0x2a, 0xbb}) - conf.public_interface = parse_ifconf(conf.public_interface, {0x3a, 0xbb}) + conf.private_interface4 = parse_ifconf(conf.private_interface4, {0x2a,0xbb}) + conf.public_interface4 = parse_ifconf(conf.public_interface4, {0x3a,0xbb}) + conf.public_interface6 = parse_ifconf(conf.public_interface6, {0x3a,0xbb}) return conf end @@ -228,32 +231,38 @@ function configure_private_router (conf, append) conf = parse_conf(conf) local c = append or config.new() - if not conf.private_interface then return c end + local interface = conf.private_interface4 + if not interface then return c end + + local routes = (conf.public_interface4 and conf.route4) or + (conf.public_interface6 and conf.route46) + + local private_links = {} config.app(c, "PrivateDispatch", dispatch.PrivateDispatch, { - node_ip4 = conf.private_interface.ip4 + node_ip4 = interface.ip4 }) config.app(c, "OutboundTTL", ttl.DecrementTTL) config.app(c, "PrivateRouter", route.PrivateRouter, { - routes = conf.route, + routes = routes, mtu = conf.mtu }) config.app(c, "PrivateICMP4", icmp.ICMP4, { - node_ip4 = conf.private_interface.ip4, + node_ip4 = interface.ip4, nexthop_mtu = conf.mtu }) config.app(c, "InboundDispatch", dispatch.InboundDispatch, { - node_ip4 = conf.private_interface.ip4 + node_ip4 = interface.ip4 }) config.app(c, "InboundTTL", ttl.DecrementTTL) config.app(c, "InboundICMP4", icmp.ICMP4, { - node_ip4 = conf.private_interface.ip4 + node_ip4 = interface.ip4 }) config.app(c, "PrivateNextHop", nexthop.NextHop4, { - node_mac = conf.private_interface.mac, - node_ip4 = conf.private_interface.ip4, - nexthop_ip4 = conf.private_interface.nexthop_ip4, - nexthop_mac = conf.private_interface.nexthop_mac + node_mac = interface.mac, + node_ip4 = interface.ip4, + nexthop_ip4 = interface.nexthop_ip4, + nexthop_mac = interface.nexthop_mac }) config.link(c, "PrivateDispatch.forward4 -> OutboundTTL.input") config.link(c, "PrivateDispatch.icmp4 -> PrivateICMP4.input") @@ -269,8 +278,10 @@ function configure_private_router (conf, append) config.link(c, "InboundTTL.output -> PrivateNextHop.forward") config.link(c, "InboundTTL.time_exceeded -> InboundICMP4.transit_ttl_exceeded") config.link(c, "InboundICMP4.output -> PrivateRouter.control") + private_links.input = "PrivateDispatch.input" + private_links.output = "PrivateNextHop.output" - for id, route in pairs(conf.route) do + for id, route in pairs(routes) do local private_in = "PrivateRouter."..id local ESP_in = "ESP_"..id.."_in" config.app(c, ESP_in.."_Tx", Transmitter, ESP_in) @@ -284,10 +295,6 @@ function configure_private_router (conf, append) config.link(c, DSP_out.."_Rx.output -> "..private_out) end - local private_links = { - input = "PrivateDispatch.input", - output = "PrivateNextHop.output" - } return c, private_links end @@ -295,40 +302,52 @@ function configure_public_router (conf, append) conf = parse_conf(conf) local c = append or config.new() - if not conf.public_interface then return c end + local interface = conf.public_interface4 or conf.public_interface6 + if not interface then return c end + + local routes = (conf.public_interface4 and conf.route4) or + (conf.public_interface6 and conf.route46) + + local public_links = {} - config.app(c, "PublicDispatch", dispatch.PublicDispatch, { - node_ip4 = conf.public_interface.ip4, - node_ip6 = conf.public_interface.ip6 - }) config.app(c, "PublicRouter", route.PublicRouter, { sa = conf.inbound_sa }) - if conf.public_interface.ip4 then + + if conf.public_interface4 then + config.app(c, "PublicDispatch", dispatch.PublicDispatch, { + node_ip4 = interface.ip4 + }) config.app(c, "PublicICMP4", icmp.ICMP4, { - node_ip4 = conf.public_interface.ip4 + node_ip4 = interface.ip4 }) config.app(c, "PublicNextHop", nexthop.NextHop4, { - node_mac = conf.public_interface.mac, - node_ip4 = conf.public_interface.ip4, - nexthop_ip4 = conf.public_interface.nexthop_ip4, - nexthop_mac = conf.public_interface.nexthop_mac + node_mac = interface.mac, + node_ip4 = interface.ip4, + nexthop_ip4 = interface.nexthop_ip4, + nexthop_mac = interface.nexthop_mac }) config.link(c, "PublicDispatch.forward4 -> PublicRouter.input") config.link(c, "PublicDispatch.icmp4 -> PublicICMP4.input") config.link(c, "PublicDispatch.arp -> PublicNextHop.arp") config.link(c, "PublicDispatch.protocol4_unreachable -> PublicICMP4.protocol_unreachable") config.link(c, "PublicICMP4.output -> PublicNextHop.icmp4") - elseif conf.public_interface.ip6 then + public_links.input = "PublicDispatch.input" + public_links.output = "PublicNextHop.output" + + elseif conf.public_interface6 then + config.app(c, "PublicDispatch", dispatch.PublicDispatch, { + node_ip6 = interface.ip6 + }) config.app(c, "PublicICMP6", icmp.ICMP6, { - node_ip6 = conf.public_interface.ip6 + node_ip6 = interface.ip6 }) config.app(c, "PublicNextHop", Join) config.app(c, "PublicND", nd_light, { - local_mac = conf.public_interface.mac, - local_ip = conf.public_interface.ip6, - next_hop = conf.public_interface.nexthop_ip6, - remote_mac = conf.public_interface.nexthop_mac + local_mac = interface.mac, + local_ip = interface.ip6, + next_hop = interface.nexthop_ip6, + remote_mac = interface.nexthop_mac }) config.link(c, "PublicDispatch.forward6 -> PublicRouter.input") config.link(c, "PublicDispatch.icmp6 -> PublicICMP6.input") @@ -336,7 +355,9 @@ function configure_public_router (conf, append) config.link(c, "PublicDispatch.protocol6_unreachable -> PublicICMP6.protocol_unreachable") config.link(c, "PublicICMP6.output -> PublicNextHop.icmp6") config.link(c, "PublicNextHop.output -> PublicND.north") - else error("Need either public_interface.ip4 or public_interface.ip6") end + public_links.input = "PublicDispatch.input" + public_links.output = "PublicND.south" + end if not conf.data_plane then config.app(c, "Protocol_in_Tx", Transmitter, "Protocol_in") @@ -345,18 +366,18 @@ function configure_public_router (conf, append) config.link(c, "Protocol_out_Rx.output -> PublicNextHop.protocol") end - for id, route in pairs(conf.route) do + for id, route in pairs(routes) do local public_out = "PublicNextHop."..id local ESP_out = "ESP_"..id.."_out" local Tunnel = "Tunnel_"..id config.app(c, ESP_out.."_Rx", Receiver, ESP_out) if route.gw_ip4 then config.app(c, Tunnel, tunnel.Tunnel4, - {src=conf.public_interface.ip4, dst=route.gw_ip4}) + {src=interface.ip4, dst=route.gw_ip4}) elseif route.gw_ip6 then config.app(c, Tunnel, tunnel.Tunnel6, - {src=conf.public_interface.ip6, dst=route.gw_ip6}) - else error("Need either route.gw_ip4 or route.gw_ip6") end + {src=interface.ip6, dst=route.gw_ip6}) + end config.link(c, ESP_out.."_Rx.output -> "..Tunnel..".input") config.link(c, Tunnel..".output -> "..public_out) end @@ -368,12 +389,6 @@ function configure_public_router (conf, append) config.link(c, public_in.." -> "..DSP_in.."_Tx.input") end - local public_links = { - input = "PublicDispatch.input", - output = (conf.public_interface.ip4 and "PublicNextHop.output") - or "PublicND.south" - } - return c, public_links end @@ -420,12 +435,16 @@ function configure_exchange (conf, append) if conf.data_plane then return end - if not conf.public_interface then return c end + local interface = conf.public_interface4 or conf.public_interface6 + if not interface then return c end + + local routes = (conf.public_interface4 and conf.route4) or + (conf.public_interface6 and conf.route46) config.app(c, "KeyExchange", exchange.KeyManager, { - node_ip4 = conf.public_interface.ip4, - node_ip6 = conf.public_interface.ip6, - routes = conf.route, + node_ip4 = interface.ip4, + node_ip6 = interface.ip6, + routes = routes, sa_db_path = sa_db_path, negotiation_ttl = conf.negotiation_ttl, sa_ttl = conf.sa_ttl From d29763d1fc499856291f17d96e42ffb97804b3c4 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 21 Jan 2019 14:17:46 +0100 Subject: [PATCH 50/81] intel_mp: fix a bug in VMDq BAM (bcast accept) handling for the i350 --- src/apps/intel_mp/intel_mp.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/intel_mp/intel_mp.lua b/src/apps/intel_mp/intel_mp.lua index 93d64ec1eb..c3c5917e62 100644 --- a/src/apps/intel_mp/intel_mp.lua +++ b/src/apps/intel_mp/intel_mp.lua @@ -1259,7 +1259,7 @@ end function Intel1g:set_vmdq_rx_pool () -- long packets enabled, multicast promiscuous, broadcast accept, accept -- untagged pkts - self.r.VMOLR[self.poolnum]:set(bits { LPE=16, MPE=28, BAM=26, AUPE=14 }) + self.r.VMOLR[self.poolnum]:set(bits { LPE=16, MPE=28, BAM=27, AUPE=24 }) -- packet splitting none self.r.PSRTYPE[self.poolnum](0) end From 6be165ea35a68da701ddde107ebdf8ef5fe90f46 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 21 Jan 2019 14:17:18 +0100 Subject: [PATCH 51/81] intel_mp: add VMDq broadcast tests --- src/apps/intel_mp/broadcast.pcap | Bin 0 -> 19250 bytes src/apps/intel_mp/test_10g_vmdq_mcast.snabb | 46 ++++++++++++++++++++ src/apps/intel_mp/test_1g_vmdq_mcast.snabb | 46 ++++++++++++++++++++ 3 files changed, 92 insertions(+) create mode 100644 src/apps/intel_mp/broadcast.pcap create mode 100755 src/apps/intel_mp/test_10g_vmdq_mcast.snabb create mode 100755 src/apps/intel_mp/test_1g_vmdq_mcast.snabb diff --git a/src/apps/intel_mp/broadcast.pcap b/src/apps/intel_mp/broadcast.pcap new file mode 100644 index 0000000000000000000000000000000000000000..6884c71760dd396f8971f5ade473b0e5db7838a3 GIT binary patch literal 19250 zcmeI)y-vbV6bJBgOPdh+bT^dA0-Z@ zlMcQIkASzdHy8*Qz_a~t(zJwd^1r`()1~x$f7D(KGD(kKk4Web>YNLXUVr#sgz1oI z&ATp$?wwsFVOJMKm`-oAY8+pdO6B$RS-Db@%l;3>mswf$%XIGNDnm;_nukG|r{}5D zc#}%2Nqx>YslLskvbh1*yaXDYO>xj`~pAaJq6X3prZQ#gvf1fsFB^;-r3#T z&*cwBue{%QV~VQ6Z7@_d+&V+mko(S1)pXw&s#@+VLsjXX|9{m8wHbC+M6?-pRz$WL z1fpa!2t=jLkkzCPLsnCE7_wUAFl4oa!?3d~!(q5*Ss{nvo@GggVYOn^Vpy%1Y%yew jQ nic0.input") +config.link(c, "nic1.output -> sink.input") + +local retries = 10 +repeat + -- This test is flaky on my hardware, apparently because sometimes the first + -- so-many packets are not delivered. This works around that. - Max + engine.configure(config.new()) + engine.configure(c) + engine.main({ duration = 1 }) + retries = retries - 1 +until link.stats(engine.app_table.sink.input.input).rxpackets > 0 + or retries == 0 + +assert(link.stats(engine.app_table.sink.input.input).rxpackets == 40, + "failed to receive mcast packets") diff --git a/src/apps/intel_mp/test_1g_vmdq_mcast.snabb b/src/apps/intel_mp/test_1g_vmdq_mcast.snabb new file mode 100755 index 0000000000..011466da20 --- /dev/null +++ b/src/apps/intel_mp/test_1g_vmdq_mcast.snabb @@ -0,0 +1,46 @@ +#!../../snabb snsh + +-- Snabb test script multicast delivery VMDq mode + +local basic_apps = require("apps.basic.basic_apps") +local intel = require("apps.intel_mp.intel_mp") +local pcap = require("apps.pcap.pcap") +local lib = require("core.lib") + +local pciaddr0 = lib.getenv("SNABB_PCI_INTEL1G0") +local pciaddr1 = lib.getenv("SNABB_PCI_INTEL1G1") + +local c = config.new() + +-- send packets on nic0 +config.app(c, "nic0", intel.Intel, + { pciaddr = pciaddr0, + txq = 0, + wait_for_link = true }) + +-- nic1 with a VMDq queue (that should receive multicast packets) +config.app(c, "nic1", intel.Intel, + { pciaddr = pciaddr1, + vmdq = true, + macaddr = "90:72:82:78:c9:7a", + wait_for_link = true }) + +config.app(c, "pcap", pcap.PcapReader, "broadcast.pcap") +config.app(c, 'sink', basic_apps.Sink) + +config.link(c, "pcap.output -> nic0.input") +config.link(c, "nic1.output -> sink.input") + +local retries = 10 +repeat + -- This test is flaky on my hardware, apparently because sometimes the first + -- so-many packets are not delivered. This works around that. - Max + engine.configure(config.new()) + engine.configure(c) + engine.main({ duration = 1 }) + retries = retries - 1 +until link.stats(engine.app_table.sink.input.input).rxpackets > 0 + or retries == 0 + +assert(link.stats(engine.app_table.sink.input.input).rxpackets == 40, + "failed to receive mcast packets") From 69b6ca1e0bb3ac7b6aa8875803dd4efac78e5269 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 21 Jan 2019 14:32:04 +0100 Subject: [PATCH 52/81] intel_mp: fix selftest from the future The commit (not yet on this branch) 373d1951a22eb434943410ad306b02b14068fa52 will eventually break test_1g_2q_blast_vmdq_auto.sh when merged. Signed-Off-By: Josh Futturman --- src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh b/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh index 8ee7c48283..79df31f9ab 100755 --- a/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh +++ b/src/apps/intel_mp/test_1g_2q_blast_vmdq_auto.sh @@ -5,9 +5,9 @@ SNABB_SEND_BLAST=true SNABB_SEND_BLAST_RATE=1e9 ./testsend.snabb $SNABB_PCI_INTEL1G1 0 source2.pcap & BLAST=$! -SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" nil nil nil > results.0 & +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "90:72:82:78:c9:7a" nil nil 0 > results.0 & PID1=$! -SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "12:34:56:78:9a:bc" nil nil nil > results.1 +SNABB_RECV_SPINUP=2 SNABB_RECV_DURATION=5 ./testvmdqrecv.snabb $SNABB_PCI_INTEL1G0 "12:34:56:78:9a:bc" nil nil 0 > results.1 wait $PID1 kill -9 $BLAST From a58cee616a06f4e6a366642c1e3e669635198dc7 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 22 Jan 2019 13:57:45 +0100 Subject: [PATCH 53/81] KeyManager: do not recover from failure to commit SA database Fail hard instead as this most likely indicates a serious problem and can have unforseeable side effects. --- src/program/vita/exchange.lua | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index beb3053a69..3239ad94a2 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -671,15 +671,11 @@ function KeyManager:commit_sa_db () end end -- Commit active SAs to SA database - local success, err = pcall( - yang.compile_config_for_schema, + yang.compile_config_for_schema( schemata['ephemeral-keys'], {outbound_sa=esp_keys, inbound_sa=dsp_keys}, self.sa_db_file ) - if not success then - audit:log("Failed to commit SA database: "..err) - end end -- Vita: simple key exchange (vita-ske, version 2a). See README.exchange From 3e7aff3eaffa0e756a9ecbb65dbc9bbe62f57b50 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 18 Jan 2019 21:19:50 +0100 Subject: [PATCH 54/81] vita/nexthop: synchronize next hop with group --- src/program/vita/nexthop.lua | 37 ++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/src/program/vita/nexthop.lua b/src/program/vita/nexthop.lua index ce8579cb41..fd82dad0de 100644 --- a/src/program/vita/nexthop.lua +++ b/src/program/vita/nexthop.lua @@ -2,6 +2,7 @@ module(...,package.seeall) +local shm = require("core.shm") local counter = require("core.counter") local lib = require("core.lib") local ethernet = require("lib.protocol.ethernet") @@ -20,7 +21,8 @@ NextHop4 = { node_mac = {required=true}, node_ip4 = {required=true}, nexthop_ip4 = {required=true}, - nexthop_mac = {} + nexthop_mac = {}, + synchronize = {default=false} }, shm = { arp_requests = {counter}, @@ -75,11 +77,13 @@ function NextHop4:new (conf) -- ...unless its supplied if conf.nexthop_mac then - print("forced") o.eth:dst(ethernet:pton(conf.nexthop_mac)) o.connected = true end + -- We can get our next hop by synchronizing with other NextHop4 instances + o.sync_interval = lib.throttle(1) + return setmetatable(o, {__index = NextHop4}) end @@ -128,6 +132,11 @@ function NextHop4:push () packet.free(p) end end + + -- Synchronize next hop + if self.synchronize and self.sync_interval() then + self:sync_nexthop() + end end function NextHop4:encapsulate (p, type) @@ -165,6 +174,7 @@ function NextHop4:handle_arp (p) -- information in the packet and set Merge_flag to true. if ip4eq(arp_ipv4:spa(), self.nexthop_ip4) and self.connected then self.eth:dst(arp_ipv4:sha()) + if self.synchronize then self:share_nexthop() end counter.add(self.shm.addresses_updated) self.connected = true end @@ -176,6 +186,7 @@ function NextHop4:handle_arp (p) -- the translation table. if not self.connected then self.eth:dst(arp_ipv4:sha()) + if self.synchronize then self:share_nexthop() end counter.add(self.shm.addresses_added) self.connected = true end @@ -199,3 +210,25 @@ function NextHop4:handle_arp (p) counter.add(self.shm.arp_errors) end end + +function NextHop4:shared_nexthop_path () + -- We share the resolved next hop with siblings of the same name. + return "group/"..self.name.."/"..self.appname +end + +function NextHop4:share_nexthop () + if not self.nexthop then + self.nexthop = shm.create(self:shared_nexthop_path(), "uint8_t[6]") + end + ffi.copy(self.nexthop, self.eth:dst(), ffi.sizeof(self.nexthop)) +end + +function NextHop4:sync_nexthop () + if not self.nexthop then + local ok, nh = pcall(shm.open, self:shared_nexthop_path(), "uint8_t[6]") + self.connected, self.nexthop = ok or self.connected, ok and nh + end + if self.connected and self.nexthop then + self.eth:dst(self.nexthop) + end +end From c7b428aadbb272d81e210a70383f23744707f9ab Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sun, 20 Jan 2019 16:17:12 +0100 Subject: [PATCH 55/81] KeyManager: delete SA database on :stop() --- src/program/vita/exchange.lua | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 3239ad94a2..ccd06cc8fb 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -291,6 +291,11 @@ function KeyManager:reconfig (conf) self.sa_ttl = conf.sa_ttl end +function KeyManager:stop () + -- make sure to remove SA database when app is stopped + S.unlink(self.sa_db_file) +end + function KeyManager:push () -- handle negotiation protocol requests local input = self.input.input From e977a24e11114bc46929c8bba6e804027d7c9eb1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 22 Jan 2019 13:58:59 +0100 Subject: [PATCH 56/81] KeyManager: qualify log messages with node ip for disambiguation --- src/program/vita/exchange.lua | 36 +++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index ccd06cc8fb..8d69959a6b 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -155,7 +155,6 @@ local lib = require("core.lib") local ipv4 = require("lib.protocol.ipv4") local yang = require("lib.yang.yang") local schemata = require("program.vita.schemata") -local audit = lib.logger_new({rate=32, module='KeyManager'}) require("program.vita.sodium_h") local C = ffi.C @@ -209,6 +208,11 @@ function KeyManager:new (conf) end function KeyManager:reconfig (conf) + self.audit = lib.logger_new({ + rate = 32, + module = ("KeyManager(%s)"):format(conf.node_ip4) + }) + local function find_route (id) for _, route in ipairs(self.routes) do if route.id == id then return route end @@ -220,12 +224,12 @@ function KeyManager:reconfig (conf) end local function free_route (route) for _, sa in ipairs(route.inbound_sa) do - audit:log(("Expiring inbound SA %d for '%s' (reconfig)") + self.audit:log(("Expiring inbound SA %d for '%s' (reconfig)") :format(sa.spi, route.id)) self.sa_db_updated = true end for _, sa in ipairs(route.outbound_sa) do - audit:log(("Expiring outbound SA %d for '%s' (reconfig)") + self.audit:log(("Expiring outbound SA %d for '%s' (reconfig)") :format(sa.spi, route.id)) self.sa_db_updated = true end @@ -243,7 +247,7 @@ function KeyManager:reconfig (conf) -- if negotation_ttl has changed, swap out old protocol fsm for a new -- one with the adjusted timeout, effectively resetting the fsm if conf.negotiation_ttl ~= self.negotiation_ttl then - audit:log("Protocol reset for '"..id.."' (reconfig)") + self.audit:log("Protocol reset for '"..id.."' (reconfig)") old_route.initiator = Protocol:new('initiator', old_route.spi, old_route.preshared_key, @@ -309,7 +313,7 @@ function KeyManager:push () -- process protocol timeouts if route.initiator:reset_if_expired() == Protocol.code.expired then counter.add(self.shm.negotiations_expired) - audit:log(("Negotiation expired for '%s' (negotiation_ttl)") + self.audit:log(("Negotiation expired for '%s' (negotiation_ttl)") :format(route.id)) end for index, sa in ipairs(route.inbound_sa) do @@ -317,7 +321,7 @@ function KeyManager:push () table.remove(route.inbound_sa, index) self.sa_db_updated = true counter.add(self.shm.inbound_sa_expired) - audit:log(("Inbound SA %d expired for '%s' (sa_ttl)") + self.audit:log(("Inbound SA %d expired for '%s' (sa_ttl)") :format(sa.spi, route.id)) end end @@ -326,7 +330,7 @@ function KeyManager:push () table.remove(route.outbound_sa, index) self.sa_db_updated = true counter.add(self.shm.outbound_sa_expired) - audit:log(("Outbound SA %d expired for '%s' (sa_ttl)") + self.audit:log(("Outbound SA %d expired for '%s' (sa_ttl)") :format(sa.spi, route.id)) end end @@ -335,7 +339,7 @@ function KeyManager:push () for index, sa in ipairs(route.outbound_sa_queue) do if sa.activation_delay() or #route.outbound_sa < self.num_outbound_sa then - audit:log(("Activating outbound SA %d for '%s'") + self.audit:log(("Activating outbound SA %d for '%s'") :format(sa.spi, route.id)) -- insert in front of SA list table.insert(route.outbound_sa, 1, sa) @@ -373,7 +377,7 @@ function KeyManager:negotiate (route) route.initiator:initiate_exchange(self.nonce_message) if not ecode then counter.add(self.shm.negotiations_initiated) - audit:log("Initiating negotiation for '"..route.id.."'") + self.audit:log("Initiating negotiation for '"..route.id.."'") link.transmit(self.output.output, self:request(route, nonce_message)) else assert(ecode == Protocol.code.protocol) end end @@ -386,7 +390,7 @@ function KeyManager:handle_negotiation (request) or self:handle_challenge_request(route, message) or self:handle_nonce_key_request(route, message)) then counter.add(self.shm.rxerrors) - audit:log(("Rejected invalid negotiation request for '%s'") + self.audit:log(("Rejected invalid negotiation request for '%s'") :format(route or "")) end end @@ -410,7 +414,7 @@ function KeyManager:handle_nonce_request (route, message) link.transmit(self.output.output, self:request(route, response)) counter.add(self.shm.challenges_offered) - audit:log("Offered challenge for '"..route.id.."'") + self.audit:log("Offered challenge for '"..route.id.."'") return true end @@ -432,7 +436,7 @@ function KeyManager:handle_challenge_request (route, message) else assert(not ecode) end counter.add(self.shm.challenges_accepted) - audit:log("Accepted challenge for '"..route.id.."'") + self.audit:log("Accepted challenge for '"..route.id.."'") -- If we got here we should offer our nonce and public key -- (offer_nonce_key -> accept_key). @@ -440,7 +444,7 @@ function KeyManager:handle_challenge_request (route, message) local ecode, response = route.initiator:offer_nonce_key(response) assert(not ecode) - audit:log("Proposing key exchange for '"..route.id.."'") + self.audit:log("Proposing key exchange for '"..route.id.."'") link.transmit(self.output.output, self:request(route, response)) return true @@ -480,7 +484,7 @@ function KeyManager:handle_nonce_key_request (route, message) local is_loopback = self.ip:src_eq(route.gw_ip4n) counter.add(self.shm.keypairs_offered) - audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %s)"): + self.audit:log(("Offered key pair for '%s' (inbound SA %d, outbound SA %s)"): format(route.id, inbound_sa.spi, (is_loopback and "-") or outbound_sa.spi)) @@ -517,7 +521,7 @@ function KeyManager:handle_key_request (route, message) local is_loopback = self.ip:src_eq(route.gw_ip4n) counter.add(self.shm.keypairs_negotiated) - audit:log(("Completed AKE for '%s' (inbound SA %s, outbound SA %d)"): + self.audit:log(("Completed AKE for '%s' (inbound SA %s, outbound SA %d)"): format(route.id, (is_loopback and "-") or inbound_sa.spi, outbound_sa.spi)) @@ -566,7 +570,7 @@ function KeyManager:upsert_outbound_sa (route, sa) end if remove_existing_sa_for_update() then counter.add(self.shm.outbound_sa_updated) - audit:log("Updating outbound SA "..sa.spi.." for '"..route.id.."'") + self.audit:log("Updating outbound SA "..sa.spi.." for '"..route.id.."'") end -- enqueue new outbound SA at the end of the queue table.insert(route.outbound_sa_queue, { From fa5ab266556ffe296983a104fb80dc8561f6eced Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 18 Jan 2019 21:24:15 +0100 Subject: [PATCH 57/81] vita: scale linearly with cores using RSS and VMDq Instead of distributing the app network across multiple cores using interlinks, spawn independent full Vita instances (per core) that share private interface address by RSS, each with its own public interface (VMDq). --- src/Makefile.vita | 4 +- src/program/vita/conftest.snabb | 14 +- src/program/vita/selftest-public-out.pcap | Bin 591 -> 591 bytes src/program/vita/selftest.snabb | 62 ++-- src/program/vita/test.lua | 64 ++-- src/program/vita/vita-esp-gateway.yang | 28 +- src/program/vita/vita-gentest.yang | 9 +- src/program/vita/vita.lua | 388 ++++++++++++++-------- 8 files changed, 343 insertions(+), 226 deletions(-) diff --git a/src/Makefile.vita b/src/Makefile.vita index ee0fe319d1..090f2cd181 100644 --- a/src/Makefile.vita +++ b/src/Makefile.vita @@ -7,12 +7,12 @@ INCLUDE = *.* core arch jit syscall pf \ lib/multi_copy.* lib/hash lib/cltable.* lib/lpm lib/interlink.* \ lib/hardware lib/macaddress.* lib/numa.* lib/cpuset.* \ lib/scheduling.* lib/timers \ - apps/interlink apps/intel_mp \ + apps/basic apps/interlink apps/intel_mp \ program/vita program/config program/ps program/pci_bind \ program/top program/shm INCLUDE_TEST = $(INCLUDE) \ - lib/pmu* apps/basic apps/test apps/packet_filter apps/ipv4 \ + lib/pmu* apps/test apps/packet_filter apps/ipv4 \ lib/pcap apps/pcap \ program/snsh program/snabbmark \ lib/virtio apps/vhost apps/ipsec apps/ipv6 \ diff --git a/src/program/vita/conftest.snabb b/src/program/vita/conftest.snabb index fe1b97ab3d..6e09284de3 100755 --- a/src/program/vita/conftest.snabb +++ b/src/program/vita/conftest.snabb @@ -6,12 +6,16 @@ local vita = require("program.vita.vita") local vita_test = require("program.vita.test") local lib = require("core.lib") +vita.init_sa_db() + --engine.log = true local function test_conf (conf) conf = lib.deepcopy(conf) -- ensure purity - engine.configure(vita_test.configure_public_router_loopback( - conf, vita.configure_exchange(conf))) + local c, private, public = vita.configure_vita_queue(conf, 1) + config.link(c, private.output.."->"..private.input) + config.link(c, public.output.."->"..public.input) + engine.configure(c) engine.main{duration=3, report={showlinks=false}} end @@ -30,12 +34,12 @@ conf.route.test1.preshared_key = string.rep("FF", 32) test_conf(conf) print("Change route gw_ip4...") -conf.route.test1.gw_ip4 = "172.17.1.10" +conf.route.test1.gateway = {["172.17.1.10"]={queue=1}} test_conf(conf) print("Change public_ip4 and route net_cidr4...") -conf.public_interface.ip4 = "172.16.0.11" -conf.public_interface.nexthop_ip4 = conf.public_interface.ip4 +conf.public_interface = {["172.16.0.11"]=conf.public_interface["172.16.0.10"]} +conf.public_interface["172.16.0.11"].nexthop_ip4 = "172.16.0.11" conf.route.test1.net_cidr4 = "172.16.0.0/16" test_conf(conf) diff --git a/src/program/vita/selftest-public-out.pcap b/src/program/vita/selftest-public-out.pcap index c32889c00b3476a852e33635b3786ecc9c7a0eb4..e1cf7b3c1d4bc9265474d8916f24b4af28af4a10 100644 GIT binary patch delta 13 VcmX@la-L;^!p6Om7$-{11pp|31)cx^ delta 24 gcmX@la-L;^0;@R#1B3ZwLq?IwmlHq)$ diff --git a/src/program/vita/selftest.snabb b/src/program/vita/selftest.snabb index f0c62f7881..ca1d9a18dc 100755 --- a/src/program/vita/selftest.snabb +++ b/src/program/vita/selftest.snabb @@ -28,6 +28,7 @@ local ffi = require("ffi") local regenerate_pcaps = main.parameters[1] local cfg = { + data_plane = true, private_interface = { pci = "00:00.0", ip4 = "192.168.10.1", @@ -35,46 +36,47 @@ local cfg = { mac = "52:54:00:00:00:00" }, public_interface = { - pci = "00:00.0", - ip4 = "203.0.113.1", - nexthop_ip4 = "203.0.0.1", - mac = "52:54:00:00:00:FF" + ["203.0.113.1"] = { + pci = "00:00.0", + nexthop_ip4 = "203.0.0.1", + mac = "52:54:00:00:00:FF", + queue = 1 + } }, mtu = 500, route = { loopback = { net_cidr4 = "192.168.10.0/24", - gw_ip4 = "203.0.113.1", + gateway = { ["203.0.113.1"] = {queue=1} }, preshared_key = string.rep("00", 32), spi = 1001 } }, - outbound_sa = { - [1001] = { - route = "loopback", - aead = "aes-gcm-16-icv", - key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", - salt = "00 00 00 00" - } - }, - inbound_sa = { - [1001] = { - route = "loopback", - aead = "aes-gcm-16-icv", - key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", - salt = "00 00 00 00", - auditing = true + sa_database = { + [1] = { + outbound_sa = { + [1001] = { + route = "loopback", + aead = "aes-gcm-16-icv", + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00" + } + }, + inbound_sa = { + [1001] = { + route = "loopback", + aead = "aes-gcm-16-icv", + key = "00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00", + salt = "00 00 00 00", + auditing = true + } + } } } } -local c = config.new() - --- Configure Vita app network (all-in-one process.) -local _, private = vita.configure_private_router(cfg, c) -local _, public = vita.configure_public_router(cfg, c) -vita.configure_esp(cfg, c) -vita.configure_dsp(cfg, c) +-- Configure Vita queue (without KeyManager.) +local c, private, public = vita.configure_vita_queue(cfg, 1) -- Add ARP resolvers. config.app(c, "private_arp", ARP, { @@ -133,7 +135,7 @@ end engine.configure(c) -- Hack to avoid ESP seq# reuse because of packets from public_in.pcap -engine.app_table.ESP_loopback.sa.seq.no = 100 +engine.app_table.OutboundSA_loopback.sa.seq.no = 100 -- Run engine until its idle (all packets have been processed). local last_frees = counter.read(engine.frees) @@ -167,7 +169,7 @@ else if #engine.app_table.match_public:errors() > 0 then engine.app_table.match_public:report() main.exit(1) - end + end for app, counters in pairs{ PrivateRouter = { rxerrors = 2, @@ -184,7 +186,7 @@ else rxerrors = 2, fragment_errors = 2 }, - DSP_loopback_1001 = { + InboundSA_loopback_1001 = { rxerrors = 2, protocol_errors = 1, decrypt_errors = 1 diff --git a/src/program/vita/test.lua b/src/program/vita/test.lua index 7bffdef583..519520ed46 100644 --- a/src/program/vita/test.lua +++ b/src/program/vita/test.lua @@ -118,10 +118,8 @@ function run_softbench (pktsize, npackets, nroutes, cpuspec) sa_ttl = 16 } - local function configure_private_router_softbench (conf) - local c, private = vita.configure_private_router(conf) - - if not conf.private_interface then return c end + local function configure_vita_softbench (conf) + local c, private, public = vita.configure_vita_queue(conf, 1) config.app(c, "bridge", basic_apps.Join) config.link(c, "bridge.output -> "..private.input) @@ -140,21 +138,17 @@ function run_softbench (pktsize, npackets, nroutes, cpuspec) config.link(c, "gauge.output -> sieve.input") config.link(c, "sieve.output -> bridge.arp") + config.link(c, public.output.." -> "..public.input) + return c end - local function softbench_workers (conf) - return { - key_manager = vita.configure_exchange(conf), - private_gauge_router = configure_private_router_softbench(conf), - public_loopback_router = configure_public_router_loopback(conf), - encapsulate = vita.configure_esp(conf), - decapsulate = vita.configure_dsp(conf) - } + local function softbench_worker (conf) + return { softbench = configure_vita_softbench(conf) } end local function wait_gauge () - if not worker.status().private_gauge_router.alive then + if not worker.status().softbench.alive then main.exit() end end @@ -166,22 +160,12 @@ function run_softbench (pktsize, npackets, nroutes, cpuspec) end vita.run_vita{ - setup_fn = softbench_workers, + setup_fn = softbench_worker, initial_configuration = gen_configuration(testconf), cpuset = cpuspec and CPUSet:new():add_from_string(cpuspec) } end -function configure_public_router_loopback (conf, append) - local c, public = vita.configure_public_router(conf, append) - - if not conf.public_interface then return c end - - config.link(c, public.output.." -> "..public.input) - - return c -end - -- Test case generation for Vita via synthetic traffic and configurations. -- Exposes configuration knobs like “number of routes” and “packet size”. @@ -192,7 +176,7 @@ end defaults = { private_interface = {}, - public_interface = {}, + public_interface = {default={["172.16.0.10"]={}}}, route_prefix = {default="172.16"}, nroutes = {default=1}, packet_size = {default="IMIX"}, @@ -209,9 +193,9 @@ private_interface_defaults = { public_interface_defaults = { pci = {default="00:00.0"}, mac = {}, - ip4 = {default="172.16.0.10"}, nexthop_ip4 = {default="172.16.0.10"}, - nexthop_mac = {} + nexthop_mac = {}, + queue = {default=1} } traffic_templates = { @@ -221,10 +205,12 @@ traffic_templates = { local function parse_gentestconf (conf) conf = lib.parse(conf, defaults) - conf.private_interface = lib.parse(conf.private_interface, - private_interface_defaults) - conf.public_interface = lib.parse(conf.public_interface, - public_interface_defaults) + conf.private_interface = + lib.parse(conf.private_interface, private_interface_defaults) + for ip4, interface in pairs(conf.public_interface) do + conf.public_interface[ip4] = + lib.parse(interface, public_interface_defaults) + end assert(conf.nroutes >= 0 and conf.nroutes <= 255, "Invalid number of routes: "..conf.nroutes) return conf @@ -235,7 +221,7 @@ function gen_packet (conf, route, size) assert(payload_size >= 0, "Negative payload_size :-(") local d = datagram:new(packet.resize(packet.allocate(), payload_size)) d:push(ipv4:new{ src = ipv4:pton(conf.private_interface.nexthop_ip4), - dst = ipv4:pton(conf.route_prefix.."."..route..".1"), + dst = ipv4:pton(conf.route_prefix.."."..route.."."..math.random(254)), total_length = ipv4:sizeof() + payload_size, ttl = 64 }) d:push(ethernet:new{ dst = ethernet:pton(conf.private_interface.mac), @@ -252,8 +238,10 @@ function gen_packets (conf) local sizes = traffic_templates[conf.packet_size] or {tonumber(conf.packet_size)} for _, size in ipairs(sizes) do - for route = 1, conf.nroutes do - table.insert(sim_packets, gen_packet(conf, route, size)) + for i = 1, math.floor(1000 / #sizes / conf.nroutes) do + for route = 1, conf.nroutes do + table.insert(sim_packets, gen_packet(conf, route, size)) + end end end return sim_packets @@ -270,12 +258,16 @@ function gen_configuration (conf) sa_ttl = conf.sa_ttl } for route = 1, conf.nroutes do - cfg.route["test"..route] = { + local r = { net_cidr4 = conf.route_prefix.."."..route..".0/24", - gw_ip4 = conf.public_interface.nexthop_ip4, + gateway = {}, preshared_key = ("%064x"):format(route), spi = 1000+route } + for _, interface in pairs(conf.public_interface) do + r.gateway[interface.nexthop_ip4] = {queue=interface.queue} + end + cfg.route["test"..route] = r end return cfg end diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index 64b2f07bfa..7c1803ed07 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -16,8 +16,15 @@ module vita-esp-gateway { leaf nexthop-mac { type yang:mac-address; } } - container private-interface { uses interface; } - container public-interface { uses interface; } + // Single private interface + container private-interface { + uses interface; + } + // One public interface per queue + list public-interface { + key ip4; unique queue; uses interface; + leaf queue { type queue; default 1; } + } leaf mtu { type uint16 { range "0..8937"; } } @@ -25,7 +32,11 @@ module vita-esp-gateway { key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; leaf id { type id; mandatory true; } leaf net-cidr4 { type inet:ipv4-prefix; mandatory true; } - leaf gw-ip4 { type inet:ipv4-address-no-zone; mandatory true; } + list gateway { + key ip4; unique queue; min-elements 1; + leaf ip4 { type inet:ipv4-address-no-zone; } + leaf queue { type queue; default 1;} + } leaf spi { type spi; mandatory true; } leaf preshared-key { type key32; mandatory true; } } @@ -43,9 +54,13 @@ module vita-esp-gateway { } // Active outbound SAs (at most one per route) and inbound SAs (possibly - // multiple per route.) - list outbound-sa { key "spi"; unique "route"; uses sa; } - list inbound-sa { key "spi"; uses sa; } + // multiple per route) per queue. + list sa-database { + key queue; + leaf queue { type queue; } + list outbound-sa { key "spi"; unique "route"; uses sa; } + list inbound-sa { key "spi"; uses sa; } + } // If true, disables KeyManager (route.spi, route.preshared-key, // negotiation-ttl, and sa-ttl are ignored.) @@ -549,4 +564,5 @@ module vita-esp-gateway { typedef key16 { type string { pattern '([0-9a-fA-F]{2}\s*){16}'; } } typedef key4 { type string { pattern '([0-9a-fA-F]{2}\s*){4}'; } } typedef time-to-live { type uint32 { range "1..max"; } } + typedef queue { type uint16 { range "1..max"; } } } diff --git a/src/program/vita/vita-gentest.yang b/src/program/vita/vita-gentest.yang index 249705b10f..f12bacf6a1 100644 --- a/src/program/vita/vita-gentest.yang +++ b/src/program/vita/vita-gentest.yang @@ -4,8 +4,13 @@ module vita-gentest { import vita-esp-gateway { prefix vita; } - container private-interface { uses vita:interface; } - container public-interface { uses vita:interface; } + container private-interface { + uses vita:interface; + } + list public-interface { + key ip4; unique queue; uses vita:interface; + leaf queue { type vita:queue; } + } leaf negotiation-ttl { type vita:time-to-live; } leaf sa-ttl { type vita:time-to-live; } diff --git a/src/program/vita/vita.lua b/src/program/vita/vita.lua index f25dbd07d0..7c68e33e8a 100644 --- a/src/program/vita/vita.lua +++ b/src/program/vita/vita.lua @@ -13,8 +13,7 @@ local nexthop = require("program.vita.nexthop") local exchange = require("program.vita.exchange") local icmp = require("program.vita.icmp") schemata = require("program.vita.schemata") -local Receiver = require("apps.interlink.receiver") -local Transmitter = require("apps.interlink.transmitter") +local basic_apps = require("apps.basic.basic_apps") local intel_mp = require("apps.intel_mp.intel_mp") local ethernet = require("lib.protocol.ethernet") local ipv4 = require("lib.protocol.ipv4") @@ -30,14 +29,13 @@ local confighelp = require("program.vita.README_config_inc") local confspec = { private_interface = {}, - public_interface = {}, + public_interface = {default={}}, mtu = {default=8937}, route = {default={}}, negotiation_ttl = {}, sa_ttl = {}, data_plane = {}, - inbound_sa = {default={}}, - outbound_sa = {default={}} + sa_database = {default={}} } local ifspec = { @@ -45,7 +43,8 @@ local ifspec = { ip4 = {required=true}, nexthop_ip4 = {required=true}, mac = {}, - nexthop_mac = {} + nexthop_mac = {}, + queue = {} } local function derive_local_unicast_mac (prefix, ip4) @@ -67,15 +66,61 @@ local function parse_ifconf (conf, mac_prefix) return conf end -local function parse_conf (conf) - conf = lib.parse(conf, confspec) +-- This takes a Vita configuration (potentially defining multiple queues) and a +-- queue id and returns the configuration for a single given queue by mutating +-- a copy of the configuration. +local function parse_conf (conf, queue) + conf = lib.parse(lib.deepcopy(conf), confspec) + conf.queue = queue + -- all queues share a single private interface conf.private_interface = parse_ifconf(conf.private_interface, {0x2a, 0xbb}) - conf.public_interface = parse_ifconf(conf.public_interface, {0x3a, 0xbb}) + -- select the public interface for the queue from the public interface list + -- (it is possible that no such interface is configured) + local public_interfaces = conf.public_interface + conf.public_interface = nil + for ip4, interface in pairs(public_interfaces) do + interface.ip4 = ip4 + local interface = parse_ifconf(interface, {0x3a, 0xbb}) + if interface.queue == conf.queue then + conf.public_interface = interface + break + end + end + -- for each route, select a single gateway ip to use for this queue + for id, route in pairs(conf.route) do + local gateways = route.gateway + route.gateway = nil + for ip4, gateway in pairs(gateways) do + if gateway.queue == 1 then -- default to the first defined gateway + route.gw_ip4 = ip4 + end + if gateway.queue == conf.queue then + route.gw_ip4 = ip4 + break + end + end + end + -- select the SAs configured for this queue, default to empty SA lists + conf.outbound_sa = {} + conf.inbound_sa = {} + local sa_database = conf.sa_database + conf.sa_database = nil + for queue, sa_db in pairs(sa_database) do + if queue == conf.queue then + conf.outbound_sa = sa_db.outbound_sa + conf.inbound_sa = sa_db.inbound_sa + break + end + end return conf end local sa_db_path = "group/sa_db" +function init_sa_db () + shm.mkdir(shm.resolve(sa_db_path)) +end + -- Vita command-line interface (CLI) function run (args) local long_opt = { @@ -118,7 +163,7 @@ end -- configuration. -- This function does not halt except for fatal error situations. function run_vita (opt) - local sa_db_path = shm.root.."/"..shm.resolve(sa_db_path) + init_sa_db() -- Schema support: because Vita configurations are generally shallow we -- choose to reliably delegate all configuration transitions to core.app by @@ -156,22 +201,47 @@ function run_vita (opt) worker_jit_flush = false } + local function absolute_sa_db_path (queue) + return shm.root.."/"..shm.resolve(sa_db_path) + end + -- Listen for SA database changes. - local sa_db_last_modified + local sa_db_last_modified = {} local function sa_db_needs_reload () - local stat = S.stat(sa_db_path) - if stat and stat.st_mtime ~= sa_db_last_modified then - sa_db_last_modified = stat.st_mtime - return true + local modified = false + for _, queue in ipairs(shm.children(sa_db_path)) do + queue = tonumber(queue) + if queue then -- ignore temp files + local stat = S.stat(absolute_sa_db_path().."/"..queue) + if stat and stat.st_mtime ~= sa_db_last_modified[queue] then + sa_db_last_modified[queue] = stat.st_mtime + modified = true + end + end + end + return modified + end + + -- Load current SA database. + local function load_sa_db () + local sa_db = {} + for _, queue in ipairs(shm.children(sa_db_path)) do + queue = tonumber(queue) + if queue then -- ignore temp files + sa_db[queue] = yang.load_configuration( + absolute_sa_db_path().."/"..queue, + {schema_name='vita-ephemeral-keys'} + ) + end end + return sa_db end -- This is how we imperatively incorporate the SA database into the -- configuration proper. NB: see schema_support and the use of purify above. local function merge_sa_db (sa_db) return function (current_config) - current_config.outbound_sa = sa_db.outbound_sa - current_config.inbound_sa = sa_db.inbound_sa + current_config.sa_database = sa_db return current_config end end @@ -187,34 +257,112 @@ function run_vita (opt) while true do supervisor:main(1) if sa_db_needs_reload() then - local success, sa_db = pcall(yang.load_configuration, sa_db_path, - {schema_name='vita-ephemeral-keys'}) - if success then - supervisor:info("Reloading SA database: %s", sa_db_path) - supervisor:update_configuration(merge_sa_db(sa_db), 'set', '/') - else - supervisor:warn("Failed to read SA database %s: %s", - sa_db_path, sa_db) - end + supervisor:info("Reloading SA database: %s", sa_db_path) + supervisor:update_configuration(merge_sa_db(load_sa_db()), 'set', '/') end end end function vita_workers (conf) - return { - key_manager = configure_exchange(conf), - private_router = configure_private_router_with_nic(conf), - public_router = configure_public_router_with_nic(conf), - encapsulate = configure_esp(conf), - decapsulate = configure_dsp(conf) + local workers = {} + -- Provision a dedicated process/queue for each address of the public + -- interface. + for _, interface in pairs(conf.public_interface or {}) do + local name = "queue"..interface.queue + workers[name] = configure_vita_queue(conf, interface.queue) + end + return workers +end + +function configure_vita_queue (conf, queue) + conf = parse_conf(conf, queue) + + local c = config.new() + local _, key_manager = configure_exchange(conf, c) + local _, private_router = configure_private_router(conf, c) + local _, public_router = configure_public_router(conf, c) + local _, outbound_sa = configure_outbound_sa(conf, c) + local _, inbound_sa = configure_inbound_sa(conf, c) + local _, interfaces = configure_interfaces(conf, c) + + local function link (from, to) config.link(c, from.." -> "..to) end + + if conf.data_plane then + config.app(c, "Null", basic_apps.Sink) + link(public_router.protocol_input, "Null.input") + else + link(public_router.protocol_input, key_manager.input) + link(key_manager.output, public_router.protocol_output) + end + + if interfaces.private then + link(interfaces.private.rx, private_router.input) + link(private_router.output, interfaces.private.tx) + end + if interfaces.public then + link(interfaces.public.rx, public_router.input) + link(public_router.output, interfaces.public.tx) + end + + for _, sa in pairs(conf.outbound_sa) do + link(private_router.outbound[sa.route], outbound_sa.input[sa.route]) + link(outbound_sa.output[sa.route], public_router.outbound[sa.route]) + end + for spi, sa in pairs(conf.inbound_sa) do + local id = sa.route.."_"..spi + link(public_router.inbound[id], inbound_sa.input[id]) + link(inbound_sa.output[id], private_router.inbound[id]) + end + + return c, private_router, public_router +end + +function configure_interfaces (conf, append) + local c = append or config.new() + + local ports = { + private = nil, -- private interface receive/transmit + public = nil -- punlic interface receive/transmit } + + if conf.private_interface and conf.private_interface.pci ~= "00:00.0" then + config.app(c, "PrivateNIC", intel_mp.Intel, { + pciaddr = conf.private_interface.pci, + rxq = conf.queue - 1, + txq = conf.queue - 1 + }) + ports.private = { + rx = "PrivateNIC.output", + tx = "PrivateNIC.input" + } + end + + if conf.public_interface and conf.public_interface.pci ~= "00:00.0" then + config.app(c, "PublicNIC", intel_mp.Intel, { + pciaddr = conf.public_interface.pci, + macaddr = conf.public_interface.mac, + vmdq = true + }) + ports.public = { + rx = "PublicNIC.output", + tx = "PublicNIC.input" + } + end + + return c, ports end function configure_private_router (conf, append) - conf = parse_conf(conf) local c = append or config.new() - if not conf.private_interface then return c end + local ports = { + input = nil, -- private input + output = nil, -- private output + outbound = {}, -- outbound SA queues (to encapsulate) + inbound = {} -- inbound SA queues (decapsulated) + } + + if not conf.private_interface then return c, ports end config.app(c, "PrivateDispatch", dispatch.PrivateDispatch, { node_ip4 = conf.private_interface.ip4 @@ -239,7 +387,8 @@ function configure_private_router (conf, append) node_mac = conf.private_interface.mac, node_ip4 = conf.private_interface.ip4, nexthop_ip4 = conf.private_interface.nexthop_ip4, - nexthop_mac = conf.private_interface.nexthop_mac + nexthop_mac = conf.private_interface.nexthop_mac, + synchronize = true }) config.link(c, "PrivateDispatch.forward4 -> OutboundTTL.input") config.link(c, "PrivateDispatch.icmp4 -> PrivateICMP4.input") @@ -256,32 +405,34 @@ function configure_private_router (conf, append) config.link(c, "InboundTTL.time_exceeded -> InboundICMP4.transit_ttl_exceeded") config.link(c, "InboundICMP4.output -> PrivateRouter.control") + ports.input = "PrivateDispatch.input" + ports.output = "PrivateNextHop.output" + for id, route in pairs(conf.route) do - local private_in = "PrivateRouter."..id - local ESP_in = "ESP_"..id.."_in" - config.app(c, ESP_in.."_Tx", Transmitter, ESP_in) - config.link(c, private_in.." -> "..ESP_in.."_Tx.input") + ports.outbound[id] = "PrivateRouter."..id end for spi, sa in pairs(conf.inbound_sa) do - local private_out = "InboundDispatch."..sa.route.."_"..spi - local DSP_out = "DSP_"..sa.route.."_"..spi.."_out" - config.app(c, DSP_out.."_Rx", Receiver, DSP_out) - config.link(c, DSP_out.."_Rx.output -> "..private_out) + local id = sa.route.."_"..spi + ports.inbound[id] = "InboundDispatch."..id end - local private_links = { - input = "PrivateDispatch.input", - output = "PrivateNextHop.output" - } - return c, private_links + return c, ports end function configure_public_router (conf, append) - conf = parse_conf(conf) local c = append or config.new() - if not conf.public_interface then return c end + local ports = { + input = nil, -- public router input + output = nil, -- public router output + protocol_input = nil, -- incoming key exchange messages + protocol_output = nil, -- outgoing key exchange messages + inbound = {}, -- inbound SA queues (to be decapsulated) + outbound = {} -- outbound SA queues (encapsulated) + } + + if not conf.public_interface then return c, ports end config.app(c, "PublicDispatch", dispatch.PublicDispatch, { node_ip4 = conf.public_interface.ip4 @@ -304,149 +455,96 @@ function configure_public_router (conf, append) config.link(c, "PublicDispatch.protocol4_unreachable -> PublicICMP4.protocol_unreachable") config.link(c, "PublicICMP4.output -> PublicNextHop.icmp4") - if not conf.data_plane then - config.app(c, "Protocol_in_Tx", Transmitter, "Protocol_in") - config.app(c, "Protocol_out_Rx", Receiver, "Protocol_out") - config.link(c, "PublicDispatch.protocol -> Protocol_in_Tx.input") - config.link(c, "Protocol_out_Rx.output -> PublicNextHop.protocol") - end + ports.input = "PublicDispatch.input" + ports.output = "PublicNextHop.output" + + ports.protocol_input = "PublicDispatch.protocol" + ports.protocol_output = "PublicNextHop.protocol" for id, route in pairs(conf.route) do - local public_out = "PublicNextHop."..id - local ESP_out = "ESP_"..id.."_out" local Tunnel = "Tunnel_"..id - config.app(c, ESP_out.."_Rx", Receiver, ESP_out) - config.app(c, Tunnel, tunnel.Tunnel4, - {src=conf.public_interface.ip4, dst=route.gw_ip4}) - config.link(c, ESP_out.."_Rx.output -> "..Tunnel..".input") - config.link(c, Tunnel..".output -> "..public_out) + config.app(c, Tunnel, tunnel.Tunnel4, { + src = conf.public_interface.ip4, + dst = route.gw_ip4 + }) + config.link(c, Tunnel..".output -> PublicNextHop."..id) + ports.outbound[id] = Tunnel..".input" end for spi, sa in pairs(conf.inbound_sa) do - local public_in = "PublicRouter."..sa.route.."_"..spi - local DSP_in = "DSP_"..sa.route.."_"..spi.."_in" - config.app(c, DSP_in.."_Tx", Transmitter, DSP_in) - config.link(c, public_in.." -> "..DSP_in.."_Tx.input") + local id = sa.route.."_"..spi + ports.inbound[id] = "PublicRouter."..id end - local public_links = { - input = "PublicDispatch.input", - output = "PublicNextHop.output" - } - - return c, public_links -end - -local function nic_config (conf, interface) - numa.check_affinity_for_pci_addresses({conf[interface].pci}) - local needs_vmdq = pci.canonical(conf.private_interface.pci) - == pci.canonical(conf.public_interface.pci) - return { - pciaddr = conf[interface].pci, - vmdq = needs_vmdq, - macaddr = needs_vmdq and conf[interface].mac - } -end - -function configure_private_router_with_nic (conf, append) - local c, private = configure_private_router(conf, append) - - if not conf.private_interface then return c end - - config.app(c, "PrivateNIC", intel_mp.Intel, - nic_config(conf, 'private_interface')) - config.link(c, "PrivateNIC.output -> "..private.input) - config.link(c, private.output.." -> PrivateNIC.input") - - return c -end - -function configure_public_router_with_nic (conf, append) - local c, public = configure_public_router(conf, append) - - if not conf.public_interface then return c end - - config.app(c, "PublicNIC", intel_mp.Intel, - nic_config(conf, 'public_interface')) - config.link(c, "PublicNIC.output -> "..public.input) - config.link(c, public.output.." -> PublicNIC.input") - - return c + return c, ports end function configure_exchange (conf, append) - conf = parse_conf(conf) local c = append or config.new() - if conf.data_plane then return end + local ports = { + input = nil, -- key exchange input + output = nil, -- key exchange output + } - if not conf.public_interface then return c end + if conf.data_plane or not conf.public_interface then return c, ports end - config.app(c, "KeyExchange", exchange.KeyManager, { + config.app(c, "KeyManager", exchange.KeyManager, { node_ip4 = conf.public_interface.ip4, routes = conf.route, - sa_db_path = sa_db_path, + sa_db_path = sa_db_path.."/"..conf.queue, negotiation_ttl = conf.negotiation_ttl, sa_ttl = conf.sa_ttl }) - config.app(c, "Protocol_in_Rx", Receiver, "Protocol_in") - config.app(c, "Protocol_out_Tx", Transmitter, "Protocol_out") - config.link(c, "Protocol_in_Rx.output -> KeyExchange.input") - config.link(c, "KeyExchange.output -> Protocol_out_Tx.input") - return c + ports.input = "KeyManager.input" + ports.output = "KeyManager.output" + + return c, ports end -- sa_db := { outbound_sa={=(SA), ...}, inbound_sa={=(SA), ...} } -- (see exchange) -function configure_esp (sa_db, append) - sa_db = parse_conf(sa_db) +function configure_outbound_sa (sa_db, append) local c = append or config.new() + local ports = { input={}, output={} } -- SA input/output pairs + for spi, sa in pairs(sa_db.outbound_sa) do - -- Configure interlink receiver/transmitter for outbound SA - local ESP_in = "ESP_"..sa.route.."_in" - local ESP_out = "ESP_"..sa.route.."_out" - config.app(c, ESP_in.."_Rx", Receiver, ESP_in) - config.app(c, ESP_out.."_Tx", Transmitter, ESP_out) - -- Configure outbound SA - local ESP = "ESP_"..sa.route - config.app(c, ESP, tunnel.Encapsulate, { + local OutboundSA = "OutboundSA_"..sa.route + config.app(c, OutboundSA, tunnel.Encapsulate, { spi = spi, aead = sa.aead, key = sa.key, salt = sa.salt }) - config.link(c, ESP_in.."_Rx.output -> "..ESP..".input4") - config.link(c, ESP..".output -> "..ESP_out.."_Tx.input") + ports.input[sa.route] = OutboundSA..".input4" + ports.output[sa.route] = OutboundSA..".output" end - return c + return c, ports end -function configure_dsp (sa_db, append) - sa_db = parse_conf(sa_db) +function configure_inbound_sa (sa_db, append) local c = append or config.new() + local ports = { input={}, output={} } -- SA input/output pairs + for spi, sa in pairs(sa_db.inbound_sa) do - -- Configure interlink receiver/transmitter for inbound SA - local DSP_in = "DSP_"..sa.route.."_"..spi.."_in" - local DSP_out = "DSP_"..sa.route.."_"..spi.."_out" - config.app(c, DSP_in.."_Rx", Receiver, DSP_in) - config.app(c, DSP_out.."_Tx", Transmitter, DSP_out) + local id = sa.route.."_"..spi -- Configure inbound SA - local DSP = "DSP_"..sa.route.."_"..spi - config.app(c, DSP, tunnel.Decapsulate, { + local InboundSA = "InboundSA_"..id + config.app(c, InboundSA, tunnel.Decapsulate, { spi = spi, aead = sa.aead, key = sa.key, salt = sa.salt, auditing = true }) - config.link(c, DSP_in.."_Rx.output -> "..DSP..".input") - config.link(c, DSP..".output4 -> "..DSP_out.."_Tx.input") + ports.input[id] = InboundSA..".input" + ports.output[id] = InboundSA..".output4" end - return c + return c, ports end From e9f12245436bfabe4b3b2478f1a671518d9493e5 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 23 Jan 2019 10:52:51 +0100 Subject: [PATCH 58/81] core.worker: amend 099cf5363 (set SNABB_WORKER_NAME in worker env) Use period (.) instead of path separator (/) to qualify name to avoid clashes with file system semantics. --- src/core/worker.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core/worker.lua b/src/core/worker.lua index 4c9f13ed1d..9cc9f90559 100644 --- a/src/core/worker.lua +++ b/src/core/worker.lua @@ -43,7 +43,7 @@ function start (name, luacode) S.setenv("SNABB_PROGRAM_LUACODE", luacode, true) -- Save this worker's qualified name in the environment. local parent = engine.program_name or S.getppid() - S.setenv("SNABB_WORKER_NAME", parent.."/"..name, true) + S.setenv("SNABB_WORKER_NAME", parent.."."..name, true) -- Restart the process with execve(). -- /proc/$$/exe is a link to the same Snabb executable that we are running local filename = ("/proc/%d/exe"):format(S.getpid()) From 7b52cf894333d3e1505f00e9abbb9fe4942ce9ee Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 15:03:34 +0100 Subject: [PATCH 59/81] lib.yang.data: fix bug that prevented top-level choices being parsed --- src/lib/yang/data.lua | 54 +++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index bc9d8c56aa..55e77365d6 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -652,22 +652,39 @@ function data_parser_from_grammar(production) function top_parsers.struct(production) local struct_t = production.ctype and typeof(production.ctype) local members = visitn(production.members) + local keys = {} + for k,v in pairs(members) do table.insert(keys, k) end return function(stream) local P = parser_mod.Parser.new(stream) local ret = {} - for k,sub in pairs(members) do ret[normalize_id(k)] = sub.init() end + local expanded_members = {} + for _,k in ipairs(keys) do + if members[k].represents then + -- Choice fields don't include the name of the choice block in the data. They + -- need to be able to provide the parser for the leaves it represents. + local member_parser = members[k].stateful_parser() + for _, node in pairs(members[k].represents()) do + -- Choice fields need to keep state around as they're called multiple times + -- and need to do some validation to comply with spec. + expanded_members[node] = member_parser + end + else + ret[normalize_id(k)] = members[k].init() + expanded_members[k] = members[k] + end + end while true do P:skip_whitespace() if P:is_eof() then break end local k = P:parse_identifier() if k == '' then P:error("Expected a keyword") end - local sub = assert(members[k], 'unrecognized parameter: '..k) + local sub = assert(expanded_members[k], 'unrecognized parameter: '..k) local id = normalize_id(k) ret[id] = sub.parse(P, ret[id], k) end - for k,sub in pairs(members) do + for k,sub in pairs(expanded_members) do local id = normalize_id(k) - ret[id] = sub.finish(ret[id]) + ret[id] = sub.finish(ret[id], k) end if struct_t then return struct_t(ret) else return ret end end @@ -1871,6 +1888,35 @@ function selftest() ]]) assert(success == false) + -- Test top-level choice. + local choice_schema = schema.load_schema([[module toplevel-choice-schema { + namespace "urn:ietf:params:xml:ns:yang:toplevel-choice-schema"; + prefix "test"; + + choice test { + case this { + leaf foo { type string; } + leaf bar { type string; } + } + case that { + leaf baz { type uint32; } + leaf qux { type uint64; } + } + } + }]]) + local choice_data = load_config_for_schema(choice_schema, + mem.open_input_string [[ + foo "hello"; + bar "world"; + ]]) + assert(choice_data.foo == "hello") + assert(choice_data.bar == "world") + local choice_data = load_config_for_schema(choice_schema, + mem.open_input_string [[ + baz 1; + ]]) + assert(choice_data.baz == 1) + -- Test range / length restrictions. local range_length_schema = schema.load_schema([[module range-length-schema { namespace "urn:ietf:params:xml:ns:yang:range-length-schema"; From 0893b1653cd5893dd7c729495828fbef1e7655d1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 19:38:55 +0100 Subject: [PATCH 60/81] lib.yang.data: fix bug when parsing list members of choice cases --- src/lib/yang/data.lua | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 55e77365d6..0311737767 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -494,6 +494,9 @@ function choice_parser(keyword, choices, members, default, mandatory) -- using different leaves from different case statements. local chosen + -- keep track of initialzed members + local inits = {} + local function init() return {} end local function parse(P, out, k) if chosen and choice_map[k] ~= chosen then @@ -501,12 +504,13 @@ function choice_parser(keyword, choices, members, default, mandatory) else chosen = choice_map[k] end - return members[chosen][k].parse(P, members[chosen][k].init(), k) + inits[k] = inits[k] or members[chosen][k].init() + return members[chosen][k].parse(P, inits[k], k) end -- This holds a copy of all the nodes so we know when we've hit the last one. local function finish(out, k) - if out ~= nil then return out end + if out ~= nil then return members[chosen][k].finish(out) end if mandatory and chosen == nil then error("missing choice value: "..keyword) end if default and default == choice_map[k] then return members[default][k].finish() @@ -1888,7 +1892,7 @@ function selftest() ]]) assert(success == false) - -- Test top-level choice. + -- Test top-level choice with list member. local choice_schema = schema.load_schema([[module toplevel-choice-schema { namespace "urn:ietf:params:xml:ns:yang:toplevel-choice-schema"; prefix "test"; @@ -1900,7 +1904,7 @@ function selftest() } case that { leaf baz { type uint32; } - leaf qux { type uint64; } + list qu-x { key id; leaf id { type string; } leaf v { type string; } } } } }]]) @@ -1914,8 +1918,12 @@ function selftest() local choice_data = load_config_for_schema(choice_schema, mem.open_input_string [[ baz 1; + qu-x { id "me"; v "hey"; } + qu-x { id "you"; v "hi"; } ]]) assert(choice_data.baz == 1) + assert(choice_data.qu_x.me.v == "hey") + assert(choice_data.qu_x.you.v == "hi") -- Test range / length restrictions. local range_length_schema = schema.load_schema([[module range-length-schema { From 5eb38578516576821f5d253683fc6e8c18e44b9d Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 21:11:59 +0100 Subject: [PATCH 61/81] lib.yang.data: correctly parse choice case short form. --- src/lib/yang/data.lua | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/lib/yang/data.lua b/src/lib/yang/data.lua index 0311737767..d2642f7f0d 100644 --- a/src/lib/yang/data.lua +++ b/src/lib/yang/data.lua @@ -184,8 +184,12 @@ function data_grammar_from_schema(schema, is_config) function handlers.choice(node) local choices = {} for choice, n in pairs(node.body) do - local members = visit_body(n) - if not is_empty(members) then choices[choice] = members end + if n.kind == 'case' then + local members = visit_body(n) + if not is_empty(members) then choices[choice] = members end + else + choices[choice] = { [choice] = visit(n) } + end end if is_empty(choices) then return end return {type="choice", default=node.default, mandatory=node.mandatory, @@ -1925,6 +1929,21 @@ function selftest() assert(choice_data.qu_x.me.v == "hey") assert(choice_data.qu_x.you.v == "hi") + -- Test choice with case short form. + local choice_schema = schema.load_schema([[module shortform-choice-schema { + namespace "urn:ietf:params:xml:ns:yang:shortform-choice-schema"; + prefix "test"; + + choice test { + default foo; + leaf foo { type string; default "something"; } + leaf bar { type string; } + } + }]]) + local choice_data = load_config_for_schema(choice_schema, + mem.open_input_string "") + assert(choice_data.foo == "something") + -- Test range / length restrictions. local range_length_schema = schema.load_schema([[module range-length-schema { namespace "urn:ietf:params:xml:ns:yang:range-length-schema"; From 1da3b4fd5bc3ecbd0c866d439771a3c7bf981a37 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 21:21:04 +0100 Subject: [PATCH 62/81] vita: fix bugs in schema --- src/program/vita/vita-esp-gateway.yang | 9 ++++----- src/program/vita/vita-gentest.yang | 13 ++++--------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index 11a8af32f9..cd61d9e376 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -29,9 +29,9 @@ module vita-esp-gateway { leaf preshared-key { type key32; mandatory true; } } - choice router { - case v4-over-v4 { - container private-interface4 { uses interface4; } + container private-interface4 { uses interface4; } + choice public-router { + case over-v4 { container public-interface4 { uses interface4; } list route4 { key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; uses route; @@ -39,8 +39,7 @@ module vita-esp-gateway { leaf gw-ip4 { type inet:ipv4-address-no-zone; } } } - case v4-over-v6 { - container private-interface4 { uses interface4; } + case over-v6 { container public-interface6 { uses interface6; } list route46 { key id; unique "net-cidr4"; unique "preshared-key"; unique "spi"; uses route; diff --git a/src/program/vita/vita-gentest.yang b/src/program/vita/vita-gentest.yang index df276884b8..ce279a40a5 100644 --- a/src/program/vita/vita-gentest.yang +++ b/src/program/vita/vita-gentest.yang @@ -4,15 +4,10 @@ module vita-gentest { import vita-esp-gateway { prefix vita; } - choice router { - case v4-over-v4 { - container private-interface4 { uses vita:interface4; } - container public-interface4 { uses vita:interface4; } - } - case v4-over-v6 { - container private-interface4 { uses vita:interface4; } - container public-interface6 { uses vita:interface6; } - } + container private-interface4 { uses vita:interface4; } + choice public-router { + container public-interface4 { uses vita:interface4; } + container public-interface6 { uses vita:interface6; } } leaf negotiation-ttl { type vita:time-to-live; } From b320469ba2dc37f13ace6e1796688327b8d42bf1 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 21:25:50 +0100 Subject: [PATCH 63/81] Makefile.vita: fixes to INCLUDE --- src/Makefile.vita | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Makefile.vita b/src/Makefile.vita index 2ab77db885..f10595d850 100644 --- a/src/Makefile.vita +++ b/src/Makefile.vita @@ -8,14 +8,14 @@ INCLUDE = *.* core arch jit syscall pf \ lib/hardware lib/macaddress.* lib/numa.* lib/cpuset.* \ lib/scheduling.* lib/timers \ apps/basic apps/interlink apps/intel_mp \ + apps/ipv6/nd_light.lua lib/pcap \ program/vita program/config program/ps program/pci_bind \ program/top program/shm INCLUDE_TEST = $(INCLUDE) \ - lib/pmu* apps/test apps/packet_filter apps/ipv4 \ - lib/pcap apps/pcap \ + lib/pmu* apps/test apps/packet_filter apps/ipv4 apps/pcap \ program/snsh program/snabbmark \ - lib/virtio apps/vhost apps/ipsec apps/ipv6 \ + lib/virtio apps/vhost apps/ipsec apps/pcap \ apps/lwaftr/lwutil.* apps/lwaftr/constants.* \ apps/lwaftr/loadgen.* program/loadtest From 5fd3fa8e1efe1612914188fab4e2fc2fa94e5d57 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 21:59:30 +0100 Subject: [PATCH 64/81] Makefile.vita: remove duplicate apps/pcap include --- src/Makefile.vita | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile.vita b/src/Makefile.vita index f10595d850..b9b5bf8e55 100644 --- a/src/Makefile.vita +++ b/src/Makefile.vita @@ -13,7 +13,7 @@ INCLUDE = *.* core arch jit syscall pf \ program/top program/shm INCLUDE_TEST = $(INCLUDE) \ - lib/pmu* apps/test apps/packet_filter apps/ipv4 apps/pcap \ + lib/pmu* apps/test apps/packet_filter apps/ipv4 \ program/snsh program/snabbmark \ lib/virtio apps/vhost apps/ipsec apps/pcap \ apps/lwaftr/lwutil.* apps/lwaftr/constants.* \ From 818024ed1b703ff2f200c42f6f7ba89f1b718395 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 21:58:44 +0100 Subject: [PATCH 65/81] vita: fix bug in schema (missing queue in public interfaces) --- src/program/vita/vita-esp-gateway.yang | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/program/vita/vita-esp-gateway.yang b/src/program/vita/vita-esp-gateway.yang index 21ad2b7bbd..a1b7b987d5 100644 --- a/src/program/vita/vita-esp-gateway.yang +++ b/src/program/vita/vita-esp-gateway.yang @@ -46,6 +46,7 @@ module vita-esp-gateway { list public-interface4 { key ip4; unique queue; uses interface; uses interface4; + leaf queue { type queue; default 1; } } list route4 { key id; unique spi; unique preshared-key; unique net-cidr4; @@ -60,6 +61,7 @@ module vita-esp-gateway { list public-interface6 { key ip6; unique queue; uses interface; uses interface6; + leaf queue { type queue; default 1; } } list route46 { key id; unique spi; unique preshared-key; unique net-cidr4; From 58ce8db2d8595b0458377cadfd049a5312de6cd4 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 26 Jan 2019 18:54:24 +0100 Subject: [PATCH 66/81] KeyManager: rate limit request handling We rate limit the handling of key exchange requests, now that we are no longer running KeyManager in a dedicated process. This way we hopefully avoid a possible DoS by spamming bogus key exchange offers. --- src/program/vita/exchange.lua | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index d526b690a6..002ece369b 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -188,7 +188,8 @@ KeyManager = { inbound_sa_expired = {counter}, outbound_sa_expired = {counter}, outbound_sa_updated = {counter} - } + }, + max_pps = 100 } function KeyManager:new (conf) @@ -202,7 +203,9 @@ function KeyManager:new (conf) challenge_message = Protocol.challenge_message:new({}), nonce_key_message = Protocol.nonce_key_message:new({}), sa_db_updated = false, - sa_db_commit_throttle = lib.throttle(1) + sa_db_commit_throttle = lib.throttle(1), + rate_bucket = KeyManager.max_pps, + rate_throttle = lib.throttle(1) } local self = setmetatable(o, { __index = KeyManager }) self:reconfig(conf) @@ -310,7 +313,7 @@ end function KeyManager:push () -- handle negotiation protocol requests local input = self.input.input - while not link.empty(input) do + while not link.empty(input) and self:rate_limit() do local request = link.receive(input) self:handle_negotiation(request) packet.free(request) @@ -377,6 +380,16 @@ function KeyManager:push () end end +function KeyManager:rate_limit () + if self.rate_throttle() then + self.rate_bucket = self.max_pps + end + if self.rate_bucket > 0 then + self.rate_bucket = self.rate_bucket - 1 + return true + end +end + function KeyManager:negotiate (route) -- Inititate AKE if the protocol fsm permits (i.e., is in the initiator -- state.) From 6ff14eff52294b72a5b62c3abb30eb1a25365810 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 5 Feb 2019 17:23:39 +0100 Subject: [PATCH 67/81] =?UTF-8?q?lib.poptrie=5Flookup:=20incorporate=20pet?= =?UTF-8?q?er=E2=80=99s=20suggestions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib/poptrie_lookup.dasl | 38 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 26 deletions(-) diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl index 3f80bd435c..6c5c2014d3 100644 --- a/src/lib/poptrie_lookup.dasl +++ b/src/lib/poptrie_lookup.dasl @@ -68,19 +68,14 @@ function lookup (Dst, Poptrie, keysize) -- v = extract(key, 0, Poptrie.s) local direct_mask = bit.lshift(1ULL, Poptrie.s) - 1 -- v = band(key, direct_mask) - | mov v, direct_mask - | and v, key + | mov v, key + | and v, direct_mask -- index = dmap[v] | mov index, dword [dmap+v*4] - -- eax = band(index, leaf_tag) - | mov eax, 0x80000000 - | and eax, index - -- is leaf_tag set? - | cmp eax, 0 - | je >1 -- leaf_tag not set, index is a node - -- eax = leaf_tag - 1 (tag inverted) - | mov eax, 0x7FFFFFFF - | and eax, index + -- eax = band(index, leaf_tag - 1) (tag inverted) + | mov eax, index + | btr eax, 31 -- is leaf_tag set? + | jnc >1 -- leaf_tag not set, index is a node | ret -- node, offset = nodes[index], s |1: @@ -90,9 +85,9 @@ function lookup (Dst, Poptrie, keysize) | mov offset, 18 else -- index, node, offset = 0, nodes[index], 0 - | mov index, 0 + | xor index, index | lea node, [nodes+0] -- nodes[0] - | mov offset, 0 + | xor offset, offset end -- while band(vec, lshift(1ULL, v)) ~= 0 |2: @@ -107,15 +102,8 @@ function lookup (Dst, Poptrie, keysize) else error("NYI") end -- vec = nodes[index].vector | mov vec, qword [node+8] - -- rax = lshift(1ULL, v) - | mov rax, 1 - | mov rcx, v - | shl rax, cl - -- rax = band(vec, rax) - | and rax, vec - -- is bit v set in vec? - | cmp rax, 0 - | je >3 -- reached leaf, exit loop + | bt vec, v -- is bit v set in vec? + | jnc >3 -- reached leaf, exit loop -- rax = lshift(2ULL, v) - 1 | mov rax, 2 | mov rcx, v @@ -126,8 +114,8 @@ function lookup (Dst, Poptrie, keysize) | popcnt rax, rax -- index = base + bc - 1 | mov index, dword [node+20] -- nodes[index].base1 - | add index, eax | sub index, 1 + | add index, eax -- node = nodes[index] | imul index, 24 -- multiply by node size | lea node, [nodes+index] @@ -151,8 +139,6 @@ function lookup (Dst, Poptrie, keysize) -- return leaves[base + bc - 1] | mov index, dword [node+16] -- nodes[index].base0 | add index, eax - | sub index, 1 - -- assumption: at most ax is set at this point because popcnt(qword) < byte - | mov ax, word [leaves+index*2] -- leaves[index] + | movzx eax, word [leaves+index*2-2] -- leaves[index] | ret end From bed7ec21d42b0b7bdadd371b9a23c374cec85a3e Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 5 Feb 2019 18:07:36 +0100 Subject: [PATCH 68/81] lib.poptrie_lookup: comment formatting --- src/lib/poptrie_lookup.dasl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl index 6c5c2014d3..9c711de977 100644 --- a/src/lib/poptrie_lookup.dasl +++ b/src/lib/poptrie_lookup.dasl @@ -74,7 +74,8 @@ function lookup (Dst, Poptrie, keysize) | mov index, dword [dmap+v*4] -- eax = band(index, leaf_tag - 1) (tag inverted) | mov eax, index - | btr eax, 31 -- is leaf_tag set? + -- is leaf_tag set? (unsets bit) + | btr eax, 31 | jnc >1 -- leaf_tag not set, index is a node | ret -- node, offset = nodes[index], s @@ -102,7 +103,8 @@ function lookup (Dst, Poptrie, keysize) else error("NYI") end -- vec = nodes[index].vector | mov vec, qword [node+8] - | bt vec, v -- is bit v set in vec? + -- is bit v set in vec? + | bt vec, v | jnc >3 -- reached leaf, exit loop -- rax = lshift(2ULL, v) - 1 | mov rax, 2 From 9d054099971282a496a8d1ce349a1c36b1a4292c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 26 Feb 2019 15:27:22 +0100 Subject: [PATCH 69/81] README.config: some corrections --- src/program/vita/README.config | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/program/vita/README.config b/src/program/vita/README.config index dc284949fb..168bf01abd 100644 --- a/src/program/vita/README.config +++ b/src/program/vita/README.config @@ -14,14 +14,14 @@ CONFIGURATION SYNTAX interface:= pci ; ip4|ip6 ; - nexthop-ip4 ; + nexthop-ip4|nexthop-ip6 ; [ mac ; ] [ nexthop-mac ; ] route:= id ; net-cidr4 ; - gw-ip4|gw-ip6 ; + [ gw-ip4|gw-ip6 ; ]+ preshared-key ; spi ; From 29d087118c9bebda53b153666db5c7f9544773aa Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 26 Jan 2019 12:27:22 +0100 Subject: [PATCH 70/81] core.main: print error to stderr in handler --- src/core/main.lua | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/core/main.lua b/src/core/main.lua index e9679efbd6..ae0677356c 100644 --- a/src/core/main.lua +++ b/src/core/main.lua @@ -168,8 +168,9 @@ function initialize () end function handler (reason) - print(reason) - print(STP.stacktrace()) + io.stderr:write(reason) + io.stderr:write("\n") + io.stderr:write((STP.stacktrace())) if debug_on_error then debug.debug() end os.exit(1) end From 47cc27188c403b878ca96f9b29856459f347aa32 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 26 Feb 2019 15:28:24 +0100 Subject: [PATCH 71/81] Support configs without interfaces and free links for soft testing --- src/program/vita/dispatch.lua | 1 + src/program/vita/test.lua | 2 +- src/program/vita/vita.lua | 17 +++++++++++------ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/program/vita/dispatch.lua b/src/program/vita/dispatch.lua index 1d2036a462..481de44b9c 100644 --- a/src/program/vita/dispatch.lua +++ b/src/program/vita/dispatch.lua @@ -142,6 +142,7 @@ function PublicDispatch:forward6 () end function PublicDispatch:protocol () + if not self.output.protocol then self:reject_protocol(); return end local p = packet.shiftleft(self.p_box[0], ethernet:sizeof() + ipv4:sizeof()) link.transmit(self.output.protocol, p) end diff --git a/src/program/vita/test.lua b/src/program/vita/test.lua index fbb047d22a..a786580305 100644 --- a/src/program/vita/test.lua +++ b/src/program/vita/test.lua @@ -124,7 +124,7 @@ function run_softbench (pktsize, npackets, nroutes, cpuspec, use_v6) end local function configure_vita_softbench (conf) - local c, private, public = vita.configure_vita_queue(conf, 1) + local c, private, public = vita.configure_vita_queue(conf, 1, 'free') config.app(c, "bridge", basic_apps.Join) config.link(c, "bridge.output -> "..private.input) diff --git a/src/program/vita/vita.lua b/src/program/vita/vita.lua index 20a67a1229..38fdb3990e 100644 --- a/src/program/vita/vita.lua +++ b/src/program/vita/vita.lua @@ -310,7 +310,7 @@ function vita_workers (conf) return workers end -function configure_vita_queue (conf, queue) +function configure_vita_queue (conf, queue, free_links) conf = parse_conf(conf, queue) local c = config.new() @@ -323,10 +323,7 @@ function configure_vita_queue (conf, queue) local function link (from, to) config.link(c, from.." -> "..to) end - if conf.data_plane then - config.app(c, "Null", basic_apps.Sink) - link(public_router.protocol_input, "Null.input") - else + if not conf.data_plane then link(public_router.protocol_input, key_manager.input) link(key_manager.output, public_router.protocol_output) end @@ -334,10 +331,18 @@ function configure_vita_queue (conf, queue) if interfaces.private then link(interfaces.private.rx, private_router.input) link(private_router.output, interfaces.private.tx) + elseif not free_links then + config.app(c, "PrivateSink", basic_apps.Sink) + link("PrivateSink.rx", private_router.input) + link(private_router.output, "PrivateSink.tx") end if interfaces.public then link(interfaces.public.rx, public_router.input) link(public_router.output, interfaces.public.tx) + elseif not free_links then + config.app(c, "PublicSink", basic_apps.Sink) + link("PublicSink.rx", public_router.input) + link(public_router.output, "PublicSink.tx") end for _, sa in pairs(conf.outbound_sa) do @@ -350,7 +355,7 @@ function configure_vita_queue (conf, queue) link(inbound_sa.output[id], private_router.inbound[id]) end - return c, private_router, public_router + return c, free_links and private_router, free_links and public_router end function configure_interfaces (conf, append) From 73f4c3d254e72849fa69ac40e934de1787821408 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 25 Jan 2019 22:00:11 +0100 Subject: [PATCH 72/81] program/vita/clitest.sh: add basic test case for Vita CLI --- .travis.yml | 1 + src/program/vita/clitest.conf | 14 +++++++++++++ src/program/vita/clitest.sh | 38 +++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) create mode 100644 src/program/vita/clitest.conf create mode 100755 src/program/vita/clitest.sh diff --git a/.travis.yml b/.travis.yml index 2a088d9496..3d7e11fbe9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,3 +15,4 @@ script: - (cd src && sudo program/vita/conftest.snabb) - (cd src && sudo program/vita/test.snabb IMIX 1e6 3) - (cd src && sudo program/vita/test6.snabb IMIX 1e6 3) + - (cd src && sudo program/vita/clitest.sh) diff --git a/src/program/vita/clitest.conf b/src/program/vita/clitest.conf new file mode 100644 index 0000000000..8ede52b9d6 --- /dev/null +++ b/src/program/vita/clitest.conf @@ -0,0 +1,14 @@ +private-interface4 { + pci 00:00.0; + ip4 172.16.0.10; + nexthop-ip4 172.16.0.1; + nexthop-mac 02:00:00:00:00:00; +} +public-interface4 { + pci 00:00.0; + ip4 172.16.0.10; + nexthop-ip4 172.17.0.10; +} +route-prefix "172.17"; +nroutes 1; +packet-size 60; \ No newline at end of file diff --git a/src/program/vita/clitest.sh b/src/program/vita/clitest.sh new file mode 100755 index 0000000000..4cec8a2a77 --- /dev/null +++ b/src/program/vita/clitest.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -e +set -x + +name=vita-clitest-test-$$ +conf=$(mktemp) + +./vita --name $name & +vita=$! + +function cleanup { kill $vita; } +trap cleanup EXIT HUP INT QUIT TERM + +sleep 1 + +program/vita/genconf.snabb < program/vita/clitest.conf > $conf + +./snabb config set $name / < $conf + +[ $(./snabb config get $name /public-interface4[ip4=172.16.0.10]/nexthop-ip4) \ + = 172.17.0.10 ] + +[ $(./snabb config get $name /route4[id=test1]/spi) = 1001 ] + +./snabb config set $name /mtu 1500 + +[ $(./snabb config get $name /mtu) = 1500 ] + +sleep 1 + +[ $(./snabb config get-state $name \ + /gateway-state/key-manager/negotiations-initiated) = 0 ] + +./snabb config set $name / < Date: Wed, 27 Feb 2019 09:13:00 +0100 Subject: [PATCH 73/81] Update tests for 47cc27188 (Support configs without interfaces ...) --- src/program/vita/conftest.snabb | 2 +- src/program/vita/selftest.snabb | 2 +- src/program/vita/selftest6.snabb | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/program/vita/conftest.snabb b/src/program/vita/conftest.snabb index fb9de0bc9b..c5efde7985 100755 --- a/src/program/vita/conftest.snabb +++ b/src/program/vita/conftest.snabb @@ -12,7 +12,7 @@ vita.init_sa_db() local function test_conf (conf) conf = lib.deepcopy(conf) -- ensure purity - local c, private, public = vita.configure_vita_queue(conf, 1) + local c, private, public = vita.configure_vita_queue(conf, 1, 'free') config.link(c, private.output.."->"..private.input) config.link(c, public.output.."->"..public.input) engine.configure(c) diff --git a/src/program/vita/selftest.snabb b/src/program/vita/selftest.snabb index c902f82724..65740ffdfc 100755 --- a/src/program/vita/selftest.snabb +++ b/src/program/vita/selftest.snabb @@ -76,7 +76,7 @@ local cfg = { } -- Configure Vita queue (without KeyManager.) -local c, private, public = vita.configure_vita_queue(cfg, 1) +local c, private, public = vita.configure_vita_queue(cfg, 1, 'free') -- Add ARP resolvers. config.app(c, "private_arp", ARP, { diff --git a/src/program/vita/selftest6.snabb b/src/program/vita/selftest6.snabb index ad0486b769..60b1db06be 100755 --- a/src/program/vita/selftest6.snabb +++ b/src/program/vita/selftest6.snabb @@ -78,7 +78,7 @@ local cfg = { } -- Configure Vita queue (without KeyManager.) -local c, private, public = vita.configure_vita_queue(cfg, 1) +local c, private, public = vita.configure_vita_queue(cfg, 1, 'free') -- Add ARP resolvers. config.app(c, "private_arp", ARP, { From 4f7f240ca886feae71cb3797f6d834c8504a0be7 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Mon, 18 Feb 2019 15:47:39 +0100 Subject: [PATCH 74/81] Vita: update README.md --- README.md | 82 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index b845e534b1..769c17b226 100644 --- a/README.md +++ b/README.md @@ -2,42 +2,17 @@ 🚧 🚧 🚧 🚧 -…is a *virtual private network* (VPN) gateway you can use to interconnect your -LANs. Vita acts as a tunnel between your local, private network and any number -of remote Vita gateways. With it, nodes spread across your outposts can -communicate with each other as if they were on the same LAN, with -confidentiality and authenticity ensured at the network layer. Vita is probably -more efficient at encapsulating traffic than your application servers. You can -free cycles for your application by offloading your packet encryption and -authentication workload to Vita. +…is a high-performance L3 VPN gateway you can use to interconnect your +networks. Vita acts as a tunnel between your local, private network and any +number of remote Vita gateways. With it, nodes spread across your outposts can +communicate with each other with confidentiality and authenticity ensured at +the network layer. -![a mesh of Vita gateways forms a VPN](vita-sketch.png) - -A Vita network can be as small as two nodes with a single route, and as large -as you like. For each pair of Vita gateways, a separate secure tunnel (*route*) -can be established—“can be” because a Vita network does not need to be a full -mesh, instead arbitrary hierarchies are supported on a route-by-route basis. -Each route uses a pre-shared super key that is installed on both ends of the -route. These keys need to be configured only once, and only need renewal when -compromised, in which case the breach will affect only the route in question. -The actual keys used to encrypt the traffic are ephemeral, and negotiated by -Vita automatically, with no manual intervention required. +Vita is probably more efficient at encapsulating traffic than your application +servers. You can free cycles for your application by offloading your packet +encryption and authentication workload to Vita. -Deploying Vita is easy, and not invasive to your existing infrastructure. It -can be as simple as adding an entry to the IP routing table of your default -gateway, to ensure that packets to destinations within your private network are -routed over an extra hop: the Vita gateway. Whether Vita forwards the -encapsulated packets back to your default gateway, or directly to your modem -depends on your setup, and is freely configurable. - -![private traffic is routed over a Vita gateway, and encapsulated before it is -transmitted over the Internet](vita-detail.png) - -To configure a Vita route, you need to specify the address prefix of the -destination subnetwork, and the public IP address of the target Vita gateway -(in addition to the pre-shared key). At the other end, you specify the source -prefix and gateway address in symmetry. You can even add and remove routes -while Vita is running, without affecting unrelated routes. +![a mesh of Vita gateways forms a VPN](vita-sketch.png) ## WARNING: @@ -45,15 +20,16 @@ while Vita is running, without affecting unrelated routes. ## Features +- ~2.5 Mpps (or ~5 Gbps of IMIX traffic) per core on a modern CPU - Runs on commodity hardware -- Implements IPsec for IPv4, specifically - *IP Encapsulating Security Payload* (ESP) in tunnel mode (audit needed) +- Implements IPsec for IPv4 and IPv6, specifically + *IP Encapsulating Security Payload* (ESP) in tunnel mode - Uses optimized AES-GCM 128-bit encryption based on a reference implementation by *Intel* for their AVX2 (generation-4) processors -- Suitable for 1-Gigabit, 10-Gigabit (and beyond?) Ethernet - Automated key exchange and rotation, with perfect forward secrecy (PFS) (audit needed) -- Dynamic reconfiguration (update routes while running) +- Can act as a pure data-plane and consume SAs established by other means +- Dynamic reconfiguration via YANG RPCs (update routes while running) - Strong observability: access relevant statistics of a running Vita node ## Documentation @@ -97,6 +73,34 @@ machine: End-to-end benchmarking procedures are documented in [vita-loadtest.md](https://github.com/inters/vita/tree/master/src/program/vita/vita-loadtest.md). +## Deployment + +A Vita network can be as small as two nodes with a single route, and as large +as you like. For each pair of Vita gateways, a separate secure tunnel (*route*) +can be established—“can be” because a Vita network does not need to be a full +mesh, instead arbitrary hierarchies are supported on a route-by-route basis. +Each route uses a pre-shared super key that is installed on both ends of the +route. These keys need to be configured only once, and only need renewal when +compromised, in which case the breach will affect only the route in question. +The actual keys used to encrypt the traffic are ephemeral, and negotiated by +Vita automatically, with no manual intervention required. + +Deploying Vita is easy, and not invasive to your existing infrastructure. It +can be as simple as adding an entry to the IP routing table of your default +gateway, to ensure that packets to destinations within your private network are +routed over an extra hop: the Vita gateway. Whether Vita forwards the +encapsulated packets back to your default gateway, or directly to your modem +depends on your setup, and is freely configurable. + +![private traffic is routed over a Vita gateway, and encapsulated before it is +transmitted over the Internet](vita-detail.png) + +To configure a Vita route, you need to specify the address prefix of the +destination subnetwork, and the public IP address of the target Vita gateway +(in addition to the pre-shared key). At the other end, you specify the source +prefix and gateway address in symmetry. You can even add and remove routes +while Vita is running, without affecting unrelated routes. + ## Powered by ![Snabb](snabb.png) @@ -109,5 +113,5 @@ networking toolkit with a wonderful community. ![NLnet](nlnet.png) -[NLnet](https://nlnet.nl) funded Vita development in 2017/2018 with their +[NLnet](https://nlnet.nl) funded Vita development in 2018/2019 with their generous donation. 🙇‍♂️ From a98d5fe6c7dfb26d23b7d50d2746d1b15b1a236c Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Sat, 2 Mar 2019 13:39:22 +0100 Subject: [PATCH 75/81] core.snabbswitch, core.memory: add lua_stacktrace C function... ...and use it to print stacktraces on segfaults. --- src/core/memory.c | 3 ++ src/core/snabbswitch.c | 74 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 1 deletion(-) diff --git a/src/core/memory.c b/src/core/memory.c index 1908732857..2e5de1e198 100644 --- a/src/core/memory.c +++ b/src/core/memory.c @@ -49,6 +49,8 @@ static void set_sigsegv_handler() assert(sigaction(SIGSEGV, &sa, NULL) != -1); } +void lua_stacktrace (); + static void memory_sigsegv_handler(int sig, siginfo_t *si, void *uc) { int fd = -1; @@ -91,6 +93,7 @@ static void memory_sigsegv_handler(int sig, siginfo_t *si, void *uc) (void *)((ucontext_t *)uc)->uc_mcontext.gregs[REG_RSP], si->si_code, si->si_errno); + lua_stacktrace(); fflush(stderr); // Fall back to the default SEGV behavior by resending the signal // now that the handler is disabled. diff --git a/src/core/snabbswitch.c b/src/core/snabbswitch.c index caf514d8a7..0e6e46812f 100644 --- a/src/core/snabbswitch.c +++ b/src/core/snabbswitch.c @@ -1,12 +1,15 @@ /* Use of this source code is governed by the Apache 2.0 license; see COPYING. */ #include +#include #include "lua.h" #include "lualib.h" #include "lauxlib.h" #include +#include +#include #if UINTPTR_MAX != UINT64_MAX #error "64-bit word size required. See doc/porting.md." @@ -15,13 +18,82 @@ int argc; char** argv; +lua_State* L; + int main(int snabb_argc, char **snabb_argv) { /* Store for use by LuaJIT code via FFI. */ argc = snabb_argc; argv = snabb_argv; - lua_State* L = luaL_newstate(); + L = luaL_newstate(); luaL_openlibs(L); return luaL_dostring(L, "require \"core.startup\""); } +#define STR_MAX 1024 +char tempstr[1024]; + +const char *lua_describe (int index) { + int type = lua_type(L, index); + const char *str; + switch (type) { + case LUA_TNONE: + return ""; + case LUA_TNIL: + return "nil"; + case LUA_TNUMBER: + snprintf(tempstr, STR_MAX, "%s: %s", + lua_typename(L, type), + lua_tostring(L, index)); + break; + case LUA_TSTRING: + str = lua_tostring(L, index); + snprintf(tempstr, STR_MAX, "%s: \"%.32s%s", + lua_typename(L, type), + str, strlen(str) > 32 ? "...\"" : "\""); + break; + case LUA_TBOOLEAN: + snprintf(tempstr, STR_MAX, "%s: %s", + lua_typename(L, type), + lua_toboolean(L, index) ? "true" : "false"); + break; + default: + snprintf(tempstr, STR_MAX, "%s: %p", + lua_typename(L, type), + lua_topointer(L, index)); + } + return tempstr; +} + +void lua_stacktrace () { + lua_Debug entry; + int depth = 0; + fprintf(stderr, "\nLua Stacktrace\n==============\n"); + while (lua_getstack(L, depth, &entry)) { + int status = lua_getinfo(L, "Slnf", &entry); + assert(status); + fprintf(stderr, "(%d) %s at %s:%d\n", + depth + 1, + entry.name ? entry.name : "?", + entry.short_src, + entry.currentline); + int local = 1; + while (1) { + const char *name = lua_getlocal(L, &entry, local); + if (!name) break; + fprintf(stderr, " %s = %s\n", name, lua_describe(lua_gettop(L))); + lua_pop(L, 1); + local++; + } + int upvalue = 1; + while (1) { + const char *name = lua_getupvalue(L, lua_gettop(L), upvalue); + if (!name) break; + fprintf(stderr, " %s = %s\n", name, lua_describe(lua_gettop(L))); + lua_pop(L, 1); + upvalue++; + } + lua_pop(L, 1); + depth++; + } +} From ecd9a3eeb78bd64c7a36930e83ad7c12cc86025d Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Fri, 8 Feb 2019 17:31:16 +0100 Subject: [PATCH 76/81] lib.poptrie_lookup: fix bug where offset was not set correctly ...when using direct pointing with s ~= 18. --- src/lib/poptrie_lookup.dasl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/poptrie_lookup.dasl b/src/lib/poptrie_lookup.dasl index 9c711de977..6664396b5b 100644 --- a/src/lib/poptrie_lookup.dasl +++ b/src/lib/poptrie_lookup.dasl @@ -83,7 +83,7 @@ function lookup (Dst, Poptrie, keysize) | imul index, 24 -- multiply by node size | lea node, [nodes+index] -- offset = s - | mov offset, 18 + | mov offset, Poptrie.s else -- index, node, offset = 0, nodes[index], 0 | xor index, index From 15bc06bd5feabcc193678f2ae25adbe4d42f84a0 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 6 Mar 2019 12:47:25 +0100 Subject: [PATCH 77/81] vita/exchange: fix packet corruption due to memory reuse This fixes an embarrassing bug where the KeyManager (of all places) mixed up used for parsing messages with memory used for generating messages. A grep for similar patterns in Vita shows that we usually keep these separate (for good reason!) --- src/program/vita/exchange.lua | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/program/vita/exchange.lua b/src/program/vita/exchange.lua index 002ece369b..cc8a967cf2 100644 --- a/src/program/vita/exchange.lua +++ b/src/program/vita/exchange.lua @@ -198,10 +198,15 @@ function KeyManager:new (conf) ip4 = ipv4:new({}), ip6 = ipv6:new({}), transport = Transport.header:new({}), + transport_in = Transport.header:new({}), nonce_message = Protocol.nonce_message:new({}), + nonce_message_in = Protocol.nonce_message:new({}), key_message = Protocol.key_message:new({}), + key_message_in = Protocol.key_message:new({}), challenge_message = Protocol.challenge_message:new({}), + challenge_message_in = Protocol.challenge_message:new({}), nonce_key_message = Protocol.nonce_key_message:new({}), + nonce_key_message_in = Protocol.nonce_key_message:new({}), sa_db_updated = false, sa_db_commit_throttle = lib.throttle(1), rate_bucket = KeyManager.max_pps, @@ -416,7 +421,7 @@ function KeyManager:handle_negotiation (request) end function KeyManager:handle_nonce_request (route, message) - if not route or message ~= self.nonce_message then return end + if not route or message ~= self.nonce_message_in then return end -- Receive nonce message if the protocol fsm permits -- (responder -> offer_challenge), otherwise reject the message and return. @@ -440,7 +445,7 @@ function KeyManager:handle_nonce_request (route, message) end function KeyManager:handle_challenge_request (route, message) - if not route or message ~= self.challenge_message then return end + if not route or message ~= self.challenge_message_in then return end -- Receive challenge message if the protocol fsm permits -- (accept_challenge -> offer_nonce_key), reject the message and return @@ -471,7 +476,7 @@ function KeyManager:handle_challenge_request (route, message) end function KeyManager:handle_nonce_key_request (route, message) - if not route or message ~= self.nonce_key_message then return end + if not route or message ~= self.nonce_key_message_in then return end -- Receive an authenticated, combined nonce and key message if the protocol -- fsm permits (responder -> offer_key), reject the message and return @@ -517,7 +522,7 @@ function KeyManager:handle_nonce_key_request (route, message) end function KeyManager:handle_key_request (route, message) - if not route or message ~= self.key_message then return end + if not route or message ~= self.key_message_in then return end -- Receive an authenticated key message if the protocol fsm permits -- (accept_key -> initiator), reject the message and return otherwise or @@ -658,7 +663,8 @@ function KeyManager:request (route, message) end function KeyManager:parse_request (request) - local transport = self.transport:new_from_mem(request.data, request.length) + local transport = + self.transport_in:new_from_mem(request.data, request.length) if not transport then counter.add(self.shm.protocol_errors) return @@ -680,13 +686,13 @@ function KeyManager:parse_request (request) local length = request.length - Transport.header:sizeof() local message = (transport:message_type() == Transport.message_type.nonce - and self.nonce_message:new_from_mem(data, length)) + and self.nonce_message_in:new_from_mem(data, length)) or (transport:message_type() == Transport.message_type.key - and self.key_message:new_from_mem(data, length)) + and self.key_message_in:new_from_mem(data, length)) or (transport:message_type() == Transport.message_type.challenge - and self.challenge_message:new_from_mem(data, length)) + and self.challenge_message_in:new_from_mem(data, length)) or (transport:message_type() == Transport.message_type.nonce_key - and self.nonce_key_message:new_from_mem(data, length)) + and self.nonce_key_message_in:new_from_mem(data, length)) if not message then counter.add(self.shm.protocol_errors) return From 6a4231f36c5653eaa9482cc325560be965340b33 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Tue, 5 Mar 2019 22:44:37 +0100 Subject: [PATCH 78/81] vita/tunnel: disable ESP resynchronization by default --- src/program/vita/tunnel.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/vita/tunnel.lua b/src/program/vita/tunnel.lua index 3e0138c9ab..bae995ea8c 100644 --- a/src/program/vita/tunnel.lua +++ b/src/program/vita/tunnel.lua @@ -60,7 +60,7 @@ Decapsulate = { key = {required=true}, salt = {required=true}, window_size = {}, - resync_threshold = {}, + resync_threshold = {default=1/0}, -- disable resynchronization resync_attempts = {}, auditing = {} }, From 3ed5659c40aa6827a563f1c8659cd71a82c7bab6 Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 6 Mar 2019 14:42:02 +0100 Subject: [PATCH 79/81] lib.tsc: avoid recalibrating rdtsc on every call to tsc:new() --- src/lib/tsc.lua | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/lib/tsc.lua b/src/lib/tsc.lua index f12b74727c..b0e91c3a75 100644 --- a/src/lib/tsc.lua +++ b/src/lib/tsc.lua @@ -27,6 +27,16 @@ rdtsc = require('dynasm').loadstring [[ return ffi.cast('uint64_t (*)()', rdtsc_code) ]]() +local function rdtsc_calibrate () + local start_ns = C.get_time_ns() + local start_ticks = rdtsc() + for _ = 1, calibration_interval do end + local end_ticks = rdtsc() + local end_ns = C.get_time_ns() + return tonumber(end_ticks - start_ticks)/tonumber(end_ns - start_ns) + * 1000000000 + 0ULL +end + local cpuinfo = lib.readfile("/proc/cpuinfo", "*a") assert(cpuinfo, "failed to read /proc/cpuinfo for tsc check") local have_usable_rdtsc = (cpuinfo:match("constant_tsc") and @@ -35,21 +45,11 @@ local have_usable_rdtsc = (cpuinfo:match("constant_tsc") and local time_sources = { rdtsc = { time_fn = rdtsc, - calibrate_fn = function () - local start_ns = C.get_time_ns() - local start_ticks = rdtsc() - for _ = 1, calibration_interval do end - local end_ticks = rdtsc() - local end_ns = C.get_time_ns() - return tonumber(end_ticks - start_ticks)/tonumber(end_ns - start_ns) - * 1000000000 + 0ULL - end + tps = have_usable_rdtsc and rdtsc_calibrate() }, system = { time_fn = C.get_time_ns, - calibrate_fn = function () - return 1000000000ULL - end + tps = 1000000000ULL } } @@ -69,7 +69,7 @@ function new (arg) "tsc: unknown time source '" .. o._source .."'") o._time_fn = source.time_fn -- Ticks per second (uint64) - o._tps = source.calibrate_fn() + o._tps = source.tps -- Nanoseconds per tick (Lua number) o._nspt = 1/tonumber(o._tps) * 1000000000 From 57a53e60f8d7dc994f229ee867e196fdc4b8a57d Mon Sep 17 00:00:00 2001 From: Max Rottenkolber Date: Wed, 6 Mar 2019 16:57:21 +0100 Subject: [PATCH 80/81] program/vita/clitest.sh: fix a bogus test assertion --- src/program/vita/clitest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/vita/clitest.sh b/src/program/vita/clitest.sh index 4cec8a2a77..18a4bb6f09 100755 --- a/src/program/vita/clitest.sh +++ b/src/program/vita/clitest.sh @@ -30,7 +30,7 @@ program/vita/genconf.snabb < program/vita/clitest.conf > $conf sleep 1 [ $(./snabb config get-state $name \ - /gateway-state/key-manager/negotiations-initiated) = 0 ] + /gateway-state/private-router/route-errors) = 0 ] ./snabb config set $name / < Date: Thu, 7 Mar 2019 12:16:46 +0100 Subject: [PATCH 81/81] vita/clitest.sh: tweak sleep duration before testing get-state --- src/program/vita/clitest.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/program/vita/clitest.sh b/src/program/vita/clitest.sh index 18a4bb6f09..8e134a2d02 100755 --- a/src/program/vita/clitest.sh +++ b/src/program/vita/clitest.sh @@ -27,7 +27,7 @@ program/vita/genconf.snabb < program/vita/clitest.conf > $conf [ $(./snabb config get $name /mtu) = 1500 ] -sleep 1 +sleep 2 [ $(./snabb config get-state $name \ /gateway-state/private-router/route-errors) = 0 ]