diff --git a/test/case/ietf_interfaces/Readme.adoc b/test/case/ietf_interfaces/Readme.adoc index cbf69d66f..72e656c9a 100644 --- a/test/case/ietf_interfaces/Readme.adoc +++ b/test/case/ietf_interfaces/Readme.adoc @@ -33,6 +33,8 @@ include::dual_bridge/Readme.adoc[] include::lag_basic/Readme.adoc[] +include::lag_failure/Readme.adoc[] + include::igmp_basic/Readme.adoc[] include::igmp_vlan/Readme.adoc[] diff --git a/test/case/ietf_interfaces/ietf_interfaces.yaml b/test/case/ietf_interfaces/ietf_interfaces.yaml index 49c1fe572..8e8e07c50 100644 --- a/test/case/ietf_interfaces/ietf_interfaces.yaml +++ b/test/case/ietf_interfaces/ietf_interfaces.yaml @@ -38,6 +38,9 @@ - name: lag_basic case: lag_basic/test.py +- name: lag_failure + case: lag_failure/test.py + - name: bridge_fwd_sgl_dut case: bridge_fwd_sgl_dut/test.py diff --git a/test/case/ietf_interfaces/lag_failure/Readme.adoc b/test/case/ietf_interfaces/lag_failure/Readme.adoc new file mode 100644 index 000000000..ca929a743 --- /dev/null +++ b/test/case/ietf_interfaces/lag_failure/Readme.adoc @@ -0,0 +1,45 @@ +=== Link Aggregation Silent Failure +==== Description +Verify communication over a link aggregate in static and LACP mode when +member links stop passing traffic without any carrier loss. In static +mode the ARP monitor is used in both ends of the lag, in LACP mode this +is not necessary, and must in fact be disabled. + +.Logical network setup, link breakers (lb1 & lb2) here managed by host PC +ifdef::topdoc[] +image::../../test/case/ietf_interfaces/lag_failure/lag-failure.svg[] +endif::topdoc[] +ifndef::topdoc[] +ifdef::testgroup[] +image::lag_failure/lag-failure.svg[] +endif::testgroup[] +ifndef::testgroup[] +image::lag-failure.svg[] +endif::testgroup[] +endif::topdoc[] + +The host verifies connectivity with dut2 via dut1 over the aggregate for +each test step using the `mon` interface. + +==== Topology +ifdef::topdoc[] +image::../../test/case/ietf_interfaces/lag_failure/topology.svg[Link Aggregation Silent Failure topology] +endif::topdoc[] +ifndef::topdoc[] +ifdef::testgroup[] +image::lag_failure/topology.svg[Link Aggregation Silent Failure topology] +endif::testgroup[] +ifndef::testgroup[] +image::topology.svg[Link Aggregation Silent Failure topology] +endif::testgroup[] +endif::topdoc[] +==== Test sequence +. Set up topology and attach to target DUTs +. Set up static link aggregate, lag0, on dut1 and dut2 +. Verify failure modes for static mode +. Set up LACP link aggregate, lag0, on dut1 and dut2 +. Verify failure modes for lacp mode + + +<<< + diff --git a/test/case/ietf_interfaces/lag_failure/foo.json b/test/case/ietf_interfaces/lag_failure/foo.json new file mode 100644 index 000000000..e30c62ce5 --- /dev/null +++ b/test/case/ietf_interfaces/lag_failure/foo.json @@ -0,0 +1,218 @@ +{ + "ietf-interfaces:interfaces": { + "interface": [ + { + "name": "br0", + "type": "infix-if-type:bridge", + "enabled": true, + "ietf-ip:ipv4": { + "address": [ + { + "ip": "192.168.2.41", + "prefix-length": 24 + } + ] + } + }, + { + "name": "e1", + "type": "infix-if-type:etherlike", + "ietf-ip:ipv6": {} + }, + { + "name": "e2", + "type": "infix-if-type:etherlike", + "ietf-ip:ipv6": {}, + "infix-interfaces:bridge-port": { + "bridge": "br0" + } + }, + { + "name": "e3", + "type": "infix-if-type:etherlike", + "enabled": true, + "ietf-ip:ipv6": {}, + "infix-interfaces:lag-port": { + "lag": "lag0" + } + }, + { + "name": "e4", + "type": "infix-if-type:etherlike", + "enabled": true, + "ietf-ip:ipv6": {}, + "infix-interfaces:lag-port": { + "lag": "lag0" + } + }, + { + "name": "e5", + "type": "infix-if-type:etherlike", + "ietf-ip:ipv6": {} + }, + { + "name": "e7", + "type": "infix-if-type:etherlike", + "ietf-ip:ipv6": {} + }, + { + "name": "e8", + "type": "infix-if-type:etherlike", + "ietf-ip:ipv6": {} + }, + { + "name": "lag0", + "type": "infix-if-type:lag", + "enabled": true, + "infix-interfaces:lag": { + "mode": "lacp", + "lacp": { + "rate": "fast" + }, + "arp-monitor": { + "interval": 100, + "peer": [ + "192.168.2.42" + ] + }, + "link-monitor": { + "interval": 100 + } + }, + "infix-interfaces:bridge-port": { + "bridge": "br0" + } + }, + { + "name": "lo", + "type": "infix-if-type:loopback", + "ietf-ip:ipv4": { + "address": [ + { + "ip": "127.0.0.1", + "prefix-length": 8 + } + ] + }, + "ietf-ip:ipv6": { + "address": [ + { + "ip": "::1", + "prefix-length": 128 + } + ] + } + } + ] + }, + "ietf-keystore:keystore": { + "asymmetric-keys": { + "asymmetric-key": [ + { + "name": "genkey", + "public-key-format": "ietf-crypto-types:ssh-public-key-format", + "public-key": "MIIBCgKCAQEAs7RX6r3flRmw6TiWpGclHrFxghsjZxhO1hjC3w0fkgprf08vXB4X+oXtqNx6wNInt1muVo6ALtZu+zBJZsL9wMJE3fbrMTKTWRY/P+1m5YgsTwihm7ucRh5LH6FPHQq3ezcS590/uOvigbkP6ofTiqvS4hrA8QB/EyBiKKPmiNOvKyBaMNGiXWy9EgQeYRyjthIBdh7zwgzpMk+P86n5dQQo1uXwimKGGkMNlUW9wilxkKFdN9gqTUoID+C1cQTNLoG4BtmZYOBvyJDn0SRGi72n2KkudAunIUUgUAjwPf9T/FjkxJBZJnFZNS2Fsxa7RBRk7Swy2RivUQPkQ6hdEQIDAQAB", + "private-key-format": "ietf-crypto-types:rsa-private-key-format", + "cleartext-private-key": "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCztFfqvd+VGbDpOJakZyUesXGCGyNnGE7WGMLfDR+SCmt/Ty9cHhf6he2o3HrA0ie3Wa5WjoAu1m77MElmwv3AwkTd9usxMpNZFj8/7WbliCxPCKGbu5xGHksfoU8dCrd7NxLn3T+46+KBuQ/qh9OKq9LiGsDxAH8TIGIoo+aI068rIFow0aJdbL0SBB5hHKO2EgF2HvPCDOkyT4/zqfl1BCjW5fCKYoYaQw2VRb3CKXGQoV032CpNSggP4LVxBM0ugbgG2Zlg4G/IkOfRJEaLvafYqS50C6chRSBQCPA9/1P8WOTEkFkmcVk1LYWzFrtEFGTtLDLZGK9RA+RDqF0RAgMBAAECggEAUVwzt8BaY0975MNtlKZsTG6rBOxThYAgZVdVlxYCdqTIEZ4gw5SOZ6rONHcKDpW3TJBKWb7vswT7vzcX7HIY3/Y0psf9qMsMojdr0H4j6YTTBr9SJ8dzk91wRrRKNMxe0ObY5OgrSwZlCTVnytfPA0gS1LKnKxX98oNlXaADJmvoRUrhF/C2hV8BhUfvAINNtsh/xllsRl7c5SmW44i9E1nvmtc6vxMBkKN5EwufVDfJfhea/tp9k12V6y+0cOzbfvvH3rjpsT6566Qun0LfpVhj03B26/c90NHKIgeyj7g/+h0qj+CozPs8yDcb1uCG1Ikeaej/uMMdzt0BzagtRwKBgQD5U/suoXyP6AQNhZZdNOQo76N95fuo8A4XtK8eCbMAeCcCRFTD18m3Q8VIT6Y5kOe+h81bzwJ0OMO8P/H3ewEw4qSWoDzrAnyoKPOEpiJB5YQjhoHcafh2J98m3dFtAInEtVccP1/9ocH/tsX7JmxKlBFgqJU+97m9xgAZrnkx4wKBgQC4g2geXX7V2Pk9xFDU7BibREZqag6sz/msgeXLe4GRmrE334qgFm+LVSfjNvDyW6KrdvGqyn467rHP9kcesMcWTs/mp3Bek8899HxT4UiGgSr9kI081k1p5C6Al/yNwhH8BOUotP5hHu/YMHCx4SRNR0rOBk+CSwLaoLPQfuYXewKBgCHOR0J9VtxUQyhqMocUwtLiGzLY2hR/6AlC0HOsMP8hS3i9NxkOyyT0JW22jv1DLojg9PE70kNb5v5BVVeO8AxmzpY1x8y9m5VZaBtWQ1LYAeCnPjhajfvHUDR+4wR6jDOFuvfzh9pl8l3vtExnW0uJZAnNEd9ly2N101GoHHqtAoGANBlep9xMeQOH9OnezRBRLl5L57ZEqIUdAZm4EgmwnzVnvtgO438SRexok96qkDRRrUqrmEcO94L4kDkBAeh2fpUIXR+AOiRQSzUieejNurT0N56+UqRMPY8hlkvUEw7uVxPmxOS+QxwiKxAacg+ZWXy84Ymkn6yghK0FuORsAcsCgYEAvimXpxeZDJeTKouofwTjNafgE/SE2BEnYPGjun6o1iWYS1ifrCXpQTXGbypXdefgrYg+qd6gu+opYstex3S11gONA/HewOZ6Tr3FKfByAH7zovUwqDfFYnphsj0njXjsHL+tqUz3ARls1K6zbHxzNUnyzSeB2byZwScXBv6s32o=" + } + ] + } + }, + "ietf-netconf-acm:nacm": { + "enable-nacm": true, + "groups": { + "group": [ + { + "name": "admin", + "user-name": [ + "admin" + ] + } + ] + }, + "rule-list": [ + { + "name": "admin-acl", + "group": [ + "admin" + ], + "rule": [ + { + "name": "permit-all", + "module-name": "*", + "access-operations": "*", + "action": "permit", + "comment": "Allow 'admin' group complete access to all operations and data." + } + ] + }, + { + "name": "default-deny-all", + "group": [ + "*" + ], + "rule": [ + { + "name": "deny-password-read", + "module-name": "ietf-system", + "path": "/ietf-system:system/authentication/user/password", + "access-operations": "*", + "action": "deny" + } + ] + } + ] + }, + "ietf-netconf-server:netconf-server": { + "listen": { + "endpoints": { + "endpoint": [ + { + "name": "default-ssh", + "ssh": { + "tcp-server-parameters": { + "local-address": "::" + }, + "ssh-server-parameters": { + "server-identity": { + "host-key": [ + { + "name": "default-key", + "public-key": { + "central-keystore-reference": "genkey" + } + } + ] + } + } + } + } + ] + } + } + }, + "ietf-system:system": { + "hostname": "dut1", + "authentication": { + "user": [ + { + "name": "admin", + "password": "$factory$", + "infix-system:shell": "infix-system:bash" + } + ] + } + }, + "infix-meta:meta": { + "version": "1.2" + }, + "infix-services:mdns": { + "enabled": true + }, + "infix-services:web": { + "enabled": true, + "restconf": { + "enabled": true + } + } +} diff --git a/test/case/ietf_interfaces/lag_failure/lag-failure.svg b/test/case/ietf_interfaces/lag_failure/lag-failure.svg new file mode 100644 index 000000000..15963fed6 --- /dev/null +++ b/test/case/ietf_interfaces/lag_failure/lag-failure.svg @@ -0,0 +1,4 @@ + + + +
lag
dut1
eth
eth
eth
bridge
eth
eth
lag
ip
pc
eth
ip
dut2
lb1
lb2
\ No newline at end of file diff --git a/test/case/ietf_interfaces/lag_failure/test.py b/test/case/ietf_interfaces/lag_failure/test.py new file mode 100755 index 000000000..061730a27 --- /dev/null +++ b/test/case/ietf_interfaces/lag_failure/test.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +r"""Link Aggregation Silent Failure + +Verify communication over a link aggregate in static and LACP mode when +member links stop passing traffic without any carrier loss. In static +mode the ARP monitor is used in both ends of the lag, in LACP mode this +is not necessary, and must in fact be disabled. + +.Logical network setup, link breakers (lb1 & lb2) here managed by host PC +image::lag-failure.svg[] + +The host verifies connectivity with dut2 via dut1 over the aggregate for +each test step using the `mon` interface. + +""" +import infamy +from infamy.netns import TPMR +from infamy.util import parallel, until + + +class LinkBreaker: + """Encapsulates basic, dumb link-breaking ops over SSH.""" + + def __init__(self, env, ns): + self.ns = ns + self.lb1 = TPMR(env.ltop.xlate("host", "lb1a")[1], + env.ltop.xlate("host", "lb1b")[1]).start() + self.lb2 = TPMR(env.ltop.xlate("host", "lb2a")[1], + env.ltop.xlate("host", "lb2b")[1]).start() + + def forward(self, lb1, lb2): + """Set link breakers in forwarding or blocking state.""" + getattr(self.lb1, lb1)() + getattr(self.lb2, lb2)() + + def fail_check(self, peer): + """Verify connectivity with a given peer during failure.""" + sequence = [ + ("forward", "forward"), + ("block", "forward"), + ("forward", "block"), + ("forward", "forward") + ] + + print(f"{'LB1':<8} | {'LB2':<8} | {'Status':<8}") + print("---------|----------|---------") + + for lb1, lb2 in sequence: + try: + print(f"{lb1:<8} | {lb2:<8} | {'...':<8}", end="\r# ") + self.forward(lb1, lb2) + self.ns.must_reach(peer, timeout=10) + print(f"{lb1:<8} | {lb2:<8} | {'OK':<8}") + except Exception as e: + print(f"{lb1:<8} | {lb2:<8} | {'FAIL':<8}") + breakpoint() + print(f"\nError encountered: {e}") + print(f"Link breakers were in state: LB1='{lb1}', LB2='{lb2}'") + raise + +def lag_init(dut, mode): + """Set up link aggregate on dut""" + _, link1 = env.ltop.xlate(dut.name, "link1") + _, link2 = env.ltop.xlate(dut.name, "link2") + + try: + _, dmon = env.ltop.xlate(dut.name, "mon") + except TypeError: + dmon = None + + if dmon: + # dut1 + extra = [ + { + "name": "br0", + "type": "infix-if-type:bridge", + "enabled": True, + "ipv4": { + "address": [ + { + "ip": "192.168.2.41", + "prefix-length": 24 + } + ] + } + }, { + "name": dmon, + "bridge-port": { + "bridge": "br0", + } + }, { + "name": "lag0", + "bridge-port": { + "bridge": "br0", + } + } + ] + if mode == "static": + extra += [ + { + "name": "lag0", + "lag": { + "arp-monitor": { + "interval": 100, + "peer": [ + "192.168.2.42" + ] + } + } + } + ] + else: + extra += [ + { + "name": "lag0", + "lag": { + "lacp": { + "rate": "fast" + }, + "arp-monitor": { + "interval": 0 + }, + "link-monitor": { + "interval": 100 + } + } + } + ] + else: + # dut2 + extra = [ + { + "name": "lag0", + "ipv4": { + "address": [ + { + "ip": "192.168.2.42", + "prefix-length": 24 + } + ] + } + } + ] + if mode == "static": + extra += [ + { + "name": "lag0", + "lag": { + "arp-monitor": { + "interval": 100, + "peer": [ + "192.168.2.41" + ] + } + } + } + ] + else: + extra += [ + { + "name": "lag0", + "lag": { + "lacp": { + "rate": "fast" + }, + "arp-monitor": { + "interval": 0 + }, + "link-monitor": { + "interval": 100 + } + } + } + ] + + dut.put_config_dicts({ + "ietf-interfaces": { + "interfaces": { + "interface": [ + { + "name": "lag0", + "type": "infix-if-type:lag", + "enabled": True, + "lag": { + "mode": mode + } + }, { + "name": link1, + "enabled": True, + "lag-port": { + "lag": "lag0" + } + }, { + "name": link2, + "enabled": True, + "lag-port": { + "lag": "lag0" + } + } + ] + extra + } + } + }) + + +with infamy.Test() as test: + with test.step("Set up topology and attach to target DUTs"): + env = infamy.Env() + dut1 = env.attach("dut1", "mgmt") + dut2 = env.attach("dut2", "mgmt") + + _, mon = env.ltop.xlate("host", "mon") + with infamy.IsolatedMacVlan(mon) as ns: + lb = LinkBreaker(env, ns) + ns.addip("192.168.2.1") + + with test.step("Set up static link aggregate, lag0, on dut1 and dut2"): + parallel(lambda: lag_init(dut1, "static"), + lambda: lag_init(dut2, "static")) + + with test.step("Verify failure modes for static mode"): + lb.fail_check("192.168.2.42") + + with test.step("Set up LACP link aggregate, lag0, on dut1 and dut2"): + parallel(lambda: lag_init(dut1, "lacp"), + lambda: lag_init(dut2, "lacp")) + + with test.step("Verify failure modes for lacp mode"): + lb.fail_check("192.168.2.42") + + test.succeed() diff --git a/test/case/ietf_interfaces/lag_failure/topology.dot b/test/case/ietf_interfaces/lag_failure/topology.dot new file mode 100644 index 000000000..3c111a7b5 --- /dev/null +++ b/test/case/ietf_interfaces/lag_failure/topology.dot @@ -0,0 +1,36 @@ +graph "lag" { + layout="neato"; + overlap="false"; + esep="+23"; + + node [shape=record, fontsize=12, fontname="DejaVu Sans Mono, Book"]; + edge [color="cornflowerblue", penwidth="2", fontname="DejaVu Serif, Book"]; + + host [ + label="{{ mgmt1 | mon | lb1a | lb2a | lb2b | lb1b | mgmt2 } | host}", + pos="9,0!", + kind="controller", + ]; + + dut1 [ + label="{ dut1\l | { mgmt | mon | link1 | link2 } }", + pos="0,6!", + kind="infix", + ]; + + dut2 [ + label="{ dut2\r | { link2 | link1 | mgmt } }", + pos="18,6!", + kind="infix", + ]; + + host:mgmt1 -- dut1:mgmt [kind=mgmt, color=lightgray] + host:mon -- dut1:mon // Monitor connection to dut2 via dut1 + host:mgmt2 -- dut2:mgmt [kind=mgmt color=lightgrey] + + dut1:link1 -- host:lb1a [color=black, fontcolor=black] + host:lb1b -- dut2:link1 [color=black, fontcolor=black] + + dut1:link2 -- host:lb2a [color=black, fontcolor=black] + host:lb2b -- dut2:link2 [color=black, fontcolor=black] +} diff --git a/test/case/ietf_interfaces/lag_failure/topology.svg b/test/case/ietf_interfaces/lag_failure/topology.svg new file mode 100644 index 000000000..44888247e --- /dev/null +++ b/test/case/ietf_interfaces/lag_failure/topology.svg @@ -0,0 +1,93 @@ + + + + + + +lag + + + +host + +mgmt1 + +mon + +lb1a + +lb2a + +lb2b + +lb1b + +mgmt2 + +host + + + +dut1 + +dut1 + +mgmt + +mon + +link1 + +link2 + + + +host:mgmt1--dut1:mgmt + + + + +host:mon--dut1:mon + + + + +dut2 + +dut2 + +link2 + +link1 + +mgmt + + + +host:mgmt2--dut2:mgmt + + + + +host:lb1b--dut2:link1 + + + + +host:lb2b--dut2:link2 + + + + +dut1:link1--host:lb1a + + + + +dut1:link2--host:lb2a + + + +