From 6f9e6159be265ca91f873576d15ccbbc061fed8d Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sun, 7 Apr 2024 14:15:36 +0000 Subject: [PATCH 001/188] T5169: Add PoC for generating CGNAT rules rfc6888 Add PoC for generating CGNAT rules https://datatracker.ietf.org/doc/html/rfc6888 Not all requirements are implemented, but some of them. Implemented: REQ-2 ``` A CGN MUST have a default "IP address pooling" behavior of "Paired" CGN must use the same external IP address mapping for all sessions associated with the same internal IP address, be they TCP, UDP, ICMP, something else, or a mix of different protocols. ``` REQ-3 ``` The CGN function SHOULD NOT have any limitations on the size or the contiguity of the external address pool ``` REQ-4 ``` A CGN MUST support limiting the number of external ports (or, equivalently, "identifiers" for ICMP) that are assigned per subscriber ``` CLI: ``` set nat cgnat pool external ext1 external-port-range '1024-65535' set nat cgnat pool external ext1 per-user-limit port '1000' set nat cgnat pool external ext1 range 192.0.2.222/32 set nat cgnat pool internal int1 range '100.64.0.0/28' set nat cgnat rule 10 source pool 'int1' set nat cgnat rule 10 translation pool 'ext1' ``` --- data/templates/firewall/nftables-cgnat.j2 | 47 ++++ interface-definitions/nat_cgnat.xml.in | 197 +++++++++++++++ src/conf_mode/nat_cgnat.py | 288 ++++++++++++++++++++++ 3 files changed, 532 insertions(+) create mode 100644 data/templates/firewall/nftables-cgnat.j2 create mode 100644 interface-definitions/nat_cgnat.xml.in create mode 100755 src/conf_mode/nat_cgnat.py diff --git a/data/templates/firewall/nftables-cgnat.j2 b/data/templates/firewall/nftables-cgnat.j2 new file mode 100644 index 00000000000..79a8e3d5a4a --- /dev/null +++ b/data/templates/firewall/nftables-cgnat.j2 @@ -0,0 +1,47 @@ +#!/usr/sbin/nft -f + +add table ip cgnat +flush table ip cgnat + +add map ip cgnat tcp_nat_map { type ipv4_addr: interval ipv4_addr . inet_service ; flags interval ;} +add map ip cgnat udp_nat_map { type ipv4_addr: interval ipv4_addr . inet_service ; flags interval ;} +add map ip cgnat icmp_nat_map { type ipv4_addr: interval ipv4_addr . inet_service ; flags interval ;} +add map ip cgnat other_nat_map { type ipv4_addr: interval ipv4_addr ; flags interval ;} +flush map ip cgnat tcp_nat_map +flush map ip cgnat udp_nat_map +flush map ip cgnat icmp_nat_map +flush map ip cgnat other_nat_map + +table ip cgnat { + map tcp_nat_map { + type ipv4_addr : interval ipv4_addr . inet_service + flags interval + elements = { {{ proto_map_elements }} } + } + + map udp_nat_map { + type ipv4_addr : interval ipv4_addr . inet_service + flags interval + elements = { {{ proto_map_elements }} } + } + + map icmp_nat_map { + type ipv4_addr : interval ipv4_addr . inet_service + flags interval + elements = { {{ proto_map_elements }} } + } + + map other_nat_map { + type ipv4_addr : interval ipv4_addr + flags interval + elements = { {{ other_map_elements }} } + } + + chain POSTROUTING { + type nat hook postrouting priority srcnat; policy accept; + ip protocol tcp counter snat ip to ip saddr map @tcp_nat_map + ip protocol udp counter snat ip to ip saddr map @udp_nat_map + ip protocol icmp counter snat ip to ip saddr map @icmp_nat_map + counter snat ip to ip saddr map @other_nat_map + } +} diff --git a/interface-definitions/nat_cgnat.xml.in b/interface-definitions/nat_cgnat.xml.in new file mode 100644 index 00000000000..caa26b4d911 --- /dev/null +++ b/interface-definitions/nat_cgnat.xml.in @@ -0,0 +1,197 @@ + + + + + + + Carrier-grade NAT (CGNAT) parameters + 221 + + + + + External and internal pool parameters + + + + + External pool name + + txt + External pool name + + + #include + + Name of pool can only contain alpha-numeric letters, hyphen and underscores + + + + + Port range + + range + Numbered port range (e.g., 1001-1005) + + + + + + 1024-65535 + + + + Per user limits for the pool + + + + + Ports per user + + u32:1-65535 + Numeric IP port + + + + + + 2000 + + + + + + Range of IP addresses + + ipv4net + IPv4 prefix + + + ipv4range + IPv4 address range + + + + + + + + + + + Sequence + + u32:1-999999 + Sequence number + + + + + Sequence number must be between 1 and 999999 + + + + + + + + + Internal pool name + + txt + Internal pool name + + + #include + + Name of pool can only contain alpha-numeric letters, hyphen and underscores + + + + + Range of IP addresses + + ipv4net + IPv4 prefix + + + ipv4range + IPv4 address range + + + + + + + + + + + + + + + Rule + + u32:1-999999 + Number for this CGNAT rule + + + + + Rule number must be between 1 and 999999 + + + + + Source parameters + + + + + Source internal pool + + nat cgnat pool internal + + + txt + Source internal pool name + + + #include + + Name of pool can only contain alpha-numeric letters, hyphen and underscores + + + + + + + Translation parameters + + + + + Translation external pool + + nat cgnat pool external + + + txt + Translation external pool name + + + #include + + Name of pool can only contain alpha-numeric letters, hyphen and underscores + + + + + + + + + + + diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py new file mode 100755 index 00000000000..f41d66c6654 --- /dev/null +++ b/src/conf_mode/nat_cgnat.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import ipaddress +import jmespath +import os + +from sys import exit + +from vyos.config import Config +from vyos.template import render +from vyos.utils.process import cmd +from vyos.utils.process import run +from vyos import ConfigError +from vyos import airbag + +airbag.enable() + + +nftables_cgnat_config = '/run/nftables-cgnat.nft' + + +class IPOperations: + def __init__(self, ip_prefix: str): + self.ip_prefix = ip_prefix + self.ip_network = ipaddress.ip_network(ip_prefix) if '/' in ip_prefix else None + + def get_ips_count(self) -> int: + """Returns the number of IPs in a prefix or range. + + Example: + % ip = IPOperations('192.0.2.0/30') + % ip.get_ips_count() + 4 + % ip = IPOperations('192.0.2.0-192.0.2.2') + % ip.get_ips_count() + 3 + """ + if '-' in self.ip_prefix: + start_ip, end_ip = self.ip_prefix.split('-') + start_ip = ipaddress.ip_address(start_ip) + end_ip = ipaddress.ip_address(end_ip) + return int(end_ip) - int(start_ip) + 1 + elif '/31' in self.ip_prefix: + return 2 + elif '/32' in self.ip_prefix: + return 1 + else: + return sum( + 1 + for _ in [self.ip_network.network_address] + + list(self.ip_network.hosts()) + + [self.ip_network.broadcast_address] + ) + + def convert_prefix_to_list_ips(self) -> list: + """Converts a prefix or IP range to a list of IPs including the network and broadcast addresses. + + Example: + % ip = IPOperations('192.0.2.0/30') + % ip.convert_prefix_to_list_ips() + ['192.0.2.0', '192.0.2.1', '192.0.2.2', '192.0.2.3'] + % + % ip = IPOperations('192.0.0.1-192.0.2.5') + % ip.convert_prefix_to_list_ips() + ['192.0.2.1', '192.0.2.2', '192.0.2.3', '192.0.2.4', '192.0.2.5'] + """ + if '-' in self.ip_prefix: + start_ip, end_ip = self.ip_prefix.split('-') + start_ip = ipaddress.ip_address(start_ip) + end_ip = ipaddress.ip_address(end_ip) + return [ + str(ipaddress.ip_address(ip)) + for ip in range(int(start_ip), int(end_ip) + 1) + ] + elif '/31' in self.ip_prefix: + return [ + str(ip) + for ip in [ + self.ip_network.network_address, + self.ip_network.broadcast_address, + ] + ] + elif '/32' in self.ip_prefix: + return [str(self.ip_network.network_address)] + else: + return [ + str(ip) + for ip in [self.ip_network.network_address] + + list(self.ip_network.hosts()) + + [self.ip_network.broadcast_address] + ] + + +def generate_port_rules( + external_hosts: list, + internal_hosts: list, + port_count: int, + global_port_range: str = '1024-65535', +) -> list: + """Generates list of nftables rules for the batch file.""" + rules = [] + proto_map_elements = [] + other_map_elements = [] + start_port, end_port = map(int, global_port_range.split('-')) + total_possible_ports = (end_port - start_port) + 1 + + # Calculate the required number of ports per host + required_ports_per_host = port_count + + # Check if there are enough external addresses for all internal hosts + if required_ports_per_host * len(internal_hosts) > total_possible_ports * len( + external_hosts + ): + raise ConfigError("Not enough ports available for the specified parameters!") + + current_port = start_port + current_external_index = 0 + + for internal_host in internal_hosts: + external_host = external_hosts[current_external_index] + next_end_port = current_port + required_ports_per_host - 1 + + # If the port range exceeds the end_port, move to the next external host + while next_end_port > end_port: + current_external_index = (current_external_index + 1) % len(external_hosts) + external_host = external_hosts[current_external_index] + current_port = start_port + next_end_port = current_port + required_ports_per_host - 1 + + # Ensure the same port is not assigned to the same external host + if any( + rule.endswith(f'{external_host}:{current_port}-{next_end_port}') + for rule in rules + ): + raise ConfigError("Not enough ports available for the specified parameters") + + proto_map_elements.append( + f'{internal_host} : {external_host} . {current_port}-{next_end_port}' + ) + other_map_elements.append(f'{internal_host} : {external_host}') + + current_port = next_end_port + 1 + if current_port > end_port: + current_port = start_port + current_external_index += 1 # Move to the next external host + + return [proto_map_elements, other_map_elements] + + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + + base = ['nat', 'cgnat'] + config = conf.get_config_dict( + base, + get_first_key=True, + key_mangling=('-', '_'), + no_tag_node_value_mangle=True, + with_recursive_defaults=True, + ) + + return config + + +def verify(config): + # bail out early - looks like removal from running config + if not config: + return None + + if 'pool' not in config: + raise ConfigError(f'Pool must be defined!') + if 'rule' not in config: + raise ConfigError(f'Rule must be defined!') + + # As PoC allow only one rule for CGNAT translations + # one internal pool and one external pool + if len(config['rule']) > 1: + raise ConfigError(f'Only one rule is allowed for translations!') + + for pool in ('external', 'internal'): + if pool not in config['pool']: + raise ConfigError(f'{pool} pool must be defined!') + for pool_name, pool_config in config['pool'][pool].items(): + if 'range' not in pool_config: + raise ConfigError( + f'Range for "{pool} pool {pool_name}" must be defined!' + ) + + for rule, rule_config in config['rule'].items(): + if 'source' not in rule_config: + raise ConfigError(f'Rule "{rule}" source pool must be defined!') + if 'pool' not in rule_config['source']: + raise ConfigError(f'Rule "{rule}" source pool must be defined!') + + if 'translation' not in rule_config: + raise ConfigError(f'Rule "{rule}" translation pool must be defined!') + + +def generate(config): + if not config: + return None + # first external pool as we allow only one as PoC + ext_pool_name = jmespath.search("rule.*.translation | [0]", config).get('pool') + int_pool_name = jmespath.search("rule.*.source | [0]", config).get('pool') + ext_query = f"pool.external.{ext_pool_name}.range | keys(@)" + int_query = f"pool.internal.{int_pool_name}.range" + external_ranges = jmespath.search(ext_query, config) + internal_ranges = [jmespath.search(int_query, config)] + + external_list_count = [] + external_list_hosts = [] + internal_list_count = [] + internal_list_hosts = [] + for ext_range in external_ranges: + # External hosts count + e_count = IPOperations(ext_range).get_ips_count() + external_list_count.append(e_count) + # External hosts list + e_hosts = IPOperations(ext_range).convert_prefix_to_list_ips() + external_list_hosts.extend(e_hosts) + for int_range in internal_ranges: + # Internal hosts count + i_count = IPOperations(int_range).get_ips_count() + internal_list_count.append(i_count) + # Internal hosts list + i_hosts = IPOperations(int_range).convert_prefix_to_list_ips() + internal_list_hosts.extend(i_hosts) + + external_host_count = sum(external_list_count) + internal_host_count = sum(internal_list_count) + ports_per_user = int( + jmespath.search(f'pool.external.{ext_pool_name}.per_user_limit.port', config) + ) + external_port_range: str = jmespath.search( + f'pool.external.{ext_pool_name}.external_port_range', config + ) + + proto_maps, other_maps = generate_port_rules( + external_list_hosts, internal_list_hosts, ports_per_user, external_port_range + ) + + config['proto_map_elements'] = ', '.join(proto_maps) + config['other_map_elements'] = ', '.join(other_maps) + + render(nftables_cgnat_config, 'firewall/nftables-cgnat.j2', config) + + # dry-run newly generated configuration + tmp = run(f'nft --check --file {nftables_cgnat_config}') + if tmp > 0: + raise ConfigError('Configuration file errors encountered!') + + +def apply(config): + if not config: + # Cleanup cgnat + cmd('nft delete table ip cgnat') + if os.path.isfile(nftables_cgnat_config): + os.unlink(nftables_cgnat_config) + return None + cmd(f'nft --file {nftables_cgnat_config}') + + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) From c455a1f71674300b8a74863ddfe6e551fe8fd252 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Tue, 9 Apr 2024 11:02:58 +0000 Subject: [PATCH 002/188] T6214: T6213: change constraint in order to not allow string starting with dot character; use such constraint in firewall group definitions. --- interface-definitions/firewall.xml.in | 26 ++++++++++++------- .../alpha-numeric-hyphen-underscore-dot.xml.i | 2 +- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in index 3219471b1aa..24e63c5eccf 100644 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -56,8 +56,9 @@ Firewall address-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot @@ -96,7 +97,7 @@ [a-zA-Z_][a-zA-Z0-9]?[\w\-\.]* - Name of domain-group can only contain alpha-numeric letters, hyphen, underscores and not start with numeric + Name of domain-group can only contain alphanumeric letters, hyphen, underscores and not start with numeric @@ -124,8 +125,9 @@ Firewall dynamic address group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot #include @@ -148,8 +150,9 @@ Firewall interface-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot @@ -177,8 +180,9 @@ Firewall ipv6-address-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot @@ -215,8 +219,9 @@ Firewall ipv6-network-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot #include @@ -248,8 +253,9 @@ Firewall mac-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot #include @@ -281,8 +287,9 @@ Firewall network-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot #include @@ -314,8 +321,9 @@ Firewall port-group - [a-zA-Z0-9][\w\-\.]* + #include + Name of firewall group can only contain alphanumeric letters, hyphen, underscores and dot #include diff --git a/interface-definitions/include/constraint/alpha-numeric-hyphen-underscore-dot.xml.i b/interface-definitions/include/constraint/alpha-numeric-hyphen-underscore-dot.xml.i index 7aeb8526024..34c94e53c11 100644 --- a/interface-definitions/include/constraint/alpha-numeric-hyphen-underscore-dot.xml.i +++ b/interface-definitions/include/constraint/alpha-numeric-hyphen-underscore-dot.xml.i @@ -1,3 +1,3 @@ -[-_a-zA-Z0-9.]+ +[-_a-zA-Z0-9][\w\-\.\+]* From 29a20ce9f9792e23137be57358ca52ddee7ac54b Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 11 Apr 2024 09:10:05 +0000 Subject: [PATCH 003/188] T6222: VRRP show prefix for long rfc3768-compatibility interfaces If we use rfc3768-compatibility with long interface names like eth1.100.200 it converts the VRRP interface name name to `v` For example `eth2.100.200v10v4` The limit for interface name is 15 symbols and it causes that interface name is ignoring by keepalived VMAC interface name 'eth2.100.200v10v4' too long or invalid characters - ignoring And it uses the default prefix `vrrp` for such cases. It works fine, but such interfaces are not displayed in the op-mode Allow prefix `vrrp` for the op-mode for `show interfaces` --- python/vyos/ifconfig/section.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/vyos/ifconfig/section.py b/python/vyos/ifconfig/section.py index 5e98cd510d4..50273cf67b5 100644 --- a/python/vyos/ifconfig/section.py +++ b/python/vyos/ifconfig/section.py @@ -97,7 +97,7 @@ def _intf_under_section (cls,section='',vlan=True): for ifname in interfaces: ifsection = cls.section(ifname) - if not ifsection: + if not ifsection and not ifname.startswith('vrrp'): continue if section and ifsection != section: From 36baf771b8ea52487bf6c913d2019f926acbc4f3 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 10 Apr 2024 13:44:07 +0000 Subject: [PATCH 004/188] T6216: firewall: add patch while migrating from 1.3 to 1.4 in order to avoid errors when using character <+> in 1.3 in firewall groups and custom firewall chains. --- src/migration-scripts/firewall/6-to-7 | 80 +++++++++++++++++++++++---- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/src/migration-scripts/firewall/6-to-7 b/src/migration-scripts/firewall/6-to-7 index 72f07880b5e..938044c6d12 100755 --- a/src/migration-scripts/firewall/6-to-7 +++ b/src/migration-scripts/firewall/6-to-7 @@ -107,6 +107,12 @@ icmpv6_translations = { 'unknown-option': [4, 2] } +v4_found = False +v6_found = False +v4_groups = ["address-group", "network-group", "port-group"] +v6_groups = ["ipv6-address-group", "ipv6-network-group", "port-group"] +translated_dict = {} + if config.exists(base + ['group']): for group_type in config.list_nodes(base + ['group']): for group_name in config.list_nodes(base + ['group', group_type]): @@ -114,6 +120,19 @@ if config.exists(base + ['group']): if config.exists(name_description): tmp = config.return_value(name_description) config.set(name_description, value=tmp[:max_len_description]) + if '+' in group_name: + replacement_string = "_" + if group_type in v4_groups and not v4_found: + v4_found = True + if group_type in v6_groups and not v6_found: + v6_found = True + new_group_name = group_name.replace('+', replacement_string) + while config.exists(base + ['group', group_type, new_group_name]): + replacement_string = replacement_string + "_" + new_group_name = group_name.replace('+', replacement_string) + translated_dict[group_name] = new_group_name + config.copy(base + ['group', group_type, group_name], base + ['group', group_type, new_group_name]) + config.delete(base + ['group', group_type, group_name]) if config.exists(base + ['name']): for name in config.list_nodes(base + ['name']): @@ -173,11 +192,31 @@ if config.exists(base + ['name']): config.set(rule_icmp + ['type'], value=translate[0]) config.set(rule_icmp + ['code'], value=translate[1]) - for src_dst in ['destination', 'source']: - pg_base = base + ['name', name, 'rule', rule, src_dst, 'group', 'port-group'] - proto_base = base + ['name', name, 'rule', rule, 'protocol'] - if config.exists(pg_base) and not config.exists(proto_base): - config.set(proto_base, value='tcp_udp') + for direction in ['destination', 'source']: + if config.exists(base + ['name', name, 'rule', rule, direction]): + if config.exists(base + ['name', name, 'rule', rule, direction, 'group']) and v4_found: + for group_type in config.list_nodes(base + ['name', name, 'rule', rule, direction, 'group']): + group_name = config.return_value(base + ['name', name, 'rule', rule, direction, 'group', group_type]) + if '+' in group_name: + if group_name[0] == "!": + new_group_name = "!" + translated_dict[group_name[1:]] + else: + new_group_name = translated_dict[group_name] + config.set(base + ['name', name, 'rule', rule, direction, 'group', group_type], value=new_group_name) + + pg_base = base + ['name', name, 'rule', rule, direction, 'group', 'port-group'] + proto_base = base + ['name', name, 'rule', rule, 'protocol'] + if config.exists(pg_base) and not config.exists(proto_base): + config.set(proto_base, value='tcp_udp') + + if '+' in name: + replacement_string = "_" + new_name = name.replace('+', replacement_string) + while config.exists(base + ['name', new_name]): + replacement_string = replacement_string + "_" + new_name = name.replace('+', replacement_string) + config.copy(base + ['name', name], base + ['name', new_name]) + config.delete(base + ['name', name]) if config.exists(base + ['ipv6-name']): for name in config.list_nodes(base + ['ipv6-name']): @@ -250,12 +289,31 @@ if config.exists(base + ['ipv6-name']): else: config.rename(rule_icmp + ['type'], 'type-name') - for src_dst in ['destination', 'source']: - pg_base = base + ['ipv6-name', name, 'rule', rule, src_dst, 'group', 'port-group'] - proto_base = base + ['ipv6-name', name, 'rule', rule, 'protocol'] - if config.exists(pg_base) and not config.exists(proto_base): - config.set(proto_base, value='tcp_udp') - + for direction in ['destination', 'source']: + if config.exists(base + ['ipv6-name', name, 'rule', rule, direction]): + if config.exists(base + ['ipv6-name', name, 'rule', rule, direction, 'group']) and v6_found: + for group_type in config.list_nodes(base + ['ipv6-name', name, 'rule', rule, direction, 'group']): + group_name = config.return_value(base + ['ipv6-name', name, 'rule', rule, direction, 'group', group_type]) + if '+' in group_name: + if group_name[0] == "!": + new_group_name = "!" + translated_dict[group_name[1:]] + else: + new_group_name = translated_dict[group_name] + config.set(base + ['ipv6-name', name, 'rule', rule, direction, 'group', group_type], value=new_group_name) + + pg_base = base + ['ipv6-name', name, 'rule', rule, direction, 'group', 'port-group'] + proto_base = base + ['ipv6-name', name, 'rule', rule, 'protocol'] + if config.exists(pg_base) and not config.exists(proto_base): + config.set(proto_base, value='tcp_udp') + + if '+' in name: + replacement_string = "_" + new_name = name.replace('+', replacement_string) + while config.exists(base + ['ipv6-name', new_name]): + replacement_string = replacement_string + "_" + new_name = name.replace('+', replacement_string) + config.copy(base + ['ipv6-name', name], base + ['ipv6-name', new_name]) + config.delete(base + ['ipv6-name', name]) try: with open(file_name, 'w') as f: f.write(config.to_string()) From ecc83562b4d756cc50910561a3f52ec260aeb478 Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Thu, 28 Dec 2023 22:08:36 -0800 Subject: [PATCH 005/188] T5871: ipsec remote access VPN: specify "cacerts" for client auth. --- data/templates/ipsec/swanctl/remote_access.j2 | 5 +++++ smoketest/scripts/cli/test_vpn_ipsec.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/data/templates/ipsec/swanctl/remote_access.j2 b/data/templates/ipsec/swanctl/remote_access.j2 index af7f2994e4c..adfa32bde79 100644 --- a/data/templates/ipsec/swanctl/remote_access.j2 +++ b/data/templates/ipsec/swanctl/remote_access.j2 @@ -34,6 +34,11 @@ {% elif rw_conf.authentication.client_mode.startswith("eap") %} auth = {{ rw_conf.authentication.client_mode }} eap_id = %any +{% endif %} +{% if rw_conf.authentication.client_mode is vyos_defined('eap-tls') or rw_conf.authentication.client_mode is vyos_defined('x509') %} +{# pass all configured CAs as filenames, separated by commas #} +{# this will produce a string like "MyCA1.pem,MyCA2.pem" #} + cacerts = {{ '.pem,'.join(rw_conf.authentication.x509.ca_certificate) ~ '.pem' }} {% endif %} } children { diff --git a/smoketest/scripts/cli/test_vpn_ipsec.py b/smoketest/scripts/cli/test_vpn_ipsec.py index 6d3a93877eb..145b5990e37 100755 --- a/smoketest/scripts/cli/test_vpn_ipsec.py +++ b/smoketest/scripts/cli/test_vpn_ipsec.py @@ -757,6 +757,7 @@ def test_remote_access_eap_tls(self): f'id = "{local_id}"', f'auth = pubkey', f'certs = peer1.pem', + f'cacerts = MyVyOS-CA.pem', f'auth = eap-tls', f'eap_id = %any', f'esp_proposals = aes256-sha512,aes256-sha384,aes256-sha256,aes256-sha1,aes128gcm128-sha256', @@ -840,6 +841,7 @@ def test_remote_access_x509(self): with self.assertRaises(ConfigSessionError): self.cli_commit() self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'x509', 'ca-certificate', ca_name]) + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'x509', 'ca-certificate', int_ca_name]) self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'esp-group', esp_group]) self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'ike-group', ike_group]) @@ -867,6 +869,7 @@ def test_remote_access_x509(self): f'id = "{local_id}"', f'auth = pubkey', f'certs = peer1.pem', + f'cacerts = MyVyOS-CA.pem,MyVyOS-IntCA.pem', f'esp_proposals = aes256-sha512,aes256-sha384,aes256-sha256,aes256-sha1,aes128gcm128-sha256', f'rekey_time = {eap_lifetime}s', f'rand_time = 540s', @@ -894,6 +897,7 @@ def test_remote_access_x509(self): # Check Root CA, Intermediate CA and Peer cert/key pair is present self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{ca_name}.pem'))) + self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{int_ca_name}.pem'))) self.assertTrue(os.path.exists(os.path.join(CERT_PATH, f'{peer_name}.pem'))) self.tearDownPKI() From 52c02ade031f165da18e6fd0542f3952f2cc9bb6 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Thu, 11 Apr 2024 11:40:07 +0300 Subject: [PATCH 006/188] T6100: Added NAT migration from IP/Netmask to Network/Netmask Added NAT migration from IP/Netmask to Network/Netmask. In 1.3 allowed using IP/Netmask in Nat rules. In 1.4 and 1.5 it is prohibited. Allowed Network/Netmask. --- src/migration-scripts/nat/5-to-6 | 120 ++++++++++++++++++++----------- 1 file changed, 79 insertions(+), 41 deletions(-) diff --git a/src/migration-scripts/nat/5-to-6 b/src/migration-scripts/nat/5-to-6 index c83b93d8470..cfe98ddcfec 100755 --- a/src/migration-scripts/nat/5-to-6 +++ b/src/migration-scripts/nat/5-to-6 @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2023 VyOS maintainers and contributors +# Copyright (C) 2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -18,46 +18,84 @@ # to # 'set nat [source|destination] rule X [inbound-interface|outbound interface] interface-name ' +# T6100: Migration from 1.3.X to 1.4 +# Change IP/netmask to Network/netmask in +# 'set nat [source|destination] rule X [source| destination| translation] address ' + +import ipaddress from sys import argv,exit from vyos.configtree import ConfigTree -if len(argv) < 2: - print("Must specify file name!") - exit(1) - -file_name = argv[1] - -with open(file_name, 'r') as f: - config_file = f.read() - -config = ConfigTree(config_file) - -if not config.exists(['nat']): - # Nothing to do - exit(0) - -for direction in ['source', 'destination']: - # If a node doesn't exist, we obviously have nothing to do. - if not config.exists(['nat', direction]): - continue - - # However, we also need to handle the case when a 'source' or 'destination' sub-node does exist, - # but there are no rules under it. - if not config.list_nodes(['nat', direction]): - continue - - for rule in config.list_nodes(['nat', direction, 'rule']): - base = ['nat', direction, 'rule', rule] - for iface in ['inbound-interface','outbound-interface']: - if config.exists(base + [iface]): - tmp = config.return_value(base + [iface]) - if tmp: - config.delete(base + [iface]) - config.set(base + [iface, 'interface-name'], value=tmp) - -try: - with open(file_name, 'w') as f: - f.write(config.to_string()) -except OSError as e: - print("Failed to save the modified config: {}".format(e)) - exit(1) + +def _func_T5643(conf, base_path): + for iface in ['inbound-interface', 'outbound-interface']: + if conf.exists(base_path + [iface]): + tmp = conf.return_value(base_path + [iface]) + if tmp: + conf.delete(base_path + [iface]) + conf.set(base_path + [iface, 'interface-name'], value=tmp) + return + + +def _func_T6100(conf, base_path): + for addr_type in ['source', 'destination', 'translation']: + base_addr_type = base_path + [addr_type] + if not conf.exists(base_addr_type) or not conf.exists( + base_addr_type + ['address']): + continue + + address = conf.return_value(base_addr_type + ['address']) + + if not address or '/' not in address: + continue + + negative = '' + network = address + if '!' in address: + negative = '!' + network = str(address.split(negative)[1]) + + network_ip = ipaddress.ip_network(network, strict=False) + if str(network_ip) != network: + network = f'{negative}{str(network_ip)}' + conf.set(base_addr_type + ['address'], value=network) + return + + +if __name__ == '__main__': + if len(argv) < 2: + print("Must specify file name!") + exit(1) + + file_name = argv[1] + + with open(file_name, 'r') as f: + config_file = f.read() + + config = ConfigTree(config_file) + + if not config.exists(['nat']): + # Nothing to do + exit(0) + + for direction in ['source', 'destination']: + # If a node doesn't exist, we obviously have nothing to do. + if not config.exists(['nat', direction]): + continue + + # However, we also need to handle the case when a 'source' or 'destination' sub-node does exist, + # but there are no rules under it. + if not config.list_nodes(['nat', direction]): + continue + + for rule in config.list_nodes(['nat', direction, 'rule']): + base = ['nat', direction, 'rule', rule] + _func_T5643(config,base) + _func_T6100(config,base) + + try: + with open(file_name, 'w') as f: + f.write(config.to_string()) + except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) From 6d8336f5ad2d9c4e0f12b54681db2924d6998d2d Mon Sep 17 00:00:00 2001 From: Nataliia Solomko <81954790+natali-rs1985@users.noreply.github.com> Date: Fri, 12 Apr 2024 12:25:21 +0300 Subject: [PATCH 007/188] pppoe-server: T6141: T5364: PPPoE-server add pado-delay without sessions fails (#3296) --- data/templates/accel-ppp/pppoe.config.j2 | 9 ++++--- .../scripts/cli/test_service_pppoe-server.py | 22 +++++++++++++++ src/conf_mode/service_pppoe-server.py | 27 ++++++++++++++++++- 3 files changed, 53 insertions(+), 5 deletions(-) diff --git a/data/templates/accel-ppp/pppoe.config.j2 b/data/templates/accel-ppp/pppoe.config.j2 index 6b01958e50d..ddf0da51812 100644 --- a/data/templates/accel-ppp/pppoe.config.j2 +++ b/data/templates/accel-ppp/pppoe.config.j2 @@ -64,12 +64,13 @@ vlan-mon={{ iface }},{{ iface_config.vlan | join(',') }} service-name={{ service_name | join(',') }} {% endif %} {% if pado_delay %} -{% set pado_delay_param = namespace(value='0') %} -{% for delay in pado_delay | sort(attribute='0') %} +{% set delay_without_sessions = pado_delay.delays_without_sessions[0] | default('0') %} +{% set pado_delay_param = namespace(value=delay_without_sessions) %} +{% for delay, sessions in pado_delay.delays_with_sessions | sort(attribute='1') %} {% if not loop.last %} -{% set pado_delay_param.value = pado_delay_param.value + ',' + delay + ':' + pado_delay[delay].sessions %} +{% set pado_delay_param.value = pado_delay_param.value + ',' + delay + ':' + sessions | string %} {% else %} -{% set pado_delay_param.value = pado_delay_param.value + ',-1:' + pado_delay[delay].sessions %} +{% set pado_delay_param.value = pado_delay_param.value + ',-1:' + sessions | string %} {% endif %} {% endfor %} pado-delay={{ pado_delay_param.value }} diff --git a/smoketest/scripts/cli/test_service_pppoe-server.py b/smoketest/scripts/cli/test_service_pppoe-server.py index d7c7aa164ca..5a48b1f584c 100755 --- a/smoketest/scripts/cli/test_service_pppoe-server.py +++ b/smoketest/scripts/cli/test_service_pppoe-server.py @@ -148,6 +148,28 @@ def test_pppoe_server_vlan(self): tmp = ','.join(vlans) self.assertIn(f'vlan-mon={interface},{tmp}', config) + def test_pppoe_server_pado_delay(self): + delay_without_sessions = '10' + delays = {'20': '200', '30': '300'} + + self.basic_config() + + self.set(['pado-delay', delay_without_sessions]) + self.cli_commit() + + conf = ConfigParser(allow_no_value=True, delimiters='=') + conf.read(self._config_file) + self.assertEqual(conf['pppoe']['pado-delay'], delay_without_sessions) + + for delay, sessions in delays.items(): + self.set(['pado-delay', delay, 'sessions', sessions]) + self.cli_commit() + + conf = ConfigParser(allow_no_value=True, delimiters='=') + conf.read(self._config_file) + + self.assertEqual(conf['pppoe']['pado-delay'], '10,20:200,-1:300') + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index c9d1e805fb2..b9d17493389 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -38,6 +38,16 @@ pppoe_conf = r'/run/accel-pppd/pppoe.conf' pppoe_chap_secrets = r'/run/accel-pppd/pppoe.chap-secrets' +def convert_pado_delay(pado_delay): + new_pado_delay = {'delays_without_sessions': [], + 'delays_with_sessions': []} + for delay, sessions in pado_delay.items(): + if not sessions: + new_pado_delay['delays_without_sessions'].append(delay) + else: + new_pado_delay['delays_with_sessions'].append((delay, int(sessions['sessions']))) + return new_pado_delay + def get_config(config=None): if config: conf = config @@ -54,6 +64,10 @@ def get_config(config=None): # Multiple named pools require ordered values T5099 pppoe['ordered_named_pools'] = get_pools_in_order(dict_search('client_ip_pool', pppoe)) + if dict_search('pado_delay', pppoe): + pado_delay = dict_search('pado_delay', pppoe) + pppoe['pado_delay'] = convert_pado_delay(pado_delay) + # reload-or-restart does not implemented in accel-ppp # use this workaround until it will be implemented # https://phabricator.accel-ppp.org/T3 @@ -65,6 +79,17 @@ def get_config(config=None): pppoe['server_type'] = 'pppoe' return pppoe +def verify_pado_delay(pppoe): + if 'pado_delay' in pppoe: + pado_delay = pppoe['pado_delay'] + + delays_without_sessions = pado_delay['delays_without_sessions'] + if len(delays_without_sessions) > 1: + raise ConfigError( + f'Cannot add more then ONE pado-delay without sessions, ' + f'but {len(delays_without_sessions)} were set' + ) + def verify(pppoe): if not pppoe: return None @@ -73,7 +98,7 @@ def verify(pppoe): verify_accel_ppp_ip_pool(pppoe) verify_accel_ppp_name_servers(pppoe) verify_accel_ppp_wins_servers(pppoe) - + verify_pado_delay(pppoe) if 'interface' not in pppoe: raise ConfigError('At least one listen interface must be defined!') From 31cd75aec6d035b36537046ae0d034c03009a3fc Mon Sep 17 00:00:00 2001 From: khramshinr Date: Fri, 12 Apr 2024 13:32:52 +0800 Subject: [PATCH 008/188] qos: T6035: QoS policy shaper queue-type random-detect requires limit avpkt Added params for configuration red on the shaper policy --- .../include/qos/queue-average-packet.xml.i | 16 +++++ .../include/qos/queue-mark-probability.xml.i | 16 +++++ .../include/qos/queue-maximum-threshold.xml.i | 16 +++++ .../include/qos/queue-minimum-threshold.xml.i | 15 +++++ interface-definitions/qos.xml.in | 67 ++++--------------- python/vyos/qos/base.py | 29 ++++++++ smoketest/scripts/cli/test_qos.py | 64 +++++++++++++++++- 7 files changed, 167 insertions(+), 56 deletions(-) create mode 100644 interface-definitions/include/qos/queue-average-packet.xml.i create mode 100644 interface-definitions/include/qos/queue-mark-probability.xml.i create mode 100644 interface-definitions/include/qos/queue-maximum-threshold.xml.i create mode 100644 interface-definitions/include/qos/queue-minimum-threshold.xml.i diff --git a/interface-definitions/include/qos/queue-average-packet.xml.i b/interface-definitions/include/qos/queue-average-packet.xml.i new file mode 100644 index 00000000000..2f8bfe2666c --- /dev/null +++ b/interface-definitions/include/qos/queue-average-packet.xml.i @@ -0,0 +1,16 @@ + + + + Average packet size (bytes) + + u32:16-10240 + Average packet size in bytes + + + + + Average packet size must be between 16 and 10240 + + 1024 + + diff --git a/interface-definitions/include/qos/queue-mark-probability.xml.i b/interface-definitions/include/qos/queue-mark-probability.xml.i new file mode 100644 index 00000000000..1a286284513 --- /dev/null +++ b/interface-definitions/include/qos/queue-mark-probability.xml.i @@ -0,0 +1,16 @@ + + + + Mark probability for random detection + + u32 + Numeric value (1/N) + + + + + Mark probability must be greater than 0 + + 10 + + diff --git a/interface-definitions/include/qos/queue-maximum-threshold.xml.i b/interface-definitions/include/qos/queue-maximum-threshold.xml.i new file mode 100644 index 00000000000..66d17ccc4fa --- /dev/null +++ b/interface-definitions/include/qos/queue-maximum-threshold.xml.i @@ -0,0 +1,16 @@ + + + + Maximum threshold for random detection + + u32:0-4096 + Maximum threshold in packets + + + + + Threshold must be between 0 and 4096 + + 18 + + diff --git a/interface-definitions/include/qos/queue-minimum-threshold.xml.i b/interface-definitions/include/qos/queue-minimum-threshold.xml.i new file mode 100644 index 00000000000..81e12d6e295 --- /dev/null +++ b/interface-definitions/include/qos/queue-minimum-threshold.xml.i @@ -0,0 +1,15 @@ + + + + Minimum threshold for random detection + + u32:0-4096 + Minimum threshold in packets + + + + + Threshold must be between 0 and 4096 + + + diff --git a/interface-definitions/qos.xml.in b/interface-definitions/qos.xml.in index 7618c3027d5..8f9ae3fa626 100644 --- a/interface-definitions/qos.xml.in +++ b/interface-definitions/qos.xml.in @@ -470,61 +470,10 @@ #include - - - Average packet size (bytes) - - u32:16-10240 - Average packet size in bytes - - - - - Average packet size must be between 16 and 10240 - - 1024 - - - - Mark probability for this precedence - - <number> - Numeric value (1/N) - - - - - Mark probability must be greater than 0 - - 10 - - - - Maximum threshold for random detection - - u32:0-4096 - Maximum Threshold in packets - - - - - Threshold must be between 0 and 4096 - - 18 - - - - Minimum threshold for random detection - - u32:0-4096 - Maximum Threshold in packets - - - - - Threshold must be between 0 and 4096 - - + #include + #include + #include + #include @@ -697,6 +646,10 @@ #include #include #include + #include + #include + #include + #include #include #include @@ -759,6 +712,10 @@ 20 + #include + #include + #include + #include #include #include diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index c8e881ee212..f9366c6b1df 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -90,6 +90,23 @@ def _get_dsfield(self, value): else: return value + def _calc_random_detect_queue_params(self, avg_pkt, max_thr, limit=None, min_thr=None, mark_probability=None): + params = dict() + avg_pkt = int(avg_pkt) + max_thr = int(max_thr) + mark_probability = int(mark_probability) + limit = int(limit) if limit else 4 * max_thr + min_thr = int(min_thr) if min_thr else (9 * max_thr) // 18 + + params['avg_pkt'] = avg_pkt + params['limit'] = limit * avg_pkt + params['min_val'] = min_thr * avg_pkt + params['max_val'] = max_thr * avg_pkt + params['burst'] = (2 * min_thr + max_thr) // 3 + params['probability'] = 1 / mark_probability + + return params + def _build_base_qdisc(self, config : dict, cls_id : int): """ Add/replace qdisc for every class (also default is a class). This is @@ -144,6 +161,18 @@ def _build_base_qdisc(self, config : dict, cls_id : int): elif queue_type == 'random-detect': default_tc += f' red' + qparams = self._calc_random_detect_queue_params( + avg_pkt=dict_search('average_packet', config), + max_thr=dict_search('maximum_threshold', config), + limit=dict_search('queue_limit', config), + min_thr=dict_search('minimum_threshold', config), + mark_probability=dict_search('mark_probability', config) + ) + + default_tc += f' limit {qparams["limit"]} avpkt {qparams["avg_pkt"]}' + default_tc += f' max {qparams["max_val"]} min {qparams["min_val"]}' + default_tc += f' burst {qparams["burst"]} probability {qparams["probability"]}' + self._cmd(default_tc) elif queue_type == 'drop-tail': diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index 1eb4b5cfb83..4f41e36cda8 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -581,7 +581,6 @@ def test_10_round_robin(self): dport = int(match_config['dport']) self.assertEqual(f'{dport:x}', filter['options']['match']['value']) - def test_11_shaper(self): bandwidth = 250 default_bandwidth = 20 @@ -635,6 +634,69 @@ def test_11_shaper(self): class_bandwidth += 1 class_ceil += 1 + def test_12_shaper_with_red_queue(self): + bandwidth = 100 + default_bandwidth = 100 + default_burst = 100 + interface = self._interfaces[0] + class_bandwidth = 50 + dst_address = '192.0.2.8/32' + + shaper_name = f'qos-shaper-{interface}' + self.cli_set(base_path + ['interface', interface, 'egress', shaper_name]) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'bandwidth', f'{bandwidth}mbit']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'bandwidth', f'{default_bandwidth}%']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'burst', f'{default_burst}']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'queue-type', 'random-detect']) + + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'bandwidth', f'{class_bandwidth}mbit']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'match', '10', 'ip', 'destination', 'address', dst_address]) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'queue-type', 'random-detect']) + + # commit changes + self.cli_commit() + + # check root htb config + output = cmd(f'tc class show dev {interface}') + + config_entries = ( + f'prio 0 rate {class_bandwidth}Mbit ceil 50Mbit burst 15Kb', # specified class + f'prio 7 rate {default_bandwidth}Mbit ceil 100Mbit burst {default_burst}b', # default class + ) + for config_entry in config_entries: + self.assertIn(config_entry, output) + + output = cmd(f'tc -d qdisc show dev {interface}') + config_entries = ( + 'qdisc red', # use random detect + 'limit 72Kb min 9Kb max 18Kb ewma 3 probability 0.1', # default config for random detect + ) + for config_entry in config_entries: + self.assertIn(config_entry, output) + + # test random detect queue params + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'queue-limit', '1024']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'average-packet', '1024']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'maximum-threshold', '32']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'minimum-threshold', '16']) + + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'queue-limit', '1024']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'average-packet', '512']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'maximum-threshold', '32']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'minimum-threshold', '16']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '2', 'mark-probability', '20']) + + self.cli_commit() + + output = cmd(f'tc -d qdisc show dev {interface}') + config_entries = ( + 'qdisc red', # use random detect + 'limit 1Mb min 16Kb max 32Kb ewma 3 probability 0.1', # default config for random detect + 'limit 512Kb min 8Kb max 16Kb ewma 3 probability 0.05', # class config for random detect + ) + for config_entry in config_entries: + self.assertIn(config_entry, output) + if __name__ == '__main__': unittest.main(verbosity=2) From 7451133671989e1207299e081cd26d3f526e25f6 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 12 Apr 2024 13:37:41 +0000 Subject: [PATCH 009/188] T6235: Git update actions-label-merge-conflict version Update `actions-label-merge-conflict` due to `Node.js 16 actions are deprecated.` --- .github/workflows/pr-conflicts.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-conflicts.yml b/.github/workflows/pr-conflicts.yml index 96040cd6025..2fd0bb42d1a 100644 --- a/.github/workflows/pr-conflicts.yml +++ b/.github/workflows/pr-conflicts.yml @@ -6,10 +6,10 @@ on: jobs: Conflict_Check: name: 'Check PR status: conflicts and resolution' - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - name: check if PRs are dirty - uses: eps1lon/actions-label-merge-conflict@releases/2.x + uses: eps1lon/actions-label-merge-conflict@v3 with: dirtyLabel: "state: conflict" removeOnDirtyLabel: "state: conflict resolved" From 840ab82e8821624aae589dec1ea86cefbcf866b7 Mon Sep 17 00:00:00 2001 From: Lucas Christian Date: Fri, 12 Apr 2024 09:34:13 -0700 Subject: [PATCH 010/188] T5872: fix ipsec dhclient hook uses "exit" instead of "return" --- src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook b/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook index ebb100e8b67..57f80305541 100755 --- a/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook +++ b/src/etc/dhcp/dhclient-exit-hooks.d/99-ipsec-dhclient-hook @@ -17,7 +17,7 @@ DHCP_HOOK_IFLIST="/tmp/ipsec_dhcp_interfaces" if ! { [ -f $DHCP_HOOK_IFLIST ] && grep -qw $interface $DHCP_HOOK_IFLIST; }; then - exit 0 + return 0 fi # Re-generate the config on the following events: @@ -26,10 +26,10 @@ fi # - REBIND: re-generate if the IP address changed if [ "$reason" == "RENEW" ] || [ "$reason" == "REBIND" ]; then if [ "$old_ip_address" == "$new_ip_address" ]; then - exit 0 + return 0 fi elif [ "$reason" != "BOUND" ]; then - exit 0 + return 0 fi # Best effort wait for any active commit to finish From 95cd743c24c6f7720af87450312fc111649db849 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 15 Apr 2024 08:40:26 +0000 Subject: [PATCH 011/188] T5734: OpenVPN check PKI DH name exists if DH configured Check if DH is configured for OpenVPN but does not exist in the PKI section ``` set pki dh dh-correct parameters 'xxxx' set interfaces openvpn vtun10 tls dh-params 'dh-fake' File "/usr/libexec/vyos/conf_mode/interfaces_openvpn.py", line 208, in verify_pki pki_dh = pki['dh'][tls['dh_params']] ~~~~~~~~~^^^^^^^^^^^^^^^^^^ KeyError: 'dh-fake' ``` --- src/conf_mode/interfaces_openvpn.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index 505ec55c6eb..0ecffd3beb0 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -198,6 +198,12 @@ def verify_pki(openvpn): raise ConfigError(f'Cannot use encrypted private key on openvpn interface {interface}') if 'dh_params' in tls: + if 'dh' not in pki: + raise ConfigError(f'pki dh is not configured') + proposed_dh = tls['dh_params'] + if proposed_dh not in pki['dh'].keys(): + raise ConfigError(f"pki dh '{proposed_dh}' is not configured") + pki_dh = pki['dh'][tls['dh_params']] dh_params = load_dh_parameters(pki_dh['parameters']) dh_numbers = dh_params.parameter_numbers() From 76dcecafca977b640dd16d8e68c4a050ca1af4fb Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Mon, 15 Apr 2024 14:15:14 +0000 Subject: [PATCH 012/188] T5535: firewall: migrate command to firewall global-optinos --- .../include/firewall/global-options.xml.i | 20 ++++++++ .../include/version/firewall-version.xml.i | 2 +- interface-definitions/system_ip.xml.in | 6 --- smoketest/scripts/cli/test_firewall.py | 1 + smoketest/scripts/cli/test_system_ip.py | 11 ----- src/conf_mode/firewall.py | 1 + src/conf_mode/system_ip.py | 5 -- src/migration-scripts/firewall/14-to-15 | 46 +++++++++++++++++++ 8 files changed, 69 insertions(+), 23 deletions(-) create mode 100755 src/migration-scripts/firewall/14-to-15 diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i index 415d85f05d7..9cd0b3239f7 100644 --- a/interface-definitions/include/firewall/global-options.xml.i +++ b/interface-definitions/include/firewall/global-options.xml.i @@ -44,6 +44,26 @@ disable + + + Policy for handling IPv4 directed broadcast forwarding on all interfaces + + enable disable + + + enable + Enable IPv4 directed broadcast forwarding on all interfaces + + + disable + Disable IPv4 directed broadcast forwarding on all interfaces + + + (enable|disable) + + + enable + Policy for handling IPv4 packets with source route option diff --git a/interface-definitions/include/version/firewall-version.xml.i b/interface-definitions/include/version/firewall-version.xml.i index 6702ee04115..fa8e26f78b2 100644 --- a/interface-definitions/include/version/firewall-version.xml.i +++ b/interface-definitions/include/version/firewall-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/system_ip.xml.in b/interface-definitions/system_ip.xml.in index 015eb270f1c..b4b5092fee5 100644 --- a/interface-definitions/system_ip.xml.in +++ b/interface-definitions/system_ip.xml.in @@ -23,12 +23,6 @@ - - - Disable IPv4 directed broadcast forwarding on all interfaces - - - IPv4 multipath settings diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index fe697725285..c4756271421 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -27,6 +27,7 @@ sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'default': '0', 'test_value': 'disable'}, 'broadcast_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_broadcasts', 'default': '1', 'test_value': 'enable'}, + 'directed_broadcast': {'sysfs': '/proc/sys/net/ipv4/conf/all/bc_forwarding', 'default': '1', 'test_value': 'disable'}, 'ip_src_route': {'sysfs': '/proc/sys/net/ipv4/conf/*/accept_source_route', 'default': '0', 'test_value': 'enable'}, 'ipv6_receive_redirects': {'sysfs': '/proc/sys/net/ipv6/conf/*/accept_redirects', 'default': '0', 'test_value': 'enable'}, 'ipv6_src_route': {'sysfs': '/proc/sys/net/ipv6/conf/*/accept_source_route', 'default': '-1', 'test_value': 'enable'}, diff --git a/smoketest/scripts/cli/test_system_ip.py b/smoketest/scripts/cli/test_system_ip.py index ac8b74236d3..5b009023704 100755 --- a/smoketest/scripts/cli/test_system_ip.py +++ b/smoketest/scripts/cli/test_system_ip.py @@ -38,17 +38,6 @@ def test_system_ip_forwarding(self): self.assertEqual(read_file(all_forwarding), '0') - def test_system_ip_directed_broadcast_forwarding(self): - # Test if IPv4 directed broadcast forwarding can be disabled globally, - # default is '1' which means forwarding enabled - bc_forwarding = '/proc/sys/net/ipv4/conf/all/bc_forwarding' - self.assertEqual(read_file(bc_forwarding), '1') - - self.cli_set(base_path + ['disable-directed-broadcast']) - self.cli_commit() - - self.assertEqual(read_file(bc_forwarding), '0') - def test_system_ip_multipath(self): # Test IPv4 multipathing options, options default to off -> '0' use_neigh = '/proc/sys/net/ipv4/fib_multipath_use_neigh' diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index 3cf6183631e..e96e57154ac 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -44,6 +44,7 @@ sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'enable': '0', 'disable': '1'}, 'broadcast_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_broadcasts', 'enable': '0', 'disable': '1'}, + 'directed_broadcast' : {'sysfs': '/proc/sys/net/ipv4/conf/all/bc_forwarding', 'enable': '1', 'disable': '0'}, 'ip_src_route': {'sysfs': '/proc/sys/net/ipv4/conf/*/accept_source_route'}, 'ipv6_receive_redirects': {'sysfs': '/proc/sys/net/ipv6/conf/*/accept_redirects'}, 'ipv6_src_route': {'sysfs': '/proc/sys/net/ipv6/conf/*/accept_source_route', 'enable': '0', 'disable': '-1'}, diff --git a/src/conf_mode/system_ip.py b/src/conf_mode/system_ip.py index b945b51f27c..2a0bda91a34 100755 --- a/src/conf_mode/system_ip.py +++ b/src/conf_mode/system_ip.py @@ -81,11 +81,6 @@ def apply(opt): value = '0' if (tmp != None) else '1' write_file('/proc/sys/net/ipv4/conf/all/forwarding', value) - # enable/disable IPv4 directed broadcast forwarding - tmp = dict_search('disable_directed_broadcast', opt) - value = '0' if (tmp != None) else '1' - write_file('/proc/sys/net/ipv4/conf/all/bc_forwarding', value) - # configure multipath tmp = dict_search('multipath.ignore_unreachable_nexthops', opt) value = '1' if (tmp != None) else '0' diff --git a/src/migration-scripts/firewall/14-to-15 b/src/migration-scripts/firewall/14-to-15 new file mode 100755 index 00000000000..735839365ec --- /dev/null +++ b/src/migration-scripts/firewall/14-to-15 @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T5535: Migrate to Date: Mon, 15 Apr 2024 14:55:44 +0200 Subject: [PATCH 013/188] pki: T6241: Fix dependency updates on PKI changes --- src/conf_mode/pki.py | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 3ab6ac5c37a..6228ff0d205 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -24,6 +24,7 @@ from vyos.configdep import set_dependents from vyos.configdep import call_dependents from vyos.configdict import node_changed +from vyos.configdiff import Diff from vyos.defaults import directories from vyos.pki import is_ca_certificate from vyos.pki import load_certificate @@ -136,32 +137,32 @@ def get_config(config=None): if len(argv) > 1 and argv[1] == 'certbot_renew': pki['certbot_renew'] = {} - tmp = node_changed(conf, base + ['ca'], recursive=True) + tmp = node_changed(conf, base + ['ca'], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) if tmp: if 'changed' not in pki: pki.update({'changed':{}}) pki['changed'].update({'ca' : tmp}) - tmp = node_changed(conf, base + ['certificate'], recursive=True) + tmp = node_changed(conf, base + ['certificate'], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) if tmp: if 'changed' not in pki: pki.update({'changed':{}}) pki['changed'].update({'certificate' : tmp}) - tmp = node_changed(conf, base + ['dh'], recursive=True) + tmp = node_changed(conf, base + ['dh'], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) if tmp: if 'changed' not in pki: pki.update({'changed':{}}) pki['changed'].update({'dh' : tmp}) - tmp = node_changed(conf, base + ['key-pair'], recursive=True) + tmp = node_changed(conf, base + ['key-pair'], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) if tmp: if 'changed' not in pki: pki.update({'changed':{}}) pki['changed'].update({'key_pair' : tmp}) - tmp = node_changed(conf, base + ['openssh'], recursive=True) + tmp = node_changed(conf, base + ['openssh'], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) if tmp: if 'changed' not in pki: pki.update({'changed':{}}) pki['changed'].update({'openssh' : tmp}) - tmp = node_changed(conf, base + ['openvpn', 'shared-secret'], recursive=True) + tmp = node_changed(conf, base + ['openvpn', 'shared-secret'], recursive=True, expand_nodes=Diff.DELETE | Diff.ADD) if tmp: if 'changed' not in pki: pki.update({'changed':{}}) pki['changed'].update({'openvpn' : tmp}) @@ -217,16 +218,21 @@ def get_config(config=None): if not search_dict: continue for found_name, found_path in dict_search_recursive(search_dict, key): - if found_name == item_name: - path = search['path'] - path_str = ' '.join(path + found_path) - print(f'PKI: Updating config: {path_str} {found_name}') - - if path[0] == 'interfaces': - ifname = found_path[0] - set_dependents(path[1], conf, ifname) - else: - set_dependents(path[1], conf) + if isinstance(found_name, list) and item_name not in found_name: + continue + + if isinstance(found_name, str) and found_name != item_name: + continue + + path = search['path'] + path_str = ' '.join(path + found_path) + print(f'PKI: Updating config: {path_str} {item_name}') + + if path[0] == 'interfaces': + ifname = found_path[0] + set_dependents(path[1], conf, ifname) + else: + set_dependents(path[1], conf) return pki From bb832acb97881d747a57da2728eab3ad138b8129 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 15 Apr 2024 15:17:14 +0000 Subject: [PATCH 014/188] T5722: Failover route add option onlink onlink pretend that the nexthop is directly attached to this link, even if it does not match any interface prefix. Useful when gateway not in the same interface network set interfaces ethernet eth0 vif 10 address '10.20.30.1/32' set protocols static route 10.20.30.0/32 interface eth0.10 set protocols failover route 192.0.2.11/32 next-hop 10.20.30.0 onlink ``` vyos@r4# sudo ip route add 192.0.2.111/32 via 10.20.30.0 dev eth0.10 metric 1 proto failover Error: Nexthop has invalid gateway. [edit] vyos@r4# [edit] vyos@r4# sudo ip route add 192.0.2.111/32 via 10.20.30.0 dev eth0.10 onlink metric 1 proto failover [edit] vyos@r4# ``` --- interface-definitions/protocols_failover.xml.in | 6 ++++++ src/helpers/vyos-failover.py | 5 +++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/interface-definitions/protocols_failover.xml.in b/interface-definitions/protocols_failover.xml.in index c0caec68e50..f709759490b 100644 --- a/interface-definitions/protocols_failover.xml.in +++ b/interface-definitions/protocols_failover.xml.in @@ -124,6 +124,12 @@ 1 + + + The next hop is directly connected to the interface, even if it does not match interface prefix + + + diff --git a/src/helpers/vyos-failover.py b/src/helpers/vyos-failover.py index f34c18916e4..34897436411 100755 --- a/src/helpers/vyos-failover.py +++ b/src/helpers/vyos-failover.py @@ -197,6 +197,7 @@ def is_target_alive(target_list=None, proto = nexthop_config.get('check').get('type') target = nexthop_config.get('check').get('target') timeout = nexthop_config.get('check').get('timeout') + onlink = 'onlink' if 'onlink' in nexthop_config else '' # Route not found in the current routing table if not is_route_exists(route, next_hop, conf_iface, conf_metric): @@ -206,14 +207,14 @@ def is_target_alive(target_list=None, if debug: print(f' [ ADD ] -- ip route add {route} via {next_hop} dev {conf_iface} ' f'metric {conf_metric} proto failover\n###') rc, command = rc_cmd(f'ip route add {route} via {next_hop} dev {conf_iface} ' - f'metric {conf_metric} proto failover') + f'{onlink} metric {conf_metric} proto failover') # If something is wrong and gateway not added # Example: Error: Next-hop has invalid gateway. if rc !=0: if debug: print(f'{command} -- return-code [RC: {rc}] {next_hop} dev {conf_iface}') else: journal.send(f'ip route add {route} via {next_hop} dev {conf_iface} ' - f'metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) + f'{onlink} metric {conf_metric} proto failover', SYSLOG_IDENTIFIER=my_name) else: if debug: print(f' [ TARGET_FAIL ] target checks fails for [{target}], do nothing') journal.send(f'Check fail for route {route} target {target} proto {proto} ' From aafe22d08bb38a579dd5075fd27a1b88beeca791 Mon Sep 17 00:00:00 2001 From: Alex W Date: Mon, 15 Apr 2024 18:23:05 +0100 Subject: [PATCH 015/188] T6242: load-balancing reverse-proxy: Ability for ssl backends to not verify server certificates --- data/templates/load-balancing/haproxy.cfg.j2 | 2 +- .../load-balancing_reverse-proxy.xml.in | 6 ++++++ .../cli/test_load-balancing_reverse-proxy.py | 18 ++++++++++++++++++ src/conf_mode/load-balancing_reverse-proxy.py | 4 ++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index 849cef74d01..feb10d24798 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -150,7 +150,7 @@ backend {{ back }} {% endfor %} {% endif %} {% if back_config.server is vyos_defined %} -{% set ssl_back = 'ssl ca-file /run/haproxy/' ~ back_config.ssl.ca_certificate ~ '.pem' if back_config.ssl.ca_certificate is vyos_defined else '' %} +{% set ssl_back = 'ssl ca-file /run/haproxy/' ~ back_config.ssl.ca_certificate ~ '.pem' if back_config.ssl.ca_certificate is vyos_defined else ('ssl verify none' if back_config.ssl.no_verify is vyos_defined else '') %} {% for server, server_config in back_config.server.items() %} server {{ server }} {{ server_config.address }}:{{ server_config.port }}{{ ' check' if server_config.check is vyos_defined }}{{ ' backup' if server_config.backup is vyos_defined }}{{ ' send-proxy' if server_config.send_proxy is vyos_defined }}{{ ' send-proxy-v2' if server_config.send_proxy_v2 is vyos_defined }} {{ ssl_back }} {% endfor %} diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index 2c2742dffbc..49d1d858e6a 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -157,6 +157,12 @@ #include + + + Do not attempt to verify SSL certificates for backend servers + + + #include diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index 97304da8bce..d21fc762ba9 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -280,6 +280,24 @@ def test_03_lb_reverse_proxy_ca_not_exists(self): self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'smoketest']) self.cli_commit() + def test_04_lb_reverse_proxy_backend_ssl_no_verify(self): + # Setup base + self.configure_pki() + self.base_config() + + # Set no-verify option + self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'no-verify']) + self.cli_commit() + + # Test no-verify option + config = read_file(HAPROXY_CONF) + self.assertIn('server bk-01 192.0.2.11:9090 send-proxy ssl verify none', config) + + # Test setting ca-certificate alongside no-verify option fails, to test config validation + self.cli_set(base_path + ['backend', 'bk-01', 'ssl', 'ca-certificate', 'smoketest']) + with self.assertRaises(ConfigSessionError) as e: + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index 694a4e1ea03..9f895c4e263 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -84,6 +84,10 @@ def verify(lb): if {'send_proxy', 'send_proxy_v2'} <= set(bk_server_conf): raise ConfigError(f'Cannot use both "send-proxy" and "send-proxy-v2" for server "{bk_server}"') + if 'ssl' in back_config: + if {'no_verify', 'ca_certificate'} <= set(back_config['ssl']): + raise ConfigError(f'backend {back} cannot have both ssl options no-verify and ca-certificate set!') + for front, front_config in lb['service'].items(): for cert in dict_search('ssl.certificate', front_config) or []: verify_pki_certificate(lb, cert) From da40bd2b2a826986de128354ea1bfc041ada0016 Mon Sep 17 00:00:00 2001 From: khramshinr Date: Tue, 16 Apr 2024 15:49:19 +0800 Subject: [PATCH 016/188] qos: T4248: Allow to remove the only rule from the qos class --- python/vyos/qos/base.py | 5 +++- smoketest/scripts/cli/test_qos.py | 39 +++++++++++++++++++++++++++++++ src/conf_mode/qos.py | 23 ++++++++++++++++++ 3 files changed, 66 insertions(+), 1 deletion(-) diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index f9366c6b1df..4173a1a438a 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -246,6 +246,7 @@ def update(self, config, direction, priority=None): filter_cmd_base += ' protocol all' if 'match' in cls_config: + is_filtered = False for index, (match, match_config) in enumerate(cls_config['match'].items(), start=1): filter_cmd = filter_cmd_base if self.qostype == 'shaper' and 'prio ' not in filter_cmd: @@ -330,11 +331,13 @@ def update(self, config, direction, priority=None): cls = int(cls) filter_cmd += f' flowid {self._parent:x}:{cls:x}' self._cmd(filter_cmd) + is_filtered = True vlan_expression = "match.*.vif" match_vlan = jmespath.search(vlan_expression, cls_config) - if any(tmp in ['exceed', 'bandwidth', 'burst'] for tmp in cls_config): + if any(tmp in ['exceed', 'bandwidth', 'burst'] for tmp in cls_config) \ + and is_filtered: # For "vif" "basic match" is used instead of "action police" T5961 if not match_vlan: filter_cmd += f' action police' diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index 4f41e36cda8..fef1ff23a20 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -697,6 +697,45 @@ def test_12_shaper_with_red_queue(self): for config_entry in config_entries: self.assertIn(config_entry, output) + def test_13_shaper_delete_only_rule(self): + default_bandwidth = 100 + default_burst = 100 + interface = self._interfaces[0] + class_bandwidth = 50 + class_ceiling = 5 + src_address = '10.1.1.0/24' + + shaper_name = f'qos-shaper-{interface}' + self.cli_set(base_path + ['interface', interface, 'egress', shaper_name]) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'bandwidth', f'10mbit']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'bandwidth', f'{default_bandwidth}mbit']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'default', 'burst', f'{default_burst}']) + + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'bandwidth', f'{class_bandwidth}mbit']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'ceiling', f'{class_ceiling}mbit']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'match', 'ADDRESS30', 'ip', 'source', 'address', src_address]) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'match', 'ADDRESS30', 'description', 'smoketest']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'priority', '5']) + self.cli_set(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'queue-type', 'fair-queue']) + + # commit changes + self.cli_commit() + # check root htb config + output = cmd(f'tc class show dev {interface}') + + config_entries = ( + f'prio 5 rate {class_bandwidth}Mbit ceil {class_ceiling}Mbit burst 15Kb', # specified class + f'prio 7 rate {default_bandwidth}Mbit ceil 100Mbit burst {default_burst}b', # default class + ) + for config_entry in config_entries: + self.assertIn(config_entry, output) + + self.assertTrue('' != cmd(f'tc filter show dev {interface}')) + # self.cli_delete(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'match', 'ADDRESS30']) + self.cli_delete(base_path + ['policy', 'shaper', shaper_name, 'class', '30', 'match', 'ADDRESS30', 'ip', 'source', 'address', src_address]) + self.cli_commit() + self.assertEqual('', cmd(f'tc filter show dev {interface}')) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 3dfb4bab85b..ccfc8f6b82a 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -70,6 +70,22 @@ def get_shaper(qos, interface_config, direction): return (map_vyops_tc[shaper_type], shaper_config) + +def _clean_conf_dict(conf): + """ + Delete empty nodes from config e.g. + match ADDRESS30 { + ip { + source {} + } + } + """ + if isinstance(conf, dict): + return {node: _clean_conf_dict(val) for node, val in conf.items() if val != {} and _clean_conf_dict(val) != {}} + else: + return conf + + def get_config(config=None): if config: conf = config @@ -120,6 +136,13 @@ def get_config(config=None): if 'queue_limit' not in qos['policy'][policy][p_name]['precedence'][precedence]: qos['policy'][policy][p_name]['precedence'][precedence]['queue_limit'] = \ str(int(4 * max_thr)) + # cleanup empty match config + if 'class' in p_config: + for cls, cls_config in p_config['class'].items(): + if 'match' in cls_config: + cls_config['match'] = _clean_conf_dict(cls_config['match']) + if cls_config['match'] == {}: + del cls_config['match'] return qos From 5ab8f9ac47d9d8d198f5ace0ffc4a0b26af098df Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Tue, 16 Apr 2024 17:40:54 +0000 Subject: [PATCH 017/188] T6191: do not append action to firewall and policy route|route6 when its not specified, in order to ensure same behavior as in Equuleus --- python/vyos/firewall.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index 946050a820b..d9d605a9d2e 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -488,9 +488,6 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if synproxy_ws: output.append(f'wscale {synproxy_ws} timestamp sack-perm') - else: - output.append('return') - output.append(f'comment "{family}-{hook}-{fw_name}-{rule_id}"') return " ".join(output) From f43edbd7cd36f52a0cd9c475b53f317882f4a6f9 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 16 Apr 2024 13:38:43 -0500 Subject: [PATCH 018/188] image-tools: T6154: installer prompts to confirm a non-default passwd --- python/vyos/utils/io.py | 5 ++++- src/op_mode/image_installer.py | 14 +++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/python/vyos/utils/io.py b/python/vyos/utils/io.py index 7e604529179..a8c430f28f7 100644 --- a/python/vyos/utils/io.py +++ b/python/vyos/utils/io.py @@ -27,7 +27,7 @@ def print_error(str='', end='\n'): sys.stderr.flush() def ask_input(question, default='', numeric_only=False, valid_responses=[], - no_echo=False): + no_echo=False, non_empty=False): from getpass import getpass question_out = question if default: @@ -48,6 +48,9 @@ def ask_input(question, default='', numeric_only=False, valid_responses=[], if valid_responses and response not in valid_responses: print("Invalid value, try again.") continue + if non_empty and not response: + print("Non-empty value required; try again.") + continue break return response diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 9f6949fb39f..b1311b6f904 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -60,7 +60,8 @@ MSG_INPUT_CONFIG_CHOOSE: str = 'Which file would you like as boot config?' MSG_INPUT_IMAGE_NAME: str = 'What would you like to name this image?' MSG_INPUT_IMAGE_DEFAULT: str = 'Would you like to set the new image as the default one for boot?' -MSG_INPUT_PASSWORD: str = 'Please enter a password for the "vyos" user' +MSG_INPUT_PASSWORD: str = 'Please enter a password for the "vyos" user:' +MSG_INPUT_PASSWORD_CONFIRM: str = 'Please confirm password for the "vyos" user:' MSG_INPUT_ROOT_SIZE_ALL: str = 'Would you like to use all the free space on the drive?' MSG_INPUT_ROOT_SIZE_SET: str = 'Please specify the size (in GB) of the root partition (min is 1.5 GB)?' MSG_INPUT_CONSOLE_TYPE: str = 'What console should be used by default? (K: KVM, S: Serial, U: USB-Serial)?' @@ -74,6 +75,7 @@ MSG_WARN_ROOT_SIZE_TOOSMALL: str = 'The size is too small. Try again' MSG_WARN_IMAGE_NAME_WRONG: str = 'The suggested name is unsupported!\n'\ 'It must be between 1 and 64 characters long and contains only the next characters: .+-_ a-z A-Z 0-9' +MSG_WARN_PASSWORD_CONFIRM: str = 'The entered values did not match. Try again' CONST_MIN_DISK_SIZE: int = 2147483648 # 2 GB CONST_MIN_ROOT_SIZE: int = 1610612736 # 1.5 GB # a reserved space: 2MB for header, 1 MB for BIOS partition, 256 MB for EFI @@ -695,8 +697,14 @@ def install_image() -> None: print(MSG_WARN_IMAGE_NAME_WRONG) # ask for password - user_password: str = ask_input(MSG_INPUT_PASSWORD, default='vyos', - no_echo=True) + while True: + user_password: str = ask_input(MSG_INPUT_PASSWORD, no_echo=True, + non_empty=True) + confirm: str = ask_input(MSG_INPUT_PASSWORD_CONFIRM, no_echo=True, + non_empty=True) + if user_password == confirm: + break + print(MSG_WARN_PASSWORD_CONFIRM) # ask for default console console_type: str = ask_input(MSG_INPUT_CONSOLE_TYPE, From 785616393557c4e3f616287de81b61a68ba177ac Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Wed, 17 Apr 2024 11:14:11 +0200 Subject: [PATCH 019/188] T6246: adds basic haproxy http-check configuration --- Pipfile.lock | 663 +++++++++++++----- data/templates/load-balancing/haproxy.cfg.j2 | 9 + .../load-balancing_reverse-proxy.xml.in | 33 + .../cli/test_load-balancing_reverse-proxy.py | 15 + 4 files changed, 539 insertions(+), 181 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index eae5ea9fe88..5a048becc3e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "5564acff86bcf8ef717129935c288ffed2631f1d795d1c44d0af36779593b89b" + "sha256": "2ec6c667a8fe2b96d82639414919deb0462744211709afe93f6a538a0b15dc1f" }, "pipfile-spec": 6, "requires": { @@ -16,46 +16,242 @@ ] }, "default": { + "bcrypt": { + "hashes": [ + "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f", + "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5", + "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb", + "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258", + "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4", + "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc", + "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2", + "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326", + "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483", + "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a", + "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966", + "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63", + "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c", + "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551", + "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d", + "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e", + "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0", + "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c", + "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb", + "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1", + "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42", + "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946", + "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab", + "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1", + "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c", + "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7", + "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369" + ], + "markers": "python_version >= '3.7'", + "version": "==4.1.2" + }, + "cffi": { + "hashes": [ + "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", + "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", + "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", + "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", + "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", + "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", + "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", + "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", + "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", + "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", + "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", + "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", + "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", + "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", + "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", + "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", + "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", + "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", + "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", + "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", + "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", + "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", + "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", + "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", + "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", + "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", + "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", + "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", + "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", + "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", + "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", + "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", + "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", + "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", + "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", + "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", + "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", + "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", + "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", + "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", + "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", + "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", + "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", + "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", + "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", + "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", + "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", + "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", + "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", + "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", + "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", + "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" + ], + "markers": "platform_python_implementation != 'PyPy'", + "version": "==1.16.0" + }, + "cryptography": { + "hashes": [ + "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee", + "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576", + "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d", + "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30", + "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413", + "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb", + "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da", + "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4", + "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd", + "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc", + "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8", + "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1", + "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc", + "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e", + "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8", + "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940", + "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400", + "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7", + "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16", + "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278", + "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74", + "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec", + "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1", + "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2", + "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c", + "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922", + "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a", + "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6", + "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1", + "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e", + "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac", + "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7" + ], + "markers": "python_version >= '3.7'", + "version": "==42.0.5" + }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", + "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" ], "index": "pypi", - "version": "==2.10.1" + "markers": "python_version >= '3.7'", + "version": "==3.1.3" }, "markupsafe": { "hashes": [ - "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", - "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", - "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", - "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", - "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", - "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", - "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", - "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", - "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", - "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", - "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", - "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", - "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", - "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", - "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", - "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", - "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", - "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", - "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", - "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", - "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", - "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", - "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", - "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", - "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", - "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", - "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", - "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" - ], - "version": "==1.1.0" + "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", + "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", + "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", + "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", + "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", + "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", + "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", + "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", + "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", + "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", + "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", + "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", + "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", + "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", + "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", + "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", + "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", + "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", + "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", + "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", + "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", + "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", + "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", + "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", + "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", + "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", + "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", + "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", + "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", + "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", + "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", + "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", + "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", + "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", + "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", + "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", + "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", + "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", + "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", + "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", + "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", + "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", + "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", + "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", + "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", + "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", + "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", + "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", + "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", + "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", + "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", + "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", + "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", + "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", + "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", + "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", + "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", + "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", + "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", + "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" + ], + "markers": "python_version >= '3.7'", + "version": "==2.1.5" + }, + "paramiko": { + "hashes": [ + "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7", + "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3" + ], + "index": "pypi", + "markers": "python_version >= '3.6'", + "version": "==3.4.0" + }, + "pycparser": { + "hashes": [ + "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", + "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc" + ], + "markers": "python_version >= '3.8'", + "version": "==2.22" + }, + "pynacl": { + "hashes": [ + "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", + "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", + "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", + "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", + "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", + "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", + "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", + "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", + "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", + "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" + ], + "markers": "python_version >= '3.6'", + "version": "==1.5.0" }, "vyos": { "file": "./python" @@ -64,128 +260,256 @@ "develop": { "astroid": { "hashes": [ - "sha256:35b032003d6a863f5dcd7ec11abd5cd5893428beaa31ab164982403bcb311f22", - "sha256:6a5d668d7dc69110de01cdf7aeec69a679ef486862a0850cc0fd5571505b6b7e" + "sha256:951798f922990137ac090c53af473db7ab4e70c770e6d7fae0cec59f74411819", + "sha256:ac248253bfa4bd924a0de213707e7ebeeb3138abeb48d798784ead1e56d419d4" ], - "version": "==2.1.0" + "markers": "python_full_version >= '3.8.0'", + "version": "==3.1.0" }, "coverage": { "hashes": [ - "sha256:09e47c529ff77bf042ecfe858fb55c3e3eb97aac2c87f0349ab5a7efd6b3939f", - "sha256:0a1f9b0eb3aa15c990c328535655847b3420231af299386cfe5efc98f9c250fe", - "sha256:0cc941b37b8c2ececfed341444a456912e740ecf515d560de58b9a76562d966d", - "sha256:10e8af18d1315de936d67775d3a814cc81d0747a1a0312d84e27ae5610e313b0", - "sha256:1b4276550b86caa60606bd3572b52769860a81a70754a54acc8ba789ce74d607", - "sha256:1e8a2627c48266c7b813975335cfdea58c706fe36f607c97d9392e61502dc79d", - "sha256:2b224052bfd801beb7478b03e8a66f3f25ea56ea488922e98903914ac9ac930b", - "sha256:447c450a093766744ab53bf1e7063ec82866f27bcb4f4c907da25ad293bba7e3", - "sha256:46101fc20c6f6568561cdd15a54018bb42980954b79aa46da8ae6f008066a30e", - "sha256:4710dc676bb4b779c4361b54eb308bc84d64a2fa3d78e5f7228921eccce5d815", - "sha256:510986f9a280cd05189b42eee2b69fecdf5bf9651d4cd315ea21d24a964a3c36", - "sha256:5535dda5739257effef56e49a1c51c71f1d37a6e5607bb25a5eee507c59580d1", - "sha256:5a7524042014642b39b1fcae85fb37556c200e64ec90824ae9ecf7b667ccfc14", - "sha256:5f55028169ef85e1fa8e4b8b1b91c0b3b0fa3297c4fb22990d46ff01d22c2d6c", - "sha256:6694d5573e7790a0e8d3d177d7a416ca5f5c150742ee703f3c18df76260de794", - "sha256:6831e1ac20ac52634da606b658b0b2712d26984999c9d93f0c6e59fe62ca741b", - "sha256:77f0d9fa5e10d03aa4528436e33423bfa3718b86c646615f04616294c935f840", - "sha256:828ad813c7cdc2e71dcf141912c685bfe4b548c0e6d9540db6418b807c345ddd", - "sha256:85a06c61598b14b015d4df233d249cd5abfa61084ef5b9f64a48e997fd829a82", - "sha256:8cb4febad0f0b26c6f62e1628f2053954ad2c555d67660f28dfb1b0496711952", - "sha256:a5c58664b23b248b16b96253880b2868fb34358911400a7ba39d7f6399935389", - "sha256:aaa0f296e503cda4bc07566f592cd7a28779d433f3a23c48082af425d6d5a78f", - "sha256:ab235d9fe64833f12d1334d29b558aacedfbca2356dfb9691f2d0d38a8a7bfb4", - "sha256:b3b0c8f660fae65eac74fbf003f3103769b90012ae7a460863010539bb7a80da", - "sha256:bab8e6d510d2ea0f1d14f12642e3f35cefa47a9b2e4c7cea1852b52bc9c49647", - "sha256:c45297bbdbc8bb79b02cf41417d63352b70bcb76f1bbb1ee7d47b3e89e42f95d", - "sha256:d19bca47c8a01b92640c614a9147b081a1974f69168ecd494687c827109e8f42", - "sha256:d64b4340a0c488a9e79b66ec9f9d77d02b99b772c8b8afd46c1294c1d39ca478", - "sha256:da969da069a82bbb5300b59161d8d7c8d423bc4ccd3b410a9b4d8932aeefc14b", - "sha256:ed02c7539705696ecb7dc9d476d861f3904a8d2b7e894bd418994920935d36bb", - "sha256:ee5b8abc35b549012e03a7b1e86c09491457dba6c94112a2482b18589cc2bdb9" + "sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c", + "sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63", + "sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7", + "sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f", + "sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8", + "sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf", + "sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0", + "sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384", + "sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76", + "sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7", + "sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d", + "sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70", + "sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f", + "sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818", + "sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b", + "sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d", + "sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec", + "sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083", + "sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2", + "sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9", + "sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd", + "sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade", + "sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e", + "sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a", + "sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227", + "sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87", + "sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c", + "sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e", + "sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c", + "sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e", + "sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd", + "sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec", + "sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562", + "sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8", + "sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677", + "sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357", + "sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c", + "sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd", + "sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49", + "sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286", + "sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1", + "sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf", + "sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51", + "sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409", + "sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384", + "sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e", + "sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978", + "sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57", + "sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e", + "sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2", + "sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48", + "sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4" ], "index": "pypi", - "version": "==4.5.2" + "markers": "python_version >= '3.8'", + "version": "==7.4.4" + }, + "dill": { + "hashes": [ + "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", + "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7" + ], + "markers": "python_version >= '3.11'", + "version": "==0.3.8" }, "isort": { "hashes": [ - "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", - "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", - "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" - ], - "version": "==4.3.4" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:0ce34342b419bd8f018e6666bfef729aec3edf62345a53b537a4dcc115746a33", - "sha256:1b668120716eb7ee21d8a38815e5eb3bb8211117d9a90b0f8e21722c0758cc39", - "sha256:209615b0fe4624d79e50220ce3310ca1a9445fd8e6d3572a896e7f9146bbf019", - "sha256:27bf62cb2b1a2068d443ff7097ee33393f8483b570b475db8ebf7e1cba64f088", - "sha256:27ea6fd1c02dcc78172a82fc37fcc0992a94e4cecf53cb6d73f11749825bd98b", - "sha256:2c1b21b44ac9beb0fc848d3993924147ba45c4ebc24be19825e57aabbe74a99e", - "sha256:2df72ab12046a3496a92476020a1a0abf78b2a7db9ff4dc2036b8dd980203ae6", - "sha256:320ffd3de9699d3892048baee45ebfbbf9388a7d65d832d7e580243ade426d2b", - "sha256:50e3b9a464d5d08cc5227413db0d1c4707b6172e4d4d915c1c70e4de0bbff1f5", - "sha256:5276db7ff62bb7b52f77f1f51ed58850e315154249aceb42e7f4c611f0f847ff", - "sha256:61a6cf00dcb1a7f0c773ed4acc509cb636af2d6337a08f362413c76b2b47a8dd", - "sha256:6ae6c4cb59f199d8827c5a07546b2ab7e85d262acaccaacd49b62f53f7c456f7", - "sha256:7661d401d60d8bf15bb5da39e4dd72f5d764c5aff5a86ef52a042506e3e970ff", - "sha256:7bd527f36a605c914efca5d3d014170b2cb184723e423d26b1fb2fd9108e264d", - "sha256:7cb54db3535c8686ea12e9535eb087d32421184eacc6939ef15ef50f83a5e7e2", - "sha256:7f3a2d740291f7f2c111d86a1c4851b70fb000a6c8883a59660d95ad57b9df35", - "sha256:81304b7d8e9c824d058087dcb89144842c8e0dea6d281c031f59f0acf66963d4", - "sha256:933947e8b4fbe617a51528b09851685138b49d511af0b6c0da2539115d6d4514", - "sha256:94223d7f060301b3a8c09c9b3bc3294b56b2188e7d8179c762a1cda72c979252", - "sha256:ab3ca49afcb47058393b0122428358d2fbe0408cf99f1b58b295cfeb4ed39109", - "sha256:bd6292f565ca46dee4e737ebcc20742e3b5be2b01556dafe169f6c65d088875f", - "sha256:cb924aa3e4a3fb644d0c463cad5bc2572649a6a3f68a7f8e4fbe44aaa6d77e4c", - "sha256:d0fc7a286feac9077ec52a927fc9fe8fe2fabab95426722be4c953c9a8bede92", - "sha256:ddc34786490a6e4ec0a855d401034cbd1242ef186c20d79d2166d6a4bd449577", - "sha256:e34b155e36fa9da7e1b7c738ed7767fc9491a62ec6af70fe9da4a057759edc2d", - "sha256:e5b9e8f6bda48460b7b143c3821b21b452cb3a835e6bbd5dd33aa0c8d3f5137d", - "sha256:e81ebf6c5ee9684be8f2c87563880f93eedd56dd2b6146d8a725b50b7e5adb0f", - "sha256:eb91be369f945f10d3a49f5f9be8b3d0b93a4c2be8f8a5b83b0571b8123e0a7a", - "sha256:f460d1ceb0e4a5dcb2a652db0904224f367c9b3c1470d5a7683c0480e582468b" - ], - "version": "==1.3.1" + "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", + "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6" + ], + "markers": "python_full_version >= '3.8.0'", + "version": "==5.13.2" }, "lxml": { "hashes": [ - "sha256:0dd6589fa75d369ba06d2b5f38dae107f76ea127f212f6a7bee134f6df2d1d21", - "sha256:1afbac344aa68c29e81ab56c1a9411c3663157b5aee5065b7fa030b398d4f7e0", - "sha256:1baad9d073692421ad5dbbd81430aba6c7f5fdc347f03537ae046ddf2c9b2297", - "sha256:1d8736421a2358becd3edf20260e41a06a0bf08a560480d3a5734a6bcbacf591", - "sha256:1e1d9bddc5afaddf0de76246d3f2152f961697ad7439c559f179002682c45801", - "sha256:1f179dc8b2643715f020f4d119d5529b02cd794c1c8f305868b73b8674d2a03f", - "sha256:241fb7bdf97cb1df1edfa8f0bcdfd80525d4023dac4523a241907c8b2f44e541", - "sha256:2f9765ee5acd3dbdcdc0d0c79309e01f7c16bc8d39b49250bf88de7b46daaf58", - "sha256:312e1e1b1c3ce0c67e0b8105317323e12807955e8186872affb667dbd67971f6", - "sha256:3273db1a8055ca70257fd3691c6d2c216544e1a70b673543e15cc077d8e9c730", - "sha256:34dfaa8c02891f9a246b17a732ca3e99c5e42802416628e740a5d1cb2f50ff49", - "sha256:3aa3f5288af349a0f3a96448ebf2e57e17332d99f4f30b02093b7948bd9f94cc", - "sha256:51102e160b9d83c1cc435162d90b8e3c8c93b28d18d87b60c56522d332d26879", - "sha256:56115fc2e2a4140e8994eb9585119a1ae9223b506826089a3ba753a62bd194a6", - "sha256:69d83de14dbe8fe51dccfd36f88bf0b40f5debeac763edf9f8325180190eba6e", - "sha256:99fdce94aeaa3ccbdfcb1e23b34273605c5853aa92ec23d84c84765178662c6c", - "sha256:a7c0cd5b8a20f3093ee4a67374ccb3b8a126743b15a4d759e2a1bf098faac2b2", - "sha256:abe12886554634ed95416a46701a917784cb2b4c77bfacac6916681d49bbf83d", - "sha256:b4f67b5183bd5f9bafaeb76ad119e977ba570d2b0e61202f534ac9b5c33b4485", - "sha256:bdd7c1658475cc1b867b36d5c4ed4bc316be8d3368abe03d348ba906a1f83b0e", - "sha256:c6f24149a19f611a415a51b9bc5f17b6c2f698e0d6b41ffb3fa9f24d35d05d73", - "sha256:d1e111b3ab98613115a208c1017f266478b0ab224a67bc8eac670fa0bad7d488", - "sha256:d6520aa965773bbab6cb7a791d5895b00d02cf9adc93ac2bf4edb9ac1a6addc5", - "sha256:dd185cde2ccad7b649593b0cda72021bc8a91667417001dbaf24cd746ecb7c11", - "sha256:de2e5b0828a9d285f909b5d2e9d43f1cf6cf21fe65bc7660bdaa1780c7b58298", - "sha256:f726444b8e909c4f41b4fde416e1071cf28fa84634bfb4befdf400933b6463af" + "sha256:04ab5415bf6c86e0518d57240a96c4d1fcfc3cb370bb2ac2a732b67f579e5a04", + "sha256:057cdc6b86ab732cf361f8b4d8af87cf195a1f6dc5b0ff3de2dced242c2015e0", + "sha256:058a1308914f20784c9f4674036527e7c04f7be6fb60f5d61353545aa7fcb739", + "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a", + "sha256:0a15438253b34e6362b2dc41475e7f80de76320f335e70c5528b7148cac253a1", + "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218", + "sha256:0e7259016bc4345a31af861fdce942b77c99049d6c2107ca07dc2bba2435c1d9", + "sha256:0ed777c1e8c99b63037b91f9d73a6aad20fd035d77ac84afcc205225f8f41188", + "sha256:0f5d65c39f16717a47c36c756af0fb36144069c4718824b7533f803ecdf91138", + "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585", + "sha256:11a04306fcba10cd9637e669fd73aa274c1c09ca64af79c041aa820ea992b637", + "sha256:1ae67b4e737cddc96c99461d2f75d218bdf7a0c3d3ad5604d1f5e7464a2f9ffe", + "sha256:1c5bb205e9212d0ebddf946bc07e73fa245c864a5f90f341d11ce7b0b854475d", + "sha256:1f7785f4f789fdb522729ae465adcaa099e2a3441519df750ebdccc481d961a1", + "sha256:200e63525948e325d6a13a76ba2911f927ad399ef64f57898cf7c74e69b71095", + "sha256:21c2e6b09565ba5b45ae161b438e033a86ad1736b8c838c766146eff8ceffff9", + "sha256:2213afee476546a7f37c7a9b4ad4d74b1e112a6fafffc9185d6d21f043128c81", + "sha256:27aa20d45c2e0b8cd05da6d4759649170e8dfc4f4e5ef33a34d06f2d79075d57", + "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536", + "sha256:2c9d147f754b1b0e723e6afb7ba1566ecb162fe4ea657f53d2139bbf894d050a", + "sha256:2ddfe41ddc81f29a4c44c8ce239eda5ade4e7fc305fb7311759dd6229a080052", + "sha256:31e9a882013c2f6bd2f2c974241bf4ba68c85eba943648ce88936d23209a2e01", + "sha256:3249cc2989d9090eeac5467e50e9ec2d40704fea9ab72f36b034ea34ee65ca98", + "sha256:3545039fa4779be2df51d6395e91a810f57122290864918b172d5dc7ca5bb433", + "sha256:394ed3924d7a01b5bd9a0d9d946136e1c2f7b3dc337196d99e61740ed4bc6fe1", + "sha256:3a6b45da02336895da82b9d472cd274b22dc27a5cea1d4b793874eead23dd14f", + "sha256:3a74c4f27167cb95c1d4af1c0b59e88b7f3e0182138db2501c353555f7ec57f4", + "sha256:3d0c3dd24bb4605439bf91068598d00c6370684f8de4a67c2992683f6c309d6b", + "sha256:3dbe858ee582cbb2c6294dc85f55b5f19c918c2597855e950f34b660f1a5ede6", + "sha256:3dc773b2861b37b41a6136e0b72a1a44689a9c4c101e0cddb6b854016acc0aa8", + "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5", + "sha256:3f7765e69bbce0906a7c74d5fe46d2c7a7596147318dbc08e4a2431f3060e306", + "sha256:417d14450f06d51f363e41cace6488519038f940676ce9664b34ebf5653433a5", + "sha256:44f6c7caff88d988db017b9b0e4ab04934f11e3e72d478031efc7edcac6c622f", + "sha256:491755202eb21a5e350dae00c6d9a17247769c64dcf62d8c788b5c135e179dc4", + "sha256:4951e4f7a5680a2db62f7f4ab2f84617674d36d2d76a729b9a8be4b59b3659be", + "sha256:52421b41ac99e9d91934e4d0d0fe7da9f02bfa7536bb4431b4c05c906c8c6919", + "sha256:530e7c04f72002d2f334d5257c8a51bf409db0316feee7c87e4385043be136af", + "sha256:533658f8fbf056b70e434dff7e7aa611bcacb33e01f75de7f821810e48d1bb66", + "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1", + "sha256:56c22432809085b3f3ae04e6e7bdd36883d7258fcd90e53ba7b2e463efc7a6af", + "sha256:58278b29cb89f3e43ff3e0c756abbd1518f3ee6adad9e35b51fb101c1c1daaec", + "sha256:588008b8497667f1ddca7c99f2f85ce8511f8f7871b4a06ceede68ab62dff64b", + "sha256:59565f10607c244bc4c05c0c5fa0c190c990996e0c719d05deec7030c2aa8289", + "sha256:59689a75ba8d7ffca577aefd017d08d659d86ad4585ccc73e43edbfc7476781a", + "sha256:5aea8212fb823e006b995c4dda533edcf98a893d941f173f6c9506126188860d", + "sha256:5c670c0406bdc845b474b680b9a5456c561c65cf366f8db5a60154088c92d102", + "sha256:5ca1e8188b26a819387b29c3895c47a5e618708fe6f787f3b1a471de2c4a94d9", + "sha256:5d077bc40a1fe984e1a9931e801e42959a1e6598edc8a3223b061d30fbd26bbc", + "sha256:5d5792e9b3fb8d16a19f46aa8208987cfeafe082363ee2745ea8b643d9cc5b45", + "sha256:5dd1537e7cc06efd81371f5d1a992bd5ab156b2b4f88834ca852de4a8ea523fa", + "sha256:5ea7b6766ac2dfe4bcac8b8595107665a18ef01f8c8343f00710b85096d1b53a", + "sha256:622020d4521e22fb371e15f580d153134bfb68d6a429d1342a25f051ec72df1c", + "sha256:627402ad8dea044dde2eccde4370560a2b750ef894c9578e1d4f8ffd54000461", + "sha256:644df54d729ef810dcd0f7732e50e5ad1bd0a135278ed8d6bcb06f33b6b6f708", + "sha256:64641a6068a16201366476731301441ce93457eb8452056f570133a6ceb15fca", + "sha256:64c2baa7774bc22dd4474248ba16fe1a7f611c13ac6123408694d4cc93d66dbd", + "sha256:6588c459c5627fefa30139be4d2e28a2c2a1d0d1c265aad2ba1935a7863a4913", + "sha256:66bc5eb8a323ed9894f8fa0ee6cb3e3fb2403d99aee635078fd19a8bc7a5a5da", + "sha256:68a2610dbe138fa8c5826b3f6d98a7cfc29707b850ddcc3e21910a6fe51f6ca0", + "sha256:6935bbf153f9a965f1e07c2649c0849d29832487c52bb4a5c5066031d8b44fd5", + "sha256:6992030d43b916407c9aa52e9673612ff39a575523c5f4cf72cdef75365709a5", + "sha256:6a014510830df1475176466b6087fc0c08b47a36714823e58d8b8d7709132a96", + "sha256:6ab833e4735a7e5533711a6ea2df26459b96f9eec36d23f74cafe03631647c41", + "sha256:6cc6ee342fb7fa2471bd9b6d6fdfc78925a697bf5c2bcd0a302e98b0d35bfad3", + "sha256:6cf58416653c5901e12624e4013708b6e11142956e7f35e7a83f1ab02f3fe456", + "sha256:70a9768e1b9d79edca17890175ba915654ee1725975d69ab64813dd785a2bd5c", + "sha256:70ac664a48aa64e5e635ae5566f5227f2ab7f66a3990d67566d9907edcbbf867", + "sha256:71e97313406ccf55d32cc98a533ee05c61e15d11b99215b237346171c179c0b0", + "sha256:7221d49259aa1e5a8f00d3d28b1e0b76031655ca74bb287123ef56c3db92f213", + "sha256:74b28c6334cca4dd704e8004cba1955af0b778cf449142e581e404bd211fb619", + "sha256:764b521b75701f60683500d8621841bec41a65eb739b8466000c6fdbc256c240", + "sha256:78bfa756eab503673991bdcf464917ef7845a964903d3302c5f68417ecdc948c", + "sha256:794f04eec78f1d0e35d9e0c36cbbb22e42d370dda1609fb03bcd7aeb458c6377", + "sha256:79bd05260359170f78b181b59ce871673ed01ba048deef4bf49a36ab3e72e80b", + "sha256:7a7efd5b6d3e30d81ec68ab8a88252d7c7c6f13aaa875009fe3097eb4e30b84c", + "sha256:7c17b64b0a6ef4e5affae6a3724010a7a66bda48a62cfe0674dabd46642e8b54", + "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b", + "sha256:853e074d4931dbcba7480d4dcab23d5c56bd9607f92825ab80ee2bd916edea53", + "sha256:857500f88b17a6479202ff5fe5f580fc3404922cd02ab3716197adf1ef628029", + "sha256:865bad62df277c04beed9478fe665b9ef63eb28fe026d5dedcb89b537d2e2ea6", + "sha256:88e22fc0a6684337d25c994381ed8a1580a6f5ebebd5ad41f89f663ff4ec2885", + "sha256:8b9c07e7a45bb64e21df4b6aa623cb8ba214dfb47d2027d90eac197329bb5e94", + "sha256:8de8f9d6caa7f25b204fc861718815d41cbcf27ee8f028c89c882a0cf4ae4134", + "sha256:8e77c69d5892cb5ba71703c4057091e31ccf534bd7f129307a4d084d90d014b8", + "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9", + "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863", + "sha256:96323338e6c14e958d775700ec8a88346014a85e5de73ac7967db0367582049b", + "sha256:9676bfc686fa6a3fa10cd4ae6b76cae8be26eb5ec6811d2a325636c460da1806", + "sha256:9b0ff53900566bc6325ecde9181d89afadc59c5ffa39bddf084aaedfe3b06a11", + "sha256:9b9ec9c9978b708d488bec36b9e4c94d88fd12ccac3e62134a9d17ddba910ea9", + "sha256:9c6ad0fbf105f6bcc9300c00010a2ffa44ea6f555df1a2ad95c88f5656104817", + "sha256:9ca66b8e90daca431b7ca1408cae085d025326570e57749695d6a01454790e95", + "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8", + "sha256:a0af35bd8ebf84888373630f73f24e86bf016642fb8576fba49d3d6b560b7cbc", + "sha256:a2b44bec7adf3e9305ce6cbfa47a4395667e744097faed97abb4728748ba7d47", + "sha256:a2dfe7e2473f9b59496247aad6e23b405ddf2e12ef0765677b0081c02d6c2c0b", + "sha256:a55ee573116ba208932e2d1a037cc4b10d2c1cb264ced2184d00b18ce585b2c0", + "sha256:a7baf9ffc238e4bf401299f50e971a45bfcc10a785522541a6e3179c83eabf0a", + "sha256:a8d5c70e04aac1eda5c829a26d1f75c6e5286c74743133d9f742cda8e53b9c2f", + "sha256:a91481dbcddf1736c98a80b122afa0f7296eeb80b72344d7f45dc9f781551f56", + "sha256:ab31a88a651039a07a3ae327d68ebdd8bc589b16938c09ef3f32a4b809dc96ef", + "sha256:abc25c3cab9ec7fcd299b9bcb3b8d4a1231877e425c650fa1c7576c5107ab851", + "sha256:adfb84ca6b87e06bc6b146dc7da7623395db1e31621c4785ad0658c5028b37d7", + "sha256:afbbdb120d1e78d2ba8064a68058001b871154cc57787031b645c9142b937a62", + "sha256:afd5562927cdef7c4f5550374acbc117fd4ecc05b5007bdfa57cc5355864e0a4", + "sha256:b070bbe8d3f0f6147689bed981d19bbb33070225373338df755a46893528104a", + "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c", + "sha256:b560e3aa4b1d49e0e6c847d72665384db35b2f5d45f8e6a5c0072e0283430533", + "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f", + "sha256:b6787b643356111dfd4032b5bffe26d2f8331556ecb79e15dacb9275da02866e", + "sha256:bcbf4af004f98793a95355980764b3d80d47117678118a44a80b721c9913436a", + "sha256:beb72935a941965c52990f3a32d7f07ce869fe21c6af8b34bf6a277b33a345d3", + "sha256:bf2e2458345d9bffb0d9ec16557d8858c9c88d2d11fed53998512504cd9df49b", + "sha256:c2d35a1d047efd68027817b32ab1586c1169e60ca02c65d428ae815b593e65d4", + "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0", + "sha256:c6f2c8372b98208ce609c9e1d707f6918cc118fea4e2c754c9f0812c04ca116d", + "sha256:c817d420c60a5183953c783b0547d9eb43b7b344a2c46f69513d5952a78cddf3", + "sha256:c8ba129e6d3b0136a0f50345b2cb3db53f6bda5dd8c7f5d83fbccba97fb5dcb5", + "sha256:c94e75445b00319c1fad60f3c98b09cd63fe1134a8a953dcd48989ef42318534", + "sha256:cc4691d60512798304acb9207987e7b2b7c44627ea88b9d77489bbe3e6cc3bd4", + "sha256:cc518cea79fd1e2f6c90baafa28906d4309d24f3a63e801d855e7424c5b34144", + "sha256:cd53553ddad4a9c2f1f022756ae64abe16da1feb497edf4d9f87f99ec7cf86bd", + "sha256:cf22b41fdae514ee2f1691b6c3cdeae666d8b7fa9434de445f12bbeee0cf48dd", + "sha256:d38c8f50ecf57f0463399569aa388b232cf1a2ffb8f0a9a5412d0db57e054860", + "sha256:d3be9b2076112e51b323bdf6d5a7f8a798de55fb8d95fcb64bd179460cdc0704", + "sha256:d4f2cc7060dc3646632d7f15fe68e2fa98f58e35dd5666cd525f3b35d3fed7f8", + "sha256:d7520db34088c96cc0e0a3ad51a4fd5b401f279ee112aa2b7f8f976d8582606d", + "sha256:d793bebb202a6000390a5390078e945bbb49855c29c7e4d56a85901326c3b5d9", + "sha256:da052e7962ea2d5e5ef5bc0355d55007407087392cf465b7ad84ce5f3e25fe0f", + "sha256:dae0ed02f6b075426accbf6b2863c3d0a7eacc1b41fb40f2251d931e50188dad", + "sha256:ddc678fb4c7e30cf830a2b5a8d869538bc55b28d6c68544d09c7d0d8f17694dc", + "sha256:df2e6f546c4df14bc81f9498bbc007fbb87669f1bb707c6138878c46b06f6510", + "sha256:e02c5175f63effbd7c5e590399c118d5db6183bbfe8e0d118bdb5c2d1b48d937", + "sha256:e196a4ff48310ba62e53a8e0f97ca2bca83cdd2fe2934d8b5cb0df0a841b193a", + "sha256:e233db59c8f76630c512ab4a4daf5a5986da5c3d5b44b8e9fc742f2a24dbd460", + "sha256:e32be23d538753a8adb6c85bd539f5fd3b15cb987404327c569dfc5fd8366e85", + "sha256:e3d30321949861404323c50aebeb1943461a67cd51d4200ab02babc58bd06a86", + "sha256:e89580a581bf478d8dcb97d9cd011d567768e8bc4095f8557b21c4d4c5fea7d0", + "sha256:e998e304036198b4f6914e6a1e2b6f925208a20e2042563d9734881150c6c246", + "sha256:ec42088248c596dbd61d4ae8a5b004f97a4d91a9fd286f632e42e60b706718d7", + "sha256:efa7b51824aa0ee957ccd5a741c73e6851de55f40d807f08069eb4c5a26b2baa", + "sha256:f0a1bc63a465b6d72569a9bba9f2ef0334c4e03958e043da1920299100bc7c08", + "sha256:f18a5a84e16886898e51ab4b1d43acb3083c39b14c8caeb3589aabff0ee0b270", + "sha256:f2a9efc53d5b714b8df2b4b3e992accf8ce5bbdfe544d74d5c6766c9e1146a3a", + "sha256:f3bbbc998d42f8e561f347e798b85513ba4da324c2b3f9b7969e9c45b10f6169", + "sha256:f42038016852ae51b4088b2862126535cc4fc85802bfe30dea3500fdfaf1864e", + "sha256:f443cdef978430887ed55112b491f670bba6462cea7a7742ff8f14b7abb98d75", + "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd", + "sha256:f8aca2e3a72f37bfc7b14ba96d4056244001ddcc18382bd0daa087fd2e68a354", + "sha256:f9737bf36262046213a28e789cc82d82c6ef19c85a0cf05e75c670a33342ac2c", + "sha256:fd6037392f2d57793ab98d9e26798f44b8b4da2f2464388588f48ac52c489ea1", + "sha256:feaa45c0eae424d3e90d78823f3828e7dc42a42f21ed420db98da2c4ecf0a2cb", + "sha256:ff097ae562e637409b429a7ac958a20aab237a0378c42dabaa1e3abf2f896e5f", + "sha256:ff46d772d5f6f73564979cd77a4fffe55c916a05f3cb70e7c9c0590059fb29ef" ], "index": "pypi", - "version": "==4.3.0" + "markers": "python_version >= '3.6'", + "version": "==5.2.1" }, "mccabe": { "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", + "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" ], - "version": "==0.6.1" + "markers": "python_version >= '3.6'", + "version": "==0.7.0" }, "nose": { "hashes": [ @@ -196,53 +520,30 @@ "index": "pypi", "version": "==1.3.7" }, + "platformdirs": { + "hashes": [ + "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068", + "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768" + ], + "markers": "python_version >= '3.8'", + "version": "==4.2.0" + }, "pylint": { "hashes": [ - "sha256:689de29ae747642ab230c6d37be2b969bf75663176658851f456619aacf27492", - "sha256:771467c434d0d9f081741fec1d64dfb011ed26e65e12a28fe06ca2f61c4d556c" + "sha256:507a5b60953874766d8a366e8e8c7af63e058b26345cfcb5f91f89d987fd6b74", + "sha256:6a69beb4a6f63debebaab0a3477ecd0f559aa726af4954fc948c51f7a2549e23" ], "index": "pypi", - "version": "==2.2.2" - }, - "six": { - "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" - ], - "version": "==1.12.0" - }, - "typed-ast": { - "hashes": [ - "sha256:023625bfa9359e29bd6e24cac2a4503495b49761d48a5f1e38333fc4ac4d93fe", - "sha256:07591f7a5fdff50e2e566c4c1e9df545c75d21e27d98d18cb405727ed0ef329c", - "sha256:153e526b0f4ffbfada72d0bb5ffe8574ba02803d2f3a9c605c8cf99dfedd72a2", - "sha256:3ad2bdcd46a4a1518d7376e9f5016d17718a9ed3c6a3f09203d832f6c165de4a", - "sha256:3ea98c84df53ada97ee1c5159bb3bc784bd734231235a1ede14c8ae0775049f7", - "sha256:51a7141ccd076fa561af107cfb7a8b6d06a008d92451a1ac7e73149d18e9a827", - "sha256:52c93cd10e6c24e7ac97e8615da9f224fd75c61770515cb323316c30830ddb33", - "sha256:6344c84baeda3d7b33e157f0b292e4dd53d05ddb57a63f738178c01cac4635c9", - "sha256:64699ca1b3bd5070bdeb043e6d43bc1d0cebe08008548f4a6bee782b0ecce032", - "sha256:74903f2e56bbffe29282ef8a5487d207d10be0f8513b41aff787d954a4cf91c9", - "sha256:7891710dba83c29ee2bd51ecaa82f60f6bede40271af781110c08be134207bf2", - "sha256:91976c56224e26c256a0de0f76d2004ab885a29423737684b4f7ebdd2f46dde2", - "sha256:9bad678a576ecc71f25eba9f1e3fd8d01c28c12a2834850b458428b3e855f062", - "sha256:b4726339a4c180a8b6ad9d8b50d2b6dc247e1b79b38fe2290549c98e82e4fd15", - "sha256:ba36f6aa3f8933edf94ea35826daf92cbb3ec248b89eccdc053d4a815d285357", - "sha256:bbc96bde544fd19e9ef168e4dfa5c3dfe704bfa78128fa76f361d64d6b0f731a", - "sha256:c0c927f1e44469056f7f2dada266c79b577da378bbde3f6d2ada726d131e4824", - "sha256:c0f9a3708008aa59f560fa1bd22385e05b79b8e38e0721a15a8402b089243442", - "sha256:f0bf6f36ff9c5643004171f11d2fdc745aa3953c5aacf2536a0685db9ceb3fb1", - "sha256:f5be39a0146be663cbf210a4d95c3c58b2d7df7b043c9047c5448e358f0550a2", - "sha256:fcd198bf19d9213e5cbf2cde2b9ef20a9856e716f76f9476157f90ae6de06cc6" - ], - "markers": "python_version < '3.7' and implementation_name == 'cpython'", - "version": "==1.2.0" - }, - "wrapt": { - "hashes": [ - "sha256:4aea003270831cceb8a90ff27c4031da6ead7ec1886023b80ce0dfe0adf61533" - ], - "version": "==1.11.1" + "markers": "python_full_version >= '3.8.0'", + "version": "==3.1.0" + }, + "tomlkit": { + "hashes": [ + "sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b", + "sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3" + ], + "markers": "python_version >= '3.7'", + "version": "==0.12.4" } } } diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index feb10d24798..83008e50a0f 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -110,6 +110,15 @@ frontend {{ front }} {% if backend is vyos_defined %} {% for back, back_config in backend.items() %} backend {{ back }} +{% if back_config.http_check is vyos_defined %} + option httpchk +{% endif %} +{% if back_config.http_check.uri is vyos_defined and back_config.http_check.method is vyos_defined %} + http-check send meth {{ back_config.http_check.method | upper }} uri {{ back_config.http_check.uri }} +{% endif %} +{% if back_config.http_check.expect is vyos_defined %} + http-check expect {{ back_config.http_check.expect }} +{% endif %} {% if back_config.balance is vyos_defined %} {% set balance_translate = {'least-connection': 'leastconn', 'round-robin': 'roundrobin', 'source-address': 'source'} %} balance {{ balance_translate[back_config.balance] }} diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index 49d1d858e6a..645fe30c766 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -102,6 +102,39 @@ + + + HTTP check configuration + + + + + HTTP method used for health check + + options|get|post|put + HTTP method used for health checking + + + options|get|post|put + + + + + + URI used for HTTP health check (Example: '/' or '/health') + + + + + Expected response for the health check to pass + + txt + expected response (Example: 'status 200-299' or 'string success') + + + + + #include diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index d21fc762ba9..8ccf2cf97c7 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -298,6 +298,21 @@ def test_04_lb_reverse_proxy_backend_ssl_no_verify(self): with self.assertRaises(ConfigSessionError) as e: self.cli_commit() + def test_05_lb_reverse_proxy_backend_http_check(self): + # Setup base + self.base_config() + + # Set http-check + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'method', 'get']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'uri', '/health']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status 200']) + self.cli_commit() + + # Test http-check + config = read_file(HAPROXY_CONF) + self.assertIn('option httpchk', config) + self.assertIn('http-check send meth GET uri /health', config) + self.assertIn('http-check expect status 200', config) if __name__ == '__main__': unittest.main(verbosity=2) From ce0bc35f8b5ff80a7b8fbfdf1b9ccc10c5c254fd Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 18 Apr 2024 07:23:38 +0000 Subject: [PATCH 020/188] T6221: Return default ip rule values after deleting VRF Fix for restoring default ip rule values after deleting VRF Defult values: ``` $ ip rule 0: from all lookup local 32766: from all lookup main 32767: from all lookup default ``` After adding and deleting a VRF we get unexpected values: ``` $ ip rule 1000: from all lookup [l3mdev-table] 2000: from all lookup [l3mdev-table] unreachable 32765: from all lookup local 32766: from all lookup main 32767: from all lookup default ``` --- src/conf_mode/vrf.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/conf_mode/vrf.py b/src/conf_mode/vrf.py index 1fc813189fd..587309005e2 100755 --- a/src/conf_mode/vrf.py +++ b/src/conf_mode/vrf.py @@ -315,6 +315,20 @@ def apply(vrf): for chain, rule in nftables_rules.items(): cmd(f'nft flush chain inet vrf_zones {chain}') + # Return default ip rule values + if 'name' not in vrf: + for afi in ['-4', '-6']: + # move lookup local to pref 0 (from 32765) + if not has_rule(afi, 0, 'local'): + call(f'ip {afi} rule add pref 0 from all lookup local') + if has_rule(afi, 32765, 'local'): + call(f'ip {afi} rule del pref 32765 table local') + + if has_rule(afi, 1000, 'l3mdev'): + call(f'ip {afi} rule del pref 1000 l3mdev protocol kernel') + if has_rule(afi, 2000, 'l3mdev'): + call(f'ip {afi} rule del pref 2000 l3mdev unreachable') + # Apply FRR filters zebra_daemon = 'zebra' # Save original configuration prior to starting any commit actions From a88b3bd344cc4a682d16681ef536c1d20e2c2c42 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 17 Apr 2024 14:14:29 -0500 Subject: [PATCH 021/188] pki: T6241: do not call dependency before its initialization --- src/conf_mode/pki.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/pki.py b/src/conf_mode/pki.py index 6228ff0d205..8deec0e85b9 100755 --- a/src/conf_mode/pki.py +++ b/src/conf_mode/pki.py @@ -25,6 +25,7 @@ from vyos.configdep import call_dependents from vyos.configdict import node_changed from vyos.configdiff import Diff +from vyos.configdiff import get_config_diff from vyos.defaults import directories from vyos.pki import is_ca_certificate from vyos.pki import load_certificate @@ -199,6 +200,7 @@ def get_config(config=None): pki['system'] = conf.get_config_dict([], key_mangling=('-', '_'), get_first_key=True, no_tag_node_value_mangle=True) + D = get_config_diff(conf) for search in sync_search: for key in search['keys']: @@ -230,9 +232,11 @@ def get_config(config=None): if path[0] == 'interfaces': ifname = found_path[0] - set_dependents(path[1], conf, ifname) + if not D.node_changed_presence(path + [ifname]): + set_dependents(path[1], conf, ifname) else: - set_dependents(path[1], conf) + if not D.node_changed_presence(path): + set_dependents(path[1], conf) return pki From a43f1c00bdc5047eb20840ebb274418362612526 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Thu, 18 Apr 2024 08:33:54 -0500 Subject: [PATCH 022/188] openvpn: T6245: return 'n/a' if client info not available --- src/op_mode/openvpn.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/op_mode/openvpn.py b/src/op_mode/openvpn.py index d54a67199bb..09287390989 100755 --- a/src/op_mode/openvpn.py +++ b/src/op_mode/openvpn.py @@ -48,9 +48,12 @@ def _get_tunnel_address(peer_host, peer_port, status_file): # 10.10.2.0/25,client1,... lst = [l for l in lst[1:] if '/' not in l.split(',')[0]] - tunnel_ip = lst[0].split(',')[0] + if lst: + tunnel_ip = lst[0].split(',')[0] - return tunnel_ip + return tunnel_ip + + return 'n/a' def _get_interface_status(mode: str, interface: str) -> dict: status_file = f'/run/openvpn/{interface}.status' From 4cde677e9e128bc9b62fad720b1b6f6cac506954 Mon Sep 17 00:00:00 2001 From: fett0 <50275740+fett0@users.noreply.github.com> Date: Sat, 20 Apr 2024 04:55:54 -0300 Subject: [PATCH 023/188] gre: T6252: allow tunnel MTU to exceed 8024 bytes --- interface-definitions/interfaces_tunnel.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/interfaces_tunnel.xml.in b/interface-definitions/interfaces_tunnel.xml.in index 99d9b34c6b5..fe1dad37194 100644 --- a/interface-definitions/interfaces_tunnel.xml.in +++ b/interface-definitions/interfaces_tunnel.xml.in @@ -20,7 +20,7 @@ #include #include #include - #include + #include 1476 From 050f24770aec7a74c1a07ba64cf2cb83afb72f1a Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Fri, 19 Apr 2024 23:59:17 +0200 Subject: [PATCH 024/188] T6246: improve haproxy http check configuration --- data/templates/load-balancing/haproxy.cfg.j2 | 17 ++++++-- .../load-balancing_reverse-proxy.xml.in | 42 +++++++++++++++---- .../cli/test_load-balancing_reverse-proxy.py | 23 +++++++++- src/conf_mode/load-balancing_reverse-proxy.py | 4 ++ 4 files changed, 73 insertions(+), 13 deletions(-) diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index 83008e50a0f..dd93afba531 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -113,11 +113,22 @@ backend {{ back }} {% if back_config.http_check is vyos_defined %} option httpchk {% endif %} -{% if back_config.http_check.uri is vyos_defined and back_config.http_check.method is vyos_defined %} - http-check send meth {{ back_config.http_check.method | upper }} uri {{ back_config.http_check.uri }} +{% set send = '' %} +{% if back_config.http_check.method is vyos_defined %} +{% set send = send + ' meth ' + back_config.http_check.method | upper %} +{% endif %} +{% if back_config.http_check.uri is vyos_defined %} +{% set send = send + ' uri ' + back_config.http_check.uri %} +{% endif %} +{% if send != '' %} + http-check send{{ send }} {% endif %} {% if back_config.http_check.expect is vyos_defined %} - http-check expect {{ back_config.http_check.expect }} +{% if back_config.http_check.expect.status is vyos_defined %} + http-check expect status {{ back_config.http_check.expect.status }} +{% elif back_config.http_check.expect.string is vyos_defined %} + http-check expect string {{ back_config.http_check.expect.string }} +{% endif %} {% endif %} {% if back_config.balance is vyos_defined %} {% set balance_translate = {'least-connection': 'leastconn', 'round-robin': 'roundrobin', 'source-address': 'source'} %} diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index 645fe30c766..eb01580da0e 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -110,29 +110,55 @@ HTTP method used for health check + + options head get post put + - options|get|post|put + options|head|get|post|put HTTP method used for health checking - options|get|post|put + (options|head|get|post|put) URI used for HTTP health check (Example: '/' or '/health') + + ^\/([^?#\s]*)(\?[^#\s]*)?$ + - + Expected response for the health check to pass - - txt - expected response (Example: 'status 200-299' or 'string success') - - + + + + Expected response status code for the health check to pass + + u32:200-399 + Expected response code + + + + + Status code must be in range 200-399 + + + + + Expected to be in response body for the health check to pass + + txt + A string expected to be in the response + + + + + #include diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index 8ccf2cf97c7..737c0740147 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -304,15 +304,34 @@ def test_05_lb_reverse_proxy_backend_http_check(self): # Set http-check self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'method', 'get']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'uri', '/health']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status 200']) self.cli_commit() # Test http-check config = read_file(HAPROXY_CONF) self.assertIn('option httpchk', config) + self.assertIn('http-check send meth GET', config) + + # Set http-check with uri and status + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'uri', '/health']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) + self.cli_commit() + + # Test http-check with uri and status + config = read_file(HAPROXY_CONF) + self.assertIn('option httpchk', config) self.assertIn('http-check send meth GET uri /health', config) self.assertIn('http-check expect status 200', config) + # Set http-check with string + self.cli_delete(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'string', 'success']) + self.cli_commit() + + # Test http-check with string + config = read_file(HAPROXY_CONF) + self.assertIn('option httpchk', config) + self.assertIn('http-check send meth GET uri /health', config) + self.assertIn('http-check expect string success', config) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index 9f895c4e263..1569d8d713e 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -75,6 +75,10 @@ def verify(lb): raise ConfigError(f'"TCP" port "{tmp_port}" is used by another service') for back, back_config in lb['backend'].items(): + if 'http-check' in back_config: + http_check = back_config['http-check'] + if 'expect' in http_check and 'status' in http_check['expect'] and 'string' in http_check['expect']: + raise ConfigError(f'"expect status" and "expect string" can not be configured together!') if 'server' not in back_config: raise ConfigError(f'"{back} server" must be configured!') for bk_server, bk_server_conf in back_config['server'].items(): From 6e9cd8821ca028b5bc05c14b0b4e3454036da6da Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 21 Apr 2024 10:33:15 +0200 Subject: [PATCH 025/188] vyos.utils: T6244: use list to build up result string When handling optional separators rather build up a list and join the list with the requested delimiter to form the resulting human readable time string. --- python/vyos/utils/convert.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/python/vyos/utils/convert.py b/python/vyos/utils/convert.py index c02f0071ea3..da207b9fb9f 100644 --- a/python/vyos/utils/convert.py +++ b/python/vyos/utils/convert.py @@ -1,4 +1,4 @@ -# Copyright 2023 VyOS maintainers and contributors +# Copyright 2023-2024 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -23,34 +23,34 @@ def seconds_to_human(s, separator=""): day = 60 * 60 * 24 hour = 60 * 60 - remainder = 0 - result = "" + result = [] + weeks = s // week if weeks > 0: - result = "{0}w".format(weeks) + result.append(f'{weeks}w') s = s % week days = s // day if days > 0: - result = "{0}{1}{2}d".format(result, separator, days) + result.append(f'{days}d') s = s % day hours = s // hour if hours > 0: - result = "{0}{1}{2}h".format(result, separator, hours) + result.append(f'{hours}h') s = s % hour minutes = s // 60 if minutes > 0: - result = "{0}{1}{2}m".format(result, separator, minutes) + result.append(f'{minutes}m') s = s % 60 seconds = s if seconds > 0: - result = "{0}{1}{2}s".format(result, separator, seconds) + result.append(f'{seconds}s') - return result + return separator.join(result) def bytes_to_human(bytes, initial_exponent=0, precision=2, int_below_exponent=0): From 8d8f3137d174a43a259cbe50dd12730805f0200c Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 21 Apr 2024 10:34:43 +0200 Subject: [PATCH 026/188] vyos.utils: T6244: add support for year timebase in seconds_to_human() We only supported calculating seconds to weeks but not seconds to years. This has been added. Testcase: from vyos.utils.convert import seconds_to_human minute = 60 hour = minute * 60 day = hour * 24 week = day * 7 year = day * 365.25 for separator in ['', ' ', '-', '/']: print(f'----- Using separator "{separator}" -----') print(seconds_to_human(10, separator)) print(seconds_to_human(5* minute, separator)) print(seconds_to_human(3* hour, separator)) print(seconds_to_human(4* day, separator)) print(seconds_to_human(7 * week, separator)) print(seconds_to_human(10 * year, separator)) print(seconds_to_human(5*year + 4*week + 3*day + 2*hour + minute + 5, separator)) print() cpo@LR1.wue3:~$ ./foo.py ----- Using separator "" ----- 10s 5m 3h 4d 7w 10y 5y4w3d2h1m5s ----- Using separator " " ----- 10s 5m 3h 4d 7w 10y 5y 4w 3d 2h 1m 5s ----- Using separator "-" ----- 10s 5m 3h 4d 7w 10y 5y-4w-3d-2h-1m-5s ----- Using separator "/" ----- 10s 5m 3h 4d 7w 10y 5y/4w/3d/2h/1m/5s --- python/vyos/utils/convert.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/vyos/utils/convert.py b/python/vyos/utils/convert.py index da207b9fb9f..41e65081f00 100644 --- a/python/vyos/utils/convert.py +++ b/python/vyos/utils/convert.py @@ -19,12 +19,17 @@ def seconds_to_human(s, separator=""): """ s = int(s) + year = 60 * 60 * 24 * 365.25 week = 60 * 60 * 24 * 7 day = 60 * 60 * 24 hour = 60 * 60 result = [] + years = s // year + if years > 0: + result.append(f'{int(years)}y') + s = int(s % year) weeks = s // week if weeks > 0: From 31b21d26751b7db7ab784486da5b8690ddd4a058 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 21 Apr 2024 10:35:08 +0200 Subject: [PATCH 027/188] op-mode: T6244: add whitespace after time unit in "show system uptime" --- src/op_mode/uptime.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/op_mode/uptime.py b/src/op_mode/uptime.py index d6adf6f4d76..059a4c3f655 100755 --- a/src/op_mode/uptime.py +++ b/src/op_mode/uptime.py @@ -49,7 +49,7 @@ def _get_raw_data(): res = {} res["uptime_seconds"] = _get_uptime_seconds() - res["uptime"] = seconds_to_human(_get_uptime_seconds()) + res["uptime"] = seconds_to_human(_get_uptime_seconds(), separator=' ') res["load_average"] = _get_load_averages() return res From 0cb4294fdfe5ae0e0e8fd06436f38b67f16413a2 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 21 Apr 2024 21:22:13 +0200 Subject: [PATCH 028/188] smoketest: support dynamic enable of smoketest debugging $ touch /tmp/vyos.smoketest.debug will enable dynamic debugging of the smoketests - showing the appropriate CLI commands on stdout --- smoketest/scripts/cli/base_vyostest_shim.py | 4 +++- src/init/vyos-router | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/smoketest/scripts/cli/base_vyostest_shim.py b/smoketest/scripts/cli/base_vyostest_shim.py index c49d3e76c5d..efaa74fe01d 100644 --- a/smoketest/scripts/cli/base_vyostest_shim.py +++ b/smoketest/scripts/cli/base_vyostest_shim.py @@ -1,4 +1,4 @@ -# Copyright (C) 2021-2023 VyOS maintainers and contributors +# Copyright (C) 2021-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -47,6 +47,8 @@ class TestCase(unittest.TestCase): def setUpClass(cls): cls._session = ConfigSession(os.getpid()) cls._session.save_config(save_config) + if os.path.exists('/tmp/vyos.smoketest.debug'): + cls.debug = True pass @classmethod diff --git a/src/init/vyos-router b/src/init/vyos-router index 06fea140da2..15e37df07cb 100755 --- a/src/init/vyos-router +++ b/src/init/vyos-router @@ -451,6 +451,7 @@ start () touch /tmp/vyos.ifconfig.debug touch /tmp/vyos.frr.debug touch /tmp/vyos.container.debug + touch /tmp/vyos.smoketest.debug fi log_action_begin_msg "Mounting VyOS Config" From 78ea623df20b44309cc6ac9848ed18e97fc4ed03 Mon Sep 17 00:00:00 2001 From: Alex W Date: Sun, 21 Apr 2024 21:59:56 +0100 Subject: [PATCH 029/188] T6237: IPSec remote access VPN: ability to set EAP ID of clients --- data/templates/ipsec/swanctl/remote_access.j2 | 2 +- interface-definitions/vpn_ipsec.xml.in | 20 +++++++++++++++++++ smoketest/scripts/cli/test_vpn_ipsec.py | 5 +++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/data/templates/ipsec/swanctl/remote_access.j2 b/data/templates/ipsec/swanctl/remote_access.j2 index adfa32bde79..6bced88c7c9 100644 --- a/data/templates/ipsec/swanctl/remote_access.j2 +++ b/data/templates/ipsec/swanctl/remote_access.j2 @@ -33,7 +33,7 @@ auth = pubkey {% elif rw_conf.authentication.client_mode.startswith("eap") %} auth = {{ rw_conf.authentication.client_mode }} - eap_id = %any + eap_id = {{ '%any' if rw_conf.authentication.eap_id == 'any' else rw_conf.authentication.eap_id }} {% endif %} {% if rw_conf.authentication.client_mode is vyos_defined('eap-tls') or rw_conf.authentication.client_mode is vyos_defined('x509') %} {# pass all configured CAs as filenames, separated by commas #} diff --git a/interface-definitions/vpn_ipsec.xml.in b/interface-definitions/vpn_ipsec.xml.in index 833019d6830..7f425d98277 100644 --- a/interface-definitions/vpn_ipsec.xml.in +++ b/interface-definitions/vpn_ipsec.xml.in @@ -768,6 +768,26 @@ #include #include + + + Remote EAP ID for client authentication + + txt + Remote EAP ID for client authentication + + + any + + + any + Allow any EAP ID + + + [[:ascii:]]{1,64} + + + any + Client authentication mode diff --git a/smoketest/scripts/cli/test_vpn_ipsec.py b/smoketest/scripts/cli/test_vpn_ipsec.py index 145b5990e37..27356d70eff 100755 --- a/smoketest/scripts/cli/test_vpn_ipsec.py +++ b/smoketest/scripts/cli/test_vpn_ipsec.py @@ -782,6 +782,11 @@ def test_remote_access_eap_tls(self): self.assertTrue(os.path.exists(os.path.join(CA_PATH, f'{ca_name}.pem'))) self.assertTrue(os.path.exists(os.path.join(CERT_PATH, f'{peer_name}.pem'))) + # Test setting of custom EAP ID + self.cli_set(base_path + ['remote-access', 'connection', conn_name, 'authentication', 'eap-id', 'eap-user@vyos.net']) + self.cli_commit() + self.assertIn(r'eap_id = eap-user@vyos.net', read_file(swanctl_file)) + self.tearDownPKI() def test_remote_access_x509(self): From 8d0aa7bfb83aecb989ab01b6d1975cf23f1c7dcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Luiz=20dos=20Santos?= Date: Sun, 21 Apr 2024 15:07:36 -0300 Subject: [PATCH 030/188] xml: T5738: fix typo in radius-additions.xml.i --- interface-definitions/include/accel-ppp/radius-additions.xml.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/include/accel-ppp/radius-additions.xml.i b/interface-definitions/include/accel-ppp/radius-additions.xml.i index cdd0bf30085..3c2eb09eb97 100644 --- a/interface-definitions/include/accel-ppp/radius-additions.xml.i +++ b/interface-definitions/include/accel-ppp/radius-additions.xml.i @@ -122,7 +122,7 @@ ipv4 - IPv4 address for aynamic authorization server + IPv4 address for dynamic authorization server From a849d913dc6a86c698dc215350a46b7af72bb99e Mon Sep 17 00:00:00 2001 From: Alex W Date: Mon, 22 Apr 2024 22:12:26 +0100 Subject: [PATCH 031/188] PKI: T6259: Support RFC822 names in certificate generation --- python/vyos/pki.py | 2 +- src/op_mode/pki.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/vyos/pki.py b/python/vyos/pki.py index 3c577db4d57..27fe793a88e 100644 --- a/python/vyos/pki.py +++ b/python/vyos/pki.py @@ -146,7 +146,7 @@ def create_certificate_request(subject, private_key, subject_alt_names=[]): if isinstance(obj, ipaddress.IPv4Address) or isinstance(obj, ipaddress.IPv6Address): alt_names.append(x509.IPAddress(obj)) elif isinstance(obj, str): - alt_names.append(x509.DNSName(obj)) + alt_names.append(x509.RFC822Name(obj) if '@' in obj else x509.DNSName(obj)) if alt_names: builder = builder.add_extension(x509.SubjectAlternativeName(alt_names), critical=False) diff --git a/src/op_mode/pki.py b/src/op_mode/pki.py index ad2c1ada0aa..b1ca6ee296d 100755 --- a/src/op_mode/pki.py +++ b/src/op_mode/pki.py @@ -306,7 +306,7 @@ def parse_san_string(san_string): output.append(ipaddress.IPv4Address(value)) elif tag == 'ipv6': output.append(ipaddress.IPv6Address(value)) - elif tag == 'dns': + elif tag == 'dns' or tag == 'rfc822': output.append(value) return output @@ -324,7 +324,7 @@ def generate_certificate_request(private_key=None, key_type=None, return_request subject_alt_names = None if ask_san and ask_yes_no('Do you want to configure Subject Alternative Names?'): - print("Enter alternative names in a comma separate list, example: ipv4:1.1.1.1,ipv6:fe80::1,dns:vyos.net") + print("Enter alternative names in a comma separate list, example: ipv4:1.1.1.1,ipv6:fe80::1,dns:vyos.net,rfc822:user@vyos.net") san_string = ask_input('Enter Subject Alternative Names:') subject_alt_names = parse_san_string(san_string) From c2fc2dba32ba861684f5e34635f810c56d551d51 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Mon, 22 Apr 2024 13:18:17 -0500 Subject: [PATCH 032/188] image-tools: T6260: remove persistence image directory if no space error --- src/op_mode/image_installer.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index b1311b6f904..ba0e3b6db2a 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -26,6 +26,7 @@ from typing import Union from urllib.parse import urlparse from passlib.hosts import linux_context +from errno import ENOSPC from psutil import disk_partitions @@ -939,6 +940,16 @@ def add_image(image_path: str, vrf: str = None, username: str = '', if set_as_default: grub.set_default(image_name, root_dir) + except OSError as e: + # if no space error, remove image dir and cleanup + if e.errno == ENOSPC: + cleanup(mounts=[str(iso_path)], + remove_items=[f'{root_dir}/boot/{image_name}']) + else: + # unmount an ISO and cleanup + cleanup([str(iso_path)]) + exit(f'Error: {e}') + except Exception as err: # unmount an ISO and cleanup cleanup([str(iso_path)]) From d2a82c30695c2f4265dc5ca2165d27d5aa3e2cef Mon Sep 17 00:00:00 2001 From: Ginko <152240782+Giggum@users.noreply.github.com> Date: Sun, 14 Apr 2024 03:15:52 -0400 Subject: [PATCH 033/188] ntp: T4909: Rewrite NTP op mode in new format ntp: T4909: Rewrite NTP op mode in new format Adapts ntp.xml.in to reference new ntp.py file Add ntp.py Adds a check to ntp.py to verify if the ntp service is configured Adds raw mode to ntp.py For raw output, replaces the original method of parsing the command line output FROM re.split+regex TO csv.reader. Separates chrony commands into equivalent functions show_tracking, show_sources, source_sourcestats and show_activity Revises the names of raw dictionary keys variables to be lowercase Corrects a comment typo and renames function name used for raw mode --- op-mode-definitions/ntp.xml.in | 16 +++- src/op_mode/ntp.py | 164 +++++++++++++++++++++++++++++++++ 2 files changed, 178 insertions(+), 2 deletions(-) create mode 100644 src/op_mode/ntp.py diff --git a/op-mode-definitions/ntp.xml.in b/op-mode-definitions/ntp.xml.in index b8d0c43ec97..17250a45e62 100644 --- a/op-mode-definitions/ntp.xml.in +++ b/op-mode-definitions/ntp.xml.in @@ -6,13 +6,25 @@ Show peer status of NTP daemon - ${vyos_op_scripts_dir}/show_ntp.sh --sourcestats + ${vyos_op_scripts_dir}/ntp.py show_sourcestats + + + Report the number of servers and peers that are online and offline + + ${vyos_op_scripts_dir}/ntp.py show_activity + + + + Show information about the current time sources being accessed + + ${vyos_op_scripts_dir}/ntp.py show_sources + Show parameters about the system clock performance - ${vyos_op_scripts_dir}/show_ntp.sh --tracking + ${vyos_op_scripts_dir}/ntp.py show_tracking diff --git a/src/op_mode/ntp.py b/src/op_mode/ntp.py new file mode 100644 index 00000000000..e14cc46d0e1 --- /dev/null +++ b/src/op_mode/ntp.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import csv +import sys +from itertools import chain + +import vyos.opmode +from vyos.configquery import ConfigTreeQuery +from vyos.utils.process import cmd + +def _get_raw_data(command: str) -> dict: + # Returns returns chronyc output as a dictionary + + # Initialize dictionary keys to align with output of + # chrony -c. From some commands, its -c switch outputs + # more parameters, make sure to include them all below. + # See to chronyc(1) for definition of key variables + match command: + case "chronyc -c activity": + keys: list = [ + 'sources_online', + 'sources_offline', + 'sources_doing_burst_return_online', + 'sources_doing_burst_return_offline', + 'sources_with_unknown_address' + ] + + case "chronyc -c sources": + keys: list = [ + 'm', + 's', + 'name_ip_address', + 'stratum', + 'poll', + 'reach', + 'last_rx', + 'last_sample_adj_offset', + 'last_sample_mes_offset', + 'last_sample_est_error' + ] + + case "chronyc -c sourcestats": + keys: list = [ + 'name_ip_address', + 'np', + 'nr', + 'span', + 'frequency', + 'freq_skew', + 'offset', + 'std_dev' + ] + + case "chronyc -c tracking": + keys: list = [ + 'ref_id', + 'ref_id_name', + 'stratum', + 'ref_time', + 'system_time', + 'last_offset', + 'rms_offset', + 'frequency', + 'residual_freq', + 'skew', + 'root_delay', + 'root_dispersion', + 'update_interval', + 'leap_status' + ] + + case _: + raise ValueError(f"Raw mode: of {command} is not implemented") + + # Get -c option command line output, splitlines, + # and save comma-separated values as a flat list + output = cmd(command).splitlines() + values = csv.reader(output) + values = list(chain.from_iterable(values)) + + # Divide values into chunks of size keys and transpose + if len(values) > len(keys): + values = _chunk_list(values,keys) + values = zip(*values) + + return dict(zip(keys, values)) + +def _chunk_list(in_list, n): + # Yields successive n-sized chunks from in_list + for i in range(0, len(in_list), len(n)): + yield in_list[i:i + len(n)] + +def _is_configured(): + # Check if ntp is configured + config = ConfigTreeQuery() + if not config.exists("service ntp"): + raise vyos.opmode.UnconfiguredSubsystem("NTP service is not enabled.") + +def show_activity(raw: bool): + _is_configured() + command = f'chronyc' + + if raw: + command += f" -c activity" + return _get_raw_data(command) + else: + command += f" activity" + return cmd(command) + +def show_sources(raw: bool): + _is_configured() + command = f'chronyc' + + if raw: + command += f" -c sources" + return _get_raw_data(command) + else: + command += f" sources -v" + return cmd(command) + +def show_tracking(raw: bool): + _is_configured() + command = f'chronyc' + + if raw: + command += f" -c tracking" + return _get_raw_data(command) + else: + command += f" tracking" + return cmd(command) + +def show_sourcestats(raw: bool): + _is_configured() + command = f'chronyc' + + if raw: + command += f" -c sourcestats" + return _get_raw_data(command) + else: + command += f" sourcestats -v" + return cmd(command) + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) From 19e0d3b74f66e082c3f131b9044e7ca2371b1d85 Mon Sep 17 00:00:00 2001 From: Ginko <152240782+Giggum@users.noreply.github.com> Date: Mon, 22 Apr 2024 21:42:13 -0400 Subject: [PATCH 034/188] connect_disconnect: T6261: correction to typo in check_ppp_running function Connect_disconnect: T6261: correction to typo in check_ppp_running function Changes include: 1. Replaces "beeing" -> being in print statement for check_ppp_running 2. Replaces "can not" -> cannot in print statement on lines 61 and 93 --- src/op_mode/connect_disconnect.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/op_mode/connect_disconnect.py b/src/op_mode/connect_disconnect.py index bd02dc6ea5f..373f9e9531c 100755 --- a/src/op_mode/connect_disconnect.py +++ b/src/op_mode/connect_disconnect.py @@ -48,7 +48,7 @@ def connect(interface): if os.path.isdir(f'/sys/class/net/{interface}'): print(f'Interface {interface}: already connected!') elif check_ppp_running(interface): - print(f'Interface {interface}: connection is beeing established!') + print(f'Interface {interface}: connection is being established!') else: print(f'Interface {interface}: connecting...') call(f'systemctl restart ppp@{interface}.service') @@ -58,7 +58,7 @@ def connect(interface): else: call(f'VYOS_TAGNODE_VALUE={interface} /usr/libexec/vyos/conf_mode/interfaces_wwan.py') else: - print(f'Unknown interface {interface}, can not connect. Aborting!') + print(f'Unknown interface {interface}, cannot connect. Aborting!') # Reaply QoS configuration config = ConfigTreeQuery() @@ -90,7 +90,7 @@ def disconnect(interface): modem = interface.lstrip('wwan') call(f'mmcli --modem {modem} --simple-disconnect', stdout=DEVNULL) else: - print(f'Unknown interface {interface}, can not disconnect. Aborting!') + print(f'Unknown interface {interface}, cannot disconnect. Aborting!') def main(): parser = argparse.ArgumentParser() From 76cc762ef4aacd1120d533e1122f56ce09a2cfa9 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 23 Apr 2024 07:30:49 +0200 Subject: [PATCH 035/188] GitHub: adjust MergifyIo regex match to both upper and lower case --- .github/workflows/mergifyio_backport.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mergifyio_backport.yml b/.github/workflows/mergifyio_backport.yml index f1f4312c479..d9f863d9ad3 100644 --- a/.github/workflows/mergifyio_backport.yml +++ b/.github/workflows/mergifyio_backport.yml @@ -13,7 +13,7 @@ jobs: id: regex-match with: text: ${{ github.event.comment.body }} - regex: '[Mm]ergifyio backport ' + regex: '@[Mm][Ee][Rr][Gg][Ii][Ff][Yy][Ii][Oo] backport ' - uses: actions-ecosystem/action-add-labels@v1 if: ${{ steps.regex-match.outputs.match != '' }} From eab0adcbf30734045cf04c140d30efae1abdf194 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 23 Apr 2024 07:38:17 +0200 Subject: [PATCH 036/188] GitHub: use ubuntu-latest for labeler action --- .github/workflows/pull-request-labels.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pull-request-labels.yml b/.github/workflows/pull-request-labels.yml index 3398af5b0f5..43856beaa1f 100644 --- a/.github/workflows/pull-request-labels.yml +++ b/.github/workflows/pull-request-labels.yml @@ -12,9 +12,9 @@ on: jobs: add-pr-label: name: Add PR Labels - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest permissions: contents: read pull-requests: write steps: - - uses: actions/labeler@v5.0.0 + - uses: actions/labeler@v5 From 984c386d11ead8371b7ac381e6c0921473e557ed Mon Sep 17 00:00:00 2001 From: Windom WU Date: Tue, 23 Apr 2024 14:39:02 +0800 Subject: [PATCH 037/188] T6226: add HAPROXY tcp-request related block to load-balancing reverse proxy config --- data/templates/load-balancing/haproxy.cfg.j2 | 12 ++ .../include/haproxy/tcp-request.xml.i | 22 ++++ .../load-balancing_reverse-proxy.xml.in | 1 + .../cli/test_load-balancing_reverse-proxy.py | 113 +++++++++++++----- 4 files changed, 115 insertions(+), 33 deletions(-) create mode 100644 interface-definitions/include/haproxy/tcp-request.xml.i diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index dd93afba531..e8622ba7b79 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -69,6 +69,18 @@ frontend {{ front }} {% endif %} {% if front_config.mode is vyos_defined %} mode {{ front_config.mode }} +{% if front_config.tcp_request.inspect_delay is vyos_defined %} + tcp-request inspect-delay {{ front_config.tcp_request.inspect_delay }} +{% endif %} +{# add tcp-request related directive if ssl is configed #} +{% if front_config.mode is vyos_defined('tcp') and front_config.rule is vyos_defined %} +{% for rule, rule_config in front_config.rule.items() %} +{% if rule_config.ssl is vyos_defined %} + tcp-request content accept if { req_ssl_hello_type 1 } +{% break %} +{% endif %} +{% endfor %} +{% endif %} {% endif %} {% if front_config.rule is vyos_defined %} {% for rule, rule_config in front_config.rule.items() %} diff --git a/interface-definitions/include/haproxy/tcp-request.xml.i b/interface-definitions/include/haproxy/tcp-request.xml.i new file mode 100644 index 00000000000..3d60bd8ad56 --- /dev/null +++ b/interface-definitions/include/haproxy/tcp-request.xml.i @@ -0,0 +1,22 @@ + + + + TCP request directive + + + + + Set the maximum allowed time to wait for data during content inspection + + u32:1-65535 + The timeout value specified in milliseconds + + + + + The timeout value must be in range 1 to 65535 milliseconds + + + + + diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index eb01580da0e..6a3b3cef1ce 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -38,6 +38,7 @@ #include #include #include + #include Redirect HTTP to HTTPS diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index 737c0740147..f9f16378286 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -299,39 +299,86 @@ def test_04_lb_reverse_proxy_backend_ssl_no_verify(self): self.cli_commit() def test_05_lb_reverse_proxy_backend_http_check(self): - # Setup base - self.base_config() - - # Set http-check - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'method', 'get']) - self.cli_commit() - - # Test http-check - config = read_file(HAPROXY_CONF) - self.assertIn('option httpchk', config) - self.assertIn('http-check send meth GET', config) - - # Set http-check with uri and status - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'uri', '/health']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) - self.cli_commit() - - # Test http-check with uri and status - config = read_file(HAPROXY_CONF) - self.assertIn('option httpchk', config) - self.assertIn('http-check send meth GET uri /health', config) - self.assertIn('http-check expect status 200', config) - - # Set http-check with string - self.cli_delete(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) - self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'string', 'success']) - self.cli_commit() - - # Test http-check with string - config = read_file(HAPROXY_CONF) - self.assertIn('option httpchk', config) - self.assertIn('http-check send meth GET uri /health', config) - self.assertIn('http-check expect string success', config) + # Setup base + self.base_config() + + # Set http-check + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'method', 'get']) + self.cli_commit() + + # Test http-check + config = read_file(HAPROXY_CONF) + self.assertIn('option httpchk', config) + self.assertIn('http-check send meth GET', config) + + # Set http-check with uri and status + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'uri', '/health']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) + self.cli_commit() + + # Test http-check with uri and status + config = read_file(HAPROXY_CONF) + self.assertIn('option httpchk', config) + self.assertIn('http-check send meth GET uri /health', config) + self.assertIn('http-check expect status 200', config) + + # Set http-check with string + self.cli_delete(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'status', '200']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-check', 'expect', 'string', 'success']) + self.cli_commit() + + # Test http-check with string + config = read_file(HAPROXY_CONF) + self.assertIn('option httpchk', config) + self.assertIn('http-check send meth GET uri /health', config) + self.assertIn('http-check expect string success', config) + + def test_06_lb_reverse_proxy_tcp_mode(self): + frontend = 'tcp_8443' + mode = 'tcp' + front_port = '8433' + tcp_request_delay = "5000" + rule_thirty = '30' + domain_bk = 'n6.example.com' + ssl_opt = "req-ssl-sni" + bk_name = 'bk-03' + bk_server = '192.0.2.11' + bk_server_port = '9090' + + back_base = base_path + ['backend'] + + self.cli_set(base_path + ['service', frontend, 'mode', mode]) + self.cli_set(base_path + ['service', frontend, 'port', front_port]) + self.cli_set(base_path + ['service', frontend, 'tcp-request', 'inspect-delay', tcp_request_delay]) + + self.cli_set(base_path + ['service', frontend, 'rule', rule_thirty, 'domain-name', domain_bk]) + self.cli_set(base_path + ['service', frontend, 'rule', rule_thirty, 'ssl', ssl_opt]) + self.cli_set(base_path + ['service', frontend, 'rule', rule_thirty, 'set', 'backend', bk_name]) + + self.cli_set(back_base + [bk_name, 'mode', mode]) + self.cli_set(back_base + [bk_name, 'server', bk_name, 'address', bk_server]) + self.cli_set(back_base + [bk_name, 'server', bk_name, 'port', bk_server_port]) + + # commit changes + self.cli_commit() + + config = read_file(HAPROXY_CONF) + + # Frontend + self.assertIn(f'frontend {frontend}', config) + self.assertIn(f'bind :::{front_port} v4v6', config) + self.assertIn(f'mode {mode}', config) + + self.assertIn(f'tcp-request inspect-delay {tcp_request_delay}', config) + self.assertIn(f"tcp-request content accept if {{ req_ssl_hello_type 1 }}", config) + self.assertIn(f'acl {rule_thirty} req_ssl_sni -i {domain_bk}', config) + self.assertIn(f'use_backend {bk_name} if {rule_thirty}', config) + + # Backend + self.assertIn(f'backend {bk_name}', config) + self.assertIn(f'balance roundrobin', config) + self.assertIn(f'mode {mode}', config) + self.assertIn(f'server {bk_name} {bk_server}:{bk_server_port}', config) if __name__ == '__main__': unittest.main(verbosity=2) From 7efe245f59bbea9f12d0c7c5a8975380efc6f2d5 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 23 Apr 2024 12:17:55 +0000 Subject: [PATCH 038/188] T6217: Conntrack-sync change the actual name of the script The actual name of the script is `vyos-vrrp-conntracksync.sh` --- src/helpers/vyos-vrrp-conntracksync.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/helpers/vyos-vrrp-conntracksync.sh b/src/helpers/vyos-vrrp-conntracksync.sh index 0cc71893857..90fa77f23bb 100755 --- a/src/helpers/vyos-vrrp-conntracksync.sh +++ b/src/helpers/vyos-vrrp-conntracksync.sh @@ -25,7 +25,7 @@ LOGCMD="logger -t $TAG -p $FACILITY.$LEVEL" VRRP_GRP="VRRP sync-group [$2]" FAILOVER_STATE="/var/run/vyatta-conntrackd-failover-state" -$LOGCMD "vyatta-vrrp-conntracksync invoked at `date`" +$LOGCMD "vyos-vrrp-conntracksync invoked at `date`" if ! systemctl is-active --quiet conntrackd.service; then echo "conntrackd service not running" @@ -148,7 +148,7 @@ case "$1" in *) echo UNKNOWN at `date` > $FAILOVER_STATE $LOGCMD "ERROR: `uname -n` unknown state transition for $VRRP_GRP" - echo "Usage: vyatta-vrrp-conntracksync.sh {master|backup|fault}" + echo "Usage: vyos-vrrp-conntracksync.sh {master|backup|fault}" exit 1 ;; esac From 7164ad40f5cc47f35c7903626d4d4da048a25113 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 23 Apr 2024 15:43:08 +0000 Subject: [PATCH 039/188] T6109: Fix remote logging for sudo commands This fix for bug when `sudo` commands were not send to the remote syslog server. They stop before the directive that includes all configurations `$IncludeConfig /etc/rsyslog.d/*.conf` --- src/etc/rsyslog.conf | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/etc/rsyslog.conf b/src/etc/rsyslog.conf index 9781f083513..b3f41acb684 100644 --- a/src/etc/rsyslog.conf +++ b/src/etc/rsyslog.conf @@ -15,21 +15,6 @@ $KLogPath /proc/kmsg #### GLOBAL DIRECTIVES #### ########################### -# The lines below cause all listed daemons/processes to be logged into -# /var/log/auth.log, then drops the message so it does not also go to the -# regular syslog so that messages are not duplicated - -$outchannel auth_log,/var/log/auth.log -if $programname == 'CRON' or - $programname == 'sudo' or - $programname == 'su' - then :omfile:$auth_log - -if $programname == 'CRON' or - $programname == 'sudo' or - $programname == 'su' - then stop - # Use traditional timestamp format. # To enable high precision timestamps, comment out the following line. # A modern-style logfile format similar to TraditionalFileFormat, buth with high-precision timestamps and timezone information @@ -60,6 +45,21 @@ $Umask 0022 # $IncludeConfig /etc/rsyslog.d/*.conf +# The lines below cause all listed daemons/processes to be logged into +# /var/log/auth.log, then drops the message so it does not also go to the +# regular syslog so that messages are not duplicated + +$outchannel auth_log,/var/log/auth.log +if $programname == 'CRON' or + $programname == 'sudo' or + $programname == 'su' + then :omfile:$auth_log + +if $programname == 'CRON' or + $programname == 'sudo' or + $programname == 'su' + then stop + ############### #### RULES #### ############### From 8602c84e1b7c0da4c4c57fc2d034ec18497303fd Mon Sep 17 00:00:00 2001 From: Alex W Date: Tue, 23 Apr 2024 18:07:18 +0100 Subject: [PATCH 040/188] T6255: static-routing: don't render whitespace from static table descriptions --- data/templates/iproute2/static.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/templates/iproute2/static.conf.j2 b/data/templates/iproute2/static.conf.j2 index 10c9bdab7e4..249483ab354 100644 --- a/data/templates/iproute2/static.conf.j2 +++ b/data/templates/iproute2/static.conf.j2 @@ -2,7 +2,7 @@ {% if table is vyos_defined %} {% for t, t_options in table.items() %} {% if t_options.description is vyos_defined %} -{{ "%-6s" | format(t) }} {{ "%-40s" | format(t_options.description) }} +{{ "%-6s" | format(t) }} {{ "%-40s" | format(t_options.description | replace(" ", "_")) }} {% endif %} {% endfor %} {% endif %} From a3713cd64f2f43f321a5138db94bb1a87edbffdd Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 24 Apr 2024 10:50:59 +0000 Subject: [PATCH 041/188] T5833: Not all AFIs compatible with VRF add verify check Not all FRR address-families compatibe with VRF ``` r4# conf t r4(config)# router bgp 65001 vrf bgp r4(config-router)# r4(config-router)# address-family ipv4 flowspec Only Unicast/Multicast/EVPN SAFIs supported in non-core instances. r4(config-router)# r4(config-router)# address-family ipv4 labeled-unicast Only Unicast/Multicast/EVPN SAFIs supported in non-core instances. r4(config-router)# r4(config-router)# address-family ipv4 vpn Only Unicast/Multicast/EVPN SAFIs supported in non-core instances. r4(config-router)# ``` Add verify AFI for VRF --- src/conf_mode/protocols_bgp.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/conf_mode/protocols_bgp.py b/src/conf_mode/protocols_bgp.py index 2b16de77569..4df97d1331a 100755 --- a/src/conf_mode/protocols_bgp.py +++ b/src/conf_mode/protocols_bgp.py @@ -473,6 +473,22 @@ def verify(bgp): if peer_group_as is None or (peer_group_as != 'internal' and peer_group_as != bgp['system_as']): raise ConfigError('route-reflector-client only supported for iBGP peers') + # T5833 not all AFIs are supported for VRF + if 'vrf' in bgp and 'address_family' in peer_config: + unsupported_vrf_afi = { + 'ipv4_flowspec', + 'ipv6_flowspec', + 'ipv4_labeled_unicast', + 'ipv6_labeled_unicast', + 'ipv4_vpn', + 'ipv6_vpn', + } + for afi in peer_config['address_family']: + if afi in unsupported_vrf_afi: + raise ConfigError( + f"VRF is not allowed for address-family '{afi.replace('_', '-')}'" + ) + # Throw an error if a peer group is not configured for allow range for prefix in dict_search('listen.range', bgp) or []: # we can not use dict_search() here as prefix contains dots ... From 0bf4b570fe2d239b9fbabd3ae801ad3f04a06bde Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 25 Apr 2024 08:23:14 +0000 Subject: [PATCH 042/188] T6258: Add sysctl base-reachable-time for IPv6 Add abiilty to change `base_reachable_time_ms` option /proc/sys/net/ipv6/neigh/{ifname}/base_reachable_time_ms --- .../interface/base-reachable-time.xml.i | 16 +++++++++++ .../include/interface/ipv6-options.xml.i | 1 + python/vyos/ifconfig/interface.py | 28 ++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) create mode 100644 interface-definitions/include/interface/base-reachable-time.xml.i diff --git a/interface-definitions/include/interface/base-reachable-time.xml.i b/interface-definitions/include/interface/base-reachable-time.xml.i new file mode 100644 index 00000000000..fb0d701014f --- /dev/null +++ b/interface-definitions/include/interface/base-reachable-time.xml.i @@ -0,0 +1,16 @@ + + + + Base reachable time in seconds + + u32:1-86400 + Base reachable time in seconds + + + + + Base reachable time must be between 1 and 86400 seconds + + 30 + + diff --git a/interface-definitions/include/interface/ipv6-options.xml.i b/interface-definitions/include/interface/ipv6-options.xml.i index edb4a74f970..ec6ec64ee93 100644 --- a/interface-definitions/include/interface/ipv6-options.xml.i +++ b/interface-definitions/include/interface/ipv6-options.xml.i @@ -5,6 +5,7 @@ #include + #include #include #include #include diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index 1b86982c4e9..f0897bc216d 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -1,4 +1,4 @@ -# Copyright 2019-2023 VyOS maintainers and contributors +# Copyright 2019-2024 VyOS maintainers and contributors # # This library is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -193,6 +193,9 @@ class Interface(Control): 'validate': assert_positive, 'location': '/proc/sys/net/ipv6/conf/{ifname}/dad_transmits', }, + 'ipv6_cache_tmo': { + 'location': '/proc/sys/net/ipv6/neigh/{ifname}/base_reachable_time_ms', + }, 'path_cost': { # XXX: we should set a maximum 'validate': assert_positive, @@ -261,6 +264,9 @@ class Interface(Control): 'ipv6_dad_transmits': { 'location': '/proc/sys/net/ipv6/conf/{ifname}/dad_transmits', }, + 'ipv6_cache_tmo': { + 'location': '/proc/sys/net/ipv6/neigh/{ifname}/base_reachable_time_ms', + }, 'proxy_arp': { 'location': '/proc/sys/net/ipv4/conf/{ifname}/proxy_arp', }, @@ -613,6 +619,21 @@ def set_arp_cache_tmo(self, tmo): return None return self.set_interface('arp_cache_tmo', tmo) + def set_ipv6_cache_tmo(self, tmo): + """ + Set IPv6 cache timeout value in seconds. Internal Kernel representation + is in milliseconds. + + Example: + >>> from vyos.ifconfig import Interface + >>> Interface('eth0').set_ipv6_cache_tmo(40) + """ + tmo = str(int(tmo) * 1000) + tmp = self.get_interface('ipv6_cache_tmo') + if tmp == tmo: + return None + return self.set_interface('ipv6_cache_tmo', tmo) + def _cleanup_mss_rules(self, table, ifname): commands = [] results = self._cmd(f'nft -a list chain {table} VYOS_TCP_MSS').split("\n") @@ -1698,6 +1719,11 @@ def update(self, config): for addr in tmp: self.add_ipv6_eui64_address(addr) + # Configure IPv6 base time in milliseconds - has default value + tmp = dict_search('ipv6.base_reachable_time', config) + value = tmp if (tmp != None) else '30' + self.set_ipv6_cache_tmo(value) + # re-add ourselves to any bridge we might have fallen out of if 'is_bridge_member' in config: tmp = config.get('is_bridge_member') From c8f9acf5d91827b0d1266d3061a5e15a82628323 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 25 Apr 2024 09:47:24 +0000 Subject: [PATCH 043/188] T6263: Groups 224.0.0.0/24 are reserved and cannot be joined The join addresses within the multicast group 224.0.0.0/24 are reserved and cannot be joined FRR ``` r4(config)# interface eth2 r4(config-if)# ip igmp join 224.0.0.0 224.0.0.10 % Configuration failed. Error type: validation Error description: Groups within 224.0.0.0/24 are reserved and cannot be joined r4(config-if)# ``` Add verify check --- src/conf_mode/protocols_pim.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/protocols_pim.py b/src/conf_mode/protocols_pim.py index 09c3be8dfe8..d450d11cae9 100755 --- a/src/conf_mode/protocols_pim.py +++ b/src/conf_mode/protocols_pim.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2023 VyOS maintainers and contributors +# Copyright (C) 2020-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -16,6 +16,7 @@ import os +from ipaddress import IPv4Address from ipaddress import IPv4Network from signal import SIGTERM from sys import exit @@ -32,6 +33,9 @@ from vyos import airbag airbag.enable() +RESERVED_MC_NET = '224.0.0.0/24' + + def get_config(config=None): if config: conf = config @@ -92,9 +96,15 @@ def verify(pim): if 'interface' not in pim: raise ConfigError('PIM require defined interfaces!') - for interface in pim['interface']: + for interface, interface_config in pim['interface'].items(): verify_interface_exists(interface) + # Check join group in reserved net + if 'igmp' in interface_config and 'join' in interface_config['igmp']: + for join_addr in interface_config['igmp']['join']: + if IPv4Address(join_addr) in IPv4Network(RESERVED_MC_NET): + raise ConfigError(f'Groups within {RESERVED_MC_NET} are reserved and cannot be joined!') + if 'rp' in pim: if 'address' not in pim['rp']: raise ConfigError('PIM rendezvous point needs to be defined!') From 107ee099e82397b31fca8cf1ac3860cbf76f0596 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Thu, 25 Apr 2024 13:27:50 +0300 Subject: [PATCH 044/188] pppoe-server: T6234: PPPoE-server pado-delay refactoring --- data/templates/accel-ppp/pppoe.config.j2 | 2 +- .../version/pppoe-server-version.xml.i | 2 +- .../service_pppoe-server.xml.in | 8 +++ .../scripts/cli/test_service_pppoe-server.py | 9 ++- src/conf_mode/service_pppoe-server.py | 17 ++++++ src/migration-scripts/pppoe-server/9-to-10 | 56 +++++++++++++++++++ 6 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 src/migration-scripts/pppoe-server/9-to-10 diff --git a/data/templates/accel-ppp/pppoe.config.j2 b/data/templates/accel-ppp/pppoe.config.j2 index ddf0da51812..42bc8440c99 100644 --- a/data/templates/accel-ppp/pppoe.config.j2 +++ b/data/templates/accel-ppp/pppoe.config.j2 @@ -67,7 +67,7 @@ service-name={{ service_name | join(',') }} {% set delay_without_sessions = pado_delay.delays_without_sessions[0] | default('0') %} {% set pado_delay_param = namespace(value=delay_without_sessions) %} {% for delay, sessions in pado_delay.delays_with_sessions | sort(attribute='1') %} -{% if not loop.last %} +{% if not delay == 'disable' %} {% set pado_delay_param.value = pado_delay_param.value + ',' + delay + ':' + sessions | string %} {% else %} {% set pado_delay_param.value = pado_delay_param.value + ',-1:' + sessions | string %} diff --git a/interface-definitions/include/version/pppoe-server-version.xml.i b/interface-definitions/include/version/pppoe-server-version.xml.i index c253c58d975..61de1277a57 100644 --- a/interface-definitions/include/version/pppoe-server-version.xml.i +++ b/interface-definitions/include/version/pppoe-server-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/service_pppoe-server.xml.in b/interface-definitions/service_pppoe-server.xml.in index 9b5e4d3fbe1..5d357c2f9af 100644 --- a/interface-definitions/service_pppoe-server.xml.in +++ b/interface-definitions/service_pppoe-server.xml.in @@ -73,12 +73,20 @@ PADO delays + + disable + Disable new connections + + + disable + u32:1-999999 Number in ms + disable Invalid PADO delay diff --git a/smoketest/scripts/cli/test_service_pppoe-server.py b/smoketest/scripts/cli/test_service_pppoe-server.py index 5a48b1f584c..97c63d4cb4b 100755 --- a/smoketest/scripts/cli/test_service_pppoe-server.py +++ b/smoketest/scripts/cli/test_service_pppoe-server.py @@ -168,7 +168,14 @@ def test_pppoe_server_pado_delay(self): conf = ConfigParser(allow_no_value=True, delimiters='=') conf.read(self._config_file) - self.assertEqual(conf['pppoe']['pado-delay'], '10,20:200,-1:300') + self.assertEqual(conf['pppoe']['pado-delay'], '10,20:200,30:300') + + self.set(['pado-delay', 'disable', 'sessions', '400']) + self.cli_commit() + + conf = ConfigParser(allow_no_value=True, delimiters='=') + conf.read(self._config_file) + self.assertEqual(conf['pppoe']['pado-delay'], '10,20:200,30:300,-1:400') if __name__ == '__main__': diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index b9d17493389..c7bdc52f0fe 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -84,12 +84,29 @@ def verify_pado_delay(pppoe): pado_delay = pppoe['pado_delay'] delays_without_sessions = pado_delay['delays_without_sessions'] + if 'disable' in delays_without_sessions: + raise ConfigError( + 'Number of sessions must be specified for "pado-delay disable"' + ) + if len(delays_without_sessions) > 1: raise ConfigError( f'Cannot add more then ONE pado-delay without sessions, ' f'but {len(delays_without_sessions)} were set' ) + if 'disable' in [delay[0] for delay in pado_delay['delays_with_sessions']]: + # need to sort delays by sessions to verify if there is no delay + # for sessions after disabling + sorted_pado_delay = sorted(pado_delay['delays_with_sessions'], key=lambda k_v: k_v[1]) + last_delay = sorted_pado_delay[-1] + + if last_delay[0] != 'disable': + raise ConfigError( + f'Cannot add pado-delay after disabled sessions, but ' + f'"pado-delay {last_delay[0]} sessions {last_delay[1]}" was set' + ) + def verify(pppoe): if not pppoe: return None diff --git a/src/migration-scripts/pppoe-server/9-to-10 b/src/migration-scripts/pppoe-server/9-to-10 new file mode 100644 index 00000000000..e0c782f0428 --- /dev/null +++ b/src/migration-scripts/pppoe-server/9-to-10 @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# Migration of pado-delay options + +from sys import argv +from sys import exit +from vyos.configtree import ConfigTree + +if len(argv) < 2: + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) +base = ['service', 'pppoe-server', 'pado-delay'] +if not config.exists(base): + exit(0) + +pado_delay = {} +for delay in config.list_nodes(base): + sessions = config.return_value(base + [delay, 'sessions']) + pado_delay[delay] = sessions + +# need to define delay for latest sessions +sorted_delays = dict(sorted(pado_delay.items(), key=lambda k_v: int(k_v[1]))) +last_delay = list(sorted_delays)[-1] + +# Rename last delay -> disable +tmp = base + [last_delay] +if config.exists(tmp): + config.rename(tmp, 'disable') + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) From 456419c7930405b80d322586736734f707affaed Mon Sep 17 00:00:00 2001 From: sarthurdev <965089+sarthurdev@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:32:52 +0200 Subject: [PATCH 045/188] firewall: T6257: Show member information for dynamic groups in op-mode --- src/op_mode/firewall.py | 78 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 70 insertions(+), 8 deletions(-) diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py index 25554b7819f..442c186cc5d 100755 --- a/src/op_mode/firewall.py +++ b/src/op_mode/firewall.py @@ -16,6 +16,7 @@ import argparse import ipaddress +import json import re import tabulate import textwrap @@ -89,10 +90,38 @@ def get_nftables_details(family, hook, priority): out[rule_id] = rule return out -def output_firewall_vertical(rules, headers): +def get_nftables_group_members(family, table, name): + prefix = 'ip6' if family == 'ipv6' else 'ip' + out = [] + + try: + results_str = cmd(f'sudo nft -j list set {prefix} {table} {name}') + results = json.loads(results_str) + except: + return out + + if 'nftables' not in results: + return out + + for obj in results['nftables']: + if 'set' not in obj: + continue + + set_obj = obj['set'] + + if 'elem' in set_obj: + for elem in set_obj['elem']: + if isinstance(elem, str): + out.append(elem) + elif isinstance(elem, dict) and 'elem' in elem: + out.append(elem['elem']) + + return out + +def output_firewall_vertical(rules, headers, adjust=True): for rule in rules: - adjusted_rule = rule + [""] * (len(headers) - len(rule)) # account for different header length, like default-action - transformed_rule = [[header, textwrap.fill(adjusted_rule[i].replace('\n', ' '), 65)] for i, header in enumerate(headers)] # create key-pair list from headers and rules lists; wrap at 100 char + adjusted_rule = rule + [""] * (len(headers) - len(rule)) if adjust else rule # account for different header length, like default-action + transformed_rule = [[header, textwrap.fill(adjusted_rule[i].replace('\n', ' '), 65)] for i, header in enumerate(headers) if i < len(adjusted_rule)] # create key-pair list from headers and rules lists; wrap at 100 char print(tabulate.tabulate(transformed_rule, tablefmt="presto")) print() @@ -453,6 +482,7 @@ def find_references(group_type, group_name): return out rows = [] + header_tail = [] for group_type, group_type_conf in firewall['group'].items(): ## @@ -479,21 +509,53 @@ def find_references(group_type, group_name): rows.append(row) else: + if not args.detail: + header_tail = ['Timeout', 'Expires'] + for dynamic_type in ['address_group', 'ipv6_address_group']: + family = 'ipv4' if dynamic_type == 'address_group' else 'ipv6' + prefix = 'DA_' if dynamic_type == 'address_group' else 'DA6_' if dynamic_type in firewall['group']['dynamic_group']: for dynamic_name, dynamic_conf in firewall['group']['dynamic_group'][dynamic_type].items(): references = find_references(dynamic_type, dynamic_name) row = [dynamic_name, textwrap.fill(dynamic_conf.get('description') or '', 50), dynamic_type + '(dynamic)', '\n'.join(references) or 'N/D'] - row.append('N/D') - rows.append(row) + + members = get_nftables_group_members(family, 'vyos_filter', f'{prefix}{dynamic_name}') + + if not members: + if args.detail: + row.append('N/D') + else: + row += ["N/D"] * 3 + rows.append(row) + continue + + for idx, member in enumerate(members): + val = member.get('val', 'N/D') + timeout = str(member.get('timeout', 'N/D')) + expires = str(member.get('expires', 'N/D')) + + if args.detail: + row.append(f'{val} (timeout: {timeout}, expires: {expires})') + continue + + if idx > 0: + row = [""] * 4 + + row += [val, timeout, expires] + rows.append(row) + + if args.detail: + header_tail += [""] * (len(members) - 1) + rows.append(row) if rows: print('Firewall Groups\n') if args.detail: - header = ['Name', 'Description','Type', 'References', 'Members'] - output_firewall_vertical(rows, header) + header = ['Name', 'Description', 'Type', 'References', 'Members'] + header_tail + output_firewall_vertical(rows, header, adjust=False) else: - header = ['Name', 'Type', 'References', 'Members'] + header = ['Name', 'Type', 'References', 'Members'] + header_tail for i in rows: rows[rows.index(i)].pop(1) print(tabulate.tabulate(rows, header)) From d518386d74ab09c7e75fdbf7f67e14839180f24b Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Fri, 26 Apr 2024 14:10:19 +0000 Subject: [PATCH 046/188] T6269: policy: ensure correct rule parsing when using, and when not using option in policy route. --- python/vyos/firewall.py | 49 +++++++++++++++++++++-------------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index d9d605a9d2e..d7b7b80a8bb 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -32,7 +32,6 @@ from vyos.utils.process import run # Conntrack - def conntrack_required(conf): required_nodes = ['nat', 'nat66', 'load-balancing wan'] @@ -454,8 +453,28 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): else: output.append(f'set update ip{def_suffix} saddr @DA{def_suffix}_{dyn_group}') + set_table = False if 'set' in rule_conf: - output.append(parse_policy_set(rule_conf['set'], def_suffix)) + # Parse set command used in policy route: + if 'connection_mark' in rule_conf['set']: + conn_mark = rule_conf['set']['connection_mark'] + output.append(f'ct mark set {conn_mark}') + if 'dscp' in rule_conf['set']: + dscp = rule_conf['set']['dscp'] + output.append(f'ip{def_suffix} dscp set {dscp}') + if 'mark' in rule_conf['set']: + mark = rule_conf['set']['mark'] + output.append(f'meta mark set {mark}') + if 'table' in rule_conf['set']: + set_table = True + table = rule_conf['set']['table'] + if table == 'main': + table = '254' + mark = 0x7FFFFFFF - int(table) + output.append(f'meta mark set {mark}') + if 'tcp_mss' in rule_conf['set']: + mss = rule_conf['set']['tcp_mss'] + output.append(f'tcp option maxseg size set {mss}') if 'action' in rule_conf: # Change action=return to action=action @@ -488,6 +507,10 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): if synproxy_ws: output.append(f'wscale {synproxy_ws} timestamp sack-perm') + else: + if set_table: + output.append('return') + output.append(f'comment "{family}-{hook}-{fw_name}-{rule_id}"') return " ".join(output) @@ -518,28 +541,6 @@ def parse_time(time): out.append(f'day {{{",".join(out_days)}}}') return " ".join(out) -def parse_policy_set(set_conf, def_suffix): - out = [] - if 'connection_mark' in set_conf: - conn_mark = set_conf['connection_mark'] - out.append(f'ct mark set {conn_mark}') - if 'dscp' in set_conf: - dscp = set_conf['dscp'] - out.append(f'ip{def_suffix} dscp set {dscp}') - if 'mark' in set_conf: - mark = set_conf['mark'] - out.append(f'meta mark set {mark}') - if 'table' in set_conf: - table = set_conf['table'] - if table == 'main': - table = '254' - mark = 0x7FFFFFFF - int(table) - out.append(f'meta mark set {mark}') - if 'tcp_mss' in set_conf: - mss = set_conf['tcp_mss'] - out.append(f'tcp option maxseg size set {mss}') - return " ".join(out) - # GeoIP nftables_geoip_conf = '/run/nftables-geoip.conf' From 9438f1f8394b7c90bb536292882571c88556ce87 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Sun, 28 Apr 2024 09:15:28 -0500 Subject: [PATCH 047/188] configdep: T6276: do not call dependencies on script error --- src/services/vyos-configd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/services/vyos-configd b/src/services/vyos-configd index 648a017d594..c89c486e59e 100755 --- a/src/services/vyos-configd +++ b/src/services/vyos-configd @@ -236,7 +236,7 @@ def process_node_data(config, data, last: bool = False) -> int: with stdout_redirected(session_out, session_mode): result = run_script(conf_mode_scripts[script_name], config, args) - if last: + if last and result == R_SUCCESS: call_dependents(dependent_func=config.dependent_func) return result From af7277c7d525c22749bc236ad2096bec5c08d998 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Mon, 29 Apr 2024 17:55:54 +0300 Subject: [PATCH 048/188] T6272: Changed interface existence verification in pppoe/ipoe to Warning Throwing Warning message instead of Error if interface which is used in pppoe/ipoe does not exist. --- src/conf_mode/service_ipoe-server.py | 2 +- src/conf_mode/service_pppoe-server.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/service_ipoe-server.py b/src/conf_mode/service_ipoe-server.py index 11e950782b1..28b7fb03cd6 100755 --- a/src/conf_mode/service_ipoe-server.py +++ b/src/conf_mode/service_ipoe-server.py @@ -66,7 +66,7 @@ def verify(ipoe): raise ConfigError('No IPoE interface configured') for interface, iface_config in ipoe['interface'].items(): - verify_interface_exists(interface) + verify_interface_exists(interface, warning_only=True) if 'client_subnet' in iface_config and 'vlan' in iface_config: raise ConfigError('Option "client-subnet" and "vlan" are mutually exclusive, ' 'use "client-ip-pool" instead!') diff --git a/src/conf_mode/service_pppoe-server.py b/src/conf_mode/service_pppoe-server.py index b9d17493389..3284879854d 100755 --- a/src/conf_mode/service_pppoe-server.py +++ b/src/conf_mode/service_pppoe-server.py @@ -105,7 +105,7 @@ def verify(pppoe): # Check is interface exists in the system for interface in pppoe['interface']: - verify_interface_exists(interface) + verify_interface_exists(interface, warning_only=True) return None From 9ff74d4370f0a5f66c303074796dab8b1ca5c4a5 Mon Sep 17 00:00:00 2001 From: Alex W Date: Mon, 29 Apr 2024 20:53:51 +0100 Subject: [PATCH 049/188] openconnect: T4982: Support defining minimum TLS version in openconnect VPN --- data/templates/ocserv/ocserv_config.j2 | 8 +++ .../include/tls-version-min.xml.i | 29 +++++++++++ .../include/version/openconnect-version.xml.i | 2 +- .../interfaces_openvpn.xml.in | 28 +---------- interface-definitions/vpn_openconnect.xml.in | 4 ++ smoketest/scripts/cli/test_vpn_openconnect.py | 11 ++++ src/migration-scripts/openconnect/2-to-3 | 50 +++++++++++++++++++ 7 files changed, 104 insertions(+), 28 deletions(-) create mode 100644 interface-definitions/include/tls-version-min.xml.i create mode 100755 src/migration-scripts/openconnect/2-to-3 diff --git a/data/templates/ocserv/ocserv_config.j2 b/data/templates/ocserv/ocserv_config.j2 index b5e890c32d0..81f77703146 100644 --- a/data/templates/ocserv/ocserv_config.j2 +++ b/data/templates/ocserv/ocserv_config.j2 @@ -61,7 +61,15 @@ keepalive = 300 dpd = 60 mobile-dpd = 300 switch-to-tcp-timeout = 30 +{% if tls_version_min == '1.0' %} tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA:-VERS-SSL3.0:-ARCFOUR-128" +{% elif tls_version_min == '1.1' %} +tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA:-VERS-SSL3.0:-ARCFOUR-128:-VERS-TLS1.0" +{% elif tls_version_min == '1.2' %} +tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA:-VERS-SSL3.0:-ARCFOUR-128:-VERS-TLS1.0:-VERS-TLS1.1" +{% elif tls_version_min == '1.3' %} +tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA:-VERS-SSL3.0:-ARCFOUR-128:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-TLS1.2" +{% endif %} auth-timeout = 240 idle-timeout = 1200 mobile-idle-timeout = 1800 diff --git a/interface-definitions/include/tls-version-min.xml.i b/interface-definitions/include/tls-version-min.xml.i new file mode 100644 index 00000000000..b3dcbad49d1 --- /dev/null +++ b/interface-definitions/include/tls-version-min.xml.i @@ -0,0 +1,29 @@ + + + + Specify the minimum required TLS version + + 1.0 1.1 1.2 1.3 + + + 1.0 + TLS v1.0 + + + 1.1 + TLS v1.1 + + + 1.2 + TLS v1.2 + + + 1.3 + TLS v1.3 + + + (1.0|1.1|1.2|1.3) + + + + diff --git a/interface-definitions/include/version/openconnect-version.xml.i b/interface-definitions/include/version/openconnect-version.xml.i index 6548062782a..15097eebe32 100644 --- a/interface-definitions/include/version/openconnect-version.xml.i +++ b/interface-definitions/include/version/openconnect-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/interfaces_openvpn.xml.in b/interface-definitions/interfaces_openvpn.xml.in index 389b5b5c99a..7b46f32b3a2 100644 --- a/interface-definitions/interfaces_openvpn.xml.in +++ b/interface-definitions/interfaces_openvpn.xml.in @@ -739,33 +739,7 @@ Peer certificate fingerprint must be a colon-separated SHA256 hex digest - - - Specify the minimum required TLS version - - 1.0 1.1 1.2 1.3 - - - 1.0 - TLS v1.0 - - - 1.1 - TLS v1.1 - - - 1.2 - TLS v1.2 - - - 1.3 - TLS v1.3 - - - (1.0|1.1|1.2|1.3) - - - + #include TLS negotiation role diff --git a/interface-definitions/vpn_openconnect.xml.in b/interface-definitions/vpn_openconnect.xml.in index 736084f8b82..7849d68864a 100644 --- a/interface-definitions/vpn_openconnect.xml.in +++ b/interface-definitions/vpn_openconnect.xml.in @@ -266,6 +266,10 @@ + #include + + 1.2 + SSL Certificate, SSL Key and CA diff --git a/smoketest/scripts/cli/test_vpn_openconnect.py b/smoketest/scripts/cli/test_vpn_openconnect.py index 96e858fdbc2..a2e426dc78d 100755 --- a/smoketest/scripts/cli/test_vpn_openconnect.py +++ b/smoketest/scripts/cli/test_vpn_openconnect.py @@ -210,6 +210,9 @@ def test_ocserv(self): # Verify configuration daemon_config = read_file(config_file) + # Verify TLS string (with default setting) + self.assertIn('tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA:-VERS-SSL3.0:-ARCFOUR-128:-VERS-TLS1.0:-VERS-TLS1.1"', daemon_config) + # authentication mode local password-otp self.assertIn(f'auth = "plain[passwd=/run/ocserv/ocpasswd,otp=/run/ocserv/users.oath]"', daemon_config) self.assertIn(f'listen-host = {listen_ip_no_cidr}', daemon_config) @@ -253,5 +256,13 @@ def test_ocserv(self): self.assertIn('included-http-headers = Pragma: no-cache', daemon_config) self.assertIn('included-http-headers = Cache-control: no-store, no-cache', daemon_config) + # Set TLS version to the highest security (v1.3 min) + self.cli_set(base_path + ['tls-version-min', '1.3']) + self.cli_commit() + + # Verify TLS string + daemon_config = read_file(config_file) + self.assertIn('tls-priorities = "NORMAL:%SERVER_PRECEDENCE:%COMPAT:-RSA:-VERS-SSL3.0:-ARCFOUR-128:-VERS-TLS1.0:-VERS-TLS1.1:-VERS-TLS1.2"', daemon_config) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/migration-scripts/openconnect/2-to-3 b/src/migration-scripts/openconnect/2-to-3 new file mode 100755 index 00000000000..e78fc8a9104 --- /dev/null +++ b/src/migration-scripts/openconnect/2-to-3 @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T4982: Retain prior default TLS version (v1.0) when upgrading installations with existing openconnect configurations + +import sys + +from vyos.configtree import ConfigTree + +if len(sys.argv) < 2: + print("Must specify file name!") + sys.exit(1) + +file_name = sys.argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + + +config = ConfigTree(config_file) +cfg_base = ['vpn', 'openconnect'] + +# bail out early if service is unconfigured +if not config.exists(cfg_base): + sys.exit(0) + +# new default is TLS 1.2 - set explicit old default value of TLS 1.0 for upgraded configurations to keep compatibility +tls_min_path = cfg_base + ['tls-version-min'] +if not config.exists(tls_min_path): + config.set(tls_min_path, value='1.0') + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + sys.exit(1) From 09c302d7e57a0fdb6c51ae8f61d5ad6371a30b67 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 26 Apr 2024 14:50:22 +0000 Subject: [PATCH 050/188] T6267: Check interface wireless module before apply config Check if the wireless device/modem exists in the system and the module `ieee802111` was loaded In cases where we do not have wireless devices, it prevents the unexpected traceback ``` set interfaces wireless wlan0 address 192.0.2.5/32 commit Traceback (most recent call last): File "/usr/libexec/vyos/conf_mode/interfaces_wireless.py", line 269, in c = get_config() ^^^^^^^^^^^^ File "/usr/libexec/vyos/conf_mode/interfaces_wireless.py", line 104, in get_cg tmp = find_other_stations(conf, base, wifi['ifname']) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/libexec/vyos/conf_mode/interfaces_wireless.py", line 54, in find_os for phy in os.listdir('/sys/class/ieee80211'): ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ FileNotFoundError: [Errno 2] No such file or directory: '/sys/class/ieee80211' ``` --- src/conf_mode/interfaces_wireless.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/interfaces_wireless.py b/src/conf_mode/interfaces_wireless.py index 02b4a2500b7..c0a17c0bc3a 100755 --- a/src/conf_mode/interfaces_wireless.py +++ b/src/conf_mode/interfaces_wireless.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2019-2020 VyOS maintainers and contributors +# Copyright (C) 2019-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -31,8 +31,9 @@ from vyos.configverify import verify_bond_bridge_member from vyos.ifconfig import WiFiIf from vyos.template import render -from vyos.utils.process import call from vyos.utils.dict import dict_search +from vyos.utils.kernel import check_kmod +from vyos.utils.process import call from vyos import ConfigError from vyos import airbag airbag.enable() @@ -118,6 +119,10 @@ def verify(wifi): if 'physical_device' not in wifi: raise ConfigError('You must specify a physical-device "phy"') + physical_device = wifi['physical_device'] + if not os.path.exists(f'/sys/class/ieee80211/{physical_device}'): + raise ConfigError(f'Wirelss interface PHY "{physical_device}" does not exist!') + if 'type' not in wifi: raise ConfigError('You must specify a WiFi mode') @@ -266,6 +271,7 @@ def apply(wifi): if __name__ == '__main__': try: + check_kmod('mac80211') c = get_config() verify(c) generate(c) From de38b01710958b7f7dababcff9557e4be98c8450 Mon Sep 17 00:00:00 2001 From: aapostoliuk Date: Mon, 29 Apr 2024 12:29:37 +0300 Subject: [PATCH 051/188] T6273: Allowed the use of "-" and "_" in PPPoE access-concentrator name Allowed the use of "-" and "_" in PPPoE access-concentrator name --- interface-definitions/include/pppoe-access-concentrator.xml.i | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface-definitions/include/pppoe-access-concentrator.xml.i b/interface-definitions/include/pppoe-access-concentrator.xml.i index ccfcc1c49ad..8a75dae0894 100644 --- a/interface-definitions/include/pppoe-access-concentrator.xml.i +++ b/interface-definitions/include/pppoe-access-concentrator.xml.i @@ -3,9 +3,9 @@ Access concentrator name - [a-zA-Z0-9]{1,100} + #include - Access-concentrator name must be alphanumerical only (max. 100 characters) + Access-concentrator name can only contain alpha-numeric letters, hyphen and underscores(max. 100 characters) From 3c37b6a44dca552da950b5288a30c7e074d58704 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 30 Apr 2024 14:50:55 +0000 Subject: [PATCH 052/188] T6169: DNS forwarding should allow underscore for srv record This srv recors looks valid: ``` set service dns forwarding authoritative-domain _tcp.db.mongors1.example.com records srv _mongodb entry 0 hostname 'mongors1.example.com' ``` But FQDN validator cannot validate it correctly, use regex to fix --- interface-definitions/service_dns_forwarding.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/service_dns_forwarding.xml.in b/interface-definitions/service_dns_forwarding.xml.in index a54618e8268..b52b4bda3d1 100644 --- a/interface-definitions/service_dns_forwarding.xml.in +++ b/interface-definitions/service_dns_forwarding.xml.in @@ -115,7 +115,7 @@ An absolute DNS domain name - + ((?!-)[-_a-zA-Z0-9.]{1,63}|@|any)(?<!\.) From 0be0cdb932ca2d7399c026f1f601b56e179cc9c3 Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Tue, 30 Apr 2024 21:49:58 +0200 Subject: [PATCH 053/188] haproxy: T6179: fix rule generation --- data/templates/load-balancing/haproxy.cfg.j2 | 10 +++++++--- .../scripts/cli/test_load-balancing_reverse-proxy.py | 5 +++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index e8622ba7b79..7917c82570e 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -85,7 +85,7 @@ frontend {{ front }} {% if front_config.rule is vyos_defined %} {% for rule, rule_config in front_config.rule.items() %} # rule {{ rule }} -{% if rule_config.domain_name is vyos_defined and rule_config.set.backend is vyos_defined %} +{% if rule_config.domain_name is vyos_defined %} {% set rule_options = 'hdr(host)' %} {% if rule_config.ssl is vyos_defined %} {% set ssl_rule_translate = {'req-ssl-sni': 'req_ssl_sni', 'ssl-fc-sni': 'ssl_fc_sni', 'ssl-fc-sni-end': 'ssl_fc_sni_end'} %} @@ -94,16 +94,20 @@ frontend {{ front }} {% for domain in rule_config.domain_name %} acl {{ rule }} {{ rule_options }} -i {{ domain }} {% endfor %} - use_backend {{ rule_config.set.backend }} if {{ rule }} {% endif %} {# path url #} -{% if rule_config.url_path is vyos_defined and rule_config.set.redirect_location is vyos_defined %} +{% if rule_config.url_path is vyos_defined %} {% set path_mod_translate = {'begin': '-i -m beg', 'end': '-i -m end', 'exact': ''} %} {% for path, path_config in rule_config.url_path.items() %} {% for url in path_config %} acl {{ rule }} path {{ path_mod_translate[path] }} {{ url }} {% endfor %} {% endfor %} +{% endif %} +{% if rule_config.set.backend is vyos_defined %} + use_backend {{ rule_config.set.backend }} if {{ rule }} +{% endif %} +{% if rule_config.set.redirect_location is vyos_defined %} http-request redirect location {{ rule_config.set.redirect_location }} code 301 if {{ rule }} {% endif %} {# endpath #} diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index f9f16378286..c8b17316f95 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -180,6 +180,7 @@ def test_01_lb_reverse_proxy_domain(self): mode = 'http' rule_ten = '10' rule_twenty = '20' + rule_thirty = '30' send_proxy = 'send-proxy' max_connections = '1000' @@ -192,6 +193,8 @@ def test_01_lb_reverse_proxy_domain(self): self.cli_set(base_path + ['service', frontend, 'rule', rule_ten, 'set', 'backend', bk_first_name]) self.cli_set(base_path + ['service', frontend, 'rule', rule_twenty, 'domain-name', domain_bk_second]) self.cli_set(base_path + ['service', frontend, 'rule', rule_twenty, 'set', 'backend', bk_second_name]) + self.cli_set(base_path + ['service', frontend, 'rule', rule_thirty, 'url-path', 'end', '/test']) + self.cli_set(base_path + ['service', frontend, 'rule', rule_thirty, 'set', 'backend', bk_second_name]) self.cli_set(back_base + [bk_first_name, 'mode', mode]) self.cli_set(back_base + [bk_first_name, 'server', bk_first_name, 'address', bk_server_first]) @@ -222,6 +225,8 @@ def test_01_lb_reverse_proxy_domain(self): self.assertIn(f'use_backend {bk_first_name} if {rule_ten}', config) self.assertIn(f'acl {rule_twenty} hdr(host) -i {domain_bk_second}', config) self.assertIn(f'use_backend {bk_second_name} if {rule_twenty}', config) + self.assertIn(f'acl {rule_thirty} path -i -m end /test', config) + self.assertIn(f'use_backend {bk_second_name} if {rule_thirty}', config) # Backend self.assertIn(f'backend {bk_first_name}', config) From a7c3f202ffea7859463f204cccf526f7517321f6 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 1 May 2024 06:48:02 +0000 Subject: [PATCH 054/188] T6287: Config-sync add the ability to configure API port Add the ability to configure the API port if the API on the secondary server works on a non-default port. The primary node will connect to configured port for config-sync ``` set service config-sync secondary address '192.0.2.11' set service config-sync secondary port '8443' ``` --- .../service_config-sync.xml.in | 4 ++++ src/helpers/vyos_config_sync.py | 23 ++++++++++--------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/interface-definitions/service_config-sync.xml.in b/interface-definitions/service_config-sync.xml.in index e9ea9aa4b9a..648c14aeebd 100644 --- a/interface-definitions/service_config-sync.xml.in +++ b/interface-definitions/service_config-sync.xml.in @@ -34,6 +34,10 @@ + #include + + 443 + Connection API timeout diff --git a/src/helpers/vyos_config_sync.py b/src/helpers/vyos_config_sync.py index 0604b28370f..9d9aec37649 100755 --- a/src/helpers/vyos_config_sync.py +++ b/src/helpers/vyos_config_sync.py @@ -93,7 +93,8 @@ def set_remote_config( key: str, op: str, mask: Dict[str, Any], - config: Dict[str, Any]) -> Optional[Dict[str, Any]]: + config: Dict[str, Any], + port: int) -> Optional[Dict[str, Any]]: """Loads the VyOS configuration in JSON format to a remote host. Args: @@ -102,6 +103,7 @@ def set_remote_config( op (str): The operation to perform (set or load). mask (dict): The dict of paths in sections. config (dict): The dict of masked config data. + port (int): The remote API port Returns: Optional[Dict[str, Any]]: The response from the remote host as a @@ -113,7 +115,7 @@ def set_remote_config( # Disable the InsecureRequestWarning urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - url = f'https://{address}/configure-section' + url = f'https://{address}:{port}/configure-section' data = json.dumps({ 'op': op, 'mask': mask, @@ -138,7 +140,8 @@ def is_section_revised(section: List[str]) -> bool: def config_sync(secondary_address: str, secondary_key: str, sections: List[list[str]], - mode: str): + mode: str, + secondary_port: int): """Retrieve a config section from primary router in JSON format and send it to secondary router """ @@ -158,7 +161,8 @@ def config_sync(secondary_address: str, key=secondary_key, op=mode, mask=mask_dict, - config=config_dict) + config=config_dict, + port=secondary_port) logger.debug(f"Set config for sections '{sections}': {set_config}") @@ -178,14 +182,12 @@ def config_sync(secondary_address: str, secondary_address = config.get('secondary', {}).get('address') secondary_address = bracketize_ipv6(secondary_address) secondary_key = config.get('secondary', {}).get('key') + secondary_port = int(config.get('secondary', {}).get('port', 443)) sections = config.get('section') timeout = int(config.get('secondary', {}).get('timeout')) - if not all([ - mode, secondary_address, secondary_key, sections - ]): - logger.error( - "Missing required configuration data for config synchronization.") + if not all([mode, secondary_address, secondary_key, sections]): + logger.error("Missing required configuration data for config synchronization.") exit(0) # Generate list_sections of sections/subsections @@ -200,5 +202,4 @@ def config_sync(secondary_address: str, else: list_sections.append([section]) - config_sync(secondary_address, secondary_key, - list_sections, mode) + config_sync(secondary_address, secondary_key, list_sections, mode, secondary_port) From 4f1db505791deed533dddf0c2f5bdedd6fba34b8 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 1 May 2024 13:36:14 +0000 Subject: [PATCH 055/188] T6056: Change static-host-mapping shold not restart snmpd We have several config XML definitions that use the same python3 script `system_host-name.py` https://github.com/vyos/vyos-1x/blob/current/interface-definitions/system_name-server.xml.in https://github.com/vyos/vyos-1x/blob/current/interface-definitions/system_host-name.xml.in https://github.com/vyos/vyos-1x/blob/current/interface-definitions/system_static-host-mapping.xml.in https://github.com/vyos/vyos-1x/blob/current/interface-definitions/system_domain-name.xml.in https://github.com/vyos/vyos-1x/blob/current/interface-definitions/system_domain-search.xml.in Any change in these scripts calls to restart the `service snmpd` The service `snmpd` should be restarted only if `host-name` or `domain-name` was changed. It is a good idea to rewrite it to `get_config_dict` in the future. --- src/conf_mode/system_host-name.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/system_host-name.py b/src/conf_mode/system_host-name.py index 8975cadb6b8..3f245f166ff 100755 --- a/src/conf_mode/system_host-name.py +++ b/src/conf_mode/system_host-name.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2018-2023 VyOS maintainers and contributors +# Copyright (C) 2018-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -22,6 +22,7 @@ from vyos.base import Warning from vyos.config import Config +from vyos.configdict import leaf_node_changed from vyos.ifconfig import Section from vyos.template import is_ip from vyos.utils.process import cmd @@ -37,6 +38,7 @@ 'domain_search': [], 'nameserver': [], 'nameservers_dhcp_interfaces': {}, + 'snmpd_restart_reqired': False, 'static_host_mapping': {} } @@ -52,6 +54,10 @@ def get_config(config=None): hosts['hostname'] = conf.return_value(['system', 'host-name']) + base = ['system'] + if leaf_node_changed(conf, base + ['host-name']) or leaf_node_changed(conf, base + ['domain-name']): + hosts['snmpd_restart_reqired'] = True + # This may happen if the config is not loaded yet, # e.g. if run by cloud-init if not hosts['hostname']: @@ -171,7 +177,7 @@ def apply(config): call("systemctl restart rsyslog.service") # If SNMP is running, restart it too - if process_named_running('snmpd'): + if process_named_running('snmpd') and config['snmpd_restart_reqired']: call('systemctl restart snmpd.service') return None From e7bb65894f86372dc0f6e8fd39b1628e0a224c68 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 1 May 2024 20:55:57 +0200 Subject: [PATCH 056/188] vrf: T6189: render FRR L3VNI configuration when creating VRF instance When adding and removing VRF instances on the fly it was noticed that the vni statement under the VRF instance in FRR vanishes. This was caused by a race condition which was previously designed to fix another bug. The wierd design of a Python helper below the VRF tree to only generate the VNI configuration nodes is now gone and all is rendered in the proper place. --- data/configd-include.json | 3 +- data/templates/frr/zebra.vrf.route-map.frr.j2 | 6 +- interface-definitions/vrf.xml.in | 15 +-- smoketest/scripts/cli/test_vrf.py | 55 ++++++++-- src/conf_mode/vrf.py | 5 - src/conf_mode/vrf_vni.py | 103 ------------------ 6 files changed, 46 insertions(+), 141 deletions(-) delete mode 100644 src/conf_mode/vrf_vni.py diff --git a/data/configd-include.json b/data/configd-include.json index dc00f069822..0c767f987a5 100644 --- a/data/configd-include.json +++ b/data/configd-include.json @@ -108,6 +108,5 @@ "vpn_openconnect.py", "vpn_pptp.py", "vpn_sstp.py", -"vrf.py", -"vrf_vni.py" +"vrf.py" ] diff --git a/data/templates/frr/zebra.vrf.route-map.frr.j2 b/data/templates/frr/zebra.vrf.route-map.frr.j2 index f1cc6fe6639..8ebb8251196 100644 --- a/data/templates/frr/zebra.vrf.route-map.frr.j2 +++ b/data/templates/frr/zebra.vrf.route-map.frr.j2 @@ -1,10 +1,6 @@ ! {% if name is vyos_defined %} {% for vrf, vrf_config in name.items() %} -{# code path required for vrf_vni.py as we will only render the required VR configuration and not all of them #} -{% if only_vrf is vyos_defined and vrf is not vyos_defined(only_vrf) %} -{% continue %} -{% endif %} vrf {{ vrf }} {% if vrf_config.ip.nht.no_resolve_via_default is vyos_defined %} no ip nht resolve-via-default @@ -25,7 +21,7 @@ vrf {{ vrf }} ipv6 protocol {{ protocol_name }} route-map {{ protocol_config.route_map }} {% endfor %} {% endif %} -{% if vrf_config.vni is vyos_defined and no_vni is not vyos_defined %} +{% if vrf_config.vni is vyos_defined %} vni {{ vrf_config.vni }} {% endif %} exit-vrf diff --git a/interface-definitions/vrf.xml.in b/interface-definitions/vrf.xml.in index 94ed96e4bc3..a20be995a99 100644 --- a/interface-definitions/vrf.xml.in +++ b/interface-definitions/vrf.xml.in @@ -120,20 +120,7 @@ VRF routing table must be in range from 100 to 65535 - - - Virtual Network Identifier - - 822 - - u32:0-16777214 - VXLAN virtual network identifier - - - - - - + #include diff --git a/smoketest/scripts/cli/test_vrf.py b/smoketest/scripts/cli/test_vrf.py index f6e4181c08a..243397dc2eb 100755 --- a/smoketest/scripts/cli/test_vrf.py +++ b/smoketest/scripts/cli/test_vrf.py @@ -18,7 +18,6 @@ import os import unittest -from netifaces import interfaces from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError @@ -27,6 +26,7 @@ from vyos.utils.file import read_file from vyos.utils.network import get_interface_config from vyos.utils.network import is_intf_addr_assigned +from vyos.utils.network import interface_exists from vyos.utils.system import sysctl_read base_path = ['vrf'] @@ -60,7 +60,7 @@ def tearDown(self): self.cli_delete(base_path) self.cli_commit() for vrf in vrfs: - self.assertNotIn(vrf, interfaces()) + self.assertFalse(interface_exists(vrf)) def test_vrf_vni_and_table_id(self): base_table = '1000' @@ -89,7 +89,7 @@ def test_vrf_vni_and_table_id(self): iproute2_config = read_file('/etc/iproute2/rt_tables.d/vyos-vrf.conf') for vrf in vrfs: description = f'VyOS-VRF-{vrf}' - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) vrf_if = Interface(vrf) # validate proper interface description self.assertEqual(vrf_if.get_alias(), description) @@ -131,7 +131,7 @@ def test_vrf_loopbacks_ips(self): loopbacks = ['127.0.0.1', '::1'] for vrf in vrfs: # Ensure VRF was created - self.assertIn(vrf, interfaces()) + self.assertTrue(interface_exists(vrf)) # Verify IP forwarding is 1 (enabled) self.assertEqual(sysctl_read(f'net.ipv4.conf.{vrf}.forwarding'), '1') self.assertEqual(sysctl_read(f'net.ipv6.conf.{vrf}.forwarding'), '1') @@ -171,7 +171,7 @@ def test_vrf_table_id_is_unalterable(self): self.cli_commit() # Check if VRF has been created - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) table = str(int(table) + 1) self.cli_set(base + ['table', table]) @@ -228,7 +228,7 @@ def test_vrf_static_route(self): next_hop = f'192.0.{table}.1' prefix = f'10.0.{table}.0/24' - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) frrconfig = self.getFRRconfig(f'vrf {vrf}') self.assertIn(f' vni {table}', frrconfig) @@ -261,7 +261,7 @@ def test_vrf_link_local_ip_addresses(self): # Apply VRF config self.cli_commit() # Ensure VRF got created - self.assertIn(vrf, interfaces()) + self.assertTrue(interface_exists(vrf)) # ... and IP addresses are still assigned for address in addresses: self.assertTrue(is_intf_addr_assigned(interface, address)) @@ -293,7 +293,7 @@ def test_vrf_disable_forwarding(self): loopbacks = ['127.0.0.1', '::1'] for vrf in vrfs: # Ensure VRF was created - self.assertIn(vrf, interfaces()) + self.assertTrue(interface_exists(vrf)) # Verify IP forwarding is 0 (disabled) self.assertEqual(sysctl_read(f'net.ipv4.conf.{vrf}.forwarding'), '0') self.assertEqual(sysctl_read(f'net.ipv6.conf.{vrf}.forwarding'), '0') @@ -425,7 +425,7 @@ def test_vrf_vni_duplicates(self): # Verify VRF configuration table = base_table for vrf in vrfs: - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) frrconfig = self.getFRRconfig(f'vrf {vrf}') self.assertIn(f' vni {table}', frrconfig) @@ -447,7 +447,7 @@ def test_vrf_vni_add_change_remove(self): # Verify VRF configuration table = base_table for vrf in vrfs: - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) frrconfig = self.getFRRconfig(f'vrf {vrf}') self.assertIn(f' vni {table}', frrconfig) @@ -470,13 +470,39 @@ def test_vrf_vni_add_change_remove(self): # Verify VRF configuration table = base_table for vrf in vrfs: - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) frrconfig = self.getFRRconfig(f'vrf {vrf}') self.assertIn(f' vni {table}', frrconfig) # Increment table ID for the next run table = str(int(table) + 2) + + # add a new VRF with VNI - this must not delete any existing VRF/VNI + purple = 'purple' + table = str(int(table) + 10) + self.cli_set(base_path + ['name', purple, 'table', table]) + self.cli_set(base_path + ['name', purple, 'vni', table]) + + # commit changes + self.cli_commit() + + # Verify VRF configuration + table = base_table + for vrf in vrfs: + self.assertTrue(interface_exists(vrf)) + + frrconfig = self.getFRRconfig(f'vrf {vrf}') + self.assertIn(f' vni {table}', frrconfig) + # Increment table ID for the next run + table = str(int(table) + 2) + + # Verify purple VRF/VNI + self.assertTrue(interface_exists(purple)) + table = str(int(table) + 10) + frrconfig = self.getFRRconfig(f'vrf {purple}') + self.assertIn(f' vni {table}', frrconfig) + # Now delete all the VNIs for vrf in vrfs: base = base_path + ['name', vrf] @@ -487,11 +513,16 @@ def test_vrf_vni_add_change_remove(self): # Verify no VNI is defined for vrf in vrfs: - self.assertTrue(vrf in interfaces()) + self.assertTrue(interface_exists(vrf)) frrconfig = self.getFRRconfig(f'vrf {vrf}') self.assertNotIn('vni', frrconfig) + # Verify purple VNI remains + self.assertTrue(interface_exists(purple)) + frrconfig = self.getFRRconfig(f'vrf {purple}') + self.assertIn(f' vni {table}', frrconfig) + def test_vrf_ip_ipv6_nht(self): table = '6910' diff --git a/src/conf_mode/vrf.py b/src/conf_mode/vrf.py index 587309005e2..8d8c234c0e5 100755 --- a/src/conf_mode/vrf.py +++ b/src/conf_mode/vrf.py @@ -130,11 +130,6 @@ def get_config(config=None): tmp = {'policy' : {'route-map' : conf.get_config_dict(['policy', 'route-map'], get_first_key=True)}} - # L3VNI setup is done via vrf_vni.py as it must be de-configured (on node - # deletetion prior to the BGP process. Tell the Jinja2 template no VNI - # setup is needed - vrf.update({'no_vni' : ''}) - # Merge policy dict into "regular" config dict vrf = dict_merge(tmp, vrf) return vrf diff --git a/src/conf_mode/vrf_vni.py b/src/conf_mode/vrf_vni.py deleted file mode 100644 index 8dab164d70d..00000000000 --- a/src/conf_mode/vrf_vni.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2023-2024 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from sys import argv -from sys import exit - -from vyos.config import Config -from vyos.template import render_to_string -from vyos import ConfigError -from vyos import frr -from vyos import airbag -airbag.enable() - -def get_config(config=None): - if config: - conf = config - else: - conf = Config() - - vrf_name = None - if len(argv) > 1: - vrf_name = argv[1] - else: - return None - - # Using duplicate L3VNIs makes no sense - it's also forbidden in FRR, - # thus VyOS CLI must deny this, too. Instead of getting only the dict for - # the requested VRF and den comparing it with depenent VRfs to not have any - # duplicate we will just grad ALL VRFs by default but only render/apply - # the configuration for the requested VRF - that makes the code easier and - # hopefully less error prone - vrf = conf.get_config_dict(['vrf'], key_mangling=('-', '_'), - no_tag_node_value_mangle=True, - get_first_key=True) - - # Store name of VRF we are interested in for FRR config rendering - vrf.update({'only_vrf' : vrf_name}) - - return vrf - -def verify(vrf): - if not vrf: - return - - if len(argv) < 2: - raise ConfigError('VRF parameter not specified when valling vrf_vni.py') - - if 'name' in vrf: - vni_ids = [] - for name, vrf_config in vrf['name'].items(): - # VRF VNI (Virtual Network Identifier) must be unique on the system - if 'vni' in vrf_config: - if vrf_config['vni'] in vni_ids: - raise ConfigError(f'VRF "{name}" VNI is not unique!') - vni_ids.append(vrf_config['vni']) - - return None - -def generate(vrf): - if not vrf: - return - - vrf['new_frr_config'] = render_to_string('frr/zebra.vrf.route-map.frr.j2', vrf) - return None - -def apply(vrf): - frr_daemon = 'zebra' - - # add configuration to FRR - frr_cfg = frr.FRRConfig() - frr_cfg.load_configuration(frr_daemon) - # There is only one VRF inside the dict as we read only one in get_config() - if vrf and 'only_vrf' in vrf: - vrf_name = vrf['only_vrf'] - frr_cfg.modify_section(f'^vrf {vrf_name}', stop_pattern='^exit-vrf', remove_stop_mark=True) - if vrf and 'new_frr_config' in vrf: - frr_cfg.add_before(frr.default_add_before, vrf['new_frr_config']) - frr_cfg.commit_configuration(frr_daemon) - - return None - -if __name__ == '__main__': - try: - c = get_config() - verify(c) - generate(c) - apply(c) - except ConfigError as e: - print(e) - exit(1) From 7b46172a4aecc714d929aecb8768ab82633de3ba Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 1 May 2024 20:58:29 +0200 Subject: [PATCH 057/188] bgp: T6189: explicitly call vtysh to remove VRF L3VNI configuration After e7bb65894 ("vrf: T6189: render FRR L3VNI configuration when creating VRF instance") we need to ensure that the VRF L3VNI configuration is removed in FRR prior to removing the BGP VRF instance. The reason is [1] where FRR only allows VRF BGP instance to be removed when there is NO VNI configured anymore. 1: https://github.com/FRRouting/frr/blob/064c3494527b9e84260410006768ed38e57e1de7/bgpd/bgp_vty.c#L1646-L1650 --- src/conf_mode/protocols_bgp.py | 46 +++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/conf_mode/protocols_bgp.py b/src/conf_mode/protocols_bgp.py index 2b16de77569..eb6d3a68408 100755 --- a/src/conf_mode/protocols_bgp.py +++ b/src/conf_mode/protocols_bgp.py @@ -31,6 +31,7 @@ from vyos.utils.network import get_interface_vrf from vyos.utils.network import is_addr_assigned from vyos.utils.process import process_named_running +from vyos.utils.process import call from vyos import ConfigError from vyos import frr from vyos import airbag @@ -50,13 +51,8 @@ def get_config(config=None): # eqivalent of the C foo ? 'a' : 'b' statement base = vrf and ['vrf', 'name', vrf, 'protocols', 'bgp'] or base_path - bgp = conf.get_config_dict( - base, - key_mangling=('-', '_'), - get_first_key=True, - no_tag_node_value_mangle=True, - with_recursive_defaults=True, - ) + bgp = conf.get_config_dict(base, key_mangling=('-', '_'), + get_first_key=True, no_tag_node_value_mangle=True) bgp['dependent_vrfs'] = conf.get_config_dict(['vrf', 'name'], key_mangling=('-', '_'), @@ -75,22 +71,29 @@ def get_config(config=None): if vrf: bgp.update({'vrf' : vrf}) # We can not delete the BGP VRF instance if there is a L3VNI configured + # FRR L3VNI must be deleted first otherwise we will see error: + # "FRR error: Please unconfigure l3vni 3000" tmp = ['vrf', 'name', vrf, 'vni'] - if conf.exists(tmp): - bgp.update({'vni' : conf.return_value(tmp)}) + if conf.exists_effective(tmp): + bgp.update({'vni' : conf.return_effective_value(tmp)}) # We can safely delete ourself from the dependent vrf list if vrf in bgp['dependent_vrfs']: del bgp['dependent_vrfs'][vrf] - bgp['dependent_vrfs'].update({'default': {'protocols': { - 'bgp': conf.get_config_dict(base_path, key_mangling=('-', '_'), - get_first_key=True, - no_tag_node_value_mangle=True)}}}) + bgp['dependent_vrfs'].update({'default': {'protocols': { + 'bgp': conf.get_config_dict(base_path, key_mangling=('-', '_'), + get_first_key=True, + no_tag_node_value_mangle=True)}}}) + if not conf.exists(base): # If bgp instance is deleted then mark it bgp.update({'deleted' : ''}) return bgp + # We have gathered the dict representation of the CLI, but there are default + # options which we need to update into the dictionary retrived. + bgp = conf.merge_defaults(bgp, recursive=True) + # We also need some additional information from the config, prefix-lists # and route-maps for instance. They will be used in verify(). # @@ -242,10 +245,6 @@ def verify(bgp): if verify_vrf_as_import(bgp['vrf'], tmp_afi, bgp['dependent_vrfs']): raise ConfigError(f'Cannot delete VRF instance "{bgp["vrf"]}", ' \ 'unconfigure "import vrf" commands!') - # We can not delete the BGP instance if a L3VNI instance exists - if 'vni' in bgp: - raise ConfigError(f'Cannot delete VRF instance "{bgp["vrf"]}", ' \ - f'unconfigure VNI "{bgp["vni"]}" first!') else: # We are running in the default VRF context, thus we can not delete # our main BGP instance if there are dependent BGP VRF instances. @@ -254,7 +253,11 @@ def verify(bgp): if vrf != 'default': if dict_search('protocols.bgp', vrf_options): raise ConfigError('Cannot delete default BGP instance, ' \ - 'dependent VRF instance(s) exist!') + 'dependent VRF instance(s) exist(s)!') + if 'vni' in vrf_options: + raise ConfigError('Cannot delete default BGP instance, ' \ + 'dependent L3VNI exists!') + return None if 'system_as' not in bgp: @@ -591,6 +594,13 @@ def generate(bgp): return None def apply(bgp): + if 'deleted' in bgp: + # We need to ensure that the L3VNI is deleted first. + # This is not possible with old config backend + # priority bug + if {'vrf', 'vni'} <= set(bgp): + call('vtysh -c "conf t" -c "vrf {vrf}" -c "no vni {vni}"'.format(**bgp)) + bgp_daemon = 'bgpd' # Save original configuration prior to starting any commit actions From 6bcb201a0e7ee9fea5874b963bd3e727ecec578f Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 1 May 2024 21:01:29 +0200 Subject: [PATCH 058/188] smoketest: T6199: remove redundant code when unpacking Kernel GZ config --- .../scripts/system/test_kernel_options.py | 76 +++++++++++-------- 1 file changed, 46 insertions(+), 30 deletions(-) diff --git a/smoketest/scripts/system/test_kernel_options.py b/smoketest/scripts/system/test_kernel_options.py index 0e3cbd0edc6..18922d93df6 100755 --- a/smoketest/scripts/system/test_kernel_options.py +++ b/smoketest/scripts/system/test_kernel_options.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2020-2023 VyOS maintainers and contributors +# Copyright (C) 2020-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -14,28 +14,38 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import gzip import re import os import platform import unittest -from vyos.utils.process import call -from vyos.utils.file import read_file - kernel = platform.release() -config = read_file(f'/boot/config-{kernel}') -CONFIG = '/proc/config.gz' class TestKernelModules(unittest.TestCase): """ VyOS makes use of a lot of Kernel drivers, modules and features. The required modules which are essential for VyOS should be tested that they are available in the Kernel that is run. """ + _config_data = None + + @classmethod + def setUpClass(cls): + import gzip + from vyos.utils.process import call + + super(TestKernelModules, cls).setUpClass() + CONFIG = '/proc/config.gz' + + if not os.path.isfile(CONFIG): + call('sudo modprobe configs') + + with gzip.open(CONFIG, 'rt') as f: + cls._config_data = f.read() + def test_bond_interface(self): # The bond/lacp interface must be enabled in the OS Kernel for option in ['CONFIG_BONDING']: - tmp = re.findall(f'{option}=(y|m)', config) + tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) def test_bridge_interface(self): @@ -43,7 +53,7 @@ def test_bridge_interface(self): for option in ['CONFIG_BRIDGE', 'CONFIG_BRIDGE_IGMP_SNOOPING', 'CONFIG_BRIDGE_VLAN_FILTERING']: - tmp = re.findall(f'{option}=(y|m)', config) + tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) def test_dropmon_enabled(self): @@ -53,47 +63,53 @@ def test_dropmon_enabled(self): 'CONFIG_BPF_EVENTS=y', 'CONFIG_TRACEPOINTS=y' ] - if not os.path.isfile(CONFIG): - call('sudo modprobe configs') - with gzip.open(CONFIG, 'rt') as f: - config_data = f.read() for option in options_to_check: - self.assertIn(option, config_data, - f"Option {option} is not present in /proc/config.gz") + self.assertIn(option, self._config_data) def test_synproxy_enabled(self): options_to_check = [ 'CONFIG_NFT_SYNPROXY', 'CONFIG_IP_NF_TARGET_SYNPROXY' ] - if not os.path.isfile(CONFIG): - call('sudo modprobe configs') - with gzip.open(CONFIG, 'rt') as f: - config_data = f.read() for option in options_to_check: - tmp = re.findall(f'{option}=(y|m)', config_data) + tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) def test_qemu_support(self): - for option in ['CONFIG_VIRTIO_BLK', 'CONFIG_SCSI_VIRTIO', - 'CONFIG_VIRTIO_NET', 'CONFIG_VIRTIO_CONSOLE', - 'CONFIG_VIRTIO', 'CONFIG_VIRTIO_PCI', - 'CONFIG_VIRTIO_BALLOON', 'CONFIG_CRYPTO_DEV_VIRTIO', - 'CONFIG_X86_PLATFORM_DEVICES']: - tmp = re.findall(f'{option}=(y|m)', config) + options_to_check = [ + 'CONFIG_VIRTIO_BLK', 'CONFIG_SCSI_VIRTIO', + 'CONFIG_VIRTIO_NET', 'CONFIG_VIRTIO_CONSOLE', + 'CONFIG_VIRTIO', 'CONFIG_VIRTIO_PCI', + 'CONFIG_VIRTIO_BALLOON', 'CONFIG_CRYPTO_DEV_VIRTIO', + 'CONFIG_X86_PLATFORM_DEVICES' + ] + for option in options_to_check: + tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) def test_vmware_support(self): for option in ['CONFIG_VMXNET3']: - tmp = re.findall(f'{option}=(y|m)', config) + tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) def test_container_cgroup_support(self): - for option in ['CONFIG_CGROUPS', 'CONFIG_MEMCG', 'CONFIG_CGROUP_PIDS', 'CONFIG_CGROUP_BPF']: - tmp = re.findall(f'{option}=(y|m)', config) + options_to_check = [ + 'CONFIG_CGROUPS', 'CONFIG_MEMCG', + 'CONFIG_CGROUP_PIDS', 'CONFIG_CGROUP_BPF' + ] + for option in options_to_check: + tmp = re.findall(f'{option}=(y|m)', self._config_data) + self.assertTrue(tmp) + + def test_ip_routing_support(self): + options_to_check = [ + 'CONFIG_IP_ADVANCED_ROUTER', 'CONFIG_IP_MULTIPLE_TABLES', + 'CONFIG_IP_ROUTE_MULTIPATH' + ] + for option in options_to_check: + tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) if __name__ == '__main__': unittest.main(verbosity=2) - From a1f4404739e0baf2f378fe7c890174350a59ffc4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 1 May 2024 21:23:03 -0500 Subject: [PATCH 059/188] pppoe-server: T6234: fix permissions on migration script --- src/migration-scripts/pppoe-server/9-to-10 | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 src/migration-scripts/pppoe-server/9-to-10 diff --git a/src/migration-scripts/pppoe-server/9-to-10 b/src/migration-scripts/pppoe-server/9-to-10 old mode 100644 new mode 100755 From 0b54c1bc411a21833ec573031cf5ad98fe709a2f Mon Sep 17 00:00:00 2001 From: khramshinr Date: Thu, 2 May 2024 17:45:57 +0600 Subject: [PATCH 060/188] qos: T6225: Fix qos random-detect policy Fix default values for random-detect Remove dsmakr qdisc from gred cofig because dsmark was deleted from kernel --- python/vyos/qos/base.py | 5 +++-- python/vyos/qos/randomdetect.py | 34 ++++++++++++------------------- smoketest/scripts/cli/test_qos.py | 8 +++++--- src/conf_mode/qos.py | 29 +++++++++++++------------- 4 files changed, 36 insertions(+), 40 deletions(-) diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index 4173a1a438a..87927ba9dfc 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -90,13 +90,14 @@ def _get_dsfield(self, value): else: return value - def _calc_random_detect_queue_params(self, avg_pkt, max_thr, limit=None, min_thr=None, mark_probability=None): + def _calc_random_detect_queue_params(self, avg_pkt, max_thr, limit=None, min_thr=None, + mark_probability=None, precedence=0): params = dict() avg_pkt = int(avg_pkt) max_thr = int(max_thr) mark_probability = int(mark_probability) limit = int(limit) if limit else 4 * max_thr - min_thr = int(min_thr) if min_thr else (9 * max_thr) // 18 + min_thr = int(min_thr) if min_thr else ((9 + precedence) * max_thr) // 18 params['avg_pkt'] = avg_pkt params['limit'] = limit * avg_pkt diff --git a/python/vyos/qos/randomdetect.py b/python/vyos/qos/randomdetect.py index d7d84260fa5..a3a39da36ee 100644 --- a/python/vyos/qos/randomdetect.py +++ b/python/vyos/qos/randomdetect.py @@ -21,33 +21,25 @@ class RandomDetect(QoSBase): # https://man7.org/linux/man-pages/man8/tc.8.html def update(self, config, direction): - tmp = f'tc qdisc add dev {self._interface} root handle {self._parent}:0 dsmark indices 8 set_tc_index' + # # Generalized Random Early Detection + handle = self._parent + tmp = f'tc qdisc add dev {self._interface} root handle {self._parent}:0 gred setup DPs 8 default 0 grio' self._cmd(tmp) - - tmp = f'tc filter add dev {self._interface} parent {self._parent}:0 protocol ip prio 1 tcindex mask 0xe0 shift 5' - self._cmd(tmp) - - # Generalized Random Early Detection - handle = self._parent +1 - tmp = f'tc qdisc add dev {self._interface} parent {self._parent}:0 handle {handle}:0 gred setup DPs 8 default 0 grio' - self._cmd(tmp) - bandwidth = self._rate_convert(config['bandwidth']) # set VQ (virtual queue) parameters for precedence, precedence_config in config['precedence'].items(): precedence = int(precedence) - avg_pkt = int(precedence_config['average_packet']) - limit = int(precedence_config['queue_limit']) * avg_pkt - min_val = int(precedence_config['minimum_threshold']) * avg_pkt - max_val = int(precedence_config['maximum_threshold']) * avg_pkt - - tmp = f'tc qdisc change dev {self._interface} handle {handle}:0 gred limit {limit} min {min_val} max {max_val} avpkt {avg_pkt} ' - - burst = (2 * int(precedence_config['minimum_threshold']) + int(precedence_config['maximum_threshold'])) // 3 - probability = 1 / int(precedence_config['mark_probability']) - tmp += f'burst {burst} bandwidth {bandwidth} probability {probability} DP {precedence} prio {8 - precedence:x}' - + qparams = self._calc_random_detect_queue_params( + avg_pkt=precedence_config.get('average_packet'), + max_thr=precedence_config.get('maximum_threshold'), + limit=precedence_config.get('queue_limit'), + min_thr=precedence_config.get('minimum_threshold'), + mark_probability=precedence_config.get('mark_probability'), + precedence=precedence + ) + tmp = f'tc qdisc change dev {self._interface} handle {handle}:0 gred limit {qparams["limit"]} min {qparams["min_val"]} max {qparams["max_val"]} avpkt {qparams["avg_pkt"]} ' + tmp += f'burst {qparams["burst"]} bandwidth {bandwidth} probability {qparams["probability"]} DP {precedence} prio {8 - precedence:x}' self._cmd(tmp) # call base class diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index fef1ff23a20..bcf5139c71b 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -441,7 +441,6 @@ def test_07_priority_queue(self): self.cli_commit() def test_08_random_detect(self): - self.skipTest('tc returns invalid JSON here - needs iproute2 fix') bandwidth = 5000 first = True @@ -467,8 +466,11 @@ def test_08_random_detect(self): bandwidth = 5000 for interface in self._interfaces: tmp = get_tc_qdisc_json(interface) - import pprint - pprint.pprint(tmp) + self.assertTrue('gred' in tmp.get('kind')) + self.assertEqual(8, len(tmp.get('options', {}).get('vqs'))) + self.assertEqual(8, tmp.get('options', {}).get('dp_cnt')) + self.assertEqual(0, tmp.get('options', {}).get('dp_default')) + self.assertTrue(tmp.get('options', {}).get('grio')) def test_09_rate_control(self): bandwidth = 5000 diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index ccfc8f6b82a..8a590cbc65a 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -39,6 +39,9 @@ from vyos.utils.process import run from vyos import ConfigError from vyos import airbag +from vyos.xml_ref import relative_defaults + + airbag.enable() map_vyops_tc = { @@ -115,8 +118,18 @@ def get_config(config=None): for rd_name in list(qos['policy'][policy]): # There are eight precedence levels - ensure all are present # to be filled later down with the appropriate default values - default_precedence = {'precedence' : { '0' : {}, '1' : {}, '2' : {}, '3' : {}, - '4' : {}, '5' : {}, '6' : {}, '7' : {} }} + default_p_val = relative_defaults( + ['qos', 'policy', 'random-detect', rd_name, 'precedence'], + {'precedence': {'0': {}}}, + get_first_key=True, recursive=True + )['0'] + default_p_val = {key.replace('-', '_'): value for key, value in default_p_val.items()} + default_precedence = { + 'precedence': {'0': default_p_val, '1': default_p_val, + '2': default_p_val, '3': default_p_val, + '4': default_p_val, '5': default_p_val, + '6': default_p_val, '7': default_p_val}} + qos['policy']['random_detect'][rd_name] = dict_merge( default_precedence, qos['policy']['random_detect'][rd_name]) @@ -124,18 +137,6 @@ def get_config(config=None): for policy in qos.get('policy', []): for p_name, p_config in qos['policy'][policy].items(): - if 'precedence' in p_config: - # precedence settings are a bit more complex as they are - # calculated under specific circumstances: - for precedence in p_config['precedence']: - max_thr = int(qos['policy'][policy][p_name]['precedence'][precedence]['maximum_threshold']) - if 'minimum_threshold' not in qos['policy'][policy][p_name]['precedence'][precedence]: - qos['policy'][policy][p_name]['precedence'][precedence]['minimum_threshold'] = str( - int((9 + int(precedence)) * max_thr) // 18); - - if 'queue_limit' not in qos['policy'][policy][p_name]['precedence'][precedence]: - qos['policy'][policy][p_name]['precedence'][precedence]['queue_limit'] = \ - str(int(4 * max_thr)) # cleanup empty match config if 'class' in p_config: for cls, cls_config in p_config['class'].items(): From 5a82aee95270737aeeee9e6facf1234b61ce436d Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 4 May 2024 16:35:27 +0200 Subject: [PATCH 061/188] smoketest: T6283: T6250: add testcases --- smoketest/scripts/cli/test_policy.py | 60 ++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/smoketest/scripts/cli/test_policy.py b/smoketest/scripts/cli/test_policy.py index ee444525136..a0c6ab05503 100755 --- a/smoketest/scripts/cli/test_policy.py +++ b/smoketest/scripts/cli/test_policy.py @@ -1923,6 +1923,66 @@ def test_multiple_commit_ipv4_table_id(self): self.assertEqual(sort_ip(tmp), sort_ip(original_second)) + def test_frr_individual_remove_T6283_T6250(self): + path = base_path + ['route-map'] + route_maps = ['RMAP-1', 'RMAP_2'] + seq = '10' + base_local_preference = 300 + base_table = 50 + + # T6250 + local_preference = base_local_preference + table = base_table + for route_map in route_maps: + self.cli_set(path + [route_map, 'rule', seq, 'action', 'permit']) + self.cli_set(path + [route_map, 'rule', seq, 'set', 'table', str(table)]) + self.cli_set(path + [route_map, 'rule', seq, 'set', 'local-preference', str(local_preference)]) + local_preference += 20 + table += 5 + + self.cli_commit() + + local_preference = base_local_preference + table = base_table + for route_map in route_maps: + config = self.getFRRconfig(f'route-map {route_map} permit {seq}', end='') + self.assertIn(f' set local-preference {local_preference}', config) + self.assertIn(f' set table {table}', config) + local_preference += 20 + table += 5 + + for route_map in route_maps: + self.cli_delete(path + [route_map, 'rule', '10', 'set', 'table']) + # we explicitly commit multiple times to be as vandal as possible to the system + self.cli_commit() + + local_preference = base_local_preference + for route_map in route_maps: + config = self.getFRRconfig(f'route-map {route_map} permit {seq}', end='') + self.assertIn(f' set local-preference {local_preference}', config) + local_preference += 20 + + # T6283 + seq = '20' + prepend = '100 100 100' + for route_map in route_maps: + self.cli_set(path + [route_map, 'rule', seq, 'action', 'permit']) + self.cli_set(path + [route_map, 'rule', seq, 'set', 'as-path', 'prepend', prepend]) + + self.cli_commit() + + for route_map in route_maps: + config = self.getFRRconfig(f'route-map {route_map} permit {seq}', end='') + self.assertIn(f' set as-path prepend {prepend}', config) + + for route_map in route_maps: + self.cli_delete(path + [route_map, 'rule', seq, 'set']) + # we explicitly commit multiple times to be as vandal as possible to the system + self.cli_commit() + + for route_map in route_maps: + config = self.getFRRconfig(f'route-map {route_map} permit {seq}', end='') + self.assertNotIn(f' set', config) def sort_ip(output): o = '\n'.join([' '.join(line.strip().split()) for line in output.strip().splitlines()]) From 0c2bf3192382cffc5ed2dcead3889c332a48820f Mon Sep 17 00:00:00 2001 From: l0crian1 <143656816+l0crian1@users.noreply.github.com> Date: Sat, 4 May 2024 10:39:09 -0400 Subject: [PATCH 062/188] op-mode: T6291: add LACP related commands show interfaces bonding lacp detail show interfaces bonding lacp detail show interfaces bonding lacp neighbors Co-authored-by: l0crian1 --- .../show-interfaces-bonding.xml.in | 34 +++++- src/op_mode/bonding.py | 103 ++++++++++++++++++ 2 files changed, 136 insertions(+), 1 deletion(-) create mode 100755 src/op_mode/bonding.py diff --git a/op-mode-definitions/show-interfaces-bonding.xml.in b/op-mode-definitions/show-interfaces-bonding.xml.in index 8ca5adb4fb7..e2950331bf6 100644 --- a/op-mode-definitions/show-interfaces-bonding.xml.in +++ b/op-mode-definitions/show-interfaces-bonding.xml.in @@ -23,8 +23,27 @@ Show detailed interface information - if [ -f "/proc/net/bonding/$4" ]; then cat "/proc/net/bonding/$4"; else echo "Interface $4 does not exist!"; fi + if [ -f "/proc/net/bonding/$4" ]; then sudo cat "/proc/net/bonding/$4"; else echo "Interface $4 does not exist!"; fi + + + Show LACP related info + + + + + Show LACP details + + sudo ${vyos_op_scripts_dir}/bonding.py show_lacp_detail --interface="$4" + + + + Show LACP Neighbors + + sudo ${vyos_op_scripts_dir}/bonding.py show_lacp_neighbors --interface="$4" + + + Show specified bonding interface information @@ -62,6 +81,19 @@ ${vyos_op_scripts_dir}/interfaces.py show --intf-type=bonding + + + Show LACP related info + + + + + Show LACP details + + sudo ${vyos_op_scripts_dir}/bonding.py show_lacp_detail + + + Show specified bonding interface information diff --git a/src/op_mode/bonding.py b/src/op_mode/bonding.py new file mode 100755 index 00000000000..07bccbd4b10 --- /dev/null +++ b/src/op_mode/bonding.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2016-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# This script will parse 'sudo cat /proc/net/bonding/' and return table output for lacp related info + +import subprocess +import re +import sys +import typing +from tabulate import tabulate + +import vyos.opmode +from vyos.configquery import ConfigTreeQuery + +def list_to_dict(data, headers, basekey): + data_list = {basekey: []} + + for row in data: + row_dict = {headers[i]: row[i] for i in range(len(headers))} + data_list[basekey].append(row_dict) + + return data_list + +def show_lacp_neighbors(raw: bool, interface: typing.Optional[str]): + headers = ["Interface", "Member", "Local ID", "Remote ID"] + data = subprocess.run(f"cat /proc/net/bonding/{interface}", stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, shell=True, text=False).stdout.decode('utf-8') + if 'Bonding Mode: IEEE 802.3ad Dynamic link aggregation' not in data: + raise vyos.opmode.DataUnavailable(f"{interface} is not present or not configured with mode 802.3ad") + + pattern = re.compile( + r"Slave Interface: (?P\w+\d+).*?" + r"system mac address: (?P[0-9a-f:]+).*?" + r"details partner lacp pdu:.*?" + r"system mac address: (?P[0-9a-f:]+)", + re.DOTALL + ) + + interfaces = [] + + for match in re.finditer(pattern, data): + member = match.group("member") + local_id = match.group("local_id") + remote_id = match.group("remote_id") + interfaces.append([interface, member, local_id, remote_id]) + + if raw: + return list_to_dict(interfaces, headers, 'lacp') + else: + return tabulate(interfaces, headers) + +def show_lacp_detail(raw: bool, interface: typing.Optional[str]): + headers = ["Interface", "Members", "Mode", "Rate", "System-MAC", "Hash"] + query = ConfigTreeQuery() + + if interface: + intList = [interface] + else: + intList = query.list_nodes(['interfaces', 'bonding']) + + bondList = [] + + for interface in intList: + data = subprocess.run(f"cat /proc/net/bonding/{interface}", stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, shell=True, text=False).stdout.decode('utf-8') + if 'Bonding Mode: IEEE 802.3ad Dynamic link aggregation' not in data: + continue + + mode_active = "active" if "LACP active: on" in data else "passive" + lacp_rate = re.search(r"LACP rate: (\w+)", data).group(1) if re.search(r"LACP rate: (\w+)", data) else "N/A" + hash_policy = re.search(r"Transmit Hash Policy: (.+?) \(\d+\)", data).group(1) if re.search(r"Transmit Hash Policy: (.+?) \(\d+\)", data) else "N/A" + system_mac = re.search(r"System MAC address: ([0-9a-f:]+)", data).group(1) if re.search(r"System MAC address: ([0-9a-f:]+)", data) else "N/A" + if raw: + members = re.findall(r"Slave Interface: ([a-zA-Z0-9:_-]+)", data) + else: + members = ",".join(set(re.findall(r"Slave Interface: ([a-zA-Z0-9:_-]+)", data))) + + bondList.append([interface, members, mode_active, lacp_rate, system_mac, hash_policy]) + + if raw: + return list_to_dict(bondList, headers, 'lacp') + else: + return tabulate(bondList, headers) + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) From 963daf62d417a3fcccf33ed93904eddd21aa6a02 Mon Sep 17 00:00:00 2001 From: l0crian1 <143656816+l0crian1@users.noreply.github.com> Date: Sat, 4 May 2024 12:50:16 -0400 Subject: [PATCH 063/188] T6291: Add bonding.py to op-mode-standardized.json --- data/op-mode-standardized.json | 1 + 1 file changed, 1 insertion(+) diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index d3685caafd9..a4ed2bcf466 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -1,6 +1,7 @@ [ "accelppp.py", "bgp.py", +"bonding.py", "bridge.py", "config_mgmt.py", "conntrack.py", From ac9f2c40c544347636973c7b84a1761f2ca330f1 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 6 May 2024 10:45:35 +0000 Subject: [PATCH 064/188] T6298: Fix TPM incorrect path for rc_cmd Fix import for `rc_cmd` --- python/vyos/tpm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/vyos/tpm.py b/python/vyos/tpm.py index b9f28546f56..a24f149fd9d 100644 --- a/python/vyos/tpm.py +++ b/python/vyos/tpm.py @@ -15,7 +15,7 @@ import os import tempfile -from vyos.util import rc_cmd +from vyos.utils.process import rc_cmd default_pcrs = ['0','2','4','7'] tpm_handle = 0x81000000 From 8041201fccffa96ca05ecd764b6d4b9d103b4c1a Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Mon, 6 May 2024 11:40:14 +0000 Subject: [PATCH 065/188] T6307: Add dependency procps to build vyos-1x --- debian/control | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/control b/debian/control index 801cce269f2..594e9e8d816 100644 --- a/debian/control +++ b/debian/control @@ -10,6 +10,7 @@ Build-Depends: iproute2, libvyosconfig0 (>= 0.0.7), libzmq3-dev, + procps, python3 (>= 3.10), # For QA pylint, From 9aa8c3cd9bb4d11d247895576ab8f90f6a0e19b5 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 6 May 2024 14:14:02 +0000 Subject: [PATCH 066/188] vyos.template: T3664: use a module-level global variable for the default template directory as a more convenient and secure alternative to environment variable --- python/vyos/template.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/python/vyos/template.py b/python/vyos/template.py index ac77e8a3d0a..fbc5f145628 100644 --- a/python/vyos/template.py +++ b/python/vyos/template.py @@ -25,6 +25,14 @@ from vyos.utils.permission import chmod from vyos.utils.permission import chown +# We use a mutable global variable for the default template directory +# to make it possible to call scripts from this repository +# outside of live VyOS systems. +# If something (like the image build scripts) +# want to call a script, they can modify the default location +# to the repository path. +DEFAULT_TEMPLATE_DIR = directories["templates"] + # Holds template filters registered via register_filter() _FILTERS = {} _TESTS = {} @@ -35,18 +43,7 @@ def _get_environment(location=None): from os import getenv if location is None: - # Sometimes functions that rely on templates need to be executed outside of VyOS installations: - # for example, installer functions are executed for image builds, - # and anything may be invoked for testing from a developer's machine. - # This environment variable allows running any unmodified code - # with a custom template location. - location_env_var = getenv("VYOS_TEMPLATE_DIR") - if location_env_var: - print(f"Using environment variable {location_env_var}") - template_dir = location_env_var - else: - template_dir = directories["templates"] - loc_loader=FileSystemLoader(template_dir) + loc_loader=FileSystemLoader(DEFAULT_TEMPLATE_DIR) else: loc_loader=FileSystemLoader(location) env = Environment( From 807784173a8b582e82741ea1a130834c6146a7f1 Mon Sep 17 00:00:00 2001 From: vijayakumar-ayyavoo_tmna Date: Tue, 7 May 2024 12:34:57 +0530 Subject: [PATCH 067/188] feat: added codeql caller workflow --- .github/workflows/codeql-caller.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .github/workflows/codeql-caller.yml diff --git a/.github/workflows/codeql-caller.yml b/.github/workflows/codeql-caller.yml new file mode 100644 index 00000000000..81ec131fe6a --- /dev/null +++ b/.github/workflows/codeql-caller.yml @@ -0,0 +1,28 @@ +# For most projects, this workflow file will not need changing; you simply need +# to commit it to your repository. +# +# You may wish to alter this file to override the set of languages analyzed, +# or to provide custom queries or build logic. +# +# ******** NOTE ******** +# We have attempted to detect the languages in your repository. Please check +# the `language` matrix defined below to confirm you have the correct set of +# supported CodeQL languages. +# +name: "Call CodeQL Analysis" + +on: + push: + branches: [ "current", crux, equuleus ] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "current" ] + schedule: + - cron: '22 10 * * 0' + +jobs: + codeql-analysis-call: + uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@feature/add-codeql-workflow + secrets: inherit + with: + languages: "['python']" From b9296a6c430928178320e269fc253734454e791e Mon Sep 17 00:00:00 2001 From: vijayakumar-ayyavoo_tmna Date: Tue, 7 May 2024 12:53:09 +0530 Subject: [PATCH 068/188] feat: added codeql caller workflow - workflow dispatch --- .github/workflows/codeql-caller.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/codeql-caller.yml b/.github/workflows/codeql-caller.yml index 81ec131fe6a..5b2ee4d8ee0 100644 --- a/.github/workflows/codeql-caller.yml +++ b/.github/workflows/codeql-caller.yml @@ -12,6 +12,7 @@ name: "Call CodeQL Analysis" on: + workflow_dispatch: push: branches: [ "current", crux, equuleus ] pull_request: From 637a73e35ff716441df0430b2308d685707b2ca0 Mon Sep 17 00:00:00 2001 From: khramshinr Date: Tue, 7 May 2024 19:10:32 +0600 Subject: [PATCH 069/188] bgp: T6082: Allow the same local-as and remote-as in one peer group --- smoketest/scripts/cli/test_protocols_bgp.py | 21 +++++++++++++++++++++ src/conf_mode/protocols_bgp.py | 2 +- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/smoketest/scripts/cli/test_protocols_bgp.py b/smoketest/scripts/cli/test_protocols_bgp.py index 03daa34aa42..ea2f561a4de 100755 --- a/smoketest/scripts/cli/test_protocols_bgp.py +++ b/smoketest/scripts/cli/test_protocols_bgp.py @@ -1330,6 +1330,27 @@ def _common_config_check(conf, include_ras=True): self.assertIn(f'neighbor {ext_neighbors[1]} remote-as external', conf) self.assertIn(f'neighbor {ext_neighbors[1]} peer-group {ext_pg_name}', conf) + def test_bgp_29_peer_group_remote_as_equal_local_as(self): + self.cli_set(base_path + ['system-as', ASN]) + self.cli_set(base_path + ['peer-group', 'OVERLAY', 'local-as', f'{int(ASN) + 1}']) + self.cli_set(base_path + ['peer-group', 'OVERLAY', 'remote-as', f'{int(ASN) + 1}']) + self.cli_set(base_path + ['peer-group', 'OVERLAY', 'address-family', 'l2vpn-evpn']) + + self.cli_set(base_path + ['peer-group', 'UNDERLAY', 'address-family', 'ipv4-unicast']) + + self.cli_set(base_path + ['neighbor', '10.177.70.62', 'peer-group', 'UNDERLAY']) + self.cli_set(base_path + ['neighbor', '10.177.70.62', 'remote-as', 'external']) + + self.cli_set(base_path + ['neighbor', '10.177.75.1', 'peer-group', 'OVERLAY']) + self.cli_set(base_path + ['neighbor', '10.177.75.2', 'peer-group', 'OVERLAY']) + + self.cli_commit() + + conf = self.getFRRconfig(f'router bgp {ASN}') + + self.assertIn(f'neighbor OVERLAY remote-as {int(ASN) + 1}', conf) + self.assertIn(f'neighbor OVERLAY local-as {int(ASN) + 1}', conf) + def test_bgp_99_bmp(self): target_name = 'instance-bmp' target_address = '127.0.0.1' diff --git a/src/conf_mode/protocols_bgp.py b/src/conf_mode/protocols_bgp.py index 44409c0e349..22f0200992b 100755 --- a/src/conf_mode/protocols_bgp.py +++ b/src/conf_mode/protocols_bgp.py @@ -333,7 +333,7 @@ def verify(bgp): raise ConfigError('Cannot have local-as same as system-as number') # Neighbor AS specified for local-as and remote-as can not be the same - if dict_search('remote_as', peer_config) == asn: + if dict_search('remote_as', peer_config) == asn and neighbor != 'peer_group': raise ConfigError(f'Neighbor "{peer}" has local-as specified which is '\ 'the same as remote-as, this is not allowed!') From 3b4e1acbd30caa4dcb9171297066ba9198025782 Mon Sep 17 00:00:00 2001 From: kumvijaya Date: Tue, 7 May 2024 18:44:40 +0530 Subject: [PATCH 070/188] T6310: added codeql workflow --- .github/workflows/codeql-caller.yml | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql-caller.yml b/.github/workflows/codeql-caller.yml index 5b2ee4d8ee0..9dd0e39416f 100644 --- a/.github/workflows/codeql-caller.yml +++ b/.github/workflows/codeql-caller.yml @@ -9,21 +9,26 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: "Call CodeQL Analysis" +name: "Perform CodeQL Analysis" on: workflow_dispatch: push: - branches: [ "current", crux, equuleus ] + branches: [ "current", "sagitta", "equuleus" ] pull_request: # The branches below must be a subset of the branches above branches: [ "current" ] schedule: - cron: '22 10 * * 0' +permissions: + actions: read + contents: read + security-events: write + jobs: codeql-analysis-call: - uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@feature/add-codeql-workflow + uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@main secrets: inherit with: languages: "['python']" From a9bb2388a1dbd3c489d81602f34285e044ff9e10 Mon Sep 17 00:00:00 2001 From: kumvijaya Date: Tue, 7 May 2024 18:54:35 +0530 Subject: [PATCH 071/188] T6310: added codeql workflow and removed old codeql workflow --- .github/workflows/codeql-caller.yml | 34 --------------- .github/workflows/codeql.yml | 65 ++++++----------------------- 2 files changed, 12 insertions(+), 87 deletions(-) delete mode 100644 .github/workflows/codeql-caller.yml diff --git a/.github/workflows/codeql-caller.yml b/.github/workflows/codeql-caller.yml deleted file mode 100644 index 9dd0e39416f..00000000000 --- a/.github/workflows/codeql-caller.yml +++ /dev/null @@ -1,34 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "Perform CodeQL Analysis" - -on: - workflow_dispatch: - push: - branches: [ "current", "sagitta", "equuleus" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "current" ] - schedule: - - cron: '22 10 * * 0' - -permissions: - actions: read - contents: read - security-events: write - -jobs: - codeql-analysis-call: - uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@main - secrets: inherit - with: - languages: "['python']" diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c39800ac84c..8b1c3c14144 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -9,66 +9,25 @@ # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # -name: "CodeQL" +name: "Perform CodeQL Analysis" on: push: - branches: [ "current", crux, equuleus ] + branches: [ "current", "sagitta", "equuleus" ] pull_request: # The branches below must be a subset of the branches above branches: [ "current" ] schedule: - cron: '22 10 * * 0' -jobs: - analyze: - name: Analyze - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 +permissions: + actions: read + contents: read + security-events: write - # ℹ️ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{matrix.language}}" +jobs: + codeql-analysis-call: + uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@main + secrets: inherit + with: + languages: "['python']" From 40b9085171ecf97f791b5f3b5cb32dd5f46d0f21 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Tue, 7 May 2024 16:48:16 +0300 Subject: [PATCH 072/188] op-mode: T6284: IPoE-server op-mode does not show IPv6 address field --- src/op_mode/ipoe-control.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/op_mode/ipoe-control.py b/src/op_mode/ipoe-control.py index 0f33beca73d..b7d6a0c434a 100755 --- a/src/op_mode/ipoe-control.py +++ b/src/op_mode/ipoe-control.py @@ -56,7 +56,11 @@ def main(): if args.selector in cmd_dict['selector'] and args.target: run(cmd_dict['cmd_base'] + "{0} {1} {2}".format(args.action, args.selector, args.target)) else: - output, err = popen(cmd_dict['cmd_base'] + cmd_dict['actions'][args.action], decode='utf-8') + if args.action == "show_sessions": + ses_pattern = " ifname,username,calling-sid,ip,ip6,ip6-dp,rate-limit,type,comp,state,uptime" + else: + ses_pattern = "" + output, err = popen(cmd_dict['cmd_base'] + cmd_dict['actions'][args.action] + ses_pattern, decode='utf-8') if not err: print(output) else: From b5f22f70006eed6c7e62700128d5034b1b95db31 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Tue, 7 May 2024 15:23:18 +0000 Subject: [PATCH 073/188] T6305: accept ipoe interfaces on firewall ruleset --- interface-definitions/include/firewall/match-interface.xml.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/include/firewall/match-interface.xml.i b/interface-definitions/include/firewall/match-interface.xml.i index 5da6f51fb9c..f25686e72d4 100644 --- a/interface-definitions/include/firewall/match-interface.xml.i +++ b/interface-definitions/include/firewall/match-interface.xml.i @@ -19,7 +19,7 @@ Inverted interface name to match - (\!?)(bond|br|dum|en|ersp|eth|gnv|ifb|lan|l2tp|l2tpeth|macsec|peth|ppp|pppoe|pptp|sstp|tun|veth|vti|vtun|vxlan|wg|wlan|wwan)([0-9]?)(\*?)(.+)?|(\!?)lo + (\!?)(bond|br|dum|en|ersp|eth|gnv|ifb|ipoe|lan|l2tp|l2tpeth|macsec|peth|ppp|pppoe|pptp|sstp|tun|veth|vti|vtun|vxlan|wg|wlan|wwan)([0-9]?)(\*?)(.+)?|(\!?)lo From 2551a741af9ac4f7dfff8dc5cbe47173a99fa478 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Tue, 7 May 2024 14:52:14 -0500 Subject: [PATCH 074/188] xml: T6319: add util for ancestor owner/priority --- python/vyos/xml_ref/__init__.py | 6 ++++++ python/vyos/xml_ref/definition.py | 27 +++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/python/vyos/xml_ref/__init__.py b/python/vyos/xml_ref/__init__.py index bf434865d5c..2ba3da4e8ae 100644 --- a/python/vyos/xml_ref/__init__.py +++ b/python/vyos/xml_ref/__init__.py @@ -53,6 +53,12 @@ def is_valueless(path: list) -> bool: def is_leaf(path: list) -> bool: return load_reference().is_leaf(path) +def owner(path: list) -> str: + return load_reference().owner(path) + +def priority(path: list) -> str: + return load_reference().priority(path) + def cli_defined(path: list, node: str, non_local=False) -> bool: return load_reference().cli_defined(path, node, non_local=non_local) diff --git a/python/vyos/xml_ref/definition.py b/python/vyos/xml_ref/definition.py index c90c5ddbc65..c85835ffd2b 100644 --- a/python/vyos/xml_ref/definition.py +++ b/python/vyos/xml_ref/definition.py @@ -135,6 +135,33 @@ def is_leaf(self, path: list) -> bool: d = self._get_ref_path(path) return self._is_leaf_node(d) + def _least_upper_data(self, path: list, name: str) -> str: + ref_path = path.copy() + d = self.ref + data = '' + while ref_path and d: + d = d.get(ref_path[0], {}) + ref_path.pop(0) + if self._is_tag_node(d) and ref_path: + ref_path.pop(0) + if self._is_leaf_node(d) and ref_path: + ref_path.pop(0) + res = self._get_ref_node_data(d, name) + if res is not None: + data = res + + return data + + def owner(self, path: list) -> str: + from pathlib import Path + data = self._least_upper_data(path, 'owner') + if data: + data = Path(data.split()[0]).name + return data + + def priority(self, path: list) -> str: + return self._least_upper_data(path, 'priority') + @staticmethod def _dict_get(d: dict, path: list) -> dict: for i in path: From 31fc5372961547bb352c56eb2f4149fd195e9be1 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 8 May 2024 21:38:34 +0200 Subject: [PATCH 075/188] bridge: T6317: call dependency when deleting bridge member --- src/conf_mode/interfaces_bridge.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/conf_mode/interfaces_bridge.py b/src/conf_mode/interfaces_bridge.py index 9789f7bd3ed..69e657830a6 100755 --- a/src/conf_mode/interfaces_bridge.py +++ b/src/conf_mode/interfaces_bridge.py @@ -56,6 +56,14 @@ def get_config(config=None): bridge['member'].update({'interface_remove' : tmp }) else: bridge.update({'member' : {'interface_remove' : tmp }}) + for interface in tmp: + # When using VXLAN member interfaces that are configured for Single + # VXLAN Device (SVD) we need to call the VXLAN conf-mode script to + # re-create VLAN to VNI mappings if required, but only if the interface + # is already live on the system - this must not be done on first commit + if interface.startswith('vxlan') and interface_exists(interface): + set_dependents('vxlan', conf, interface) + if dict_search('member.interface', bridge) is not None: for interface in list(bridge['member']['interface']): @@ -168,12 +176,19 @@ def apply(bridge): else: br.update(bridge) - for interface in dict_search('member.interface', bridge) or []: - if interface.startswith('vxlan') and interface_exists(interface): + tmp = [] + if 'member' in bridge: + if 'interface_remove' in bridge['member']: + tmp.extend(bridge['member']['interface_remove']) + if 'interface' in bridge['member']: + tmp.extend(bridge['member']['interface']) + + for interface in tmp: + if interface.startswith(tuple(['vxlan'])) and interface_exists(interface): try: call_dependents() except ConfigError: - raise ConfigError('Error in updating VXLAN interface after changing bridge!') + raise ConfigError('Error updating member interface configuration after changing bridge!') return None From 431443ab3f663a6617008536d2d6d96407aebfcb Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 8 May 2024 21:40:35 +0200 Subject: [PATCH 076/188] bridge: T6317: add dependency call for wireless interfaces --- data/config-mode-dependencies/vyos-1x.json | 3 ++- data/templates/wifi/hostapd.conf.j2 | 7 ++++++- smoketest/scripts/cli/test_interfaces_wireless.py | 11 +++++++++++ src/conf_mode/interfaces_bridge.py | 14 +++++++++----- 4 files changed, 28 insertions(+), 7 deletions(-) diff --git a/data/config-mode-dependencies/vyos-1x.json b/data/config-mode-dependencies/vyos-1x.json index afe3dd838d5..13de434bd06 100644 --- a/data/config-mode-dependencies/vyos-1x.json +++ b/data/config-mode-dependencies/vyos-1x.json @@ -11,7 +11,8 @@ "ethernet": ["interfaces_ethernet"] }, "interfaces_bridge": { - "vxlan": ["interfaces_vxlan"] + "vxlan": ["interfaces_vxlan"], + "wlan": ["interfaces_wireless"] }, "load_balancing_wan": { "conntrack": ["system_conntrack"] diff --git a/data/templates/wifi/hostapd.conf.j2 b/data/templates/wifi/hostapd.conf.j2 index 83009242bc2..769325b4969 100644 --- a/data/templates/wifi/hostapd.conf.j2 +++ b/data/templates/wifi/hostapd.conf.j2 @@ -28,6 +28,12 @@ interface={{ ifname }} {% for bridge in is_bridge_member %} bridge={{ bridge }} {% endfor %} + +# WDS (4-address frame) mode with per-station virtual interfaces +# (only supported with driver=nl80211) +# This mode allows associated stations to use 4-address frames to allow layer 2 +# bridging to be used. +wds_sta=1 {% endif %} # Driver interface type (hostap/wired/none/nl80211/bsd); @@ -739,4 +745,3 @@ wmm_ac_vo_cwmin=2 wmm_ac_vo_cwmax=3 wmm_ac_vo_txop_limit=47 wmm_ac_vo_acm=0 - diff --git a/smoketest/scripts/cli/test_interfaces_wireless.py b/smoketest/scripts/cli/test_interfaces_wireless.py index 83b00ac0c45..b45754cae43 100755 --- a/smoketest/scripts/cli/test_interfaces_wireless.py +++ b/smoketest/scripts/cli/test_interfaces_wireless.py @@ -236,6 +236,17 @@ def test_wireless_access_point_bridge(self): self.assertIn(interface, bridge_members) + # Now generate a VLAN on the bridge + self.cli_set(bridge_path + ['enable-vlan']) + self.cli_set(bridge_path + ['vif', '20', 'address', '10.0.0.1/24']) + + self.cli_commit() + + tmp = get_config_value(interface, 'bridge') + self.assertEqual(tmp, bridge) + tmp = get_config_value(interface, 'wds_sta') + self.assertEqual(tmp, '1') + self.cli_delete(bridge_path) def test_wireless_security_station_address(self): diff --git a/src/conf_mode/interfaces_bridge.py b/src/conf_mode/interfaces_bridge.py index 69e657830a6..7b2c1ee0bce 100755 --- a/src/conf_mode/interfaces_bridge.py +++ b/src/conf_mode/interfaces_bridge.py @@ -63,7 +63,10 @@ def get_config(config=None): # is already live on the system - this must not be done on first commit if interface.startswith('vxlan') and interface_exists(interface): set_dependents('vxlan', conf, interface) - + # When using Wireless member interfaces we need to inform hostapd + # to properly set-up the bridge + elif interface.startswith('wlan') and interface_exists(interface): + set_dependents('wlan', conf, interface) if dict_search('member.interface', bridge) is not None: for interface in list(bridge['member']['interface']): @@ -99,6 +102,10 @@ def get_config(config=None): # is already live on the system - this must not be done on first commit if interface.startswith('vxlan') and interface_exists(interface): set_dependents('vxlan', conf, interface) + # When using Wireless member interfaces we need to inform hostapd + # to properly set-up the bridge + elif interface.startswith('wlan') and interface_exists(interface): + set_dependents('wlan', conf, interface) # delete empty dictionary keys - no need to run code paths if nothing is there to do if 'member' in bridge: @@ -148,9 +155,6 @@ def verify(bridge): if 'enable_vlan' in bridge: if 'has_vlan' in interface_config: raise ConfigError(error_msg + 'it has VLAN subinterface(s) assigned!') - - if 'wlan' in interface: - raise ConfigError(error_msg + 'VLAN aware cannot be set!') else: for option in ['allowed_vlan', 'native_vlan']: if option in interface_config: @@ -184,7 +188,7 @@ def apply(bridge): tmp.extend(bridge['member']['interface']) for interface in tmp: - if interface.startswith(tuple(['vxlan'])) and interface_exists(interface): + if interface.startswith(tuple(['vxlan', 'wlan'])) and interface_exists(interface): try: call_dependents() except ConfigError: From d8a0bb52c396298c1ab61c6d9dc2bb2464bf2a72 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Thu, 9 May 2024 12:45:08 +0530 Subject: [PATCH 077/188] T6315: updated codeql branch name --- .github/workflows/codeql.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 8b1c3c14144..9e2e4bf0f07 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,7 +27,7 @@ permissions: jobs: codeql-analysis-call: - uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@main + uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@current secrets: inherit with: languages: "['python']" From 890391885d66b517817c003648584ccd7e30d127 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Thu, 9 May 2024 15:09:58 +0530 Subject: [PATCH 078/188] T6316: remove reviewers assignment in workflow as it will done by codeowners file T6316: remove reviewers yml as it is controlled in global level --- .github/reviewers.yml | 3 --- .github/workflows/auto-author-assign.yml | 10 ---------- 2 files changed, 13 deletions(-) delete mode 100644 .github/reviewers.yml diff --git a/.github/reviewers.yml b/.github/reviewers.yml deleted file mode 100644 index a1647d20d83..00000000000 --- a/.github/reviewers.yml +++ /dev/null @@ -1,3 +0,0 @@ ---- -"**/*": - - team: reviewers diff --git a/.github/workflows/auto-author-assign.yml b/.github/workflows/auto-author-assign.yml index 1a7f8ef0b8a..0bfe972c08e 100644 --- a/.github/workflows/auto-author-assign.yml +++ b/.github/workflows/auto-author-assign.yml @@ -15,13 +15,3 @@ jobs: uses: toshimaru/auto-author-assign@v1.6.2 with: repo-token: ${{ secrets.GITHUB_TOKEN }} - - # https://github.com/shufo/auto-assign-reviewer-by-files - assign_reviewer: - runs-on: ubuntu-latest - steps: - - name: Request review based on files changes and/or groups the author belongs to - uses: shufo/auto-assign-reviewer-by-files@v1.1.4 - with: - token: ${{ secrets.PR_ACTION_ASSIGN_REVIEWERS }} - config: .github/reviewers.yml From 7dab763df070dd5138d6428450496f54b1f33d44 Mon Sep 17 00:00:00 2001 From: srividya0208 Date: Thu, 9 May 2024 07:08:08 -0400 Subject: [PATCH 079/188] T6323: openvpn: Correction of auto-completion description of "mfa totp digits" --- interface-definitions/interfaces_openvpn.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/interfaces_openvpn.xml.in b/interface-definitions/interfaces_openvpn.xml.in index 7b46f32b3a2..23cc83e9ab6 100644 --- a/interface-definitions/interfaces_openvpn.xml.in +++ b/interface-definitions/interfaces_openvpn.xml.in @@ -663,7 +663,7 @@ Number of digits to use for totp hash 1-65535 - Seconds + Digits From a1aa0a8a663ef9e575bbdbae0ced84f85fe81c25 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 9 May 2024 16:33:15 +0200 Subject: [PATCH 080/188] T6199: remove unused Python pip files --- .gitignore | 4 + Pipfile | 18 -- Pipfile.lock | 549 --------------------------------------------------- 3 files changed, 4 insertions(+), 567 deletions(-) delete mode 100644 Pipfile delete mode 100644 Pipfile.lock diff --git a/.gitignore b/.gitignore index 7707e94ca3f..507daceeef1 100644 --- a/.gitignore +++ b/.gitignore @@ -145,3 +145,7 @@ data/component-versions.json # vyos-1x XML cache python/vyos/xml_ref/cache.py python/vyos/xml_ref/pkg_cache/*_cache.py + +# We do not use pip +Pipfile +Pipfile.lock diff --git a/Pipfile b/Pipfile deleted file mode 100644 index 5a4fb6fe935..00000000000 --- a/Pipfile +++ /dev/null @@ -1,18 +0,0 @@ -[[source]] -name = "pypi" -url = "https://pypi.org/simple" -verify_ssl = true - -[dev-packages] -lxml = "*" -pylint = "*" -nose = "*" -coverage = "*" - -[packages] -vyos = {file = "./python"} -jinja2 = "*" -paramiko = "*" - -[requires] -python_version = "3.6" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 5a048becc3e..00000000000 --- a/Pipfile.lock +++ /dev/null @@ -1,549 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "2ec6c667a8fe2b96d82639414919deb0462744211709afe93f6a538a0b15dc1f" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.6" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "bcrypt": { - "hashes": [ - "sha256:02d9ef8915f72dd6daaef40e0baeef8a017ce624369f09754baf32bb32dba25f", - "sha256:1c28973decf4e0e69cee78c68e30a523be441972c826703bb93099868a8ff5b5", - "sha256:2a298db2a8ab20056120b45e86c00a0a5eb50ec4075b6142db35f593b97cb3fb", - "sha256:33313a1200a3ae90b75587ceac502b048b840fc69e7f7a0905b5f87fac7a1258", - "sha256:3566a88234e8de2ccae31968127b0ecccbb4cddb629da744165db72b58d88ca4", - "sha256:387e7e1af9a4dd636b9505a465032f2f5cb8e61ba1120e79a0e1cd0b512f3dfc", - "sha256:44290ccc827d3a24604f2c8bcd00d0da349e336e6503656cb8192133e27335e2", - "sha256:57fa9442758da926ed33a91644649d3e340a71e2d0a5a8de064fb621fd5a3326", - "sha256:68e3c6642077b0c8092580c819c1684161262b2e30c4f45deb000c38947bf483", - "sha256:69057b9fc5093ea1ab00dd24ede891f3e5e65bee040395fb1e66ee196f9c9b4a", - "sha256:6cad43d8c63f34b26aef462b6f5e44fdcf9860b723d2453b5d391258c4c8e966", - "sha256:71b8be82bc46cedd61a9f4ccb6c1a493211d031415a34adde3669ee1b0afbb63", - "sha256:732b3920a08eacf12f93e6b04ea276c489f1c8fb49344f564cca2adb663b3e4c", - "sha256:9800ae5bd5077b13725e2e3934aa3c9c37e49d3ea3d06318010aa40f54c63551", - "sha256:a97e07e83e3262599434816f631cc4c7ca2aa8e9c072c1b1a7fec2ae809a1d2d", - "sha256:ac621c093edb28200728a9cca214d7e838529e557027ef0581685909acd28b5e", - "sha256:b8df79979c5bae07f1db22dcc49cc5bccf08a0380ca5c6f391cbb5790355c0b0", - "sha256:b90e216dc36864ae7132cb151ffe95155a37a14e0de3a8f64b49655dd959ff9c", - "sha256:ba4e4cc26610581a6329b3937e02d319f5ad4b85b074846bf4fef8a8cf51e7bb", - "sha256:ba55e40de38a24e2d78d34c2d36d6e864f93e0d79d0b6ce915e4335aa81d01b1", - "sha256:be3ab1071662f6065899fe08428e45c16aa36e28bc42921c4901a191fda6ee42", - "sha256:d75fc8cd0ba23f97bae88a6ec04e9e5351ff3c6ad06f38fe32ba50cbd0d11946", - "sha256:e51c42750b7585cee7892c2614be0d14107fad9581d1738d954a262556dd1aab", - "sha256:ea505c97a5c465ab8c3ba75c0805a102ce526695cd6818c6de3b1a38f6f60da1", - "sha256:eb3bd3321517916696233b5e0c67fd7d6281f0ef48e66812db35fc963a422a1c", - "sha256:f70d9c61f9c4ca7d57f3bfe88a5ccf62546ffbadf3681bb1e268d9d2e41c91a7", - "sha256:fbe188b878313d01b7718390f31528be4010fed1faa798c5a1d0469c9c48c369" - ], - "markers": "python_version >= '3.7'", - "version": "==4.1.2" - }, - "cffi": { - "hashes": [ - "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc", - "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a", - "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417", - "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab", - "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520", - "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36", - "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743", - "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8", - "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed", - "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684", - "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56", - "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324", - "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d", - "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235", - "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e", - "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088", - "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000", - "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7", - "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e", - "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673", - "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c", - "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe", - "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2", - "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098", - "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8", - "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a", - "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0", - "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b", - "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896", - "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e", - "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9", - "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2", - "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b", - "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6", - "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404", - "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f", - "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0", - "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4", - "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc", - "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936", - "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba", - "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872", - "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb", - "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614", - "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1", - "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d", - "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969", - "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b", - "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4", - "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627", - "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956", - "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357" - ], - "markers": "platform_python_implementation != 'PyPy'", - "version": "==1.16.0" - }, - "cryptography": { - "hashes": [ - "sha256:0270572b8bd2c833c3981724b8ee9747b3ec96f699a9665470018594301439ee", - "sha256:111a0d8553afcf8eb02a4fea6ca4f59d48ddb34497aa8706a6cf536f1a5ec576", - "sha256:16a48c23a62a2f4a285699dba2e4ff2d1cff3115b9df052cdd976a18856d8e3d", - "sha256:1b95b98b0d2af784078fa69f637135e3c317091b615cd0905f8b8a087e86fa30", - "sha256:1f71c10d1e88467126f0efd484bd44bca5e14c664ec2ede64c32f20875c0d413", - "sha256:2424ff4c4ac7f6b8177b53c17ed5d8fa74ae5955656867f5a8affaca36a27abb", - "sha256:2bce03af1ce5a5567ab89bd90d11e7bbdff56b8af3acbbec1faded8f44cb06da", - "sha256:329906dcc7b20ff3cad13c069a78124ed8247adcac44b10bea1130e36caae0b4", - "sha256:37dd623507659e08be98eec89323469e8c7b4c1407c85112634ae3dbdb926fdd", - "sha256:3eaafe47ec0d0ffcc9349e1708be2aaea4c6dd4978d76bf6eb0cb2c13636c6fc", - "sha256:5e6275c09d2badf57aea3afa80d975444f4be8d3bc58f7f80d2a484c6f9485c8", - "sha256:6fe07eec95dfd477eb9530aef5bead34fec819b3aaf6c5bd6d20565da607bfe1", - "sha256:7367d7b2eca6513681127ebad53b2582911d1736dc2ffc19f2c3ae49997496bc", - "sha256:7cde5f38e614f55e28d831754e8a3bacf9ace5d1566235e39d91b35502d6936e", - "sha256:9481ffe3cf013b71b2428b905c4f7a9a4f76ec03065b05ff499bb5682a8d9ad8", - "sha256:98d8dc6d012b82287f2c3d26ce1d2dd130ec200c8679b6213b3c73c08b2b7940", - "sha256:a011a644f6d7d03736214d38832e030d8268bcff4a41f728e6030325fea3e400", - "sha256:a2913c5375154b6ef2e91c10b5720ea6e21007412f6437504ffea2109b5a33d7", - "sha256:a30596bae9403a342c978fb47d9b0ee277699fa53bbafad14706af51fe543d16", - "sha256:b03c2ae5d2f0fc05f9a2c0c997e1bc18c8229f392234e8a0194f202169ccd278", - "sha256:b6cd2203306b63e41acdf39aa93b86fb566049aeb6dc489b70e34bcd07adca74", - "sha256:b7ffe927ee6531c78f81aa17e684e2ff617daeba7f189f911065b2ea2d526dec", - "sha256:b8cac287fafc4ad485b8a9b67d0ee80c66bf3574f655d3b97ef2e1082360faf1", - "sha256:ba334e6e4b1d92442b75ddacc615c5476d4ad55cc29b15d590cc6b86efa487e2", - "sha256:ba3e4a42397c25b7ff88cdec6e2a16c2be18720f317506ee25210f6d31925f9c", - "sha256:c41fb5e6a5fe9ebcd58ca3abfeb51dffb5d83d6775405305bfa8715b76521922", - "sha256:cd2030f6650c089aeb304cf093f3244d34745ce0cfcc39f20c6fbfe030102e2a", - "sha256:cd65d75953847815962c84a4654a84850b2bb4aed3f26fadcc1c13892e1e29f6", - "sha256:e4985a790f921508f36f81831817cbc03b102d643b5fcb81cd33df3fa291a1a1", - "sha256:e807b3188f9eb0eaa7bbb579b462c5ace579f1cedb28107ce8b48a9f7ad3679e", - "sha256:f12764b8fffc7a123f641d7d049d382b73f96a34117e0b637b80643169cec8ac", - "sha256:f8837fe1d6ac4a8052a9a8ddab256bc006242696f03368a4009be7ee3075cdb7" - ], - "markers": "python_version >= '3.7'", - "version": "==42.0.5" - }, - "jinja2": { - "hashes": [ - "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa", - "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90" - ], - "index": "pypi", - "markers": "python_version >= '3.7'", - "version": "==3.1.3" - }, - "markupsafe": { - "hashes": [ - "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf", - "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff", - "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f", - "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3", - "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532", - "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f", - "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617", - "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df", - "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4", - "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906", - "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f", - "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4", - "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8", - "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371", - "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2", - "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465", - "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52", - "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6", - "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169", - "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad", - "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2", - "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0", - "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029", - "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f", - "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a", - "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced", - "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5", - "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c", - "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf", - "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9", - "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb", - "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad", - "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3", - "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1", - "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46", - "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc", - "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a", - "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee", - "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900", - "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5", - "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea", - "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f", - "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5", - "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e", - "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a", - "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f", - "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50", - "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a", - "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b", - "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4", - "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff", - "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2", - "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46", - "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b", - "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf", - "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5", - "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5", - "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab", - "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd", - "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68" - ], - "markers": "python_version >= '3.7'", - "version": "==2.1.5" - }, - "paramiko": { - "hashes": [ - "sha256:43f0b51115a896f9c00f59618023484cb3a14b98bbceab43394a39c6739b7ee7", - "sha256:aac08f26a31dc4dffd92821527d1682d99d52f9ef6851968114a8728f3c274d3" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==3.4.0" - }, - "pycparser": { - "hashes": [ - "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", - "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc" - ], - "markers": "python_version >= '3.8'", - "version": "==2.22" - }, - "pynacl": { - "hashes": [ - "sha256:06b8f6fa7f5de8d5d2f7573fe8c863c051225a27b61e6860fd047b1775807858", - "sha256:0c84947a22519e013607c9be43706dd42513f9e6ae5d39d3613ca1e142fba44d", - "sha256:20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93", - "sha256:401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1", - "sha256:52cb72a79269189d4e0dc537556f4740f7f0a9ec41c1322598799b0bdad4ef92", - "sha256:61f642bf2378713e2c2e1de73444a3778e5f0a38be6fee0fe532fe30060282ff", - "sha256:8ac7448f09ab85811607bdd21ec2464495ac8b7c66d146bf545b0f08fb9220ba", - "sha256:a36d4a9dda1f19ce6e03c9a784a2921a4b726b02e1c736600ca9c22029474394", - "sha256:a422368fc821589c228f4c49438a368831cb5bbc0eab5ebe1d7fac9dded6567b", - "sha256:e46dae94e34b085175f8abb3b0aaa7da40767865ac82c928eeb9e57e1ea8a543" - ], - "markers": "python_version >= '3.6'", - "version": "==1.5.0" - }, - "vyos": { - "file": "./python" - } - }, - "develop": { - "astroid": { - "hashes": [ - "sha256:951798f922990137ac090c53af473db7ab4e70c770e6d7fae0cec59f74411819", - "sha256:ac248253bfa4bd924a0de213707e7ebeeb3138abeb48d798784ead1e56d419d4" - ], - "markers": "python_full_version >= '3.8.0'", - "version": "==3.1.0" - }, - "coverage": { - "hashes": [ - "sha256:00838a35b882694afda09f85e469c96367daa3f3f2b097d846a7216993d37f4c", - "sha256:0513b9508b93da4e1716744ef6ebc507aff016ba115ffe8ecff744d1322a7b63", - "sha256:09c3255458533cb76ef55da8cc49ffab9e33f083739c8bd4f58e79fecfe288f7", - "sha256:09ef9199ed6653989ebbcaacc9b62b514bb63ea2f90256e71fea3ed74bd8ff6f", - "sha256:09fa497a8ab37784fbb20ab699c246053ac294d13fc7eb40ec007a5043ec91f8", - "sha256:0f9f50e7ef2a71e2fae92774c99170eb8304e3fdf9c8c3c7ae9bab3e7229c5cf", - "sha256:137eb07173141545e07403cca94ab625cc1cc6bc4c1e97b6e3846270e7e1fea0", - "sha256:1f384c3cc76aeedce208643697fb3e8437604b512255de6d18dae3f27655a384", - "sha256:201bef2eea65e0e9c56343115ba3814e896afe6d36ffd37bab783261db430f76", - "sha256:38dd60d7bf242c4ed5b38e094baf6401faa114fc09e9e6632374388a404f98e7", - "sha256:3b799445b9f7ee8bf299cfaed6f5b226c0037b74886a4e11515e569b36fe310d", - "sha256:3ea79bb50e805cd6ac058dfa3b5c8f6c040cb87fe83de10845857f5535d1db70", - "sha256:40209e141059b9370a2657c9b15607815359ab3ef9918f0196b6fccce8d3230f", - "sha256:41c9c5f3de16b903b610d09650e5e27adbfa7f500302718c9ffd1c12cf9d6818", - "sha256:54eb8d1bf7cacfbf2a3186019bcf01d11c666bd495ed18717162f7eb1e9dd00b", - "sha256:598825b51b81c808cb6f078dcb972f96af96b078faa47af7dfcdf282835baa8d", - "sha256:5fc1de20b2d4a061b3df27ab9b7c7111e9a710f10dc2b84d33a4ab25065994ec", - "sha256:623512f8ba53c422fcfb2ce68362c97945095b864cda94a92edbaf5994201083", - "sha256:690db6517f09336559dc0b5f55342df62370a48f5469fabf502db2c6d1cffcd2", - "sha256:69eb372f7e2ece89f14751fbcbe470295d73ed41ecd37ca36ed2eb47512a6ab9", - "sha256:73bfb9c09951125d06ee473bed216e2c3742f530fc5acc1383883125de76d9cd", - "sha256:742a76a12aa45b44d236815d282b03cfb1de3b4323f3e4ec933acfae08e54ade", - "sha256:7c95949560050d04d46b919301826525597f07b33beba6187d04fa64d47ac82e", - "sha256:8130a2aa2acb8788e0b56938786c33c7c98562697bf9f4c7d6e8e5e3a0501e4a", - "sha256:8a2b2b78c78293782fd3767d53e6474582f62443d0504b1554370bde86cc8227", - "sha256:8ce1415194b4a6bd0cdcc3a1dfbf58b63f910dcb7330fe15bdff542c56949f87", - "sha256:9ca28a302acb19b6af89e90f33ee3e1906961f94b54ea37de6737b7ca9d8827c", - "sha256:a4cdc86d54b5da0df6d3d3a2f0b710949286094c3a6700c21e9015932b81447e", - "sha256:aa5b1c1bfc28384f1f53b69a023d789f72b2e0ab1b3787aae16992a7ca21056c", - "sha256:aadacf9a2f407a4688d700e4ebab33a7e2e408f2ca04dbf4aef17585389eff3e", - "sha256:ae71e7ddb7a413dd60052e90528f2f65270aad4b509563af6d03d53e979feafd", - "sha256:b14706df8b2de49869ae03a5ccbc211f4041750cd4a66f698df89d44f4bd30ec", - "sha256:b1a93009cb80730c9bca5d6d4665494b725b6e8e157c1cb7f2db5b4b122ea562", - "sha256:b2991665420a803495e0b90a79233c1433d6ed77ef282e8e152a324bbbc5e0c8", - "sha256:b2c5edc4ac10a7ef6605a966c58929ec6c1bd0917fb8c15cb3363f65aa40e677", - "sha256:b4d33f418f46362995f1e9d4f3a35a1b6322cb959c31d88ae56b0298e1c22357", - "sha256:b91cbc4b195444e7e258ba27ac33769c41b94967919f10037e6355e998af255c", - "sha256:c74880fc64d4958159fbd537a091d2a585448a8f8508bf248d72112723974cbd", - "sha256:c901df83d097649e257e803be22592aedfd5182f07b3cc87d640bbb9afd50f49", - "sha256:cac99918c7bba15302a2d81f0312c08054a3359eaa1929c7e4b26ebe41e9b286", - "sha256:cc4f1358cb0c78edef3ed237ef2c86056206bb8d9140e73b6b89fbcfcbdd40e1", - "sha256:ccd341521be3d1b3daeb41960ae94a5e87abe2f46f17224ba5d6f2b8398016cf", - "sha256:ce4b94265ca988c3f8e479e741693d143026632672e3ff924f25fab50518dd51", - "sha256:cf271892d13e43bc2b51e6908ec9a6a5094a4df1d8af0bfc360088ee6c684409", - "sha256:d5ae728ff3b5401cc320d792866987e7e7e880e6ebd24433b70a33b643bb0384", - "sha256:d71eec7d83298f1af3326ce0ff1d0ea83c7cb98f72b577097f9083b20bdaf05e", - "sha256:d898fe162d26929b5960e4e138651f7427048e72c853607f2b200909794ed978", - "sha256:d89d7b2974cae412400e88f35d86af72208e1ede1a541954af5d944a8ba46c57", - "sha256:dfa8fe35a0bb90382837b238fff375de15f0dcdb9ae68ff85f7a63649c98527e", - "sha256:e0be5efd5127542ef31f165de269f77560d6cdef525fffa446de6f7e9186cfb2", - "sha256:fdfafb32984684eb03c2d83e1e51f64f0906b11e64482df3c5db936ce3839d48", - "sha256:ff7687ca3d7028d8a5f0ebae95a6e4827c5616b31a4ee1192bdfde697db110d4" - ], - "index": "pypi", - "markers": "python_version >= '3.8'", - "version": "==7.4.4" - }, - "dill": { - "hashes": [ - "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca", - "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7" - ], - "markers": "python_version >= '3.11'", - "version": "==0.3.8" - }, - "isort": { - "hashes": [ - "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109", - "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6" - ], - "markers": "python_full_version >= '3.8.0'", - "version": "==5.13.2" - }, - "lxml": { - "hashes": [ - "sha256:04ab5415bf6c86e0518d57240a96c4d1fcfc3cb370bb2ac2a732b67f579e5a04", - "sha256:057cdc6b86ab732cf361f8b4d8af87cf195a1f6dc5b0ff3de2dced242c2015e0", - "sha256:058a1308914f20784c9f4674036527e7c04f7be6fb60f5d61353545aa7fcb739", - "sha256:08802f0c56ed150cc6885ae0788a321b73505d2263ee56dad84d200cab11c07a", - "sha256:0a15438253b34e6362b2dc41475e7f80de76320f335e70c5528b7148cac253a1", - "sha256:0c3f67e2aeda739d1cc0b1102c9a9129f7dc83901226cc24dd72ba275ced4218", - "sha256:0e7259016bc4345a31af861fdce942b77c99049d6c2107ca07dc2bba2435c1d9", - "sha256:0ed777c1e8c99b63037b91f9d73a6aad20fd035d77ac84afcc205225f8f41188", - "sha256:0f5d65c39f16717a47c36c756af0fb36144069c4718824b7533f803ecdf91138", - "sha256:0f8c09ed18ecb4ebf23e02b8e7a22a05d6411911e6fabef3a36e4f371f4f2585", - "sha256:11a04306fcba10cd9637e669fd73aa274c1c09ca64af79c041aa820ea992b637", - "sha256:1ae67b4e737cddc96c99461d2f75d218bdf7a0c3d3ad5604d1f5e7464a2f9ffe", - "sha256:1c5bb205e9212d0ebddf946bc07e73fa245c864a5f90f341d11ce7b0b854475d", - "sha256:1f7785f4f789fdb522729ae465adcaa099e2a3441519df750ebdccc481d961a1", - "sha256:200e63525948e325d6a13a76ba2911f927ad399ef64f57898cf7c74e69b71095", - "sha256:21c2e6b09565ba5b45ae161b438e033a86ad1736b8c838c766146eff8ceffff9", - "sha256:2213afee476546a7f37c7a9b4ad4d74b1e112a6fafffc9185d6d21f043128c81", - "sha256:27aa20d45c2e0b8cd05da6d4759649170e8dfc4f4e5ef33a34d06f2d79075d57", - "sha256:2a66bf12fbd4666dd023b6f51223aed3d9f3b40fef06ce404cb75bafd3d89536", - "sha256:2c9d147f754b1b0e723e6afb7ba1566ecb162fe4ea657f53d2139bbf894d050a", - "sha256:2ddfe41ddc81f29a4c44c8ce239eda5ade4e7fc305fb7311759dd6229a080052", - "sha256:31e9a882013c2f6bd2f2c974241bf4ba68c85eba943648ce88936d23209a2e01", - "sha256:3249cc2989d9090eeac5467e50e9ec2d40704fea9ab72f36b034ea34ee65ca98", - "sha256:3545039fa4779be2df51d6395e91a810f57122290864918b172d5dc7ca5bb433", - "sha256:394ed3924d7a01b5bd9a0d9d946136e1c2f7b3dc337196d99e61740ed4bc6fe1", - "sha256:3a6b45da02336895da82b9d472cd274b22dc27a5cea1d4b793874eead23dd14f", - "sha256:3a74c4f27167cb95c1d4af1c0b59e88b7f3e0182138db2501c353555f7ec57f4", - "sha256:3d0c3dd24bb4605439bf91068598d00c6370684f8de4a67c2992683f6c309d6b", - "sha256:3dbe858ee582cbb2c6294dc85f55b5f19c918c2597855e950f34b660f1a5ede6", - "sha256:3dc773b2861b37b41a6136e0b72a1a44689a9c4c101e0cddb6b854016acc0aa8", - "sha256:3e183c6e3298a2ed5af9d7a356ea823bccaab4ec2349dc9ed83999fd289d14d5", - "sha256:3f7765e69bbce0906a7c74d5fe46d2c7a7596147318dbc08e4a2431f3060e306", - "sha256:417d14450f06d51f363e41cace6488519038f940676ce9664b34ebf5653433a5", - "sha256:44f6c7caff88d988db017b9b0e4ab04934f11e3e72d478031efc7edcac6c622f", - "sha256:491755202eb21a5e350dae00c6d9a17247769c64dcf62d8c788b5c135e179dc4", - "sha256:4951e4f7a5680a2db62f7f4ab2f84617674d36d2d76a729b9a8be4b59b3659be", - "sha256:52421b41ac99e9d91934e4d0d0fe7da9f02bfa7536bb4431b4c05c906c8c6919", - "sha256:530e7c04f72002d2f334d5257c8a51bf409db0316feee7c87e4385043be136af", - "sha256:533658f8fbf056b70e434dff7e7aa611bcacb33e01f75de7f821810e48d1bb66", - "sha256:5670fb70a828663cc37552a2a85bf2ac38475572b0e9b91283dc09efb52c41d1", - "sha256:56c22432809085b3f3ae04e6e7bdd36883d7258fcd90e53ba7b2e463efc7a6af", - "sha256:58278b29cb89f3e43ff3e0c756abbd1518f3ee6adad9e35b51fb101c1c1daaec", - "sha256:588008b8497667f1ddca7c99f2f85ce8511f8f7871b4a06ceede68ab62dff64b", - "sha256:59565f10607c244bc4c05c0c5fa0c190c990996e0c719d05deec7030c2aa8289", - "sha256:59689a75ba8d7ffca577aefd017d08d659d86ad4585ccc73e43edbfc7476781a", - "sha256:5aea8212fb823e006b995c4dda533edcf98a893d941f173f6c9506126188860d", - "sha256:5c670c0406bdc845b474b680b9a5456c561c65cf366f8db5a60154088c92d102", - "sha256:5ca1e8188b26a819387b29c3895c47a5e618708fe6f787f3b1a471de2c4a94d9", - "sha256:5d077bc40a1fe984e1a9931e801e42959a1e6598edc8a3223b061d30fbd26bbc", - "sha256:5d5792e9b3fb8d16a19f46aa8208987cfeafe082363ee2745ea8b643d9cc5b45", - "sha256:5dd1537e7cc06efd81371f5d1a992bd5ab156b2b4f88834ca852de4a8ea523fa", - "sha256:5ea7b6766ac2dfe4bcac8b8595107665a18ef01f8c8343f00710b85096d1b53a", - "sha256:622020d4521e22fb371e15f580d153134bfb68d6a429d1342a25f051ec72df1c", - "sha256:627402ad8dea044dde2eccde4370560a2b750ef894c9578e1d4f8ffd54000461", - "sha256:644df54d729ef810dcd0f7732e50e5ad1bd0a135278ed8d6bcb06f33b6b6f708", - "sha256:64641a6068a16201366476731301441ce93457eb8452056f570133a6ceb15fca", - "sha256:64c2baa7774bc22dd4474248ba16fe1a7f611c13ac6123408694d4cc93d66dbd", - "sha256:6588c459c5627fefa30139be4d2e28a2c2a1d0d1c265aad2ba1935a7863a4913", - "sha256:66bc5eb8a323ed9894f8fa0ee6cb3e3fb2403d99aee635078fd19a8bc7a5a5da", - "sha256:68a2610dbe138fa8c5826b3f6d98a7cfc29707b850ddcc3e21910a6fe51f6ca0", - "sha256:6935bbf153f9a965f1e07c2649c0849d29832487c52bb4a5c5066031d8b44fd5", - "sha256:6992030d43b916407c9aa52e9673612ff39a575523c5f4cf72cdef75365709a5", - "sha256:6a014510830df1475176466b6087fc0c08b47a36714823e58d8b8d7709132a96", - "sha256:6ab833e4735a7e5533711a6ea2df26459b96f9eec36d23f74cafe03631647c41", - "sha256:6cc6ee342fb7fa2471bd9b6d6fdfc78925a697bf5c2bcd0a302e98b0d35bfad3", - "sha256:6cf58416653c5901e12624e4013708b6e11142956e7f35e7a83f1ab02f3fe456", - "sha256:70a9768e1b9d79edca17890175ba915654ee1725975d69ab64813dd785a2bd5c", - "sha256:70ac664a48aa64e5e635ae5566f5227f2ab7f66a3990d67566d9907edcbbf867", - "sha256:71e97313406ccf55d32cc98a533ee05c61e15d11b99215b237346171c179c0b0", - "sha256:7221d49259aa1e5a8f00d3d28b1e0b76031655ca74bb287123ef56c3db92f213", - "sha256:74b28c6334cca4dd704e8004cba1955af0b778cf449142e581e404bd211fb619", - "sha256:764b521b75701f60683500d8621841bec41a65eb739b8466000c6fdbc256c240", - "sha256:78bfa756eab503673991bdcf464917ef7845a964903d3302c5f68417ecdc948c", - "sha256:794f04eec78f1d0e35d9e0c36cbbb22e42d370dda1609fb03bcd7aeb458c6377", - "sha256:79bd05260359170f78b181b59ce871673ed01ba048deef4bf49a36ab3e72e80b", - "sha256:7a7efd5b6d3e30d81ec68ab8a88252d7c7c6f13aaa875009fe3097eb4e30b84c", - "sha256:7c17b64b0a6ef4e5affae6a3724010a7a66bda48a62cfe0674dabd46642e8b54", - "sha256:804f74efe22b6a227306dd890eecc4f8c59ff25ca35f1f14e7482bbce96ef10b", - "sha256:853e074d4931dbcba7480d4dcab23d5c56bd9607f92825ab80ee2bd916edea53", - "sha256:857500f88b17a6479202ff5fe5f580fc3404922cd02ab3716197adf1ef628029", - "sha256:865bad62df277c04beed9478fe665b9ef63eb28fe026d5dedcb89b537d2e2ea6", - "sha256:88e22fc0a6684337d25c994381ed8a1580a6f5ebebd5ad41f89f663ff4ec2885", - "sha256:8b9c07e7a45bb64e21df4b6aa623cb8ba214dfb47d2027d90eac197329bb5e94", - "sha256:8de8f9d6caa7f25b204fc861718815d41cbcf27ee8f028c89c882a0cf4ae4134", - "sha256:8e77c69d5892cb5ba71703c4057091e31ccf534bd7f129307a4d084d90d014b8", - "sha256:9123716666e25b7b71c4e1789ec829ed18663152008b58544d95b008ed9e21e9", - "sha256:958244ad566c3ffc385f47dddde4145088a0ab893504b54b52c041987a8c1863", - "sha256:96323338e6c14e958d775700ec8a88346014a85e5de73ac7967db0367582049b", - "sha256:9676bfc686fa6a3fa10cd4ae6b76cae8be26eb5ec6811d2a325636c460da1806", - "sha256:9b0ff53900566bc6325ecde9181d89afadc59c5ffa39bddf084aaedfe3b06a11", - "sha256:9b9ec9c9978b708d488bec36b9e4c94d88fd12ccac3e62134a9d17ddba910ea9", - "sha256:9c6ad0fbf105f6bcc9300c00010a2ffa44ea6f555df1a2ad95c88f5656104817", - "sha256:9ca66b8e90daca431b7ca1408cae085d025326570e57749695d6a01454790e95", - "sha256:9e2addd2d1866fe112bc6f80117bcc6bc25191c5ed1bfbcf9f1386a884252ae8", - "sha256:a0af35bd8ebf84888373630f73f24e86bf016642fb8576fba49d3d6b560b7cbc", - "sha256:a2b44bec7adf3e9305ce6cbfa47a4395667e744097faed97abb4728748ba7d47", - "sha256:a2dfe7e2473f9b59496247aad6e23b405ddf2e12ef0765677b0081c02d6c2c0b", - "sha256:a55ee573116ba208932e2d1a037cc4b10d2c1cb264ced2184d00b18ce585b2c0", - "sha256:a7baf9ffc238e4bf401299f50e971a45bfcc10a785522541a6e3179c83eabf0a", - "sha256:a8d5c70e04aac1eda5c829a26d1f75c6e5286c74743133d9f742cda8e53b9c2f", - "sha256:a91481dbcddf1736c98a80b122afa0f7296eeb80b72344d7f45dc9f781551f56", - "sha256:ab31a88a651039a07a3ae327d68ebdd8bc589b16938c09ef3f32a4b809dc96ef", - "sha256:abc25c3cab9ec7fcd299b9bcb3b8d4a1231877e425c650fa1c7576c5107ab851", - "sha256:adfb84ca6b87e06bc6b146dc7da7623395db1e31621c4785ad0658c5028b37d7", - "sha256:afbbdb120d1e78d2ba8064a68058001b871154cc57787031b645c9142b937a62", - "sha256:afd5562927cdef7c4f5550374acbc117fd4ecc05b5007bdfa57cc5355864e0a4", - "sha256:b070bbe8d3f0f6147689bed981d19bbb33070225373338df755a46893528104a", - "sha256:b0b58fbfa1bf7367dde8a557994e3b1637294be6cf2169810375caf8571a085c", - "sha256:b560e3aa4b1d49e0e6c847d72665384db35b2f5d45f8e6a5c0072e0283430533", - "sha256:b6241d4eee5f89453307c2f2bfa03b50362052ca0af1efecf9fef9a41a22bb4f", - "sha256:b6787b643356111dfd4032b5bffe26d2f8331556ecb79e15dacb9275da02866e", - "sha256:bcbf4af004f98793a95355980764b3d80d47117678118a44a80b721c9913436a", - "sha256:beb72935a941965c52990f3a32d7f07ce869fe21c6af8b34bf6a277b33a345d3", - "sha256:bf2e2458345d9bffb0d9ec16557d8858c9c88d2d11fed53998512504cd9df49b", - "sha256:c2d35a1d047efd68027817b32ab1586c1169e60ca02c65d428ae815b593e65d4", - "sha256:c38d7b9a690b090de999835f0443d8aa93ce5f2064035dfc48f27f02b4afc3d0", - "sha256:c6f2c8372b98208ce609c9e1d707f6918cc118fea4e2c754c9f0812c04ca116d", - "sha256:c817d420c60a5183953c783b0547d9eb43b7b344a2c46f69513d5952a78cddf3", - "sha256:c8ba129e6d3b0136a0f50345b2cb3db53f6bda5dd8c7f5d83fbccba97fb5dcb5", - "sha256:c94e75445b00319c1fad60f3c98b09cd63fe1134a8a953dcd48989ef42318534", - "sha256:cc4691d60512798304acb9207987e7b2b7c44627ea88b9d77489bbe3e6cc3bd4", - "sha256:cc518cea79fd1e2f6c90baafa28906d4309d24f3a63e801d855e7424c5b34144", - "sha256:cd53553ddad4a9c2f1f022756ae64abe16da1feb497edf4d9f87f99ec7cf86bd", - "sha256:cf22b41fdae514ee2f1691b6c3cdeae666d8b7fa9434de445f12bbeee0cf48dd", - "sha256:d38c8f50ecf57f0463399569aa388b232cf1a2ffb8f0a9a5412d0db57e054860", - "sha256:d3be9b2076112e51b323bdf6d5a7f8a798de55fb8d95fcb64bd179460cdc0704", - "sha256:d4f2cc7060dc3646632d7f15fe68e2fa98f58e35dd5666cd525f3b35d3fed7f8", - "sha256:d7520db34088c96cc0e0a3ad51a4fd5b401f279ee112aa2b7f8f976d8582606d", - "sha256:d793bebb202a6000390a5390078e945bbb49855c29c7e4d56a85901326c3b5d9", - "sha256:da052e7962ea2d5e5ef5bc0355d55007407087392cf465b7ad84ce5f3e25fe0f", - "sha256:dae0ed02f6b075426accbf6b2863c3d0a7eacc1b41fb40f2251d931e50188dad", - "sha256:ddc678fb4c7e30cf830a2b5a8d869538bc55b28d6c68544d09c7d0d8f17694dc", - "sha256:df2e6f546c4df14bc81f9498bbc007fbb87669f1bb707c6138878c46b06f6510", - "sha256:e02c5175f63effbd7c5e590399c118d5db6183bbfe8e0d118bdb5c2d1b48d937", - "sha256:e196a4ff48310ba62e53a8e0f97ca2bca83cdd2fe2934d8b5cb0df0a841b193a", - "sha256:e233db59c8f76630c512ab4a4daf5a5986da5c3d5b44b8e9fc742f2a24dbd460", - "sha256:e32be23d538753a8adb6c85bd539f5fd3b15cb987404327c569dfc5fd8366e85", - "sha256:e3d30321949861404323c50aebeb1943461a67cd51d4200ab02babc58bd06a86", - "sha256:e89580a581bf478d8dcb97d9cd011d567768e8bc4095f8557b21c4d4c5fea7d0", - "sha256:e998e304036198b4f6914e6a1e2b6f925208a20e2042563d9734881150c6c246", - "sha256:ec42088248c596dbd61d4ae8a5b004f97a4d91a9fd286f632e42e60b706718d7", - "sha256:efa7b51824aa0ee957ccd5a741c73e6851de55f40d807f08069eb4c5a26b2baa", - "sha256:f0a1bc63a465b6d72569a9bba9f2ef0334c4e03958e043da1920299100bc7c08", - "sha256:f18a5a84e16886898e51ab4b1d43acb3083c39b14c8caeb3589aabff0ee0b270", - "sha256:f2a9efc53d5b714b8df2b4b3e992accf8ce5bbdfe544d74d5c6766c9e1146a3a", - "sha256:f3bbbc998d42f8e561f347e798b85513ba4da324c2b3f9b7969e9c45b10f6169", - "sha256:f42038016852ae51b4088b2862126535cc4fc85802bfe30dea3500fdfaf1864e", - "sha256:f443cdef978430887ed55112b491f670bba6462cea7a7742ff8f14b7abb98d75", - "sha256:f51969bac61441fd31f028d7b3b45962f3ecebf691a510495e5d2cd8c8092dbd", - "sha256:f8aca2e3a72f37bfc7b14ba96d4056244001ddcc18382bd0daa087fd2e68a354", - "sha256:f9737bf36262046213a28e789cc82d82c6ef19c85a0cf05e75c670a33342ac2c", - "sha256:fd6037392f2d57793ab98d9e26798f44b8b4da2f2464388588f48ac52c489ea1", - "sha256:feaa45c0eae424d3e90d78823f3828e7dc42a42f21ed420db98da2c4ecf0a2cb", - "sha256:ff097ae562e637409b429a7ac958a20aab237a0378c42dabaa1e3abf2f896e5f", - "sha256:ff46d772d5f6f73564979cd77a4fffe55c916a05f3cb70e7c9c0590059fb29ef" - ], - "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==5.2.1" - }, - "mccabe": { - "hashes": [ - "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", - "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e" - ], - "markers": "python_version >= '3.6'", - "version": "==0.7.0" - }, - "nose": { - "hashes": [ - "sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac", - "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", - "sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98" - ], - "index": "pypi", - "version": "==1.3.7" - }, - "platformdirs": { - "hashes": [ - "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068", - "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768" - ], - "markers": "python_version >= '3.8'", - "version": "==4.2.0" - }, - "pylint": { - "hashes": [ - "sha256:507a5b60953874766d8a366e8e8c7af63e058b26345cfcb5f91f89d987fd6b74", - "sha256:6a69beb4a6f63debebaab0a3477ecd0f559aa726af4954fc948c51f7a2549e23" - ], - "index": "pypi", - "markers": "python_full_version >= '3.8.0'", - "version": "==3.1.0" - }, - "tomlkit": { - "hashes": [ - "sha256:5cd82d48a3dd89dee1f9d64420aa20ae65cfbd00668d6f094d7578a78efbb77b", - "sha256:7ca1cfc12232806517a8515047ba66a19369e71edf2439d0f5824f91032b6cc3" - ], - "markers": "python_version >= '3.7'", - "version": "==0.12.4" - } - } -} From 92b468b9a0d5eee8484601568227f7c56e71b119 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Thu, 9 May 2024 16:35:48 +0300 Subject: [PATCH 081/188] sstp: T4393: Add support to configure host-name (SNI) --- data/templates/accel-ppp/sstp.config.j2 | 3 +++ interface-definitions/vpn_sstp.xml.in | 9 +++++++++ smoketest/scripts/cli/test_vpn_sstp.py | 10 ++++++++++ 3 files changed, 22 insertions(+) diff --git a/data/templates/accel-ppp/sstp.config.j2 b/data/templates/accel-ppp/sstp.config.j2 index b624f83a3a0..22fb5550637 100644 --- a/data/templates/accel-ppp/sstp.config.j2 +++ b/data/templates/accel-ppp/sstp.config.j2 @@ -42,6 +42,9 @@ accept=ssl ssl-ca-file=/run/accel-pppd/sstp-ca.pem ssl-pemfile=/run/accel-pppd/sstp-cert.pem ssl-keyfile=/run/accel-pppd/sstp-cert.key +{% if host_name is vyos_defined %} +host-name={{ host_name }} +{% endif %} {% if default_pool is vyos_defined %} ip-pool={{ default_pool }} {% endif %} diff --git a/interface-definitions/vpn_sstp.xml.in b/interface-definitions/vpn_sstp.xml.in index d23a001d5b1..d9ed1c04051 100644 --- a/interface-definitions/vpn_sstp.xml.in +++ b/interface-definitions/vpn_sstp.xml.in @@ -53,6 +53,15 @@ #include #include #include + + + Only allow connection to specified host with the same TLS SNI + + #include + + Host-name must be alphanumeric and can contain hyphens + + diff --git a/smoketest/scripts/cli/test_vpn_sstp.py b/smoketest/scripts/cli/test_vpn_sstp.py index f0695d5776d..1a3e1df6ea7 100755 --- a/smoketest/scripts/cli/test_vpn_sstp.py +++ b/smoketest/scripts/cli/test_vpn_sstp.py @@ -75,6 +75,16 @@ def test_accel_local_authentication(self): config = read_file(self._config_file) self.assertIn(f'port={port}', config) + def test_sstp_host_name(self): + host_name = 'test.vyos.io' + self.set(['host-name', host_name]) + + self.basic_config() + self.cli_commit() + + config = read_file(self._config_file) + self.assertIn(f'host-name={host_name}', config) + if __name__ == '__main__': unittest.main(verbosity=2) From 234f35d8bae71b5d33ad97cdabc236ec6b13c3a2 Mon Sep 17 00:00:00 2001 From: fett0 Date: Sat, 4 May 2024 17:07:26 +0000 Subject: [PATCH 082/188] bond: T6303: add system mac address on bond --- .../interfaces_bonding.xml.in | 12 +++++++++ python/vyos/ifconfig/bond.py | 27 +++++++++++++++++++ .../scripts/cli/test_interfaces_bonding.py | 16 +++++++++++ 3 files changed, 55 insertions(+) diff --git a/interface-definitions/interfaces_bonding.xml.in b/interface-definitions/interfaces_bonding.xml.in index 92c0911dbbc..e6baed590f6 100644 --- a/interface-definitions/interfaces_bonding.xml.in +++ b/interface-definitions/interfaces_bonding.xml.in @@ -176,6 +176,18 @@ 0 + + + System MAC address for 802.3ad + + macaddr + MAC address + + + + + + Rate in which we will ask our link partner to transmit LACPDU packets diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index c6d0f1cff15..b381ba0e1d9 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -18,6 +18,7 @@ from vyos.ifconfig.interface import Interface from vyos.utils.dict import dict_search from vyos.utils.assertion import assert_list +from vyos.utils.assertion import assert_mac from vyos.utils.assertion import assert_positive @Interface.register @@ -54,6 +55,10 @@ class BondIf(Interface): 'validate': lambda v: assert_list(v, ['slow', 'fast']), 'location': '/sys/class/net/{ifname}/bonding/lacp_rate', }, + 'bond_system_mac': { + 'validate': assert_mac, + 'location': '/sys/class/net/{ifname}/bonding/ad_actor_system', + }, 'bond_miimon': { 'validate': assert_positive, 'location': '/sys/class/net/{ifname}/bonding/miimon' @@ -385,6 +390,24 @@ def set_mode(self, mode): """ return self.set_interface('bond_mode', mode) + def set_system_mac(self, mac): + """ + In an AD system, this specifies the mac-address for the actor in + protocol packet exchanges (LACPDUs). The value cannot be NULL or + multicast. It is preferred to have the local-admin bit set for this + mac but driver does not enforce it. If the value is not given then + system defaults to using the masters' mac address as actors' system + address. + + This parameter has effect only in 802.3ad mode and is available through + SysFs interface. + + Example: + >>> from vyos.ifconfig import BondIf + >>> BondIf('bond0').set_system_mac('00:50:ab:cd:ef:01') + """ + return self.set_interface('bond_system_mac', mac) + def update(self, config): """ General helper function which works on a dictionary retrived by get_config_dict(). It's main intention is to consolidate the scattered @@ -433,6 +456,10 @@ def update(self, config): if mode == '802.3ad': self.set_lacp_rate(config.get('lacp_rate')) + # Add system mac address for 802.3ad + if mode == '802.3ad' and 'system_mac' in config: + self.set_system_mac(config.get('system_mac')) + if mode not in ['802.3ad', 'balance-tlb', 'balance-alb']: tmp = dict_search('arp_monitor.interval', config) value = tmp if (tmp != None) else '0' diff --git a/smoketest/scripts/cli/test_interfaces_bonding.py b/smoketest/scripts/cli/test_interfaces_bonding.py index 419de774ac5..091a7a3c59e 100755 --- a/smoketest/scripts/cli/test_interfaces_bonding.py +++ b/smoketest/scripts/cli/test_interfaces_bonding.py @@ -241,6 +241,22 @@ def test_bonding_uniq_member_description(self): for member in self._members: self.assertIn(member, slaves) + def test_bonding_system_mac(self): + # configure member interfaces and system-mac + system_mac = '00:50:ab:cd:ef:11' + for interface in self._interfaces: + for option in self._options.get(interface, []): + self.cli_set(self._base_path + [interface] + option.split()) + + self.cli_set(self._base_path + [interface, 'system-mac', system_mac]) + + self.cli_commit() + + # verify config + for interface in self._interfaces: + defined_mac = read_file(f'/sys/class/net/{interface}/bonding/ad_actor_system') + self.assertIn(defined_mac, system_mac) + def test_bonding_evpn_multihoming(self): id = '5' for interface in self._interfaces: From d8ddd7191d3004e886fa45a2cf9bd8dd5e7f5e14 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Fri, 10 May 2024 15:15:53 +0200 Subject: [PATCH 083/188] bond: T6303: system-mac is not allowed to be a multicast MAC address --- python/vyos/ifconfig/bond.py | 2 +- python/vyos/utils/assertion.py | 4 ++-- src/conf_mode/interfaces_bonding.py | 11 +++++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index b381ba0e1d9..f2642691578 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -56,7 +56,7 @@ class BondIf(Interface): 'location': '/sys/class/net/{ifname}/bonding/lacp_rate', }, 'bond_system_mac': { - 'validate': assert_mac, + 'validate': lambda v: assert_mac(v, test_all_zero=False), 'location': '/sys/class/net/{ifname}/bonding/ad_actor_system', }, 'bond_miimon': { diff --git a/python/vyos/utils/assertion.py b/python/vyos/utils/assertion.py index 1aaa54dffdd..c7fa220c38f 100644 --- a/python/vyos/utils/assertion.py +++ b/python/vyos/utils/assertion.py @@ -53,7 +53,7 @@ def assert_mtu(mtu, ifname): if (max_mtu and cur_mtu > max_mtu) or cur_mtu > 65536: raise ValueError(f'MTU is too small for interface "{ifname}": {mtu} > {max_mtu}') -def assert_mac(m): +def assert_mac(m, test_all_zero=True): split = m.split(':') size = len(split) @@ -74,7 +74,7 @@ def assert_mac(m): raise ValueError(f'{m} is a multicast MAC address') # overall mac address is not allowed to be 00:00:00:00:00:00 - if sum(octets) == 0: + if test_all_zero and sum(octets) == 0: raise ValueError('00:00:00:00:00:00 is not a valid MAC address') if octets[:5] == (0, 0, 94, 0, 1): diff --git a/src/conf_mode/interfaces_bonding.py b/src/conf_mode/interfaces_bonding.py index 371b219c0d9..5e5d5fba165 100755 --- a/src/conf_mode/interfaces_bonding.py +++ b/src/conf_mode/interfaces_bonding.py @@ -33,6 +33,7 @@ from vyos.ifconfig.ethernet import EthernetIf from vyos.ifconfig import Section from vyos.template import render_to_string +from vyos.utils.assertion import assert_mac from vyos.utils.dict import dict_search from vyos.utils.dict import dict_to_paths_values from vyos.utils.network import interface_exists @@ -244,6 +245,16 @@ def verify(bond): raise ConfigError('primary interface only works for mode active-backup, ' \ 'transmit-load-balance or adaptive-load-balance') + if 'system_mac' in bond: + if bond['mode'] != '802.3ad': + raise ConfigError('Actor MAC address only available in 802.3ad mode!') + + system_mac = bond['system_mac'] + try: + assert_mac(system_mac, test_all_zero=False) + except: + raise ConfigError(f'Cannot use a multicast MAC address "{system_mac}" as system-mac!') + return None def generate(bond): From 314901e7b45782fb6266b35b0e788ab7ea1404b8 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Fri, 10 May 2024 15:17:22 +0200 Subject: [PATCH 084/188] bond: T6303: must reset system-mac to 00:00:00:00:00:00 on deletion --- python/vyos/ifconfig/bond.py | 19 +++++++++++-------- .../scripts/cli/test_interfaces_bonding.py | 16 ++++++++++++++-- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/python/vyos/ifconfig/bond.py b/python/vyos/ifconfig/bond.py index f2642691578..b8ea9004965 100644 --- a/python/vyos/ifconfig/bond.py +++ b/python/vyos/ifconfig/bond.py @@ -449,18 +449,13 @@ def update(self, config): Interface(interface).set_admin_state('up') # Bonding policy/mode - default value, always present - mode = config.get('mode') - self.set_mode(mode) + self.set_mode(config['mode']) # LACPDU transmission rate - default value - if mode == '802.3ad': + if config['mode'] == '802.3ad': self.set_lacp_rate(config.get('lacp_rate')) - # Add system mac address for 802.3ad - if mode == '802.3ad' and 'system_mac' in config: - self.set_system_mac(config.get('system_mac')) - - if mode not in ['802.3ad', 'balance-tlb', 'balance-alb']: + if config['mode'] not in ['802.3ad', 'balance-tlb', 'balance-alb']: tmp = dict_search('arp_monitor.interval', config) value = tmp if (tmp != None) else '0' self.set_arp_interval(value) @@ -495,6 +490,14 @@ def update(self, config): Interface(interface).flush_addrs() self.add_port(interface) + # Add system mac address for 802.3ad - default address is all zero + # mode is always present (defaultValue) + if config['mode'] == '802.3ad': + mac = '00:00:00:00:00:00' + if 'system_mac' in config: + mac = config['system_mac'] + self.set_system_mac(mac) + # Primary device interface - must be set after 'mode' value = config.get('primary') if value: self.set_primary(value) diff --git a/smoketest/scripts/cli/test_interfaces_bonding.py b/smoketest/scripts/cli/test_interfaces_bonding.py index 091a7a3c59e..f436424b87d 100755 --- a/smoketest/scripts/cli/test_interfaces_bonding.py +++ b/smoketest/scripts/cli/test_interfaces_bonding.py @@ -243,7 +243,9 @@ def test_bonding_uniq_member_description(self): def test_bonding_system_mac(self): # configure member interfaces and system-mac + default_system_mac = '00:00:00:00:00:00' # default MAC is all zeroes system_mac = '00:50:ab:cd:ef:11' + for interface in self._interfaces: for option in self._options.get(interface, []): self.cli_set(self._base_path + [interface] + option.split()) @@ -254,8 +256,18 @@ def test_bonding_system_mac(self): # verify config for interface in self._interfaces: - defined_mac = read_file(f'/sys/class/net/{interface}/bonding/ad_actor_system') - self.assertIn(defined_mac, system_mac) + tmp = read_file(f'/sys/class/net/{interface}/bonding/ad_actor_system') + self.assertIn(tmp, system_mac) + + for interface in self._interfaces: + self.cli_delete(self._base_path + [interface, 'system-mac']) + + self.cli_commit() + + # verify default value + for interface in self._interfaces: + tmp = read_file(f'/sys/class/net/{interface}/bonding/ad_actor_system') + self.assertIn(tmp, default_system_mac) def test_bonding_evpn_multihoming(self): id = '5' From 72c95ec1df8ad7be8a715b3338001349684cafa9 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Fri, 10 May 2024 13:28:03 +0000 Subject: [PATCH 085/188] T6329: firewall: add a patch for op-mode command --- src/op_mode/firewall.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py index 442c186cc5d..3f8ba6fa169 100755 --- a/src/op_mode/firewall.py +++ b/src/op_mode/firewall.py @@ -531,9 +531,15 @@ def find_references(group_type, group_name): continue for idx, member in enumerate(members): - val = member.get('val', 'N/D') - timeout = str(member.get('timeout', 'N/D')) - expires = str(member.get('expires', 'N/D')) + if type(member) == str: + # Only member, and no timeout: + val = member + timeout = "N/D" + expires = "N/D" + else: + val = member.get('val', 'N/D') + timeout = str(member.get('timeout', 'N/D')) + expires = str(member.get('expires', 'N/D')) if args.detail: row.append(f'{val} (timeout: {timeout}, expires: {expires})') From 32658e981babffb5b7149534bd50a64d11f7c74f Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 10 May 2024 08:31:15 -0500 Subject: [PATCH 086/188] image-tools: T6327: drop boot console type ttyUSB --- data/templates/grub/grub_compat.j2 | 4 ---- data/templates/grub/grub_options.j2 | 6 ------ python/vyos/system/compat.py | 10 +++------- src/op_mode/image_installer.py | 6 +++--- 4 files changed, 6 insertions(+), 20 deletions(-) diff --git a/data/templates/grub/grub_compat.j2 b/data/templates/grub/grub_compat.j2 index d1085eec826..8fb4f71dc3e 100644 --- a/data/templates/grub/grub_compat.j2 +++ b/data/templates/grub/grub_compat.j2 @@ -14,8 +14,6 @@ KVM {%- elif type == 'ttyS' -%} Serial -{%- elif type == 'ttyUSB' -%} - USB {%- else -%} Unknown {%- endif %} @@ -25,8 +23,6 @@ console=ttyS0,{{ console_speed }} console=tty0 {%- elif type == 'ttyS' -%} console=tty0 console=ttyS0,{{ console_speed }} -{%- elif type == 'ttyUSB' -%} - console=tty0 console=ttyUSB0,115200 {%- else -%} console=tty0 console=ttyS0,{{ console_speed }} {%- endif %} diff --git a/data/templates/grub/grub_options.j2 b/data/templates/grub/grub_options.j2 index c8a1472e16b..a00bf4e37a1 100644 --- a/data/templates/grub/grub_options.j2 +++ b/data/templates/grub/grub_options.j2 @@ -33,12 +33,6 @@ submenu "Boot options" { setup_serial configfile ${prefix}/grub.cfg.d/*vyos-menu*.cfg } - menuentry "ttyUSB (USB serial)" { - set console_type="ttyUSB" - export console_type - setup_serial - configfile ${prefix}/grub.cfg.d/*vyos-menu*.cfg - } } menuentry "Enter console number" { read console_num diff --git a/python/vyos/system/compat.py b/python/vyos/system/compat.py index 1b487c1d22a..94e40d26859 100644 --- a/python/vyos/system/compat.py +++ b/python/vyos/system/compat.py @@ -220,14 +220,8 @@ def get_default(data: dict, root_dir: str = '') -> Union[int, None]: sublist = list(filter(lambda x: (x.get('version') == image_name and x.get('console_type') == console_type and - x.get('console_num') == console_num and x.get('bootmode') == 'normal'), menu_entries)) - # legacy images added with legacy tools omitted 'ttyUSB'; if entry not - # available, default to initial entry of version - if not sublist: - sublist = list(filter(lambda x: x.get('version') == image_name, - menu_entries)) if sublist: return menu_entries.index(sublist[0]) @@ -268,7 +262,9 @@ def update_version_list(root_dir: str = '') -> list[dict]: add = list(set(current_versions) - set(menu_versions)) for ver in add: last = menu_entries[0].get('version') - new = deepcopy(list(filter(lambda x: x.get('version') == last, + # copy legacy format of menu entries; ignore deprecated ttyUSB + new = deepcopy(list(filter(lambda x: (x.get('version') == last and + x.get('console_type') != 'ttyUSB'), menu_entries))) for e in new: boot_opts = grub.get_boot_opts(ver) diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index ba0e3b6db2a..fdfaa1e1781 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -65,7 +65,7 @@ MSG_INPUT_PASSWORD_CONFIRM: str = 'Please confirm password for the "vyos" user:' MSG_INPUT_ROOT_SIZE_ALL: str = 'Would you like to use all the free space on the drive?' MSG_INPUT_ROOT_SIZE_SET: str = 'Please specify the size (in GB) of the root partition (min is 1.5 GB)?' -MSG_INPUT_CONSOLE_TYPE: str = 'What console should be used by default? (K: KVM, S: Serial, U: USB-Serial)?' +MSG_INPUT_CONSOLE_TYPE: str = 'What console should be used by default? (K: KVM, S: Serial)?' MSG_INPUT_COPY_DATA: str = 'Would you like to copy data to the new image?' MSG_INPUT_CHOOSE_COPY_DATA: str = 'From which image would you like to save config information?' MSG_INPUT_COPY_ENC_DATA: str = 'Would you like to copy the encrypted config to the new image?' @@ -710,8 +710,8 @@ def install_image() -> None: # ask for default console console_type: str = ask_input(MSG_INPUT_CONSOLE_TYPE, default='K', - valid_responses=['K', 'S', 'U']) - console_dict: dict[str, str] = {'K': 'tty', 'S': 'ttyS', 'U': 'ttyUSB'} + valid_responses=['K', 'S']) + console_dict: dict[str, str] = {'K': 'tty', 'S': 'ttyS'} config_boot_list = ['/opt/vyatta/etc/config/config.boot', '/opt/vyatta/etc/config.boot.default'] From eb281199ba35de52a8a97146dfc063e557755648 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 10 May 2024 09:06:39 -0500 Subject: [PATCH 087/188] image-tools: T6184: add op-mode set boot-console --- op-mode-definitions/system-image.xml.in | 9 ++++++++ src/op_mode/image_manager.py | 28 +++++++++++++++++++++++-- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/op-mode-definitions/system-image.xml.in b/op-mode-definitions/system-image.xml.in index 7b5260b4e3e..44b055be6f7 100644 --- a/op-mode-definitions/system-image.xml.in +++ b/op-mode-definitions/system-image.xml.in @@ -72,6 +72,15 @@ Set system operational parameters + + + Set system console type at boot + + + + + sudo ${vyos_op_scripts_dir}/image_manager.py --action set_console_type --console-type "${4}" + Set system image parameters diff --git a/src/op_mode/image_manager.py b/src/op_mode/image_manager.py index 1cfb5f5a1aa..fb4286dbc2d 100755 --- a/src/op_mode/image_manager.py +++ b/src/op_mode/image_manager.py @@ -21,7 +21,7 @@ from pathlib import Path from shutil import rmtree from sys import exit -from typing import Optional +from typing import Optional, Literal, TypeAlias, get_args from vyos.system import disk, grub, image, compat from vyos.utils.io import ask_yes_no, select_entry @@ -33,6 +33,8 @@ MSG_DELETE_IMAGE_RUNNING: str = 'Currently running image cannot be deleted; reboot into another image first' MSG_DELETE_IMAGE_DEFAULT: str = 'Default image cannot be deleted; set another image as default first' +ConsoleType: TypeAlias = Literal['tty', 'ttyS'] + def annotate_list(images_list: list[str]) -> list[str]: """Annotate list of images with additional info @@ -202,6 +204,15 @@ def rename_image(name_old: str, name_new: str) -> None: exit(f'Unable to rename the encrypted config for "{name_old}" to "{name_new}": {err}') +@compat.grub_cfg_update +def set_console_type(console_type: ConsoleType) -> None: + console_choice = get_args(ConsoleType) + if console_type not in console_choice: + exit(f'console type \'{console_type}\' not available') + + grub.set_console_type(console_type) + + def list_images() -> None: """Print list of available images for CLI hints""" images_list: list[str] = grub.version_list() @@ -209,6 +220,13 @@ def list_images() -> None: print(image_name) +def list_console_types() -> None: + """Print list of console types for CLI hints""" + console_types: list[str] = list(get_args(ConsoleType)) + for console_type in console_types: + print(console_type) + + def parse_arguments() -> Namespace: """Parse arguments @@ -217,7 +235,8 @@ def parse_arguments() -> Namespace: """ parser: ArgumentParser = ArgumentParser(description='Manage system images') parser.add_argument('--action', - choices=['delete', 'set', 'rename', 'list'], + choices=['delete', 'set', 'set_console_type', + 'rename', 'list', 'list_console_types'], required=True, help='action to perform with an image') parser.add_argument('--no-prompt', action='store_true', @@ -227,6 +246,7 @@ def parse_arguments() -> Namespace: help= 'a name of an image to add, delete, install, rename, or set as default') parser.add_argument('--image-new-name', help='a new name for image') + parser.add_argument('--console-type', help='console type for boot') args: Namespace = parser.parse_args() # Validate arguments if args.action == 'rename' and (not args.image_name or @@ -243,10 +263,14 @@ def parse_arguments() -> Namespace: delete_image(args.image_name, args.no_prompt) if args.action == 'set': set_image(args.image_name) + if args.action == 'set_console_type': + set_console_type(args.console_type) if args.action == 'rename': rename_image(args.image_name, args.image_new_name) if args.action == 'list': list_images() + if args.action == 'list_console_types': + list_console_types() exit() From 428d03e47e7d01b08ccb8cf1acc0ab8a53275286 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 10 May 2024 09:20:38 -0500 Subject: [PATCH 088/188] image-tools: T6176: add console hint during image install --- src/op_mode/image_installer.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index fdfaa1e1781..22bdc26fbd2 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -23,6 +23,8 @@ from glob import glob from sys import exit from os import environ +from os import readlink +from os import getpid, getppid from typing import Union from urllib.parse import urlparse from passlib.hosts import linux_context @@ -614,6 +616,20 @@ def copy_ssh_host_keys() -> bool: return False +def console_hint() -> str: + pid = getppid() if 'SUDO_USER' in environ else getpid() + try: + path = readlink(f'/proc/{pid}/fd/1') + except OSError: + path = '/dev/tty' + + name = Path(path).name + if name == 'ttyS0': + return 'S' + else: + return 'K' + + def cleanup(mounts: list[str] = [], remove_items: list[str] = []) -> None: """Clean up after installation From ecee10d0512daff3034499d1163463d1390e35d4 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 10 May 2024 14:31:50 -0500 Subject: [PATCH 089/188] image-tools: T6327: prune, instead of ignore, menu entries with ttyUSB --- python/vyos/system/compat.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/vyos/system/compat.py b/python/vyos/system/compat.py index 94e40d26859..d35bddea2d2 100644 --- a/python/vyos/system/compat.py +++ b/python/vyos/system/compat.py @@ -247,6 +247,10 @@ def update_version_list(root_dir: str = '') -> list[dict]: menu_entries = parse_menuentries(grub_cfg_main) menu_versions = find_versions(menu_entries) + # remove deprecated console-type ttyUSB + menu_entries = list(filter(lambda x: x.get('console_type') != 'ttyUSB', + menu_entries)) + # get list of versions added/removed by image-tools current_versions = grub.version_list(root_dir) @@ -262,9 +266,7 @@ def update_version_list(root_dir: str = '') -> list[dict]: add = list(set(current_versions) - set(menu_versions)) for ver in add: last = menu_entries[0].get('version') - # copy legacy format of menu entries; ignore deprecated ttyUSB - new = deepcopy(list(filter(lambda x: (x.get('version') == last and - x.get('console_type') != 'ttyUSB'), + new = deepcopy(list(filter(lambda x: x.get('version') == last, menu_entries))) for e in new: boot_opts = grub.get_boot_opts(ver) From 0eb09b81f763a62684a7be905267f081f9d6aeb1 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 10 May 2024 14:33:40 -0500 Subject: [PATCH 090/188] image-tools: T6176: use console_hint as default --- src/op_mode/image_installer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 22bdc26fbd2..0d2d7076c50 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -725,7 +725,7 @@ def install_image() -> None: # ask for default console console_type: str = ask_input(MSG_INPUT_CONSOLE_TYPE, - default='K', + default=console_hint(), valid_responses=['K', 'S']) console_dict: dict[str, str] = {'K': 'tty', 'S': 'ttyS'} From 5565f27d15c5e7378e94aae8db8a894a12e25d7b Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 11 May 2024 14:12:31 +0200 Subject: [PATCH 091/188] ethernet: T6306: add support for EVPN MH uplink/core tracking When all the underlay links go down the PE no longer has access to the VxLAN +overlay. To prevent blackholing of traffic the server/ES links are protodowned on the PE. A link can be setup for uplink tracking via the following configuration: set interfaces ethernet eth0 evpn uplink --- .../include/interface/evpn-mh-uplink.xml.i | 8 ++++++++ interface-definitions/interfaces_bonding.xml.in | 7 +------ .../interfaces_ethernet.xml.in | 8 ++++++++ .../scripts/cli/test_interfaces_ethernet.py | 10 ++++++++++ src/conf_mode/interfaces_ethernet.py | 17 +++++++++++++++++ 5 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 interface-definitions/include/interface/evpn-mh-uplink.xml.i diff --git a/interface-definitions/include/interface/evpn-mh-uplink.xml.i b/interface-definitions/include/interface/evpn-mh-uplink.xml.i new file mode 100644 index 00000000000..5f7fe1b7f43 --- /dev/null +++ b/interface-definitions/include/interface/evpn-mh-uplink.xml.i @@ -0,0 +1,8 @@ + + + + Uplink to the VXLAN core + + + + diff --git a/interface-definitions/interfaces_bonding.xml.in b/interface-definitions/interfaces_bonding.xml.in index 92c0911dbbc..d503760e6b5 100644 --- a/interface-definitions/interfaces_bonding.xml.in +++ b/interface-definitions/interfaces_bonding.xml.in @@ -102,12 +102,7 @@ - - - Uplink to the VXLAN core - - - + #include diff --git a/interface-definitions/interfaces_ethernet.xml.in b/interface-definitions/interfaces_ethernet.xml.in index 4e55bac7cf1..89f990d411d 100644 --- a/interface-definitions/interfaces_ethernet.xml.in +++ b/interface-definitions/interfaces_ethernet.xml.in @@ -57,6 +57,14 @@ auto #include + + + EVPN Multihoming + + + #include + + #include #include #include diff --git a/smoketest/scripts/cli/test_interfaces_ethernet.py b/smoketest/scripts/cli/test_interfaces_ethernet.py index 8f387b23dd7..4843a40da1b 100755 --- a/smoketest/scripts/cli/test_interfaces_ethernet.py +++ b/smoketest/scripts/cli/test_interfaces_ethernet.py @@ -354,5 +354,15 @@ def test_ethtool_flow_control(self): out = loads(out) self.assertFalse(out[0]['autonegotiate']) + def test_ethtool_evpn_uplink_tarcking(self): + for interface in self._interfaces: + self.cli_set(self._base_path + [interface, 'evpn', 'uplink']) + + self.cli_commit() + + for interface in self._interfaces: + frrconfig = self.getFRRconfig(f'interface {interface}', daemon='zebra') + self.assertIn(f' evpn mh uplink', frrconfig) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/interfaces_ethernet.py b/src/conf_mode/interfaces_ethernet.py index 6da7e6a6980..54d0669cb18 100755 --- a/src/conf_mode/interfaces_ethernet.py +++ b/src/conf_mode/interfaces_ethernet.py @@ -41,6 +41,7 @@ from vyos.pki import load_certificate from vyos.pki import wrap_private_key from vyos.template import render +from vyos.template import render_to_string from vyos.utils.process import call from vyos.utils.dict import dict_search from vyos.utils.dict import dict_to_paths_values @@ -48,6 +49,7 @@ from vyos.utils.dict import dict_delete from vyos.utils.file import write_file from vyos import ConfigError +from vyos import frr from vyos import airbag airbag.enable() @@ -389,6 +391,10 @@ def generate(ethernet): write_file(ca_cert_file_path, '\n'.join(ca_chains)) + ethernet['frr_zebra_config'] = '' + if 'deleted' not in ethernet: + ethernet['frr_zebra_config'] = render_to_string('frr/evpn.mh.frr.j2', ethernet) + return None def apply(ethernet): @@ -407,6 +413,17 @@ def apply(ethernet): call(f'systemctl {eapol_action} wpa_supplicant-wired@{ifname}') + zebra_daemon = 'zebra' + # Save original configuration prior to starting any commit actions + frr_cfg = frr.FRRConfig() + + # The route-map used for the FIB (zebra) is part of the zebra daemon + frr_cfg.load_configuration(zebra_daemon) + frr_cfg.modify_section(f'^interface {ifname}', stop_pattern='^exit', remove_stop_mark=True) + if 'frr_zebra_config' in ethernet: + frr_cfg.add_before(frr.default_add_before, ethernet['frr_zebra_config']) + frr_cfg.commit_configuration(zebra_daemon) + if __name__ == '__main__': try: c = get_config() From b705adc40b761e338026b938d80398fdb281a197 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 12 May 2024 08:56:54 +0200 Subject: [PATCH 092/188] T6329: firewall: use isinstance() in op-mode script --- src/op_mode/firewall.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/op_mode/firewall.py b/src/op_mode/firewall.py index 3f8ba6fa169..15fbb65a2ce 100755 --- a/src/op_mode/firewall.py +++ b/src/op_mode/firewall.py @@ -531,7 +531,7 @@ def find_references(group_type, group_name): continue for idx, member in enumerate(members): - if type(member) == str: + if isinstance(member, str): # Only member, and no timeout: val = member timeout = "N/D" From 549089a970e39d1ea09c10af5eaf8f696dd19d40 Mon Sep 17 00:00:00 2001 From: Maxime THIEBAUT <46688461+0xThiebaut@users.noreply.github.com> Date: Wed, 1 May 2024 22:16:03 +0200 Subject: [PATCH 093/188] suricata: T751: Initial support for suricata --- data/templates/ids/suricata.j2 | 1280 +++++++++++++++++ data/templates/ids/suricata_logrotate.j2 | 17 + debian/control | 2 + .../service_ids_suricata.xml.in | 250 ++++ op-mode-definitions/suricata.xml.in | 23 + python/vyos/base.py | 2 +- src/conf_mode/service_ids_suricata.py | 190 +++ .../suricata.service.d/10-override.conf | 9 + src/validators/port-range-exclude | 7 + 9 files changed, 1779 insertions(+), 1 deletion(-) create mode 100644 data/templates/ids/suricata.j2 create mode 100644 data/templates/ids/suricata_logrotate.j2 create mode 100644 interface-definitions/service_ids_suricata.xml.in create mode 100644 op-mode-definitions/suricata.xml.in create mode 100755 src/conf_mode/service_ids_suricata.py create mode 100644 src/etc/systemd/system/suricata.service.d/10-override.conf create mode 100755 src/validators/port-range-exclude diff --git a/data/templates/ids/suricata.j2 b/data/templates/ids/suricata.j2 new file mode 100644 index 00000000000..1bd90b67f21 --- /dev/null +++ b/data/templates/ids/suricata.j2 @@ -0,0 +1,1280 @@ +%YAML 1.1 +--- + +# Suricata configuration file. In addition to the comments describing all +# options in this file, full documentation can be found at: +# https://suricata.readthedocs.io/en/latest/configuration/suricata-yaml.html +# +# This configuration file generated by: +# Suricata 6.0.10 + +## +## Step 1: Inform Suricata about your network +## + +vars: + # more specific is better for alert accuracy and performance + address-groups: +{% for (name, value) in suricata['address-group'] %} + {{ name }}: "[{{ value | join(',') }}]" +{% endfor %} + + port-groups: +{% for (name, value) in suricata['port-group'] %} + {{ name }}: "[{{ value | join(',') }}]" +{% endfor %} + +## +## Step 2: Select outputs to enable +## + +# The default logging directory. Any log or output file will be +# placed here if it's not specified with a full path name. This can be +# overridden with the -l command line parameter. +default-log-dir: /var/log/suricata/ + +# Configure the type of alert (and other) logging you would like. +{% if suricata.log is vyos_defined %} +outputs: +{% if suricata.log.eve is vyos_defined %} + # Extensible Event Format (nicknamed EVE) event log in JSON format + - eve-log: + enabled: yes + filetype: {{ suricata.log.eve.filetype }} #regular|syslog|unix_dgram|unix_stream|redis + filename: {{ suricata.log.eve.filename }} + + types: +{% if suricata.log.eve.type is not vyos_defined or "alert" in suricata.log.eve.type %} + - alert: + tagged-packets: yes +{% endif %} +{% if "http" in suricata.log.eve.type %} + - http: + enabled: yes + extended: yes +{% endif %} +{% if "tls" in suricata.log.eve.type %} + - tls: + enabled: yes + extended: yes # enable this for extended logging information +{% endif %} +{% for protocol in suricata.log.eve.type %} +{% if protocol not in ["alert","http","tls"] %} + - {{ protocol }}: + enabled: yes +{% endif %} +{% endfor %} +{% endif %} +{% endif %} + +## +## Step 3: Configure common capture settings +## +## See "Advanced Capture Options" below for more options, including Netmap +## and PF_RING. +## + +# Linux high speed capture support +af-packet: +{% for interface in suricata.interface %} + - interface: {{ interface }} + # Default clusterid. AF_PACKET will load balance packets based on flow. + cluster-id: 99 + # Default AF_PACKET cluster type. AF_PACKET can load balance per flow or per hash. + # This is only supported for Linux kernel > 3.1 + # possible value are: + # * cluster_flow: all packets of a given flow are sent to the same socket + # * cluster_cpu: all packets treated in kernel by a CPU are sent to the same socket + # * cluster_qm: all packets linked by network card to a RSS queue are sent to the same + # socket. Requires at least Linux 3.14. + # * cluster_ebpf: eBPF file load balancing. See doc/userguide/capture-hardware/ebpf-xdp.rst for + # more info. + # Recommended modes are cluster_flow on most boxes and cluster_cpu or cluster_qm on system + # with capture card using RSS (requires cpu affinity tuning and system IRQ tuning) + cluster-type: cluster_flow + # In some fragmentation cases, the hash can not be computed. If "defrag" is set + # to yes, the kernel will do the needed defragmentation before sending the packets. + defrag: yes +{% endfor %} + +# Cross platform libpcap capture support +pcap: +{% for interface in suricata.interface %} + - interface: {{ interface }} +{% endfor %} + +# Settings for reading pcap files +pcap-file: + # Possible values are: + # - yes: checksum validation is forced + # - no: checksum validation is disabled + # - auto: Suricata uses a statistical approach to detect when + # checksum off-loading is used. (default) + # Warning: 'checksum-validation' must be set to yes to have checksum tested + checksum-checks: auto + +# See "Advanced Capture Options" below for more options, including Netmap +# and PF_RING. + + +## +## Step 4: App Layer Protocol configuration +## + +# Configure the app-layer parsers. +# +# The error-policy setting applies to all app-layer parsers. Values can be +# "drop-flow", "pass-flow", "bypass", "drop-packet", "pass-packet", "reject" or +# "ignore" (the default). +# +# The protocol's section details each protocol. +# +# The option "enabled" takes 3 values - "yes", "no", "detection-only". +# "yes" enables both detection and the parser, "no" disables both, and +# "detection-only" enables protocol detection only (parser disabled). +app-layer: + # error-policy: ignore + protocols: + rfb: + enabled: yes + detection-ports: + dp: 5900, 5901, 5902, 5903, 5904, 5905, 5906, 5907, 5908, 5909 + # MQTT, disabled by default. + mqtt: + enabled: yes + # max-msg-length: 1mb + # subscribe-topic-match-limit: 100 + # unsubscribe-topic-match-limit: 100 + # Maximum number of live MQTT transactions per flow + # max-tx: 4096 + krb5: + enabled: yes + snmp: + enabled: yes + ikev2: + enabled: yes + tls: + enabled: yes + detection-ports: + dp: 443 + + # Generate JA3 fingerprint from client hello. If not specified it + # will be disabled by default, but enabled if rules require it. + #ja3-fingerprints: auto + + # What to do when the encrypted communications start: + # - default: keep tracking TLS session, check for protocol anomalies, + # inspect tls_* keywords. Disables inspection of unmodified + # 'content' signatures. + # - bypass: stop processing this flow as much as possible. No further + # TLS parsing and inspection. Offload flow bypass to kernel + # or hardware if possible. + # - full: keep tracking and inspection as normal. Unmodified content + # keyword signatures are inspected as well. + # + # For best performance, select 'bypass'. + # + #encryption-handling: default + + dcerpc: + enabled: yes + ftp: + enabled: yes + # memcap: 64mb + rdp: + enabled: yes + ssh: + enabled: yes + #hassh: yes + # HTTP2: Experimental HTTP 2 support. Disabled by default. + http2: + enabled: no + # use http keywords on HTTP2 traffic + http1-rules: no + smtp: + enabled: yes + raw-extraction: no + # Configure SMTP-MIME Decoder + mime: + # Decode MIME messages from SMTP transactions + # (may be resource intensive) + # This field supersedes all others because it turns the entire + # process on or off + decode-mime: yes + + # Decode MIME entity bodies (ie. Base64, quoted-printable, etc.) + decode-base64: yes + decode-quoted-printable: yes + + # Maximum bytes per header data value stored in the data structure + # (default is 2000) + header-value-depth: 2000 + + # Extract URLs and save in state data structure + extract-urls: yes + # Set to yes to compute the md5 of the mail body. You will then + # be able to journalize it. + body-md5: no + # Configure inspected-tracker for file_data keyword + inspected-tracker: + content-limit: 100000 + content-inspect-min-size: 32768 + content-inspect-window: 4096 + imap: + enabled: detection-only + smb: + enabled: yes + detection-ports: + dp: 139, 445 + + # Stream reassembly size for SMB streams. By default track it completely. + #stream-depth: 0 + + nfs: + enabled: yes + tftp: + enabled: yes + dns: + tcp: + enabled: yes + detection-ports: + dp: 53 + udp: + enabled: yes + detection-ports: + dp: 53 + http: + enabled: yes + # memcap: Maximum memory capacity for HTTP + # Default is unlimited, values can be 64mb, e.g. + + # default-config: Used when no server-config matches + # personality: List of personalities used by default + # request-body-limit: Limit reassembly of request body for inspection + # by http_client_body & pcre /P option. + # response-body-limit: Limit reassembly of response body for inspection + # by file_data, http_server_body & pcre /Q option. + # + # For advanced options, see the user guide + + + # server-config: List of server configurations to use if address matches + # address: List of IP addresses or networks for this block + # personality: List of personalities used by this block + # + # Then, all the fields from default-config can be overloaded + # + # Currently Available Personalities: + # Minimal, Generic, IDS (default), IIS_4_0, IIS_5_0, IIS_5_1, IIS_6_0, + # IIS_7_0, IIS_7_5, Apache_2 + libhtp: + default-config: + personality: IDS + + # Can be specified in kb, mb, gb. Just a number indicates + # it's in bytes. + request-body-limit: 100kb + response-body-limit: 100kb + + # inspection limits + request-body-minimal-inspect-size: 32kb + request-body-inspect-window: 4kb + response-body-minimal-inspect-size: 40kb + response-body-inspect-window: 16kb + + # response body decompression (0 disables) + response-body-decompress-layer-limit: 2 + + # auto will use http-body-inline mode in IPS mode, yes or no set it statically + http-body-inline: auto + + # Decompress SWF files. + # Two types: 'deflate', 'lzma', 'both' will decompress deflate and lzma + # compress-depth: + # Specifies the maximum amount of data to decompress, + # set 0 for unlimited. + # decompress-depth: + # Specifies the maximum amount of decompressed data to obtain, + # set 0 for unlimited. + swf-decompression: + enabled: yes + type: both + compress-depth: 100kb + decompress-depth: 100kb + + # Use a random value for inspection sizes around the specified value. + # This lowers the risk of some evasion techniques but could lead + # to detection change between runs. It is set to 'yes' by default. + #randomize-inspection-sizes: yes + # If "randomize-inspection-sizes" is active, the value of various + # inspection size will be chosen from the [1 - range%, 1 + range%] + # range + # Default value of "randomize-inspection-range" is 10. + #randomize-inspection-range: 10 + + # decoding + double-decode-path: no + double-decode-query: no + + # Can enable LZMA decompression + #lzma-enabled: false + # Memory limit usage for LZMA decompression dictionary + # Data is decompressed until dictionary reaches this size + #lzma-memlimit: 1mb + # Maximum decompressed size with a compression ratio + # above 2048 (only LZMA can reach this ratio, deflate cannot) + #compression-bomb-limit: 1mb + # Maximum time spent decompressing a single transaction in usec + #decompression-time-limit: 100000 + + server-config: + + #- apache: + # address: [192.168.1.0/24, 127.0.0.0/8, "::1"] + # personality: Apache_2 + # # Can be specified in kb, mb, gb. Just a number indicates + # # it's in bytes. + # request-body-limit: 4096 + # response-body-limit: 4096 + # double-decode-path: no + # double-decode-query: no + + #- iis7: + # address: + # - 192.168.0.0/24 + # - 192.168.10.0/24 + # personality: IIS_7_0 + # # Can be specified in kb, mb, gb. Just a number indicates + # # it's in bytes. + # request-body-limit: 4096 + # response-body-limit: 4096 + # double-decode-path: no + # double-decode-query: no + + # Note: Modbus probe parser is minimalist due to the limited usage in the field. + # Only Modbus message length (greater than Modbus header length) + # and protocol ID (equal to 0) are checked in probing parser + # It is important to enable detection port and define Modbus port + # to avoid false positives + modbus: + # How many unanswered Modbus requests are considered a flood. + # If the limit is reached, the app-layer-event:modbus.flooded; will match. + #request-flood: 500 + + enabled: no + detection-ports: + dp: 502 + # According to MODBUS Messaging on TCP/IP Implementation Guide V1.0b, it + # is recommended to keep the TCP connection opened with a remote device + # and not to open and close it for each MODBUS/TCP transaction. In that + # case, it is important to set the depth of the stream reassembling as + # unlimited (stream.reassembly.depth: 0) + + # Stream reassembly size for modbus. By default track it completely. + stream-depth: 0 + + # DNP3 + dnp3: + enabled: no + detection-ports: + dp: 20000 + + # SCADA EtherNet/IP and CIP protocol support + enip: + enabled: no + detection-ports: + dp: 44818 + sp: 44818 + + ntp: + enabled: yes + + dhcp: + enabled: yes + + sip: + enabled: yes + +# Limit for the maximum number of asn1 frames to decode (default 256) +asn1-max-frames: 256 + +# Datasets default settings +# datasets: +# # Default fallback memcap and hashsize values for datasets in case these +# # were not explicitly defined. +# defaults: +# memcap: 100mb +# hashsize: 2048 + +############################################################################## +## +## Advanced settings below +## +############################################################################## + +## +## Run Options +## + +# Run Suricata with a specific user-id and group-id: +#run-as: +# user: suri +# group: suri + +# Some logging modules will use that name in event as identifier. The default +# value is the hostname +#sensor-name: suricata + +# Default location of the pid file. The pid file is only used in +# daemon mode (start Suricata with -D). If not running in daemon mode +# the --pidfile command line option must be used to create a pid file. +#pid-file: /var/run/suricata.pid + +# Daemon working directory +# Suricata will change directory to this one if provided +# Default: "/" +#daemon-directory: "/" + +# Umask. +# Suricata will use this umask if it is provided. By default it will use the +# umask passed on by the shell. +#umask: 022 + +# Suricata core dump configuration. Limits the size of the core dump file to +# approximately max-dump. The actual core dump size will be a multiple of the +# page size. Core dumps that would be larger than max-dump are truncated. On +# Linux, the actual core dump size may be a few pages larger than max-dump. +# Setting max-dump to 0 disables core dumping. +# Setting max-dump to 'unlimited' will give the full core dump file. +# On 32-bit Linux, a max-dump value >= ULONG_MAX may cause the core dump size +# to be 'unlimited'. + +coredump: + max-dump: unlimited + +# If the Suricata box is a router for the sniffed networks, set it to 'router'. If +# it is a pure sniffing setup, set it to 'sniffer-only'. +# If set to auto, the variable is internally switched to 'router' in IPS mode +# and 'sniffer-only' in IDS mode. +# This feature is currently only used by the reject* keywords. +host-mode: auto + +# Number of packets preallocated per thread. The default is 1024. A higher number +# will make sure each CPU will be more easily kept busy, but may negatively +# impact caching. +#max-pending-packets: 1024 + +# Runmode the engine should use. Please check --list-runmodes to get the available +# runmodes for each packet acquisition method. Default depends on selected capture +# method. 'workers' generally gives best performance. +#runmode: autofp + +# Specifies the kind of flow load balancer used by the flow pinned autofp mode. +# +# Supported schedulers are: +# +# hash - Flow assigned to threads using the 5-7 tuple hash. +# ippair - Flow assigned to threads using addresses only. +# +#autofp-scheduler: hash + +# Preallocated size for each packet. Default is 1514 which is the classical +# size for pcap on Ethernet. You should adjust this value to the highest +# packet size (MTU + hardware header) on your system. +#default-packet-size: 1514 + +# Unix command socket that can be used to pass commands to Suricata. +# An external tool can then connect to get information from Suricata +# or trigger some modifications of the engine. Set enabled to yes +# to activate the feature. In auto mode, the feature will only be +# activated in live capture mode. You can use the filename variable to set +# the file name of the socket. +unix-command: + enabled: yes + filename: /run/suricata/suricata.socket + +# Magic file. The extension .mgc is added to the value here. +#magic-file: /usr/share/file/magic +#magic-file: + +# GeoIP2 database file. Specify path and filename of GeoIP2 database +# if using rules with "geoip" rule option. +#geoip-database: /usr/local/share/GeoLite2/GeoLite2-Country.mmdb + +legacy: + uricontent: enabled + +## +## Detection settings +## + +# Set the order of alerts based on actions +# The default order is pass, drop, reject, alert +# action-order: +# - pass +# - drop +# - reject +# - alert + +# Define maximum number of possible alerts that can be triggered for the same +# packet. Default is 15 +#packet-alert-max: 15 + +# IP Reputation +#reputation-categories-file: /etc/suricata/iprep/categories.txt +#default-reputation-path: /etc/suricata/iprep +#reputation-files: +# - reputation.list + +# When run with the option --engine-analysis, the engine will read each of +# the parameters below, and print reports for each of the enabled sections +# and exit. The reports are printed to a file in the default log dir +# given by the parameter "default-log-dir", with engine reporting +# subsection below printing reports in its own report file. +engine-analysis: + # enables printing reports for fast-pattern for every rule. + rules-fast-pattern: yes + # enables printing reports for each rule + rules: yes + +#recursion and match limits for PCRE where supported +pcre: + match-limit: 3500 + match-limit-recursion: 1500 + +## +## Advanced Traffic Tracking and Reconstruction Settings +## + +# Host specific policies for defragmentation and TCP stream +# reassembly. The host OS lookup is done using a radix tree, just +# like a routing table so the most specific entry matches. +host-os-policy: + # Make the default policy windows. + windows: [0.0.0.0/0] + bsd: [] + bsd-right: [] + old-linux: [] + linux: [] + old-solaris: [] + solaris: [] + hpux10: [] + hpux11: [] + irix: [] + macos: [] + vista: [] + windows2k3: [] + +# Defrag settings: + +# The memcap-policy value can be "drop-flow", "pass-flow", "bypass", +# "drop-packet", "pass-packet", "reject" or "ignore" (which is the default). +defrag: + memcap: 32mb + # memcap-policy: ignore + hash-size: 65536 + trackers: 65535 # number of defragmented flows to follow + max-frags: 65535 # number of fragments to keep (higher than trackers) + prealloc: yes + timeout: 60 + +# Enable defrag per host settings +# host-config: +# +# - dmz: +# timeout: 30 +# address: [192.168.1.0/24, 127.0.0.0/8, 1.1.1.0/24, 2.2.2.0/24, "1.1.1.1", "2.2.2.2", "::1"] +# +# - lan: +# timeout: 45 +# address: +# - 192.168.0.0/24 +# - 192.168.10.0/24 +# - 172.16.14.0/24 + +# Flow settings: +# By default, the reserved memory (memcap) for flows is 32MB. This is the limit +# for flow allocation inside the engine. You can change this value to allow +# more memory usage for flows. +# The hash-size determines the size of the hash used to identify flows inside +# the engine, and by default the value is 65536. +# At startup, the engine can preallocate a number of flows, to get better +# performance. The number of flows preallocated is 10000 by default. +# emergency-recovery is the percentage of flows that the engine needs to +# prune before clearing the emergency state. The emergency state is activated +# when the memcap limit is reached, allowing new flows to be created, but +# pruning them with the emergency timeouts (they are defined below). +# If the memcap is reached, the engine will try to prune flows +# with the default timeouts. If it doesn't find a flow to prune, it will set +# the emergency bit and it will try again with more aggressive timeouts. +# If that doesn't work, then it will try to kill the oldest flows using +# last time seen flows. +# The memcap can be specified in kb, mb, gb. Just a number indicates it's +# in bytes. +# The memcap-policy can be "drop-flow", "pass-flow", "bypass", "drop-packet", +# "pass-packet", "reject" or "ignore" (which is the default). + +flow: + memcap: 128mb + #memcap-policy: ignore + hash-size: 65536 + prealloc: 10000 + emergency-recovery: 30 + #managers: 1 # default to one flow manager + #recyclers: 1 # default to one flow recycler thread + +# This option controls the use of VLAN ids in the flow (and defrag) +# hashing. Normally this should be enabled, but in some (broken) +# setups where both sides of a flow are not tagged with the same VLAN +# tag, we can ignore the VLAN id's in the flow hashing. +vlan: + use-for-tracking: true + +# Specific timeouts for flows. Here you can specify the timeouts that the +# active flows will wait to transit from the current state to another, on each +# protocol. The value of "new" determines the seconds to wait after a handshake or +# stream startup before the engine frees the data of that flow it doesn't +# change the state to established (usually if we don't receive more packets +# of that flow). The value of "established" is the amount of +# seconds that the engine will wait to free the flow if that time elapses +# without receiving new packets or closing the connection. "closed" is the +# amount of time to wait after a flow is closed (usually zero). "bypassed" +# timeout controls locally bypassed flows. For these flows we don't do any other +# tracking. If no packets have been seen after this timeout, the flow is discarded. +# +# There's an emergency mode that will become active under attack circumstances, +# making the engine to check flow status faster. This configuration variables +# use the prefix "emergency-" and work similar as the normal ones. +# Some timeouts doesn't apply to all the protocols, like "closed", for udp and +# icmp. + +flow-timeouts: + + default: + new: 30 + established: 300 + closed: 0 + bypassed: 100 + emergency-new: 10 + emergency-established: 100 + emergency-closed: 0 + emergency-bypassed: 50 + tcp: + new: 60 + established: 600 + closed: 60 + bypassed: 100 + emergency-new: 5 + emergency-established: 100 + emergency-closed: 10 + emergency-bypassed: 50 + udp: + new: 30 + established: 300 + bypassed: 100 + emergency-new: 10 + emergency-established: 100 + emergency-bypassed: 50 + icmp: + new: 30 + established: 300 + bypassed: 100 + emergency-new: 10 + emergency-established: 100 + emergency-bypassed: 50 + +# Stream engine settings. Here the TCP stream tracking and reassembly +# engine is configured. +# +# stream: +# memcap: 64mb # Can be specified in kb, mb, gb. Just a +# # number indicates it's in bytes. +# memcap-policy: ignore # Can be "drop-flow", "pass-flow", "bypass", +# # "drop-packet", "pass-packet", "reject" or +# # "ignore" default is "ignore" +# checksum-validation: yes # To validate the checksum of received +# # packet. If csum validation is specified as +# # "yes", then packets with invalid csum values will not +# # be processed by the engine stream/app layer. +# # Warning: locally generated traffic can be +# # generated without checksum due to hardware offload +# # of checksum. You can control the handling of checksum +# # on a per-interface basis via the 'checksum-checks' +# # option +# prealloc-sessions: 2k # 2k sessions prealloc'd per stream thread +# midstream: false # don't allow midstream session pickups +# midstream-policy: ignore # Can be "drop-flow", "pass-flow", "bypass", +# # "drop-packet", "pass-packet", "reject" or +# # "ignore" default is "ignore" +# async-oneside: false # don't enable async stream handling +# inline: no # stream inline mode +# drop-invalid: yes # in inline mode, drop packets that are invalid with regards to streaming engine +# max-synack-queued: 5 # Max different SYN/ACKs to queue +# bypass: no # Bypass packets when stream.reassembly.depth is reached. +# # Warning: first side to reach this triggers +# # the bypass. +# +# reassembly: +# memcap: 256mb # Can be specified in kb, mb, gb. Just a number +# # indicates it's in bytes. +# memcap-policy: ignore # Can be "drop-flow", "pass-flow", "bypass", +# # "drop-packet", "pass-packet", "reject" or +# # "ignore" default is "ignore" +# depth: 1mb # Can be specified in kb, mb, gb. Just a number +# # indicates it's in bytes. +# toserver-chunk-size: 2560 # inspect raw stream in chunks of at least +# # this size. Can be specified in kb, mb, +# # gb. Just a number indicates it's in bytes. +# toclient-chunk-size: 2560 # inspect raw stream in chunks of at least +# # this size. Can be specified in kb, mb, +# # gb. Just a number indicates it's in bytes. +# randomize-chunk-size: yes # Take a random value for chunk size around the specified value. +# # This lowers the risk of some evasion techniques but could lead +# # to detection change between runs. It is set to 'yes' by default. +# randomize-chunk-range: 10 # If randomize-chunk-size is active, the value of chunk-size is +# # a random value between (1 - randomize-chunk-range/100)*toserver-chunk-size +# # and (1 + randomize-chunk-range/100)*toserver-chunk-size and the same +# # calculation for toclient-chunk-size. +# # Default value of randomize-chunk-range is 10. +# +# raw: yes # 'Raw' reassembly enabled or disabled. +# # raw is for content inspection by detection +# # engine. +# +# segment-prealloc: 2048 # number of segments preallocated per thread +# +# check-overlap-different-data: true|false +# # check if a segment contains different data +# # than what we've already seen for that +# # position in the stream. +# # This is enabled automatically if inline mode +# # is used or when stream-event:reassembly_overlap_different_data; +# # is used in a rule. +# +stream: + memcap: 64mb + #memcap-policy: ignore + checksum-validation: yes # reject incorrect csums + #midstream: false + #midstream-policy: ignore + inline: auto # auto will use inline mode in IPS mode, yes or no set it statically + reassembly: + memcap: 256mb + #memcap-policy: ignore + depth: 1mb # reassemble 1mb into a stream + toserver-chunk-size: 2560 + toclient-chunk-size: 2560 + randomize-chunk-size: yes + #randomize-chunk-range: 10 + #raw: yes + #segment-prealloc: 2048 + #check-overlap-different-data: true + +# Host table: +# +# Host table is used by the tagging and per host thresholding subsystems. +# +host: + hash-size: 4096 + prealloc: 1000 + memcap: 32mb + +# IP Pair table: +# +# Used by xbits 'ippair' tracking. +# +#ippair: +# hash-size: 4096 +# prealloc: 1000 +# memcap: 32mb + +# Decoder settings + +decoder: + # Teredo decoder is known to not be completely accurate + # as it will sometimes detect non-teredo as teredo. + teredo: + enabled: true + # ports to look for Teredo. Max 4 ports. If no ports are given, or + # the value is set to 'any', Teredo detection runs on _all_ UDP packets. + ports: $TEREDO_PORTS # syntax: '[3544, 1234]' or '3533' or 'any'. + + # VXLAN decoder is assigned to up to 4 UDP ports. By default only the + # IANA assigned port 4789 is enabled. + vxlan: + enabled: true + ports: $VXLAN_PORTS # syntax: '[8472, 4789]' or '4789'. + + # VNTag decode support + vntag: + enabled: false + + # Geneve decoder is assigned to up to 4 UDP ports. By default only the + # IANA assigned port 6081 is enabled. + geneve: + enabled: true + ports: $GENEVE_PORTS # syntax: '[6081, 1234]' or '6081'. + + # maximum number of decoder layers for a packet + # max-layers: 16 + +## +## Performance tuning and profiling +## + +# The detection engine builds internal groups of signatures. The engine +# allows us to specify the profile to use for them, to manage memory in an +# efficient way keeping good performance. For the profile keyword you +# can use the words "low", "medium", "high" or "custom". If you use custom, +# make sure to define the values in the "custom-values" section. +# Usually you would prefer medium/high/low. +# +# "sgh mpm-context", indicates how the staging should allot mpm contexts for +# the signature groups. "single" indicates the use of a single context for +# all the signature group heads. "full" indicates a mpm-context for each +# group head. "auto" lets the engine decide the distribution of contexts +# based on the information the engine gathers on the patterns from each +# group head. +# +# The option inspection-recursion-limit is used to limit the recursive calls +# in the content inspection code. For certain payload-sig combinations, we +# might end up taking too much time in the content inspection code. +# If the argument specified is 0, the engine uses an internally defined +# default limit. When a value is not specified, there are no limits on the recursion. +detect: + profile: medium + custom-values: + toclient-groups: 3 + toserver-groups: 25 + sgh-mpm-context: auto + inspection-recursion-limit: 3000 + # If set to yes, the loading of signatures will be made after the capture + # is started. This will limit the downtime in IPS mode. + #delayed-detect: yes + + prefilter: + # default prefiltering setting. "mpm" only creates MPM/fast_pattern + # engines. "auto" also sets up prefilter engines for other keywords. + # Use --list-keywords=all to see which keywords support prefiltering. + default: mpm + + # the grouping values above control how many groups are created per + # direction. Port whitelisting forces that port to get its own group. + # Very common ports will benefit, as well as ports with many expensive + # rules. + grouping: + #tcp-whitelist: 53, 80, 139, 443, 445, 1433, 3306, 3389, 6666, 6667, 8080 + #udp-whitelist: 53, 135, 5060 + + profiling: + # Log the rules that made it past the prefilter stage, per packet + # default is off. The threshold setting determines how many rules + # must have made it past pre-filter for that rule to trigger the + # logging. + #inspect-logging-threshold: 200 + grouping: + dump-to-disk: false + include-rules: false # very verbose + include-mpm-stats: false + +# Select the multi pattern algorithm you want to run for scan/search the +# in the engine. +# +# The supported algorithms are: +# "ac" - Aho-Corasick, default implementation +# "ac-bs" - Aho-Corasick, reduced memory implementation +# "ac-ks" - Aho-Corasick, "Ken Steele" variant +# "hs" - Hyperscan, available when built with Hyperscan support +# +# The default mpm-algo value of "auto" will use "hs" if Hyperscan is +# available, "ac" otherwise. +# +# The mpm you choose also decides the distribution of mpm contexts for +# signature groups, specified by the conf - "detect.sgh-mpm-context". +# Selecting "ac" as the mpm would require "detect.sgh-mpm-context" +# to be set to "single", because of ac's memory requirements, unless the +# ruleset is small enough to fit in memory, in which case one can +# use "full" with "ac". The rest of the mpms can be run in "full" mode. + +mpm-algo: auto + +# Select the matching algorithm you want to use for single-pattern searches. +# +# Supported algorithms are "bm" (Boyer-Moore) and "hs" (Hyperscan, only +# available if Suricata has been built with Hyperscan support). +# +# The default of "auto" will use "hs" if available, otherwise "bm". + +spm-algo: auto + +# Suricata is multi-threaded. Here the threading can be influenced. +threading: + set-cpu-affinity: no + # Tune cpu affinity of threads. Each family of threads can be bound + # to specific CPUs. + # + # These 2 apply to the all runmodes: + # management-cpu-set is used for flow timeout handling, counters + # worker-cpu-set is used for 'worker' threads + # + # Additionally, for autofp these apply: + # receive-cpu-set is used for capture threads + # verdict-cpu-set is used for IPS verdict threads + # + cpu-affinity: + - management-cpu-set: + cpu: [ 0 ] # include only these CPUs in affinity settings + - receive-cpu-set: + cpu: [ 0 ] # include only these CPUs in affinity settings + - worker-cpu-set: + cpu: [ "all" ] + mode: "exclusive" + # Use explicitly 3 threads and don't compute number by using + # detect-thread-ratio variable: + # threads: 3 + prio: + low: [ 0 ] + medium: [ "1-2" ] + high: [ 3 ] + default: "medium" + #- verdict-cpu-set: + # cpu: [ 0 ] + # prio: + # default: "high" + # + # By default Suricata creates one "detect" thread per available CPU/CPU core. + # This setting allows controlling this behaviour. A ratio setting of 2 will + # create 2 detect threads for each CPU/CPU core. So for a dual core CPU this + # will result in 4 detect threads. If values below 1 are used, less threads + # are created. So on a dual core CPU a setting of 0.5 results in 1 detect + # thread being created. Regardless of the setting at a minimum 1 detect + # thread will always be created. + # + detect-thread-ratio: 1.0 + # + # By default, the per-thread stack size is left to its default setting. If + # the default thread stack size is too small, use the following configuration + # setting to change the size. Note that if any thread's stack size cannot be + # set to this value, a fatal error occurs. + # + # Generally, the per-thread stack-size should not exceed 8MB. + #stack-size: 8mb + +# Luajit has a strange memory requirement, its 'states' need to be in the +# first 2G of the process' memory. +# +# 'luajit.states' is used to control how many states are preallocated. +# State use: per detect script: 1 per detect thread. Per output script: 1 per +# script. +luajit: + states: 128 + +# Profiling settings. Only effective if Suricata has been built with +# the --enable-profiling configure flag. +# +profiling: + # Run profiling for every X-th packet. The default is 1, which means we + # profile every packet. If set to 1000, one packet is profiled for every + # 1000 received. + #sample-rate: 1000 + + # rule profiling + rules: + + # Profiling can be disabled here, but it will still have a + # performance impact if compiled in. + enabled: yes + filename: rule_perf.log + append: yes + + # Sort options: ticks, avgticks, checks, matches, maxticks + # If commented out all the sort options will be used. + #sort: avgticks + + # Limit the number of sids for which stats are shown at exit (per sort). + limit: 10 + + # output to json + json: yes + + # per keyword profiling + keywords: + enabled: yes + filename: keyword_perf.log + append: yes + + prefilter: + enabled: yes + filename: prefilter_perf.log + append: yes + + # per rulegroup profiling + rulegroups: + enabled: yes + filename: rule_group_perf.log + append: yes + + # packet profiling + packets: + + # Profiling can be disabled here, but it will still have a + # performance impact if compiled in. + enabled: yes + filename: packet_stats.log + append: yes + + # per packet csv output + csv: + + # Output can be disabled here, but it will still have a + # performance impact if compiled in. + enabled: no + filename: packet_stats.csv + + # profiling of locking. Only available when Suricata was built with + # --enable-profiling-locks. + locks: + enabled: no + filename: lock_stats.log + append: yes + + pcap-log: + enabled: no + filename: pcaplog_stats.log + append: yes + +## +## Netfilter integration +## + +# When running in NFQ inline mode, it is possible to use a simulated +# non-terminal NFQUEUE verdict. +# This permits sending all needed packet to Suricata via this rule: +# iptables -I FORWARD -m mark ! --mark $MARK/$MASK -j NFQUEUE +# And below, you can have your standard filtering ruleset. To activate +# this mode, you need to set mode to 'repeat' +# If you want a packet to be sent to another queue after an ACCEPT decision +# set the mode to 'route' and set next-queue value. +# On Linux >= 3.1, you can set batchcount to a value > 1 to improve performance +# by processing several packets before sending a verdict (worker runmode only). +# On Linux >= 3.6, you can set the fail-open option to yes to have the kernel +# accept the packet if Suricata is not able to keep pace. +# bypass mark and mask can be used to implement NFQ bypass. If bypass mark is +# set then the NFQ bypass is activated. Suricata will set the bypass mark/mask +# on packet of a flow that need to be bypassed. The Nefilter ruleset has to +# directly accept all packets of a flow once a packet has been marked. +nfq: +# mode: accept +# repeat-mark: 1 +# repeat-mask: 1 +# bypass-mark: 1 +# bypass-mask: 1 +# route-queue: 2 +# batchcount: 20 +# fail-open: yes + +#nflog support +nflog: + # netlink multicast group + # (the same as the iptables --nflog-group param) + # Group 0 is used by the kernel, so you can't use it + - group: 2 + # netlink buffer size + buffer-size: 18432 + # put default value here + - group: default + # set number of packets to queue inside kernel + qthreshold: 1 + # set the delay before flushing packet in the kernel's queue + qtimeout: 100 + # netlink max buffer size + max-size: 20000 + +## +## Advanced Capture Options +## + +# General settings affecting packet capture +capture: + # disable NIC offloading. It's restored when Suricata exits. + # Enabled by default. + #disable-offloading: false + # + # disable checksum validation. Same as setting '-k none' on the + # commandline. + #checksum-validation: none + +# Netmap support +# +# Netmap operates with NIC directly in driver, so you need FreeBSD 11+ which has +# built-in Netmap support or compile and install the Netmap module and appropriate +# NIC driver for your Linux system. +# To reach maximum throughput disable all receive-, segmentation-, +# checksum- offloading on your NIC (using ethtool or similar). +# Disabling TX checksum offloading is *required* for connecting OS endpoint +# with NIC endpoint. +# You can find more information at https://github.com/luigirizzo/netmap +# +netmap: + - interface: default + +# PF_RING configuration: for use with native PF_RING support +# for more info see http://www.ntop.org/products/pf_ring/ +pfring: + - interface: default + #threads: 2 + +# For FreeBSD ipfw(8) divert(4) support. +# Please make sure you have ipfw_load="YES" and ipdivert_load="YES" +# in /etc/loader.conf or kldload'ing the appropriate kernel modules. +# Additionally, you need to have an ipfw rule for the engine to see +# the packets from ipfw. For Example: +# +# ipfw add 100 divert 8000 ip from any to any +# +# N.B. This example uses "8000" -- this number must mach the values +# you passed on the command line, i.e., -d 8000 +# +ipfw: + + # Reinject packets at the specified ipfw rule number. This config + # option is the ipfw rule number AT WHICH rule processing continues + # in the ipfw processing system after the engine has finished + # inspecting the packet for acceptance. If no rule number is specified, + # accepted packets are reinjected at the divert rule which they entered + # and IPFW rule processing continues. No check is done to verify + # this will rule makes sense so care must be taken to avoid loops in ipfw. + # + ## The following example tells the engine to reinject packets + # back into the ipfw firewall AT rule number 5500: + # + # ipfw-reinjection-rule-number: 5500 + + +napatech: + # When use_all_streams is set to "yes" the initialization code will query + # the Napatech service for all configured streams and listen on all of them. + # When set to "no" the streams config array will be used. + # + # This option necessitates running the appropriate NTPL commands to create + # the desired streams prior to running Suricata. + #use-all-streams: no + + # The streams to listen on when auto-config is disabled or when and threading + # cpu-affinity is disabled. This can be either: + # an individual stream (e.g. streams: [0]) + # or + # a range of streams (e.g. streams: ["0-3"]) + # + streams: ["0-3"] + + # Stream stats can be enabled to provide fine grain packet and byte counters + # for each thread/stream that is configured. + # + enable-stream-stats: no + + # When auto-config is enabled the streams will be created and assigned + # automatically to the NUMA node where the thread resides. If cpu-affinity + # is enabled in the threading section. Then the streams will be created + # according to the number of worker threads specified in the worker-cpu-set. + # Otherwise, the streams array is used to define the streams. + # + # This option is intended primarily to support legacy configurations. + # + # This option cannot be used simultaneously with either "use-all-streams" + # or "hardware-bypass". + # + auto-config: yes + + # Enable hardware level flow bypass. + # + hardware-bypass: yes + + # Enable inline operation. When enabled traffic arriving on a given port is + # automatically forwarded out its peer port after analysis by Suricata. + # + inline: no + + # Ports indicates which Napatech ports are to be used in auto-config mode. + # these are the port IDs of the ports that will be merged prior to the + # traffic being distributed to the streams. + # + # When hardware-bypass is enabled the ports must be configured as a segment. + # specify the port(s) on which upstream and downstream traffic will arrive. + # This information is necessary for the hardware to properly process flows. + # + # When using a tap configuration one of the ports will receive inbound traffic + # for the network and the other will receive outbound traffic. The two ports on a + # given segment must reside on the same network adapter. + # + # When using a SPAN-port configuration the upstream and downstream traffic + # arrives on a single port. This is configured by setting the two sides of the + # segment to reference the same port. (e.g. 0-0 to configure a SPAN port on + # port 0). + # + # port segments are specified in the form: + # ports: [0-1,2-3,4-5,6-6,7-7] + # + # For legacy systems when hardware-bypass is disabled this can be specified in any + # of the following ways: + # + # a list of individual ports (e.g. ports: [0,1,2,3]) + # + # a range of ports (e.g. ports: [0-3]) + # + # "all" to indicate that all ports are to be merged together + # (e.g. ports: [all]) + # + # This parameter has no effect if auto-config is disabled. + # + ports: [0-1,2-3] + + # When auto-config is enabled the hashmode specifies the algorithm for + # determining to which stream a given packet is to be delivered. + # This can be any valid Napatech NTPL hashmode command. + # + # The most common hashmode commands are: hash2tuple, hash2tuplesorted, + # hash5tuple, hash5tuplesorted and roundrobin. + # + # See Napatech NTPL documentation other hashmodes and details on their use. + # + # This parameter has no effect if auto-config is disabled. + # + hashmode: hash5tuplesorted + +## +## Configure Suricata to load Suricata-Update managed rules. +## + +# As VyOS leverages suricata-update, the default rule path points to the +# generated rules instead of the built-in rules. +# +# default-rule-path: /etc/suricata/rules +default-rule-path: /var/lib/suricata/rules + +rule-files: + - suricata.rules + +## +## Auxiliary configuration files. +## + +# As VyOS leverages suricata-update, the classification file points to the +# generated classification instead of the built-in one. +# +# classification-file: /etc/suricata/classification.config +classification-file: /var/lib/suricata/rules/classification.config +reference-config-file: /etc/suricata/reference.config +# threshold-file: /etc/suricata/threshold.config + +## +## Include other configs +## + +# Includes: Files included here will be handled as if they were in-lined +# in this configuration file. Files with relative pathnames will be +# searched for in the same directory as this configuration file. You may +# use absolute pathnames too. +# You can specify more than 2 configuration files, if needed. +#include: include1.yaml +#include: include2.yaml diff --git a/data/templates/ids/suricata_logrotate.j2 b/data/templates/ids/suricata_logrotate.j2 new file mode 100644 index 00000000000..62773fc6872 --- /dev/null +++ b/data/templates/ids/suricata_logrotate.j2 @@ -0,0 +1,17 @@ +{% for filename in [(log.eve.filename | default("eve.json"))] %} +{{ filename if filename.startswith("/") else ("/var/log/suricata/" + filename) }} +{% endfor %}{ + weekly + dateext + dateformat _%Y-%m-%d_%H-%M-%S + maxsize 10M + rotate 10 + missingok + nocompress + nocreate + nomail + sharedscripts + postrotate + /bin/kill -HUP `cat /run/suricata/suricata.pid 2>/dev/null` 2>/dev/null || true + endscript +} diff --git a/debian/control b/debian/control index 801cce269f2..4151d83bb29 100644 --- a/debian/control +++ b/debian/control @@ -163,6 +163,8 @@ Depends: # End "service dns dynamic" # # For "service ids" fastnetmon [amd64], + suricata, + suricata-update, # End "service ids" # # For "service ndp-proxy" ndppd, diff --git a/interface-definitions/service_ids_suricata.xml.in b/interface-definitions/service_ids_suricata.xml.in new file mode 100644 index 00000000000..8c1973567c1 --- /dev/null +++ b/interface-definitions/service_ids_suricata.xml.in @@ -0,0 +1,250 @@ + + + + + + + + + Network IDS, IPS and Network Security Monitoring + 740 + + + #include + + + Address group name + + home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server + + + [a-z0-9-]+ + + + + + + IP address or subnet + + ipv4 + IPv4 address to match + + + ipv6 + IPv6 address to match + + + ipv4net + IPv4 prefix to match + + + ipv6net + IPv6 prefix to match + + + !ipv4 + Exclude the specified IPv4 address from matches + + + !ipv6 + Exclude the specified IPv6 address from matches + + + !ipv4net + Exclude the specified IPv6 prefix from matches + + + !ipv6net + Exclude the specified IPv6 prefix from matches + + + + + + + + + + + + + + + + + Address group + + service ids suricata address-group + home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server + + + string + Address group to match + + + !string + Exclude the specified address group from matches + + + !?[a-z0-9-]+ + + + + + + + + + Port group name + + http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports + + + [a-z0-9-]+ + + + + + + Port number + + u32:1-65535 + Numeric port to match + + + !u32:1-65535 + Numeric port to exclude from matches + + + start-end + Numbered port range (e.g. 1001-1005) to match + + + !start-end + Numbered port range (e.g. !1001-1005) to exclude from matches + + + + + + + + + + + Port group + + service ids suricata port-group + http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports + + + string + Port group to match + + + !string + Exclude the specified port group from matches + + + !?[a-z0-9-]+ + + + + + + + + + Suricata log outputs + + + + + Extensible Event Format (EVE) + + + + + EVE logging destination + + regular syslog + + + regular + Log to filename + + + syslog + Log to syslog + + + (regular|syslog) + + + regular + + + + Log file + + filename + File name in default Suricata log directory + + + /path + Absolute file path + + + eve.json + + + + Log types + + alert anomaly drop files http dns tls smtp dnp3 ftp rdp nfs smb tftp ikev2 dcerpc krb5 snmp rfb sip dhcp ssh mqtt http2 flow netflow + + + alert + Record events for rule matches + + + anomaly + Record unexpected conditions such as truncated packets, packets with invalid IP/UDP/TCP length values, and other events that render the packet invalid for further processing or describe unexpected behavior on an established stream + + + drop + Record events for dropped packets + + + file + Record file details (e.g., MD5) for files extracted from application protocols (e.g., HTTP) + + + application (http, dns, tls, ...) + Record application-level transactions + + + flow + Record bi-directional flows + + + netflow + Record uni-directional flows + + + (alert|anomaly|http|dns|tls|files|drop|smtp|dnp3|ftp|rdp|nfs|smb|tftp|ikev2|dcerpc|krb5|snmp|rfb|sip|dhcp|ssh|mqtt|http2|flow|netflow) + + + + + + + + + + + + + + + diff --git a/op-mode-definitions/suricata.xml.in b/op-mode-definitions/suricata.xml.in new file mode 100644 index 00000000000..a5025afba8b --- /dev/null +++ b/op-mode-definitions/suricata.xml.in @@ -0,0 +1,23 @@ + + + + + + + Update Suricata + + if test -f /run/suricata/suricata.yaml; then sudo suricata-update --suricata-conf /run/suricata/suricata.yaml; sudo systemctl restart suricata; else echo "Service Suricata not configured"; fi + + + + + + + + Restart Suricata service + + if systemctl is-active --quiet suricata; then sudo systemctl restart suricata.service; else echo "Service Suricata not configured"; fi + + + + diff --git a/python/vyos/base.py b/python/vyos/base.py index 9b93cb2f27f..054b1d83781 100644 --- a/python/vyos/base.py +++ b/python/vyos/base.py @@ -41,7 +41,7 @@ def print(self): isfirstmessage = False initial_indent = self.standardindent print(f'{mes}') - print('') + print('', flush=True) class Warning(): diff --git a/src/conf_mode/service_ids_suricata.py b/src/conf_mode/service_ids_suricata.py new file mode 100755 index 00000000000..49fbce244c4 --- /dev/null +++ b/src/conf_mode/service_ids_suricata.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +from sys import exit + +from vyos.base import Warning +from vyos.config import Config +from vyos.template import render +from vyos.utils.process import call +from vyos import ConfigError +from vyos import airbag +airbag.enable() + +config_file = '/run/suricata/suricata.yaml' +rotate_file = '/etc/logrotate.d/suricata' + +address_group_defaults = { + 'home-net': {'address': ['192.168.0.0/16','10.0.0.0/8','172.16.0.0/12']}, + 'external-net': {'group': ['!home-net']}, + 'http-servers': {'group': ['home-net']}, + 'smtp-servers': {'group': ['home-net']}, + 'sql-servers': {'group': ['home-net']}, + 'dns-servers': {'group': ['home-net']}, + 'telnet-servers': {'group': ['home-net']}, + 'aim-servers': {'group': ['external-net']}, + 'dc-servers': {'group': ['home-net']}, + 'dnp3-server': {'group': ['home-net']}, + 'modbus-client': {'group': ['home-net']}, + 'modbus-server': {'group': ['home-net']}, + 'enip-client': {'group': ['home-net']}, + 'enip-server': {'group': ['home-net']}, +} + +port_group_defaults = { + 'http-ports': {'port': ['80']}, + 'shellcode-ports': {'port': ['!80']}, + 'oracle-ports': {'port': ['1521']}, + 'ssh-ports': {'port': ['22']}, + 'dnp3-ports': {'port': ['20000']}, + 'modbus-ports': {'port': ['502']}, + 'file-data-ports': {'port': ['110', '143'], 'group': ['http-ports']}, + 'ftp-ports': {'port': ['21']}, + 'geneve-ports': {'port': ['6081']}, + 'vxlan-ports': {'port': ['4789']}, + 'teredo-ports': {'port': ['3544']}, +} + +def get_config(config=None): + if config: + conf = config + else: + conf = Config() + base = ['service', 'ids', 'suricata'] + if not conf.exists(base): + return None + + suricata = conf.get_config_dict(base, + get_first_key=True, + with_recursive_defaults=True) + + # Ensure minimal defaults are present + suricata['address-group'] = address_group_defaults | suricata.get('address-group', {}) + suricata['port-group'] = port_group_defaults | suricata.get('port-group', {}) + + return suricata + +# https://en.wikipedia.org/wiki/Topological_sorting#Depth-first_search +def topological_sort(source): + sorted_nodes = [] + permanent_marks = set() + temporary_marks = set() + + def visit(n, v): + if n in permanent_marks: + return + if n in temporary_marks: + raise ConfigError('At least one cycle exists in the referenced groups') + + temporary_marks.add(n) + + for m in v.get('group', []): + m = m.lstrip('!') + if m not in source: + raise ConfigError(f'Undefined referenced group "{m}"') + visit(m, source[m]) + + temporary_marks.remove(n) + permanent_marks.add(n) + sorted_nodes.append((n, v)) + + while len(permanent_marks) < len(source): + n = next(n for n in source.keys() if n not in permanent_marks) + visit(n, source[n]) + + return sorted_nodes + +def verify(suricata): + if not suricata: + return None + + if 'interface' not in suricata: + raise ConfigError('No interfaces configured') + + try: + topological_sort(suricata['address-group']) + except (ConfigError,StopIteration) as e: + raise ConfigError(f'Invalid address-group: {e}') + + try: + topological_sort(suricata['port-group']) + except (ConfigError,StopIteration) as e: + raise ConfigError(f'Invalid port-group: {e}') + +def generate(suricata): + if not suricata: + for file in [config_file, rotate_file]: + if os.path.isfile(file): + os.unlink(file) + + return None + + # Config-related formatters + def to_var(s:str): + return s.replace('-','_').upper() + + def to_val(s:str): + return s.replace('-',':') + + def to_ref(s:str): + if s[0] == '!': + return '!$' + to_var(s[1:]) + return '$' + to_var(s) + + def to_config(kind:str): + def format_group(group): + (name, value) = group + property = [to_val(property) for property in value.get(kind,[])] + group = [to_ref(group) for group in value.get('group',[])] + return (to_var(name), property + group) + return format_group + + # Format the address group + suricata['address-group'] = map(to_config('address'), + topological_sort(suricata['address-group'])) + + # Format the port group + suricata['port-group'] = map(to_config('port'), + topological_sort(suricata['port-group'])) + + render(config_file, 'ids/suricata.j2', {'suricata': suricata}) + render(rotate_file, 'ids/suricata_logrotate.j2', suricata) + return None + +def apply(suricata): + systemd_service = 'suricata.service' + if not suricata or 'interface' not in suricata: + # Stop suricata service if removed + call(f'systemctl stop {systemd_service}') + else: + Warning('To fetch the latest rules, use "update suricata"; ' + 'To periodically fetch the latest rules, ' + 'use the task scheduler!') + call(f'systemctl restart {systemd_service}') + + return None + +if __name__ == '__main__': + try: + c = get_config() + verify(c) + generate(c) + apply(c) + except ConfigError as e: + print(e) + exit(1) diff --git a/src/etc/systemd/system/suricata.service.d/10-override.conf b/src/etc/systemd/system/suricata.service.d/10-override.conf new file mode 100644 index 00000000000..781256cf5c9 --- /dev/null +++ b/src/etc/systemd/system/suricata.service.d/10-override.conf @@ -0,0 +1,9 @@ +[Service] +ExecStart= +ExecStart=/usr/bin/suricata -D --af-packet -c /run/suricata/suricata.yaml --pidfile /run/suricata/suricata.pid +PIDFile= +PIDFile=/run/suricata/suricata.pid +ExecReload= +ExecReload=/usr/bin/suricatasc -c reload-rules /run/suricata/suricata.socket ; /bin/kill -HUP $MAINPID +ExecStop= +ExecStop=/usr/bin/suricatasc -c shutdown /run/suricata/suricata.socket diff --git a/src/validators/port-range-exclude b/src/validators/port-range-exclude new file mode 100755 index 00000000000..4c049e98f16 --- /dev/null +++ b/src/validators/port-range-exclude @@ -0,0 +1,7 @@ +#!/bin/sh +arg="$1" +if [ "${arg:0:1}" != "!" ]; then + exit 1 +fi +path=$(dirname "$0") +${path}/port-range "${arg:1}" From 7e0e988c474cc649e9086f1146b676e1863767b7 Mon Sep 17 00:00:00 2001 From: khramshinr Date: Mon, 13 May 2024 19:17:07 +0600 Subject: [PATCH 094/188] T6251: Extend table number limits for policy route-map set table --- interface-definitions/policy.xml.in | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/interface-definitions/policy.xml.in b/interface-definitions/policy.xml.in index 791fa1d8712..eb907cb9e40 100644 --- a/interface-definitions/policy.xml.in +++ b/interface-definitions/policy.xml.in @@ -1546,11 +1546,11 @@ Set prefixes to table - u32:1-200 + u32:1-4294967295 Table value - + From 7c438caa2c21101cbefc2eec21935ab55af19c46 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 14 May 2024 16:47:29 +0000 Subject: [PATCH 095/188] T3420: Remove service upnp Remove `service upnp` as it never worked as expected, nft rules do not integrated and custom patches do not seem like a suitable solution for now. Security: UPnP has been historically associated with security risks due to its automatic and potentially unauthenticated nature. UPnP devices might be vulnerable to unauthorized access or exploitation. --- data/configd-include.json | 1 - data/templates/firewall/upnpd.conf.j2 | 227 -------------------- debian/control | 3 - interface-definitions/service_upnp.xml.in | 229 --------------------- smoketest/scripts/cli/test_service_upnp.py | 103 --------- src/conf_mode/service_upnp.py | 157 -------------- src/systemd/miniupnpd.service | 13 -- 7 files changed, 733 deletions(-) delete mode 100644 data/templates/firewall/upnpd.conf.j2 delete mode 100644 interface-definitions/service_upnp.xml.in delete mode 100755 smoketest/scripts/cli/test_service_upnp.py delete mode 100755 src/conf_mode/service_upnp.py delete mode 100644 src/systemd/miniupnpd.service diff --git a/data/configd-include.json b/data/configd-include.json index 0c767f987a5..dcee5030647 100644 --- a/data/configd-include.json +++ b/data/configd-include.json @@ -81,7 +81,6 @@ "service_sla.py", "service_ssh.py", "service_tftp-server.py", -"service_upnp.py", "service_webproxy.py", "system_acceleration.py", "system_config-management.py", diff --git a/data/templates/firewall/upnpd.conf.j2 b/data/templates/firewall/upnpd.conf.j2 deleted file mode 100644 index 616e8869f7e..00000000000 --- a/data/templates/firewall/upnpd.conf.j2 +++ /dev/null @@ -1,227 +0,0 @@ -# This is the UPNP configuration file - -# WAN network interface -ext_ifname={{ wan_interface }} -{% if wan_ip is vyos_defined %} - -# if the WAN network interface for IPv6 is different than for IPv4, -# set ext_ifname6 -#ext_ifname6=eth2 - -# If the WAN interface has several IP addresses, you -# can specify the one to use below. -# Setting ext_ip is also useful in double NAT setup, you can declare here -# the public IP address. -{% for addr in wan_ip %} -ext_ip={{ addr }} -{% endfor %} -{% endif %} - -{% if stun is vyos_defined %} -# WAN interface must have public IP address. Otherwise it is behind NAT -# and port forwarding is impossible. In some cases WAN interface can be -# behind unrestricted full-cone NAT 1:1 when all incoming traffic is NAT-ed and -# routed to WAN interfaces without any filtering. In this cases miniupnpd -# needs to know public IP address and it can be learnt by asking external -# server via STUN protocol. Following option enable retrieving external -# public IP address from STUN server and detection of NAT type. You need -# to specify also external STUN server in stun_host option below. -# This option is disabled by default. -ext_perform_stun=yes -# Specify STUN server, either hostname or IP address -# Some public STUN servers: -# stun.stunprotocol.org -# stun.sipgate.net -# stun.xten.com -# stun.l.google.com (on non standard port 19302) -ext_stun_host={{ stun.host }} -# Specify STUN UDP port, by default it is standard port 3478. -ext_stun_port={{ stun.port }} -{% endif %} - -# LAN network interfaces IPs / networks -{% if listen is vyos_defined %} -# There can be multiple listening IPs for SSDP traffic, in that case -# use multiple 'listening_ip=...' lines, one for each network interface. -# It can be IP address or network interface name (ie. "eth0") -# It is mandatory to use the network interface name in order to enable IPv6 -# HTTP is available on all interfaces. -# When MULTIPLE_EXTERNAL_IP is enabled, the external IP -# address associated with the subnet follows. For example: -# listening_ip=192.168.0.1/24 88.22.44.13 -# When MULTIPLE_EXTERNAL_IP is disabled, you can list associated network -# interfaces (for bridges) -# listening_ip=bridge0 em0 wlan0 -{% for addr in listen %} -{% if addr | is_ipv4 %} -listening_ip={{ addr }} -{% elif addr | is_ipv6 %} -ipv6_listening_ip={{ addr }} -{% else %} -listening_ip={{ addr }} -{% endif %} -{% endfor %} -{% endif %} - -# CAUTION: mixing up WAN and LAN interfaces may introduce security risks! -# Be sure to assign the correct interfaces to LAN and WAN and consider -# implementing UPnP permission rules at the bottom of this configuration file - -# Port for HTTP (descriptions and SOAP) traffic. Set to 0 for autoselect. -#http_port=0 -# Port for HTTPS. Set to 0 for autoselect (default) -#https_port=0 - -# Path to the UNIX socket used to communicate with MiniSSDPd -# If running, MiniSSDPd will manage M-SEARCH answering. -# default is /var/run/minissdpd.sock -#minissdpdsocket=/var/run/minissdpd.sock - -{% if nat_pmp is vyos_defined %} -# Enable NAT-PMP support (default is no) -enable_natpmp=yes -{% endif %} - -# Enable UPNP support (default is yes) -enable_upnp=yes - -{% if pcp_lifetime is vyos_defined %} -# PCP -# Configure the minimum and maximum lifetime of a port mapping in seconds -# 120s and 86400s (24h) are suggested values from PCP-base -{% if pcp_lifetime.max is vyos_defined %} -max_lifetime={{ pcp_lifetime.max }} -{% endif %} -{% if pcp_lifetime.min is vyos_defined %} -min_lifetime={{ pcp_lifetime.min }} -{% endif %} -{% endif %} - -# table names for netfilter nft. Default is "filter" for both -#upnp_table_name= -#upnp_nat_table_name= -# chain names for netfilter and netfilter nft -# netfilter : default are MINIUPNPD, MINIUPNPD, MINIUPNPD-POSTROUTING -# netfilter nft : default are miniupnpd, prerouting_miniupnpd, postrouting_miniupnpd -#upnp_forward_chain=forwardUPnP -#upnp_nat_chain=UPnP -#upnp_nat_postrouting_chain=UPnP-Postrouting - -# Lease file location -lease_file=/config/upnp.leases - -# To enable the next few runtime options, see compile time -# ENABLE_MANUFACTURER_INFO_CONFIGURATION (config.h) - -{% if friendly_name is vyos_defined %} -# Name of this service, default is "`uname -s` router" -friendly_name={{ friendly_name }} -{% endif %} - -# Manufacturer name, default is "`uname -s`" -manufacturer_name=VyOS - -# Manufacturer URL, default is URL of OS vendor -manufacturer_url=https://vyos.io/ - -# Model name, default is "`uname -s` router" -model_name=VyOS Router Model - -# Model description, default is "`uname -s` router" -model_description=Vyos open source enterprise router/firewall operating system - -# Model URL, default is URL of OS vendor -model_url=https://vyos.io/ - -# Bitrates reported by daemon in bits per second -# by default miniupnpd tries to get WAN interface speed -#bitrate_up=1000000 -#bitrate_down=10000000 - -{% if secure_mode is vyos_defined %} -# Secure Mode, UPnP clients can only add mappings to their own IP -secure_mode=yes -{% else %} -# Secure Mode, UPnP clients can only add mappings to their own IP -secure_mode=no -{% endif %} - -{% if presentation_url is vyos_defined %} -# Default presentation URL is HTTP address on port 80 -# If set to an empty string, no presentationURL element will appear -# in the XML description of the device, which prevents MS Windows -# from displaying an icon in the "Network Connections" panel. -#presentation_url= {{ presentation_url }} -{% endif %} - -# Report system uptime instead of daemon uptime -system_uptime=yes - -# Notify interval in seconds. default is 30 seconds. -#notify_interval=240 -notify_interval=60 - -# Unused rules cleaning. -# never remove any rule before this threshold for the number -# of redirections is exceeded. default to 20 -clean_ruleset_threshold=10 -# Clean process work interval in seconds. default to 0 (disabled). -# a 600 seconds (10 minutes) interval makes sense -clean_ruleset_interval=600 - -############################################################################ -## The next 5 config parameters (packet_log, anchor, queue, tag, quickrules) -## are specific to BSD's pf(4) packet filter and hence cannot be enabled in -## VyOS. -# Log packets in pf (default is no) -#packet_log=no - -# Anchor name in pf (default is miniupnpd) -#anchor=miniupnpd - -# ALTQ queue in pf -# Filter rules must be used for this to be used. -# compile with PF_ENABLE_FILTER_RULES (see config.h file) -#queue=queue_name1 - -# Tag name in pf -#tag=tag_name1 - -# Make filter rules in pf quick or not. default is yes -# active when compiled with PF_ENABLE_FILTER_RULES (see config.h file) -#quickrules=no -## -## End of pf(4)-specific configuration not to be set in VyOS. -############################################################################ - -# UUID, generate your own UUID with "make genuuid" -uuid={{ uuid }} - -# Daemon's serial and model number when reporting to clients -# (in XML description) -#serial=12345678 -#model_number=1 - -# If compiled with IGD_V2 defined, force reporting IGDv1 in rootDesc (default -# is no) -#force_igd_desc_v1=no - -{% if rule is vyos_defined %} -# UPnP permission rules (also enforced for NAT-PMP and PCP) -# (allow|deny) (external port range) IP/mask (internal port range) (optional regex filter) -# A port range is - or if there is only -# one port in the range. -# IP/mask format must be nnn.nnn.nnn.nnn/nn -# It is advised to only allow redirection of port >= 1024 -# and end the rule set with "deny 0-65535 0.0.0.0/0 0-65535" -# The following default ruleset allows specific LAN side IP addresses -# to request only ephemeral ports. It is recommended that users -# modify the IP ranges to match their own internal networks, and -# also consider implementing network-specific restrictions -# CAUTION: failure to enforce any rules may permit insecure requests to be made! -{% for rule, config in rule.items() %} -{% if config.disable is not vyos_defined %} -{{ config.action }} {{ config.external_port_range }} {{ config.ip }}{{ '/32' if '/' not in config.ip else '' }} {{ config.internal_port_range }} -{% endif %} -{% endfor %} -{% endif %} diff --git a/debian/control b/debian/control index 594e9e8d816..329ed09eb61 100644 --- a/debian/control +++ b/debian/control @@ -196,9 +196,6 @@ Depends: snmp, snmpd, # End "service snmp" -# For "service upnp" - miniupnpd-nftables, -# End "service upnp" # For "service webproxy" squid, squidclient, diff --git a/interface-definitions/service_upnp.xml.in b/interface-definitions/service_upnp.xml.in deleted file mode 100644 index 064386ee5a9..00000000000 --- a/interface-definitions/service_upnp.xml.in +++ /dev/null @@ -1,229 +0,0 @@ - - - - - - - Universal Plug and Play (UPnP) service - 900 - - - - - Name of this service - - txt - Friendly name - - - - - - WAN network interface - - - - - #include - - - - - - WAN network IP - - ipv4 - IPv4 address - - - ipv6 - IPv6 address - - - - - - - - - - - Enable NAT-PMP support - - - - - - Enable Secure Mode - - - - - - Presentation Url - - txt - Presentation Url - - - - - - PCP-base lifetime Option - - - - - Max lifetime time - - - - - - - - Min lifetime time - - - - - - - - - - Local IP addresses for service to listen on - - - - - - <interface> - Monitor interface address - - - ipv4 - IPv4 address to listen for incoming connections - - - ipv4net - IPv4 prefix to listen for incoming connections - - - ipv6 - IPv6 address to listen for incoming connections - - - ipv6net - IPv6 prefix to listen for incoming connections - - - - #include - - - - - - - - - Enable STUN probe support (can be used with NAT 1:1 support for WAN interfaces) - - - - - The STUN server address - - txt - The STUN server host address - - - - - - - #include - - - - - UPnP Rule - - u32:0-65535 - Rule number - - - - - - - #include - - - Port range (REQUIRE) - - <port> - single port - - - <portN>-<portM> - Port range (use '-' as delimiter) - - - - - - - - - Port range (REQUIRE) - - <port> - single port - - - <portN>-<portM> - Port range (use '-' as delimiter) - - - - - - - - - The IP to which this rule applies (REQUIRE) - - ipv4 - The IPv4 address to which this rule applies - - - ipv4net - The IPv4 to which this rule applies - - - - - - - - - - - Actions against the rule (REQUIRE) - - allow deny - - - (allow|deny) - - - - - - - - - - diff --git a/smoketest/scripts/cli/test_service_upnp.py b/smoketest/scripts/cli/test_service_upnp.py deleted file mode 100755 index fd67b0ced17..00000000000 --- a/smoketest/scripts/cli/test_service_upnp.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021-2024 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import unittest - -from base_vyostest_shim import VyOSUnitTestSHIM - -from vyos.configsession import ConfigSessionError -from vyos.template import ip_from_cidr -from vyos.utils.file import read_file -from vyos.utils.process import process_named_running - -UPNP_CONF = '/run/upnp/miniupnp.conf' -DAEMON = 'miniupnpd' -interface = 'eth0' -base_path = ['service', 'upnp'] -address_base = ['interfaces', 'ethernet', interface, 'address'] - -ipv4_addr = '100.64.0.1/24' -ipv6_addr = '2001:db8::1/64' - -class TestServiceUPnP(VyOSUnitTestSHIM.TestCase): - @classmethod - def setUpClass(cls): - super(TestServiceUPnP, cls).setUpClass() - - # ensure we can also run this test on a live system - so lets clean - # out the current configuration :) - cls.cli_delete(cls, base_path) - - cls.cli_set(cls, address_base + [ipv4_addr]) - cls.cli_set(cls, address_base + [ipv6_addr]) - - @classmethod - def tearDownClass(cls): - cls.cli_delete(cls, address_base) - cls._session.commit() - - super(TestServiceUPnP, cls).tearDownClass() - - def tearDown(self): - # Check for running process - self.assertTrue(process_named_running(DAEMON)) - - self.cli_delete(base_path) - self.cli_commit() - - # Check for running process - self.assertFalse(process_named_running(DAEMON)) - - def test_ipv4_base(self): - self.cli_set(base_path + ['nat-pmp']) - self.cli_set(base_path + ['listen', interface]) - - # check validate() - WAN interface is mandatory - with self.assertRaises(ConfigSessionError): - self.cli_commit() - self.cli_set(base_path + ['wan-interface', interface]) - - self.cli_commit() - - config = read_file(UPNP_CONF) - self.assertIn(f'ext_ifname={interface}', config) - self.assertIn(f'listening_ip={interface}', config) - self.assertIn(f'enable_natpmp=yes', config) - self.assertIn(f'enable_upnp=yes', config) - - def test_ipv6_base(self): - v6_addr = ip_from_cidr(ipv6_addr) - - self.cli_set(base_path + ['nat-pmp']) - self.cli_set(base_path + ['listen', interface]) - self.cli_set(base_path + ['listen', v6_addr]) - - # check validate() - WAN interface is mandatory - with self.assertRaises(ConfigSessionError): - self.cli_commit() - self.cli_set(base_path + ['wan-interface', interface]) - - self.cli_commit() - - config = read_file(UPNP_CONF) - self.assertIn(f'ext_ifname={interface}', config) - self.assertIn(f'listening_ip={interface}', config) - self.assertIn(f'ipv6_listening_ip={v6_addr}', config) - self.assertIn(f'enable_natpmp=yes', config) - self.assertIn(f'enable_upnp=yes', config) - -if __name__ == '__main__': - unittest.main(verbosity=2) diff --git a/src/conf_mode/service_upnp.py b/src/conf_mode/service_upnp.py deleted file mode 100755 index 0df8dc09ef6..00000000000 --- a/src/conf_mode/service_upnp.py +++ /dev/null @@ -1,157 +0,0 @@ -#!/usr/bin/env python3 -# -# Copyright (C) 2021-2022 VyOS maintainers and contributors -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License version 2 or later as -# published by the Free Software Foundation. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os - -from sys import exit -import uuid -import netifaces -from ipaddress import IPv4Network -from ipaddress import IPv6Network - -from vyos.config import Config -from vyos.utils.process import call -from vyos.template import render -from vyos.template import is_ipv4 -from vyos.template import is_ipv6 -from vyos import ConfigError -from vyos import airbag -airbag.enable() - -config_file = r'/run/upnp/miniupnp.conf' - -def get_config(config=None): - if config: - conf = config - else: - conf = Config() - - base = ['service', 'upnp'] - upnpd = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True) - - if not upnpd: - return None - - upnpd = conf.merge_defaults(upnpd, recursive=True) - - uuidgen = uuid.uuid1() - upnpd.update({'uuid': uuidgen}) - - return upnpd - -def get_all_interface_addr(prefix, filter_dev, filter_family): - list_addr = [] - for interface in netifaces.interfaces(): - if filter_dev and interface in filter_dev: - continue - addrs = netifaces.ifaddresses(interface) - if netifaces.AF_INET in addrs.keys(): - if netifaces.AF_INET in filter_family: - for addr in addrs[netifaces.AF_INET]: - if prefix: - # we need to manually assemble a list of IPv4 address/prefix - prefix = '/' + \ - str(IPv4Network('0.0.0.0/' + addr['netmask']).prefixlen) - list_addr.append(addr['addr'] + prefix) - else: - list_addr.append(addr['addr']) - if netifaces.AF_INET6 in addrs.keys(): - if netifaces.AF_INET6 in filter_family: - for addr in addrs[netifaces.AF_INET6]: - if prefix: - # we need to manually assemble a list of IPv4 address/prefix - bits = bin(int(addr['netmask'].replace(':', '').split('/')[0], 16)).count('1') - prefix = '/' + str(bits) - list_addr.append(addr['addr'] + prefix) - else: - list_addr.append(addr['addr']) - - return list_addr - -def verify(upnpd): - if not upnpd: - return None - - if 'wan_interface' not in upnpd: - raise ConfigError('To enable UPNP, you must have the "wan-interface" option!') - - if 'rule' in upnpd: - for rule, rule_config in upnpd['rule'].items(): - for option in ['external_port_range', 'internal_port_range', 'ip', 'action']: - if option not in rule_config: - tmp = option.replace('_', '-') - raise ConfigError(f'Every UPNP rule requires "{tmp}" to be set!') - - if 'stun' in upnpd: - for option in ['host', 'port']: - if option not in upnpd['stun']: - raise ConfigError(f'A UPNP stun support must have an "{option}" option!') - - # Check the validity of the IP address - listen_dev = [] - system_addrs_cidr = get_all_interface_addr(True, [], [netifaces.AF_INET, netifaces.AF_INET6]) - system_addrs = get_all_interface_addr(False, [], [netifaces.AF_INET, netifaces.AF_INET6]) - if 'listen' not in upnpd: - raise ConfigError(f'Listen address or interface is required!') - for listen_if_or_addr in upnpd['listen']: - if listen_if_or_addr not in netifaces.interfaces(): - listen_dev.append(listen_if_or_addr) - if (listen_if_or_addr not in system_addrs) and (listen_if_or_addr not in system_addrs_cidr) and \ - (listen_if_or_addr not in netifaces.interfaces()): - if is_ipv4(listen_if_or_addr) and IPv4Network(listen_if_or_addr).is_multicast: - raise ConfigError(f'The address "{listen_if_or_addr}" is an address that is not allowed' - f'to listen on. It is not an interface address nor a multicast address!') - if is_ipv6(listen_if_or_addr) and IPv6Network(listen_if_or_addr).is_multicast: - raise ConfigError(f'The address "{listen_if_or_addr}" is an address that is not allowed' - f'to listen on. It is not an interface address nor a multicast address!') - - system_listening_dev_addrs_cidr = get_all_interface_addr(True, listen_dev, [netifaces.AF_INET6]) - system_listening_dev_addrs = get_all_interface_addr(False, listen_dev, [netifaces.AF_INET6]) - for listen_if_or_addr in upnpd['listen']: - if listen_if_or_addr not in netifaces.interfaces() and \ - (listen_if_or_addr not in system_listening_dev_addrs_cidr) and \ - (listen_if_or_addr not in system_listening_dev_addrs) and \ - is_ipv6(listen_if_or_addr) and \ - (not IPv6Network(listen_if_or_addr).is_multicast): - raise ConfigError(f'{listen_if_or_addr} must listen on the interface of the network card') - -def generate(upnpd): - if not upnpd: - return None - - if os.path.isfile(config_file): - os.unlink(config_file) - - render(config_file, 'firewall/upnpd.conf.j2', upnpd) - -def apply(upnpd): - systemd_service_name = 'miniupnpd.service' - if not upnpd: - # Stop the UPNP service - call(f'systemctl stop {systemd_service_name}') - else: - # Start the UPNP service - call(f'systemctl restart {systemd_service_name}') - -if __name__ == '__main__': - try: - c = get_config() - verify(c) - generate(c) - apply(c) - except ConfigError as e: - print(e) - exit(1) diff --git a/src/systemd/miniupnpd.service b/src/systemd/miniupnpd.service deleted file mode 100644 index 51cb2eed81c..00000000000 --- a/src/systemd/miniupnpd.service +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=UPnP service -ConditionPathExists=/run/upnp/miniupnp.conf -After=vyos-router.service -StartLimitIntervalSec=0 - -[Service] -WorkingDirectory=/run/upnp -Type=simple -ExecStart=/usr/sbin/miniupnpd -d -f /run/upnp/miniupnp.conf -PrivateTmp=yes -PIDFile=/run/miniupnpd.pid -Restart=on-failure From fbc846725d7d581f54b2db48a584e4580be3ad00 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Tue, 14 May 2024 19:23:16 +0200 Subject: [PATCH 096/188] smoketest: ospf: T4739: add timeout in ldp test --- smoketest/scripts/cli/test_protocols_ospf.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/smoketest/scripts/cli/test_protocols_ospf.py b/smoketest/scripts/cli/test_protocols_ospf.py index 1b9cc50fe56..585c1dc89e0 100755 --- a/smoketest/scripts/cli/test_protocols_ospf.py +++ b/smoketest/scripts/cli/test_protocols_ospf.py @@ -16,6 +16,7 @@ import unittest +from time import sleep from base_vyostest_shim import VyOSUnitTestSHIM from vyos.configsession import ConfigSessionError @@ -480,6 +481,8 @@ def test_ospf_15_ldp_sync(self): # Commit main OSPF changes self.cli_commit() + sleep(10) + # Verify main OSPF changes frrconfig = self.getFRRconfig('router ospf', daemon=PROCESS_NAME) self.assertIn(f'router ospf', frrconfig) From 75d553932504c55e710265776e4865a238223e1f Mon Sep 17 00:00:00 2001 From: khramshinr Date: Mon, 13 May 2024 17:26:55 +0600 Subject: [PATCH 097/188] T5756: L2TP RADIUS backup and weight settings --- .../accel-ppp/config_chap_secrets_radius.j2 | 15 +++++++++++- .../include/accel-ppp/radius-additions.xml.i | 7 ++++++ .../include/radius-priority.xml.i | 14 +++++++++++ interface-definitions/system_login.xml.in | 11 +-------- smoketest/scripts/cli/base_accel_ppp_test.py | 23 +++++++++++++++++++ smoketest/scripts/cli/test_vpn_l2tp.py | 23 +++++++++++++++++++ 6 files changed, 82 insertions(+), 11 deletions(-) create mode 100644 interface-definitions/include/radius-priority.xml.i diff --git a/data/templates/accel-ppp/config_chap_secrets_radius.j2 b/data/templates/accel-ppp/config_chap_secrets_radius.j2 index 595e3a565f7..e343ce461ac 100644 --- a/data/templates/accel-ppp/config_chap_secrets_radius.j2 +++ b/data/templates/accel-ppp/config_chap_secrets_radius.j2 @@ -5,7 +5,20 @@ chap-secrets={{ chap_secrets_file }} [radius] verbose=1 {% for server, options in authentication.radius.server.items() if not options.disable is vyos_defined %} -server={{ server }},{{ options.key }},auth-port={{ options.port }},acct-port={{ options.acct_port }},req-limit=0,fail-time={{ options.fail_time }} +{% set _server_cfg = "server=" %} +{% set _server_cfg = _server_cfg + server %} +{% set _server_cfg = _server_cfg + "," + options.key %} +{% set _server_cfg = _server_cfg + ",auth-port=" + options.port %} +{% set _server_cfg = _server_cfg + ",acct-port=" + options.acct_port %} +{% set _server_cfg = _server_cfg + ",req-limit=0" %} +{% set _server_cfg = _server_cfg + ",fail-time=" + options.fail_time %} +{% if options.priority is vyos_defined %} +{% set _server_cfg = _server_cfg + ",weight=" + options.priority %} +{% endif %} +{% if options.backup is vyos_defined %} +{% set _server_cfg = _server_cfg + ",backup" %} +{% endif %} +{{ _server_cfg }} {% endfor %} {% if authentication.radius.accounting_interim_interval is vyos_defined %} acct-interim-interval={{ authentication.radius.accounting_interim_interval }} diff --git a/interface-definitions/include/accel-ppp/radius-additions.xml.i b/interface-definitions/include/accel-ppp/radius-additions.xml.i index 3c2eb09eb97..5222ba86410 100644 --- a/interface-definitions/include/accel-ppp/radius-additions.xml.i +++ b/interface-definitions/include/accel-ppp/radius-additions.xml.i @@ -57,6 +57,13 @@ 0 + #include + + + Use backup server if other servers are not available + + + diff --git a/interface-definitions/include/radius-priority.xml.i b/interface-definitions/include/radius-priority.xml.i new file mode 100644 index 00000000000..f77f5016e9a --- /dev/null +++ b/interface-definitions/include/radius-priority.xml.i @@ -0,0 +1,14 @@ + + + + Server priority + + u32:1-255 + Server priority + + + + + + + diff --git a/interface-definitions/system_login.xml.in b/interface-definitions/system_login.xml.in index e94bb721956..f6c8021d325 100644 --- a/interface-definitions/system_login.xml.in +++ b/interface-definitions/system_login.xml.in @@ -202,17 +202,8 @@ #include + #include - - Server priority - - u32:1-255 - Server priority - - - - - 255 diff --git a/smoketest/scripts/cli/base_accel_ppp_test.py b/smoketest/scripts/cli/base_accel_ppp_test.py index 383adc445f7..ab723e707f4 100644 --- a/smoketest/scripts/cli/base_accel_ppp_test.py +++ b/smoketest/scripts/cli/base_accel_ppp_test.py @@ -367,6 +367,27 @@ def test_accel_radius_authentication(self): ] ) + self.set( + [ + "authentication", + "radius", + "server", + radius_server, + "backup", + ] + ) + + self.set( + [ + "authentication", + "radius", + "server", + radius_server, + "priority", + "10", + ] + ) + # commit changes self.cli_commit() @@ -379,6 +400,8 @@ def test_accel_radius_authentication(self): self.assertEqual(f"acct-port=0", server[3]) self.assertEqual(f"req-limit=0", server[4]) self.assertEqual(f"fail-time=0", server[5]) + self.assertIn('weight=10', server) + self.assertIn('backup', server) def test_accel_ipv4_pool(self): self.basic_config(is_gateway=False, is_client_pool=False) diff --git a/smoketest/scripts/cli/test_vpn_l2tp.py b/smoketest/scripts/cli/test_vpn_l2tp.py index 8c4e538955c..07a7e29066c 100755 --- a/smoketest/scripts/cli/test_vpn_l2tp.py +++ b/smoketest/scripts/cli/test_vpn_l2tp.py @@ -95,6 +95,29 @@ def test_vpn_l2tp_dependence_ipsec_swanctl(self): self.cli_set(base_path + ['authentication', 'protocols', 'chap']) self.cli_commit() + def test_l2tp_radius_server(self): + base_path = ['vpn', 'l2tp', 'remote-access'] + radius_server = "192.0.2.22" + radius_key = "secretVyOS" + + self.cli_set(base_path + ['authentication', 'mode', 'radius']) + self.cli_set(base_path + ['gateway-address', '192.0.2.1']) + self.cli_set(base_path + ['client-ip-pool', 'SIMPLE-POOL', 'range', '192.0.2.0/24']) + self.cli_set(base_path + ['default-pool', 'SIMPLE-POOL']) + self.cli_set(base_path + ['authentication', 'radius', 'server', radius_server, 'key', radius_key]) + self.cli_set(base_path + ['authentication', 'radius', 'server', radius_server, 'priority', '10']) + self.cli_set(base_path + ['authentication', 'radius', 'server', radius_server, 'backup']) + + # commit changes + self.cli_commit() + + # Validate configuration values + conf = ConfigParser(allow_no_value=True) + conf.read(self._config_file) + server = conf["radius"]["server"].split(",") + self.assertIn('weight=10', server) + self.assertIn('backup', server) + if __name__ == '__main__': unittest.main(verbosity=2) From cc0573a78aac4d6ac4479fdf951d151a36b88cbc Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 15 May 2024 13:38:21 +0100 Subject: [PATCH 098/188] op mode: T6339: display build flavor and comment in "show version" --- src/op_mode/version.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/op_mode/version.py b/src/op_mode/version.py index ad0293aca97..09d69ad1d21 100755 --- a/src/op_mode/version.py +++ b/src/op_mode/version.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# Copyright (C) 2016-2022 VyOS maintainers and contributors +# Copyright (C) 2016-2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -30,11 +30,15 @@ version_output_tmpl = """ Version: VyOS {{version}} Release train: {{release_train}} +Release flavor: {{flavor}} Built by: {{built_by}} Built on: {{built_on}} Build UUID: {{build_uuid}} Build commit ID: {{build_git}} +{%- if build_comment %} +Build comment: {{build_comment}} +{% endif %} Architecture: {{system_arch}} Boot via: {{boot_via}} From d8223d50ab627d2b58e14101891bf8ffb8111234 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 15 May 2024 14:15:37 +0100 Subject: [PATCH 099/188] op mode: T3355: remove the mention of legacy non-image installations They were never supported by VyOS, that was just for very old systens upgraded from Vyatta Core --- python/vyos/version.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/python/vyos/version.py b/python/vyos/version.py index b5ed2705b1b..47a10e2014c 100644 --- a/python/vyos/version.py +++ b/python/vyos/version.py @@ -81,16 +81,14 @@ def get_full_version_data(fname=version_file): else: version_data['system_type'] = f"{hypervisor} guest" - # Get boot type, it can be livecd, installed image, or, possible, a system installed - # via legacy "install system" mechanism + # Get boot type, it can be livecd or installed image # In installed images, the squashfs image file is named after its image version, # while on livecd it's just "filesystem.squashfs", that's how we tell a livecd boot # from an installed image - boot_via = "installed image" - if run(""" grep -e '^overlay.*/filesystem.squashfs' /proc/mounts >/dev/null""") == 0: + if run(""" grep -e '^overlay.*/filesystem.squashfs' /proc/mounts >/dev/null """) == 0: boot_via = "livecd" - elif run(""" grep '^overlay /' /proc/mounts >/dev/null """) != 0: - boot_via = "legacy non-image installation" + else: + boot_via = "installed image" version_data['boot_via'] = boot_via # Get hardware details from DMI From 6871c5541c1962e63d7a9b75d2bb43df2a8d372b Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Wed, 15 May 2024 17:09:16 +0000 Subject: [PATCH 100/188] T3900: add support for raw table in firewall. --- data/templates/firewall/nftables.j2 | 46 ++- interface-definitions/firewall.xml.in | 2 + .../include/firewall/action-and-notrack.xml.i | 8 +- .../firewall/add-addr-to-group-ipv4.xml.i | 25 ++ .../firewall/add-addr-to-group-ipv6.xml.i | 25 ++ .../include/firewall/common-rule-inet.xml.i | 239 +------------- .../firewall/common-rule-ipv4-raw.xml.i | 309 +----------------- .../include/firewall/common-rule-ipv4.xml.i | 57 +--- .../firewall/common-rule-ipv6-raw.xml.i | 50 +++ .../include/firewall/common-rule-ipv6.xml.i | 57 +--- .../include/firewall/connection-status.xml.i | 28 ++ .../include/firewall/fragment.xml.i | 21 ++ .../include/firewall/icmp.xml.i | 34 ++ .../include/firewall/icmpv6.xml.i | 34 ++ .../include/firewall/ipv4-hook-output.xml.i | 27 ++ .../firewall/ipv4-hook-prerouting.xml.i | 34 -- .../include/firewall/ipv6-hook-output.xml.i | 27 ++ .../firewall/ipv6-hook-prerouting.xml.i | 51 +++ .../include/firewall/limit.xml.i | 33 ++ .../include/firewall/protocol.xml.i | 34 ++ .../include/firewall/recent.xml.i | 44 +++ .../include/firewall/time.xml.i | 70 ++++ .../include/policy/route-common.xml.i | 72 +--- .../include/policy/route-ipv4.xml.i | 33 +- python/vyos/firewall.py | 6 +- smoketest/scripts/cli/test_firewall.py | 40 ++- src/conf_mode/firewall.py | 4 +- src/conf_mode/system_conntrack.py | 3 + 28 files changed, 625 insertions(+), 788 deletions(-) create mode 100644 interface-definitions/include/firewall/add-addr-to-group-ipv4.xml.i create mode 100644 interface-definitions/include/firewall/add-addr-to-group-ipv6.xml.i create mode 100644 interface-definitions/include/firewall/common-rule-ipv6-raw.xml.i create mode 100644 interface-definitions/include/firewall/connection-status.xml.i create mode 100644 interface-definitions/include/firewall/fragment.xml.i create mode 100644 interface-definitions/include/firewall/icmp.xml.i create mode 100644 interface-definitions/include/firewall/icmpv6.xml.i create mode 100644 interface-definitions/include/firewall/ipv6-hook-prerouting.xml.i create mode 100644 interface-definitions/include/firewall/limit.xml.i create mode 100644 interface-definitions/include/firewall/protocol.xml.i create mode 100644 interface-definitions/include/firewall/recent.xml.i create mode 100644 interface-definitions/include/firewall/time.xml.i diff --git a/data/templates/firewall/nftables.j2 b/data/templates/firewall/nftables.j2 index 833df3a6701..343917feeb7 100644 --- a/data/templates/firewall/nftables.j2 +++ b/data/templates/firewall/nftables.j2 @@ -57,7 +57,7 @@ table ip vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('FWD-filter', 'ipv4') }} + {{ conf | nft_default_rule('FWD-' + prior, 'ipv4') }} } {% endfor %} {% endif %} @@ -77,7 +77,7 @@ table ip vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('INP-filter', 'ipv4') }} + {{ conf | nft_default_rule('INP-' + prior, 'ipv4') }} } {% endfor %} {% endif %} @@ -97,14 +97,11 @@ table ip vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('OUT-filter', 'ipv4') }} + {{ conf | nft_default_rule('OUT-' + prior, 'ipv4') }} } {% endfor %} {% endif %} - chain VYOS_FRAG_MARK { - type filter hook prerouting priority -450; policy accept; - ip frag-off & 0x3fff != 0 meta mark set 0xffff1 return - } + {% if ipv4.prerouting is vyos_defined %} {% for prior, conf in ipv4.prerouting.items() %} chain VYOS_PREROUTING_{{ prior }} { @@ -117,11 +114,16 @@ table ip vyos_filter { {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('PRE-filter', 'ipv4') }} + {{ conf | nft_default_rule('PRE-' + prior, 'ipv4') }} } {% endfor %} {% endif %} + chain VYOS_FRAG_MARK { + type filter hook prerouting priority -450; policy accept; + ip frag-off & 0x3fff != 0 meta mark set 0xffff1 return + } + {% if ipv4.name is vyos_defined %} {% for name_text, conf in ipv4.name.items() %} chain NAME_{{ name_text }} { @@ -202,13 +204,13 @@ table ip6 vyos_filter { {% endif %} {% if conf.rule is vyos_defined %} {% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} - {{ rule_conf | nft_rule('FWD', prior, rule_id ,'ip6') }} + {{ rule_conf | nft_rule('FWD', prior, rule_id, 'ip6') }} {% if rule_conf.recent is vyos_defined %} {% set ns.sets = ns.sets + ['FWD_' + prior + '_' + rule_id] %} {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('FWD-filter', 'ipv6') }} + {{ conf | nft_default_rule('FWD-' + prior, 'ipv6') }} } {% endfor %} {% endif %} @@ -222,13 +224,13 @@ table ip6 vyos_filter { {% endif %} {% if conf.rule is vyos_defined %} {% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} - {{ rule_conf | nft_rule('INP', prior, rule_id ,'ip6') }} + {{ rule_conf | nft_rule('INP', prior, rule_id, 'ip6') }} {% if rule_conf.recent is vyos_defined %} {% set ns.sets = ns.sets + ['INP_' + prior + '_' + rule_id] %} {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('INP-filter', 'ipv6') }} + {{ conf | nft_default_rule('INP-' + prior, 'ipv6') }} } {% endfor %} {% endif %} @@ -242,17 +244,33 @@ table ip6 vyos_filter { {% endif %} {% if conf.rule is vyos_defined %} {% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} - {{ rule_conf | nft_rule('OUT', prior, rule_id ,'ip6') }} + {{ rule_conf | nft_rule('OUT', prior, rule_id, 'ip6') }} {% if rule_conf.recent is vyos_defined %} {% set ns.sets = ns.sets + ['OUT_ ' + prior + '_' + rule_id] %} {% endif %} {% endfor %} {% endif %} - {{ conf | nft_default_rule('OUT-filter', 'ipv6') }} + {{ conf | nft_default_rule('OUT-' + prior, 'ipv6') }} } {% endfor %} {% endif %} +{% if ipv6.prerouting is vyos_defined %} +{% for prior, conf in ipv6.prerouting.items() %} + chain VYOS_IPV6_PREROUTING_{{ prior }} { + type filter hook prerouting priority {{ prior }}; policy accept; +{% if conf.rule is vyos_defined %} +{% for rule_id, rule_conf in conf.rule.items() if rule_conf.disable is not vyos_defined %} + {{ rule_conf | nft_rule('PRE', prior, rule_id, 'ip6') }} +{% if rule_conf.recent is vyos_defined %} +{% set ns.sets = ns.sets + ['PRE_' + prior + '_' + rule_id] %} +{% endif %} +{% endfor %} +{% endif %} + {{ conf | nft_default_rule('PRE-' + prior, 'ipv6') }} + } +{% endfor %} +{% endif %} chain VYOS_FRAG6_MARK { type filter hook prerouting priority -450; policy accept; exthdr frag exists meta mark set 0xffff1 return diff --git a/interface-definitions/firewall.xml.in b/interface-definitions/firewall.xml.in index 24e63c5eccf..dc4625af0be 100644 --- a/interface-definitions/firewall.xml.in +++ b/interface-definitions/firewall.xml.in @@ -378,6 +378,7 @@ #include #include #include + #include #include @@ -389,6 +390,7 @@ #include #include #include + #include #include diff --git a/interface-definitions/include/firewall/action-and-notrack.xml.i b/interface-definitions/include/firewall/action-and-notrack.xml.i index 5f81a1451e5..e063c58d546 100644 --- a/interface-definitions/include/firewall/action-and-notrack.xml.i +++ b/interface-definitions/include/firewall/action-and-notrack.xml.i @@ -3,12 +3,16 @@ Rule action - accept jump notrack reject return drop queue + accept continue jump notrack reject return drop queue accept Accept matching entries + + continue + Continue parsing next rule + jump Jump to another chain @@ -34,7 +38,7 @@ Igone connection tracking - (accept|jump|notrack|reject|return|drop|queue) + (accept|continue|jump|notrack|reject|return|drop|queue) diff --git a/interface-definitions/include/firewall/add-addr-to-group-ipv4.xml.i b/interface-definitions/include/firewall/add-addr-to-group-ipv4.xml.i new file mode 100644 index 00000000000..a47cadd5557 --- /dev/null +++ b/interface-definitions/include/firewall/add-addr-to-group-ipv4.xml.i @@ -0,0 +1,25 @@ + + + + Add ip address to dynamic address-group + + + + + Add source ip addresses to dynamic address-group + + + #include + + + + + Add destination ip addresses to dynamic address-group + + + #include + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/add-addr-to-group-ipv6.xml.i b/interface-definitions/include/firewall/add-addr-to-group-ipv6.xml.i new file mode 100644 index 00000000000..2cb07745065 --- /dev/null +++ b/interface-definitions/include/firewall/add-addr-to-group-ipv6.xml.i @@ -0,0 +1,25 @@ + + + + Add ipv6 address to dynamic ipv6-address-group + + + + + Add source ipv6 addresses to dynamic ipv6-address-group + + + #include + + + + + Add destination ipv6 addresses to dynamic ipv6-address-group + + + #include + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/common-rule-inet.xml.i b/interface-definitions/include/firewall/common-rule-inet.xml.i index bef1c3da551..55ffa3a8bb8 100644 --- a/interface-definitions/include/firewall/common-rule-inet.xml.i +++ b/interface-definitions/include/firewall/common-rule-inet.xml.i @@ -1,235 +1,24 @@ #include -#include -#include -#include -#include -#include #include -#include +#include +#include +#include #include - - - IP fragment match - - - - - Second and further fragments of fragmented packets - - - - - - Head fragments or unfragmented packets - - - - - - - - Rate limit using a token bucket filter - - - - - Maximum number of packets to allow in excess of rate - - u32:0-4294967295 - Maximum number of packets to allow in excess of rate - - - - - - - - - Maximum average matching rate - - txt - integer/unit (Example: 5/minute) - - - \d+/(second|minute|hour|day) - - - - - +#include +#include +#include +#include #include #include - - - Connection status - - - - - NAT connection status - - destination source - - - destination - Match connections that are subject to destination NAT - - - source - Match connections that are subject to source NAT - - - (destination|source) - - - - - - - - Protocol to match (protocol name, number, or "all") - - - all tcp_udp - - - all - All IP protocols - - - tcp_udp - Both TCP and UDP - - - u32:0-255 - IP protocol number - - - <protocol> - IP protocol name - - - !<protocol> - IP protocol name - - - - - - - - - Parameters for matching recently seen sources - - - - - Source addresses seen more than N times - - u32:1-255 - Source addresses seen more than N times - - - - - - - - - Source addresses seen in the last second/minute/hour - - second minute hour - - - second - Source addresses seen COUNT times in the last second - - - minute - Source addresses seen COUNT times in the last minute - - - hour - Source addresses seen COUNT times in the last hour - - - (second|minute|hour) - - - - - -#include +#include +#include +#include +#include +#include #include +#include #include #include - - - Time to match rule - - - - - Date to start matching rule - - txt - Enter date using following notation - YYYY-MM-DD - - - (\d{4}\-\d{2}\-\d{2}) - - - - - - Time of day to start matching rule - - txt - Enter time using using 24 hour notation - hh:mm:ss - - - ([0-2][0-9](\:[0-5][0-9]){1,2}) - - - - - - Date to stop matching rule - - txt - Enter date using following notation - YYYY-MM-DD - - - (\d{4}\-\d{2}\-\d{2}) - - - - - - Time of day to stop matching rule - - txt - Enter time using using 24 hour notation - hh:mm:ss - - - ([0-2][0-9](\:[0-5][0-9]){1,2}) - - - - - - Comma separated weekdays to match rule on - - txt - Name of day (Monday, Tuesday, Wednesday, Thursdays, Friday, Saturday, Sunday) - - - u32:0-6 - Day number (0 = Sunday ... 6 = Saturday) - - - - - +#include diff --git a/interface-definitions/include/firewall/common-rule-ipv4-raw.xml.i b/interface-definitions/include/firewall/common-rule-ipv4-raw.xml.i index e7468bfba46..960c960dbb5 100644 --- a/interface-definitions/include/firewall/common-rule-ipv4-raw.xml.i +++ b/interface-definitions/include/firewall/common-rule-ipv4-raw.xml.i @@ -1,9 +1,22 @@ +#include #include #include #include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include Destination parameters @@ -18,228 +31,6 @@ #include -#include - - - IP fragment match - - - - - Second and further fragments of fragmented packets - - - - - - Head fragments or unfragmented packets - - - - - - - - ICMP type and code information - - - - - ICMP code - - u32:0-255 - ICMP code (0-255) - - - - - - - - - ICMP type - - u32:0-255 - ICMP type (0-255) - - - - - - - #include - - - - - Inbound IPsec packets - - - - - Inbound IPsec packets - - - - - - Inbound non-IPsec packets - - - - - - - - Rate limit using a token bucket filter - - - - - Maximum number of packets to allow in excess of rate - - u32:0-4294967295 - Maximum number of packets to allow in excess of rate - - - - - - - - - Maximum average matching rate - - txt - integer/unit (Example: 5/minute) - - - \d+/(second|minute|hour|day) - - - - - - - - Option to log packets matching rule - - enable disable - - - enable - Enable log - - - disable - Disable log - - - (enable|disable) - - - -#include - - - Connection status - - - - - NAT connection status - - destination source - - - destination - Match connections that are subject to destination NAT - - - source - Match connections that are subject to source NAT - - - (destination|source) - - - - - - - - Protocol to match (protocol name, number, or "all") - - - all tcp_udp - - - all - All IP protocols - - - tcp_udp - Both TCP and UDP - - - u32:0-255 - IP protocol number - - - <protocol> - IP protocol name - - - !<protocol> - IP protocol name - - - - - - - - - Parameters for matching recently seen sources - - - - - Source addresses seen more than N times - - u32:1-255 - Source addresses seen more than N times - - - - - - - - - Source addresses seen in the last second/minute/hour - - second minute hour - - - second - Source addresses seen COUNT times in the last second - - - minute - Source addresses seen COUNT times in the last minute - - - hour - Source addresses seen COUNT times in the last hour - - - (second|minute|hour) - - - - - Source parameters @@ -254,74 +45,4 @@ #include -#include -#include - - - Time to match rule - - - - - Date to start matching rule - - txt - Enter date using following notation - YYYY-MM-DD - - - (\d{4}\-\d{2}\-\d{2}) - - - - - - Time of day to start matching rule - - txt - Enter time using using 24 hour notation - hh:mm:ss - - - ([0-2][0-9](\:[0-5][0-9]){1,2}) - - - - - - Date to stop matching rule - - txt - Enter date using following notation - YYYY-MM-DD - - - (\d{4}\-\d{2}\-\d{2}) - - - - - - Time of day to stop matching rule - - txt - Enter time using using 24 hour notation - hh:mm:ss - - - ([0-2][0-9](\:[0-5][0-9]){1,2}) - - - - - - Comma separated weekdays to match rule on - - txt - Name of day (Monday, Tuesday, Wednesday, Thursdays, Friday, Saturday, Sunday) - - - u32:0-6 - Day number (0 = Sunday ... 6 = Saturday) - - - - - - + \ No newline at end of file diff --git a/interface-definitions/include/firewall/common-rule-ipv4.xml.i b/interface-definitions/include/firewall/common-rule-ipv4.xml.i index 158c7a662b9..803b94b06b3 100644 --- a/interface-definitions/include/firewall/common-rule-ipv4.xml.i +++ b/interface-definitions/include/firewall/common-rule-ipv4.xml.i @@ -1,29 +1,8 @@ +#include #include +#include #include - - - Add ip address to dynamic address-group - - - - - Add source ip addresses to dynamic address-group - - - #include - - - - - Add destination ip addresses to dynamic address-group - - - #include - - - - Destination parameters @@ -39,38 +18,6 @@ #include - - - ICMP type and code information - - - - - ICMP code - - u32:0-255 - ICMP code (0-255) - - - - - - - - - ICMP type - - u32:0-255 - ICMP type (0-255) - - - - - - - #include - - Set jump target. Action jump must be defined to use this setting diff --git a/interface-definitions/include/firewall/common-rule-ipv6-raw.xml.i b/interface-definitions/include/firewall/common-rule-ipv6-raw.xml.i new file mode 100644 index 00000000000..958167b8915 --- /dev/null +++ b/interface-definitions/include/firewall/common-rule-ipv6-raw.xml.i @@ -0,0 +1,50 @@ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + + Destination parameters + + + #include + #include + #include + #include + #include + #include + #include + #include + + + + + Source parameters + + + #include + #include + #include + #include + #include + #include + #include + #include + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/common-rule-ipv6.xml.i b/interface-definitions/include/firewall/common-rule-ipv6.xml.i index 78eeb361ed6..bb176fe712a 100644 --- a/interface-definitions/include/firewall/common-rule-ipv6.xml.i +++ b/interface-definitions/include/firewall/common-rule-ipv6.xml.i @@ -1,29 +1,8 @@ +#include #include #include - - - Add ipv6 address to dynamic ipv6-address-group - - - - - Add source ipv6 addresses to dynamic ipv6-address-group - - - #include - - - - - Add destination ipv6 addresses to dynamic ipv6-address-group - - - #include - - - - +#include Destination parameters @@ -39,38 +18,6 @@ #include - - - ICMPv6 type and code information - - - - - ICMPv6 code - - u32:0-255 - ICMPv6 code (0-255) - - - - - - - - - ICMPv6 type - - u32:0-255 - ICMPv6 type (0-255) - - - - - - - #include - - Set jump target. Action jump must be defined to use this setting diff --git a/interface-definitions/include/firewall/connection-status.xml.i b/interface-definitions/include/firewall/connection-status.xml.i new file mode 100644 index 00000000000..5236c2f4f62 --- /dev/null +++ b/interface-definitions/include/firewall/connection-status.xml.i @@ -0,0 +1,28 @@ + + + + Connection status + + + + + NAT connection status + + destination source + + + destination + Match connections that are subject to destination NAT + + + source + Match connections that are subject to source NAT + + + (destination|source) + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/fragment.xml.i b/interface-definitions/include/firewall/fragment.xml.i new file mode 100644 index 00000000000..1f4c1105588 --- /dev/null +++ b/interface-definitions/include/firewall/fragment.xml.i @@ -0,0 +1,21 @@ + + + + IP fragment match + + + + + Second and further fragments of fragmented packets + + + + + + Head fragments or unfragmented packets + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/icmp.xml.i b/interface-definitions/include/firewall/icmp.xml.i new file mode 100644 index 00000000000..deb50a41026 --- /dev/null +++ b/interface-definitions/include/firewall/icmp.xml.i @@ -0,0 +1,34 @@ + + + + ICMP type and code information + + + + + ICMP code + + u32:0-255 + ICMP code (0-255) + + + + + + + + + ICMP type + + u32:0-255 + ICMP type (0-255) + + + + + + + #include + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/icmpv6.xml.i b/interface-definitions/include/firewall/icmpv6.xml.i new file mode 100644 index 00000000000..c0118626ef2 --- /dev/null +++ b/interface-definitions/include/firewall/icmpv6.xml.i @@ -0,0 +1,34 @@ + + + + ICMPv6 type and code information + + + + + ICMPv6 code + + u32:0-255 + ICMPv6 code (0-255) + + + + + + + + + ICMPv6 type + + u32:0-255 + ICMPv6 type (0-255) + + + + + + + #include + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/ipv4-hook-output.xml.i b/interface-definitions/include/firewall/ipv4-hook-output.xml.i index 2b537ce5e1a..ca47ae09b1e 100644 --- a/interface-definitions/include/firewall/ipv4-hook-output.xml.i +++ b/interface-definitions/include/firewall/ipv4-hook-output.xml.i @@ -31,6 +31,33 @@ + + + IPv4 firewall output raw + + + #include + #include + #include + + + IPv4 Firewall output raw rule number + + u32:1-999999 + Number for this firewall rule + + + + + Firewall rule number must be between 1 and 999999 + + + #include + #include + + + + diff --git a/interface-definitions/include/firewall/ipv4-hook-prerouting.xml.i b/interface-definitions/include/firewall/ipv4-hook-prerouting.xml.i index c38918375c8..17ecfe82448 100644 --- a/interface-definitions/include/firewall/ipv4-hook-prerouting.xml.i +++ b/interface-definitions/include/firewall/ipv4-hook-prerouting.xml.i @@ -4,40 +4,6 @@ IPv4 prerouting firewall - - - IPv4 firewall prerouting filter - - - #include - #include - - - IPv4 Firewall prerouting filter rule number - - u32:1-999999 - Number for this firewall rule - - - - - Firewall rule number must be between 1 and 999999 - - - #include - #include - - - Set jump target. Action jump must be defined to use this setting - - firewall ipv4 name - - - - - - - IPv4 firewall prerouting raw diff --git a/interface-definitions/include/firewall/ipv6-hook-output.xml.i b/interface-definitions/include/firewall/ipv6-hook-output.xml.i index ffe1c72b8aa..f877cfaafb6 100644 --- a/interface-definitions/include/firewall/ipv6-hook-output.xml.i +++ b/interface-definitions/include/firewall/ipv6-hook-output.xml.i @@ -30,6 +30,33 @@ + + + + IPv6 firewall output raw + + + #include + #include + #include + + + IPv6 Firewall output raw rule number + + u32:1-999999 + Number for this firewall rule + + + + + Firewall rule number must be between 1 and 999999 + + + #include + #include + + + diff --git a/interface-definitions/include/firewall/ipv6-hook-prerouting.xml.i b/interface-definitions/include/firewall/ipv6-hook-prerouting.xml.i new file mode 100644 index 00000000000..3f384828da3 --- /dev/null +++ b/interface-definitions/include/firewall/ipv6-hook-prerouting.xml.i @@ -0,0 +1,51 @@ + + + + IPv6 prerouting firewall + + + + + IPv6 firewall prerouting raw + + + #include + #include + + + Set jump target. Action jump must be defined in default-action to use this setting + + firewall ipv6 name + + + + + + IPv6 Firewall prerouting raw rule number + + u32:1-999999 + Number for this firewall rule + + + + + Firewall rule number must be between 1 and 999999 + + + #include + #include + + + Set jump target. Action jump must be defined to use this setting + + firewall ipv6 name + + + + + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/limit.xml.i b/interface-definitions/include/firewall/limit.xml.i new file mode 100644 index 00000000000..21068dec2b0 --- /dev/null +++ b/interface-definitions/include/firewall/limit.xml.i @@ -0,0 +1,33 @@ + + + + Rate limit using a token bucket filter + + + + + Maximum number of packets to allow in excess of rate + + u32:0-4294967295 + Maximum number of packets to allow in excess of rate + + + + + + + + + Maximum average matching rate + + txt + integer/unit (Example: 5/minute) + + + \d+/(second|minute|hour|day) + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/protocol.xml.i b/interface-definitions/include/firewall/protocol.xml.i new file mode 100644 index 00000000000..e391cae4159 --- /dev/null +++ b/interface-definitions/include/firewall/protocol.xml.i @@ -0,0 +1,34 @@ + + + + Protocol to match (protocol name, number, or "all") + + + all tcp_udp + + + all + All IP protocols + + + tcp_udp + Both TCP and UDP + + + u32:0-255 + IP protocol number + + + <protocol> + IP protocol name + + + !<protocol> + IP protocol name + + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/recent.xml.i b/interface-definitions/include/firewall/recent.xml.i new file mode 100644 index 00000000000..38f40b91699 --- /dev/null +++ b/interface-definitions/include/firewall/recent.xml.i @@ -0,0 +1,44 @@ + + + + Parameters for matching recently seen sources + + + + + Source addresses seen more than N times + + u32:1-255 + Source addresses seen more than N times + + + + + + + + + Source addresses seen in the last second/minute/hour + + second minute hour + + + second + Source addresses seen COUNT times in the last second + + + minute + Source addresses seen COUNT times in the last minute + + + hour + Source addresses seen COUNT times in the last hour + + + (second|minute|hour) + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/firewall/time.xml.i b/interface-definitions/include/firewall/time.xml.i new file mode 100644 index 00000000000..7bd7374502e --- /dev/null +++ b/interface-definitions/include/firewall/time.xml.i @@ -0,0 +1,70 @@ + + + + Time to match rule + + + + + Date to start matching rule + + txt + Enter date using following notation - YYYY-MM-DD + + + (\d{4}\-\d{2}\-\d{2}) + + + + + + Time of day to start matching rule + + txt + Enter time using using 24 hour notation - hh:mm:ss + + + ([0-2][0-9](\:[0-5][0-9]){1,2}) + + + + + + Date to stop matching rule + + txt + Enter date using following notation - YYYY-MM-DD + + + (\d{4}\-\d{2}\-\d{2}) + + + + + + Time of day to stop matching rule + + txt + Enter time using using 24 hour notation - hh:mm:ss + + + ([0-2][0-9](\:[0-5][0-9]){1,2}) + + + + + + Comma separated weekdays to match rule on + + txt + Name of day (Monday, Tuesday, Wednesday, Thursdays, Friday, Saturday, Sunday) + + + u32:0-6 + Day number (0 = Sunday ... 6 = Saturday) + + + + + + \ No newline at end of file diff --git a/interface-definitions/include/policy/route-common.xml.i b/interface-definitions/include/policy/route-common.xml.i index e412fe58ef6..97795601eed 100644 --- a/interface-definitions/include/policy/route-common.xml.i +++ b/interface-definitions/include/policy/route-common.xml.i @@ -3,75 +3,9 @@ #include #include #include - - - IP fragment match - - - - - Second and further fragments of fragmented packets - - - - - - Head fragments or unfragmented packets - - - - - - - - Inbound IPsec packets - - - - - Inbound IPsec packets - - - - - - Inbound non-IPsec packets - - - - - - - - Rate limit using a token bucket filter - - - - - Maximum number of packets to allow in excess of rate - - u32:0-4294967295 - Maximum number of packets to allow in excess of rate - - - - - - - - - Maximum average matching rate - - u32:0-4294967295 - Maximum average matching rate - - - - - - - - +#include +#include +#include #include diff --git a/interface-definitions/include/policy/route-ipv4.xml.i b/interface-definitions/include/policy/route-ipv4.xml.i index 1f717a1a433..c12abcae2c0 100644 --- a/interface-definitions/include/policy/route-ipv4.xml.i +++ b/interface-definitions/include/policy/route-ipv4.xml.i @@ -10,36 +10,5 @@ #include - - - ICMP type and code information - - - - - ICMP code (0-255) - - u32:0-255 - ICMP code (0-255) - - - - - - - - - ICMP type (0-255) - - u32:0-255 - ICMP type (0-255) - - - - - - - #include - - +#include diff --git a/python/vyos/firewall.py b/python/vyos/firewall.py index d7b7b80a8bb..664df28cc6d 100644 --- a/python/vyos/firewall.py +++ b/python/vyos/firewall.py @@ -178,6 +178,8 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): hook_name = 'input' if hook == 'OUT': hook_name = 'output' + if hook == 'PRE': + hook_name = 'prerouting' if hook == 'NAM': hook_name = f'name{def_suffix}' output.append(f'{ip_name} {prefix}addr {operator} @FQDN_{hook_name}_{fw_name}_{rule_id}_{prefix}') @@ -193,6 +195,8 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): hook_name = 'input' if hook == 'OUT': hook_name = 'output' + if hook == 'PRE': + hook_name = 'prerouting' if hook == 'NAM': hook_name = f'name' output.append(f'{ip_name} {prefix}addr {operator} @GEOIP_CC{def_suffix}_{hook_name}_{fw_name}_{rule_id}') @@ -477,8 +481,6 @@ def parse_rule(rule_conf, hook, fw_name, rule_id, ip_name): output.append(f'tcp option maxseg size set {mss}') if 'action' in rule_conf: - # Change action=return to action=action - # #output.append(nft_action(rule_conf['action'])) if rule_conf['action'] == 'offload': offload_target = rule_conf['offload_target'] output.append(f'flow add @VYOS_FLOWTABLE_{offload_target}') diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index c4756271421..5cf2b214675 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -236,6 +236,14 @@ def test_ipv4_basic_rules(self): self.cli_set(['firewall', 'ipv4', 'output', 'filter', 'rule', '6', 'protocol', 'icmp']) self.cli_set(['firewall', 'ipv4', 'output', 'filter', 'rule', '6', 'connection-mark', conn_mark]) + self.cli_set(['firewall', 'ipv4', 'output', 'raw', 'default-action', 'drop']) + self.cli_set(['firewall', 'ipv4', 'output', 'raw', 'rule', '1', 'action', 'accept']) + self.cli_set(['firewall', 'ipv4', 'output', 'raw', 'rule', '1', 'protocol', 'udp']) + + self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'action', 'drop']) + self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'destination', 'port', '23']) + self.cli_commit() mark_hex = "{0:#010x}".format(int(conn_mark)) @@ -256,6 +264,14 @@ def test_ipv4_basic_rules(self): ['meta l4proto gre', f'oifname != "{interface}"', 'drop'], ['meta l4proto icmp', f'ct mark {mark_hex}', 'return'], ['log prefix "[ipv4-OUT-filter-default-D]"','OUT-filter default-action drop', 'drop'], + ['chain VYOS_OUTPUT_raw'], + ['type filter hook output priority raw; policy accept;'], + ['udp', 'accept'], + ['OUT-raw default-action drop', 'drop'], + ['chain VYOS_PREROUTING_raw'], + ['type filter hook prerouting priority raw; policy accept;'], + ['tcp dport 23', 'drop'], + ['PRE-raw default-action accept', 'accept'], ['chain NAME_smoketest'], ['saddr 172.16.20.10', 'daddr 172.16.10.10', 'log prefix "[ipv4-NAM-smoketest-1-A]" log level debug', 'ip ttl 15', 'accept'], ['tcp flags syn / syn,ack', 'tcp dport 8888', 'log prefix "[ipv4-NAM-smoketest-2-R]" log level err', 'ip ttl > 102', 'reject'], @@ -446,16 +462,24 @@ def test_ipv6_basic_rules(self): self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '2', 'destination', 'port', '8888']) self.cli_set(['firewall', 'ipv6', 'forward', 'filter', 'rule', '2', 'inbound-interface', 'name', interface]) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'action', 'accept']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'protocol', 'udp']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'source', 'address', '2002::1:2']) + self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'inbound-interface', 'name', interface]) + self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'default-action', 'drop']) self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'default-log']) self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '3', 'action', 'return']) self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '3', 'protocol', 'gre']) self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '3', 'outbound-interface', 'name', interface]) - self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'action', 'accept']) - self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'protocol', 'udp']) - self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'source', 'address', '2002::1:2']) - self.cli_set(['firewall', 'ipv6', 'input', 'filter', 'rule', '3', 'inbound-interface', 'name', interface]) + self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'default-action', 'drop']) + self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'rule', '1', 'action', 'accept']) + self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'rule', '1', 'protocol', 'udp']) + + self.cli_set(['firewall', 'ipv6', 'prerouting', 'raw', 'rule', '1', 'action', 'drop']) + self.cli_set(['firewall', 'ipv6', 'prerouting', 'raw', 'rule', '1', 'protocol', 'tcp']) + self.cli_set(['firewall', 'ipv6', 'prerouting', 'raw', 'rule', '1', 'destination', 'port', '23']) self.cli_commit() @@ -472,6 +496,14 @@ def test_ipv6_basic_rules(self): ['type filter hook output priority filter; policy accept;'], ['meta l4proto gre', f'oifname "{interface}"', 'return'], ['log prefix "[ipv6-OUT-filter-default-D]"','OUT-filter default-action drop', 'drop'], + ['chain VYOS_IPV6_OUTPUT_raw'], + ['type filter hook output priority raw; policy accept;'], + ['udp', 'accept'], + ['OUT-raw default-action drop', 'drop'], + ['chain VYOS_IPV6_PREROUTING_raw'], + ['type filter hook prerouting priority raw; policy accept;'], + ['tcp dport 23', 'drop'], + ['PRE-raw default-action accept', 'accept'], [f'chain NAME6_{name}'], ['saddr 2002::1', 'daddr 2002::1:1', 'log prefix "[ipv6-NAM-v6-smoketest-1-A]" log level crit', 'accept'], [f'"{name} default-action drop"', f'log prefix "[ipv6-{name}-default-D]"', 'drop'], diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index e96e57154ac..acf3805d2cb 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -351,7 +351,7 @@ def verify(firewall): verify_nested_group(group_name, group, groups, []) if 'ipv4' in firewall: - for name in ['name','forward','input','output']: + for name in ['name','forward','input','output', 'prerouting']: if name in firewall['ipv4']: for name_id, name_conf in firewall['ipv4'][name].items(): if 'jump' in name_conf['default_action'] and 'default_jump_target' not in name_conf: @@ -371,7 +371,7 @@ def verify(firewall): verify_rule(firewall, rule_conf, False) if 'ipv6' in firewall: - for name in ['name','forward','input','output']: + for name in ['name','forward','input','output', 'prerouting']: if name in firewall['ipv6']: for name_id, name_conf in firewall['ipv6'][name].items(): if 'jump' in name_conf['default_action'] and 'default_jump_target' not in name_conf: diff --git a/src/conf_mode/system_conntrack.py b/src/conf_mode/system_conntrack.py index 031fe63b0a5..d9c38fd956d 100755 --- a/src/conf_mode/system_conntrack.py +++ b/src/conf_mode/system_conntrack.py @@ -18,6 +18,7 @@ from sys import exit +from vyos.base import Warning from vyos.config import Config from vyos.configdep import set_dependents, call_dependents from vyos.utils.dict import dict_search @@ -165,6 +166,8 @@ def verify(conntrack): if not group_obj: Warning(f'{error_group} "{group_name}" has no members!') + Warning(f'It is prefered to defined {inet} conntrack ignore rules in the section') + if dict_search_args(conntrack, 'timeout', 'custom', inet, 'rule') != None: for rule, rule_config in conntrack['timeout']['custom'][inet]['rule'].items(): if 'protocol' not in rule_config: From c6be441c86bc8fe2e938e2bd3c85f99071cbfb49 Mon Sep 17 00:00:00 2001 From: l0crian1 Date: Wed, 15 May 2024 14:54:18 -0400 Subject: [PATCH 101/188] T6335: Add/Update EVPN op commands Added the following commands: show evpn show evpn es show evpn es show evpn es detail show evpn es-evi show evpn es-evi detail show evpn es-evi vni show evpn vni show evpn vni detail show evpn vni Updated the following commands: show evpn access-vlan show evpn arp-cache show evpn mac show evpn next-hops show evpn rmac --- data/op-mode-standardized.json | 1 + .../include/vni-tagnode-all.xml.i | 5 +- op-mode-definitions/include/vni-tagnode.xml.i | 5 +- op-mode-definitions/show-evpn.xml.in | 59 ++++++++++++++++++- src/completion/list_esi.py | 31 ++++++++++ src/completion/list_vni.py | 27 +++++++++ src/op_mode/evpn.py | 46 +++++++++++++++ 7 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 src/completion/list_esi.py create mode 100644 src/completion/list_vni.py create mode 100644 src/op_mode/evpn.py diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index a4ed2bcf466..911869d093c 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -9,6 +9,7 @@ "cpu.py", "dhcp.py", "dns.py", +"evpn.py", "interfaces.py", "ipsec.py", "lldp.py", diff --git a/op-mode-definitions/include/vni-tagnode-all.xml.i b/op-mode-definitions/include/vni-tagnode-all.xml.i index 0fedb937162..18f74653ba6 100644 --- a/op-mode-definitions/include/vni-tagnode-all.xml.i +++ b/op-mode-definitions/include/vni-tagnode-all.xml.i @@ -3,9 +3,10 @@ VXLAN network identifier (VNI) number - 1-16777215 all + <1-16777215> all + - ${vyos_op_scripts_dir}/vtysh_wrapper.sh $@ + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" diff --git a/op-mode-definitions/include/vni-tagnode.xml.i b/op-mode-definitions/include/vni-tagnode.xml.i index 22f2d33bd9c..363da8d5e89 100644 --- a/op-mode-definitions/include/vni-tagnode.xml.i +++ b/op-mode-definitions/include/vni-tagnode.xml.i @@ -3,9 +3,10 @@ VXLAN network identifier (VNI) number - 1-16777215 + <1-16777215> + - ${vyos_op_scripts_dir}/vtysh_wrapper.sh $@ + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" diff --git a/op-mode-definitions/show-evpn.xml.in b/op-mode-definitions/show-evpn.xml.in index a005cbc304e..ba3b104cdd8 100644 --- a/op-mode-definitions/show-evpn.xml.in +++ b/op-mode-definitions/show-evpn.xml.in @@ -14,7 +14,7 @@ #include - ${vyos_op_scripts_dir}/vtysh_wrapper.sh $@ + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" @@ -31,7 +31,7 @@ <1-4094> - ${vyos_op_scripts_dir}/vtysh_wrapper.sh $@ + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" @@ -43,6 +43,45 @@ #include + + + Show ESI information for specified ESI + + <esi> + + + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + + + Show ESI information + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + + + Show ESI details + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + + + + + Show ESI information per EVI + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + + + Show ESI per EVI details + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + #include + + MAC addresses @@ -67,7 +106,23 @@ #include + #include + + + Show VNI information + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + + + Show VNI details + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" + + + + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" diff --git a/src/completion/list_esi.py b/src/completion/list_esi.py new file mode 100644 index 00000000000..fc40f37b699 --- /dev/null +++ b/src/completion/list_esi.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2016-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# This script is completion helper to list all configured VNIs that are visible to FRR + +import json +from vyos.utils.process import cmd + +def get_esi(): + esiDict = json.loads(cmd(f"vtysh -c 'show evpn es json'")) + esiList = [] + for i in esiDict: + esiList.append(i['esi']) + + print(' '.join(esiList)) + +if __name__ == '__main__': + get_esi() diff --git a/src/completion/list_vni.py b/src/completion/list_vni.py new file mode 100644 index 00000000000..ef78eb16fa1 --- /dev/null +++ b/src/completion/list_vni.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2016-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# This script is completion helper to list all configured VNIs that are visible to FRR + +import json +from vyos.utils.process import cmd + +def get_vni(): + vniDict = json.loads(cmd(f"vtysh -c 'show evpn vni json'")) + print(' '.join(vniDict.keys())) + +if __name__ == '__main__': + get_vni() diff --git a/src/op_mode/evpn.py b/src/op_mode/evpn.py new file mode 100644 index 00000000000..cae4ab9f5b6 --- /dev/null +++ b/src/op_mode/evpn.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2016-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# This script is a helper to run VTYSH commands for "show evpn", allowing for the --raw flag to output JSON + +import sys +import typing +import json + +import vyos.opmode +from vyos.utils.process import cmd + +def show_evpn(raw: bool, command: typing.Optional[str]): + if raw: + command = f"{command} json" + evpnDict = {} + try: + evpnDict['evpn'] = json.loads(cmd(f"vtysh -c '{command}'")) + except: + raise vyos.opmode.DataUnavailable(f"\"{command.replace(' json', '')}\" is invalid or has no JSON option") + + return evpnDict + else: + return cmd(f"vtysh -c '{command}'") + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) From 3917e3e9f985063ab7419c903f6019116224f640 Mon Sep 17 00:00:00 2001 From: l0crian1 Date: Thu, 16 May 2024 08:07:02 -0400 Subject: [PATCH 102/188] T6335: Add/Update EVPN op commands Converted completion helpers from python to bash for performance Previous commit: Added the following commands: show evpn show evpn es show evpn es show evpn es detail show evpn es-evi show evpn es-evi detail show evpn es-evi vni show evpn vni show evpn vni detail show evpn vni Updated the following commands: show evpn access-vlan show evpn arp-cache show evpn mac show evpn next-hops show evpn rmac --- .../include/vni-tagnode-all.xml.i | 2 +- op-mode-definitions/include/vni-tagnode.xml.i | 2 +- op-mode-definitions/show-evpn.xml.in | 2 +- src/completion/{list_esi.py => list_esi.sh} | 23 +++++-------------- src/completion/{list_vni.py => list_vni.sh} | 17 ++++---------- 5 files changed, 14 insertions(+), 32 deletions(-) rename src/completion/{list_esi.py => list_esi.sh} (50%) mode change 100644 => 100755 rename src/completion/{list_vni.py => list_vni.sh} (63%) mode change 100644 => 100755 diff --git a/op-mode-definitions/include/vni-tagnode-all.xml.i b/op-mode-definitions/include/vni-tagnode-all.xml.i index 18f74653ba6..fabab19d7fd 100644 --- a/op-mode-definitions/include/vni-tagnode-all.xml.i +++ b/op-mode-definitions/include/vni-tagnode-all.xml.i @@ -4,7 +4,7 @@ VXLAN network identifier (VNI) number <1-16777215> all - + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" diff --git a/op-mode-definitions/include/vni-tagnode.xml.i b/op-mode-definitions/include/vni-tagnode.xml.i index 363da8d5e89..f5b99dcc866 100644 --- a/op-mode-definitions/include/vni-tagnode.xml.i +++ b/op-mode-definitions/include/vni-tagnode.xml.i @@ -4,7 +4,7 @@ VXLAN network identifier (VNI) number <1-16777215> - + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" diff --git a/op-mode-definitions/show-evpn.xml.in b/op-mode-definitions/show-evpn.xml.in index ba3b104cdd8..3c1e5c7d665 100644 --- a/op-mode-definitions/show-evpn.xml.in +++ b/op-mode-definitions/show-evpn.xml.in @@ -48,7 +48,7 @@ Show ESI information for specified ESI <esi> - + ${vyos_op_scripts_dir}/evpn.py show_evpn --command "$*" diff --git a/src/completion/list_esi.py b/src/completion/list_esi.sh old mode 100644 new mode 100755 similarity index 50% rename from src/completion/list_esi.py rename to src/completion/list_esi.sh index fc40f37b699..b8373fa5706 --- a/src/completion/list_esi.py +++ b/src/completion/list_esi.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env python3 +#!/bin/bash # -# Copyright (C) 2016-2024 VyOS maintainers and contributors +# Copyright (C) 2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -8,24 +8,13 @@ # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # -# This script is completion helper to list all configured VNIs that are visible to FRR +# This script is completion helper to list all valid ESEs that are visible to FRR -import json -from vyos.utils.process import cmd - -def get_esi(): - esiDict = json.loads(cmd(f"vtysh -c 'show evpn es json'")) - esiList = [] - for i in esiDict: - esiList.append(i['esi']) - - print(' '.join(esiList)) - -if __name__ == '__main__': - get_esi() +esiJson=$(vtysh -c 'show evpn es json') +echo "$(echo "$esiJson" | jq -r '.[] | .esi')" diff --git a/src/completion/list_vni.py b/src/completion/list_vni.sh old mode 100644 new mode 100755 similarity index 63% rename from src/completion/list_vni.py rename to src/completion/list_vni.sh index ef78eb16fa1..f8bd4a993b7 --- a/src/completion/list_vni.py +++ b/src/completion/list_vni.sh @@ -1,6 +1,6 @@ -#!/usr/bin/env python3 +#!/bin/bash # -# Copyright (C) 2016-2024 VyOS maintainers and contributors +# Copyright (C) 2024 VyOS maintainers and contributors # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License version 2 or later as @@ -8,7 +8,7 @@ # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License @@ -16,12 +16,5 @@ # # This script is completion helper to list all configured VNIs that are visible to FRR -import json -from vyos.utils.process import cmd - -def get_vni(): - vniDict = json.loads(cmd(f"vtysh -c 'show evpn vni json'")) - print(' '.join(vniDict.keys())) - -if __name__ == '__main__': - get_vni() +vniJson=$(vtysh -c 'show evpn vni json') +echo "$(echo "$vniJson" | jq -r 'keys | .[]')" From e337030f31a9162738748d246dae83ad46125439 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 16 May 2024 12:08:34 +0000 Subject: [PATCH 103/188] T6347: CGNAT fix error if pool contain dashes in the name --- src/conf_mode/nat_cgnat.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index f41d66c6654..8292f23a453 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -219,8 +219,8 @@ def generate(config): # first external pool as we allow only one as PoC ext_pool_name = jmespath.search("rule.*.translation | [0]", config).get('pool') int_pool_name = jmespath.search("rule.*.source | [0]", config).get('pool') - ext_query = f"pool.external.{ext_pool_name}.range | keys(@)" - int_query = f"pool.internal.{int_pool_name}.range" + ext_query = f'pool.external."{ext_pool_name}".range | keys(@)' + int_query = f'pool.internal."{int_pool_name}".range' external_ranges = jmespath.search(ext_query, config) internal_ranges = [jmespath.search(int_query, config)] @@ -246,10 +246,10 @@ def generate(config): external_host_count = sum(external_list_count) internal_host_count = sum(internal_list_count) ports_per_user = int( - jmespath.search(f'pool.external.{ext_pool_name}.per_user_limit.port', config) + jmespath.search(f'pool.external."{ext_pool_name}".per_user_limit.port', config) ) external_port_range: str = jmespath.search( - f'pool.external.{ext_pool_name}.external_port_range', config + f'pool.external."{ext_pool_name}".external_port_range', config ) proto_maps, other_maps = generate_port_rules( From c4cee2b7c51567350943a0387068f57d04456d12 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 16 May 2024 16:25:18 +0000 Subject: [PATCH 104/188] T6351: CGNAT add verification if the pool exists Add verification if the external/internal pools are exists before we can use them in the source and translation rules --- src/conf_mode/nat_cgnat.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index f41d66c6654..9c582451225 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -203,6 +203,11 @@ def verify(config): f'Range for "{pool} pool {pool_name}" must be defined!' ) + external_pools_query = "keys(pool.external)" + external_pools: list = jmespath.search(external_pools_query, config) + internal_pools_query = "keys(pool.internal)" + internal_pools: list = jmespath.search(internal_pools_query, config) + for rule, rule_config in config['rule'].items(): if 'source' not in rule_config: raise ConfigError(f'Rule "{rule}" source pool must be defined!') @@ -212,6 +217,14 @@ def verify(config): if 'translation' not in rule_config: raise ConfigError(f'Rule "{rule}" translation pool must be defined!') + internal_pool = rule_config['source']['pool'] + if internal_pool not in internal_pools: + raise ConfigError(f'Internal pool "{internal_pool}" does not exist!') + + external_pool = rule_config['translation']['pool'] + if external_pool not in external_pools: + raise ConfigError(f'External pool "{external_pool}" does not exist!') + def generate(config): if not config: From d5c4b758ba466119782887a73f5c2701b227c1a6 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 16 May 2024 18:37:34 +0000 Subject: [PATCH 105/188] T6350: CGNAT add op-mode to show allocation Add op-mode command `show nat cgnat allocation` to get CGNAT allocations (internal address, external address, port-range) --- data/op-mode-standardized.json | 1 + op-mode-definitions/nat.xml.in | 13 ++++++ src/op_mode/cgnat.py | 75 ++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+) create mode 100755 src/op_mode/cgnat.py diff --git a/data/op-mode-standardized.json b/data/op-mode-standardized.json index a4ed2bcf466..eea8244ec2d 100644 --- a/data/op-mode-standardized.json +++ b/data/op-mode-standardized.json @@ -3,6 +3,7 @@ "bgp.py", "bonding.py", "bridge.py", +"cgnat.py", "config_mgmt.py", "conntrack.py", "container.py", diff --git a/op-mode-definitions/nat.xml.in b/op-mode-definitions/nat.xml.in index 307a9133728..6398c0e0789 100644 --- a/op-mode-definitions/nat.xml.in +++ b/op-mode-definitions/nat.xml.in @@ -7,6 +7,19 @@ Show IPv4 Network Address Translation (NAT) information + + + Show Carrier-Grade Network Address Translation (CGNAT) + + + + + Show allocated CGNAT parameters + + sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation + + + Show source IPv4 to IPv4 Network Address Translation (NAT) information diff --git a/src/op_mode/cgnat.py b/src/op_mode/cgnat.py new file mode 100755 index 00000000000..a98269a15f3 --- /dev/null +++ b/src/op_mode/cgnat.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import json +import sys +import typing + +from tabulate import tabulate + +import vyos.opmode + +from vyos.configquery import ConfigTreeQuery +from vyos.utils.process import cmd + +CGNAT_TABLE = 'cgnat' + + +def _get_raw_data(): + """ Get CGNAT dictionary + """ + cmd_output = cmd(f'nft --json list table ip {CGNAT_TABLE}') + data = json.loads(cmd_output) + return data + + +def _get_formatted_output(data): + elements = data['nftables'][2]['map']['elem'] + allocations = [] + for elem in elements: + internal = elem[0] # internal + external = elem[1]['concat'][0] # external + start_port = elem[1]['concat'][1]['range'][0] + end_port = elem[1]['concat'][1]['range'][1] + port_range = f'{start_port}-{end_port}' + allocations.append((internal, external, port_range)) + + headers = ['Internal IP', 'External IP', 'Port range'] + output = tabulate(allocations, headers, numalign="left") + return output + + +def show_allocation(raw: bool): + config = ConfigTreeQuery() + if not config.exists('nat cgnat'): + raise vyos.opmode.UnconfiguredSubsystem('CGNAT is not configured') + + if raw: + return _get_raw_data() + + else: + raw_data = _get_raw_data() + return _get_formatted_output(raw_data) + + +if __name__ == '__main__': + try: + res = vyos.opmode.run(sys.modules[__name__]) + if res: + print(res) + except (ValueError, vyos.opmode.Error) as e: + print(e) + sys.exit(1) From 1cba74f91a67348bc8e8ad3e2ef4325dc9f9d6e0 Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Fri, 17 May 2024 10:56:52 +0300 Subject: [PATCH 106/188] op mode: T6348: SNAT op-mode fails with flowtable offload entries --- src/op_mode/nat.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/op_mode/nat.py b/src/op_mode/nat.py index 2bc7e24fe40..4ab524fb7ac 100755 --- a/src/op_mode/nat.py +++ b/src/op_mode/nat.py @@ -263,7 +263,7 @@ def _get_formatted_translation(dict_data, nat_direction, family, verbose): proto = meta['layer4']['protoname'] if direction == 'independent': conn_id = meta['id'] - timeout = meta['timeout'] + timeout = meta.get('timeout', 'n/a') orig_src = f'{orig_src}:{orig_sport}' if orig_sport else orig_src orig_dst = f'{orig_dst}:{orig_dport}' if orig_dport else orig_dst reply_src = f'{reply_src}:{reply_sport}' if reply_sport else reply_src From 240f199cdfadbc12ce713dae74c8db3af44a398c Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Fri, 17 May 2024 08:45:45 +0200 Subject: [PATCH 107/188] T6358: Remove duplicate host name handling --- src/conf_mode/container.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index a73a18ffa22..c3b661ac516 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -339,11 +339,6 @@ def generate_run_arguments(name, container_config): entrypoint = json_write(container_config['entrypoint'].split()).replace('"', """) entrypoint = f'--entrypoint '{entrypoint}'' - hostname = '' - if 'host_name' in container_config: - hostname = container_config['host_name'] - hostname = f'--hostname {hostname}' - command = '' if 'command' in container_config: command = container_config['command'].strip() From f5051de4fc034bd95677ef142423e59eae47cd2f Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Fri, 17 May 2024 08:59:55 +0200 Subject: [PATCH 108/188] T6358: Add config option for host process namespace --- interface-definitions/container.xml.in | 8 +++++++- smoketest/config-tests/container-simple | 1 + smoketest/configs/container-simple | 1 + src/conf_mode/container.py | 6 +++++- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/interface-definitions/container.xml.in b/interface-definitions/container.xml.in index e7dacea367b..2296a3e9e8f 100644 --- a/interface-definitions/container.xml.in +++ b/interface-definitions/container.xml.in @@ -15,9 +15,15 @@ Container name must be alphanumeric and can contain hyphens + + + Allow sharing host process namespace with container + + + - Allow host networks in container + Allow sharing host networking with container diff --git a/smoketest/config-tests/container-simple b/smoketest/config-tests/container-simple index 299af64cbe0..cc80ef4cf52 100644 --- a/smoketest/config-tests/container-simple +++ b/smoketest/config-tests/container-simple @@ -8,5 +8,6 @@ set container name c01 capability 'net-bind-service' set container name c01 capability 'net-raw' set container name c01 image 'busybox:stable' set container name c02 allow-host-networks +set container name c02 allow-host-pid set container name c02 capability 'sys-time' set container name c02 image 'busybox:stable' diff --git a/smoketest/configs/container-simple b/smoketest/configs/container-simple index 05efe05e9c2..82983afb708 100644 --- a/smoketest/configs/container-simple +++ b/smoketest/configs/container-simple @@ -7,6 +7,7 @@ container { } name c02 { allow-host-networks + allow-host-pid cap-add sys-time image busybox:stable } diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index c3b661ac516..91a10e89138 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -329,9 +329,13 @@ def generate_run_arguments(name, container_config): prop = vol_config['propagation'] volume += f' --volume {svol}:{dvol}:{mode},{prop}' + host_pid = '' + if 'allow_host_pid' in container_config: + host_pid = '--pid host' + container_base_cmd = f'--detach --interactive --tty --replace {capabilities} ' \ f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \ - f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label} {uid}' + f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label} {uid} {host_pid}' entrypoint = '' if 'entrypoint' in container_config: From 783edc98f82f8718ccc856976d9a8f59bc6822e9 Mon Sep 17 00:00:00 2001 From: khramshinr Date: Fri, 17 May 2024 15:22:16 +0600 Subject: [PATCH 109/188] T6354: Get rid of the custom boot type check in version.py --- python/vyos/version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/vyos/version.py b/python/vyos/version.py index 47a10e2014c..86e96d0ece5 100644 --- a/python/vyos/version.py +++ b/python/vyos/version.py @@ -33,11 +33,11 @@ import requests import vyos.defaults +from vyos.system.image import is_live_boot from vyos.utils.file import read_file from vyos.utils.file import read_json from vyos.utils.process import popen -from vyos.utils.process import run from vyos.utils.process import DEVNULL version_file = os.path.join(vyos.defaults.directories['data'], 'version.json') @@ -85,7 +85,7 @@ def get_full_version_data(fname=version_file): # In installed images, the squashfs image file is named after its image version, # while on livecd it's just "filesystem.squashfs", that's how we tell a livecd boot # from an installed image - if run(""" grep -e '^overlay.*/filesystem.squashfs' /proc/mounts >/dev/null """) == 0: + if is_live_boot(): boot_via = "livecd" else: boot_via = "installed image" From 3339a4f990976d9b04df01d6c7f0777ed8e71e74 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Fri, 17 May 2024 14:03:31 +0000 Subject: [PATCH 110/188] T5169: Add smoketest for CGNAT --- smoketest/scripts/cli/test_cgnat.py | 99 +++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100755 smoketest/scripts/cli/test_cgnat.py diff --git a/smoketest/scripts/cli/test_cgnat.py b/smoketest/scripts/cli/test_cgnat.py new file mode 100755 index 00000000000..c65c58820fc --- /dev/null +++ b/smoketest/scripts/cli/test_cgnat.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import unittest + +from base_vyostest_shim import VyOSUnitTestSHIM +from vyos.configsession import ConfigSessionError + + +base_path = ['nat', 'cgnat'] +nftables_cgnat_config = '/run/nftables-cgnat.nft' + + +class TestCGNAT(VyOSUnitTestSHIM.TestCase): + @classmethod + def setUpClass(cls): + super(TestCGNAT, cls).setUpClass() + + # ensure we can also run this test on a live system - so lets clean + # out the current configuration :) + cls.cli_delete(cls, base_path) + + def tearDown(self): + self.cli_delete(base_path) + self.cli_commit() + self.assertFalse(os.path.exists(nftables_cgnat_config)) + + def test_cgnat(self): + internal_name = 'vyos-int-01' + external_name = 'vyos-ext-01' + internal_net = '100.64.0.0/29' + external_net = '192.0.2.1-192.0.2.2' + external_ports = '40000-60000' + ports_per_subscriber = '5000' + rule = '100' + + nftables_search = [ + ['map tcp_nat_map'], + ['map udp_nat_map'], + ['map icmp_nat_map'], + ['map other_nat_map'], + ['100.64.0.0 : 192.0.2.1 . 40000-44999'], + ['100.64.0.1 : 192.0.2.1 . 45000-49999'], + ['100.64.0.2 : 192.0.2.1 . 50000-54999'], + ['100.64.0.3 : 192.0.2.1 . 55000-59999'], + ['100.64.0.4 : 192.0.2.2 . 40000-44999'], + ['100.64.0.5 : 192.0.2.2 . 45000-49999'], + ['100.64.0.6 : 192.0.2.2 . 50000-54999'], + ['100.64.0.7 : 192.0.2.2 . 55000-59999'], + ['chain POSTROUTING'], + ['type nat hook postrouting priority srcnat'], + ['ip protocol tcp counter snat ip to ip saddr map @tcp_nat_map'], + ['ip protocol udp counter snat ip to ip saddr map @udp_nat_map'], + ['ip protocol icmp counter snat ip to ip saddr map @icmp_nat_map'], + ['counter snat ip to ip saddr map @other_nat_map'], + ] + + self.cli_set(base_path + ['pool', 'external', external_name, 'external-port-range', external_ports]) + self.cli_set(base_path + ['pool', 'external', external_name, 'range', external_net]) + + # allocation out of the available ports + with self.assertRaises(ConfigSessionError): + self.cli_set(base_path + ['pool', 'external', external_name, 'per-user-limit', 'port', '8000']) + self.cli_commit() + self.cli_set(base_path + ['pool', 'external', external_name, 'per-user-limit', 'port', ports_per_subscriber]) + + # internal pool not set + with self.assertRaises(ConfigSessionError): + self.cli_commit() + self.cli_set(base_path + ['pool', 'internal', internal_name, 'range', internal_net]) + + self.cli_set(base_path + ['rule', rule, 'source', 'pool', internal_name]) + # non-exist translation pool + with self.assertRaises(ConfigSessionError): + self.cli_set(base_path + ['rule', rule, 'translation', 'pool', 'fake-pool']) + self.cli_commit() + + self.cli_set(base_path + ['rule', rule, 'translation', 'pool', external_name]) + self.cli_commit() + + self.verify_nftables(nftables_search, 'ip cgnat', inverse=False, args='-s') + + +if __name__ == '__main__': + unittest.main(verbosity=2) From e0105ef380f1575613982f3b43c8ea3856654208 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Fri, 17 May 2024 11:56:50 -0500 Subject: [PATCH 111/188] T6354: do an explicit read from version file to avoid circular reference --- python/vyos/system/image.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/python/vyos/system/image.py b/python/vyos/system/image.py index ba9a6dfa711..aae52e770f0 100644 --- a/python/vyos/system/image.py +++ b/python/vyos/system/image.py @@ -18,8 +18,9 @@ from functools import wraps from tempfile import TemporaryDirectory from typing import TypedDict +from json import loads -from vyos import version +from vyos.defaults import directories from vyos.system import disk, grub # Define variables @@ -201,9 +202,12 @@ def get_running_image() -> str: if running_image_result: running_image: str = running_image_result.groupdict().get( 'image_version', '') - # we need to have a fallback for live systems + # we need to have a fallback for live systems: + # explicit read from version file if not running_image: - running_image: str = version.get_version() + json_data: str = Path(directories['data']).joinpath('version.json').read_text() + dict_data: dict = loads(json_data) + running_image: str = dict_data['version'] return running_image From 807791a01f946784a7e192b0e827ec891c112a77 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Sat, 18 May 2024 08:10:58 +0530 Subject: [PATCH 112/188] T6349: updated conflict check workflow (#3468) * T6349: updated conflict workflow * T6349: updated conflict workflow * T6349: updated all workflows to use reusable workflows * T6349: updated all workflows to use reusable workflows --- ...l-request-labels.yml => add-pr-labels.yml} | 9 +--- .github/workflows/auto-author-assign.yml | 9 +--- .github/workflows/chceck-pr-message.yml | 14 +++++ .github/workflows/check-pr-conflicts.yml | 13 +++++ .github/workflows/check-stale.yml | 9 ++++ .github/workflows/check-unused-imports.yml | 11 ++++ .github/workflows/codeql.yml | 13 +---- .github/workflows/label-backport.yml | 8 +++ .github/workflows/linit-j2.yml | 14 +++++ .github/workflows/mergifyio_backport.yml | 22 -------- .github/workflows/pr-conflicts.yml | 18 ------- .github/workflows/pull-request-management.yml | 25 --------- .../workflows/pull-request-message-check.yml | 23 --------- .github/workflows/stale.yml | 22 -------- .github/workflows/unused-imports.yml | 22 -------- scripts/check-pr-title-and-commit-messages.py | 51 ------------------- 16 files changed, 74 insertions(+), 209 deletions(-) rename .github/workflows/{pull-request-labels.yml => add-pr-labels.yml} (50%) create mode 100644 .github/workflows/chceck-pr-message.yml create mode 100644 .github/workflows/check-pr-conflicts.yml create mode 100644 .github/workflows/check-stale.yml create mode 100644 .github/workflows/check-unused-imports.yml create mode 100644 .github/workflows/label-backport.yml create mode 100644 .github/workflows/linit-j2.yml delete mode 100644 .github/workflows/mergifyio_backport.yml delete mode 100644 .github/workflows/pr-conflicts.yml delete mode 100644 .github/workflows/pull-request-management.yml delete mode 100644 .github/workflows/pull-request-message-check.yml delete mode 100644 .github/workflows/stale.yml delete mode 100644 .github/workflows/unused-imports.yml delete mode 100755 scripts/check-pr-title-and-commit-messages.py diff --git a/.github/workflows/pull-request-labels.yml b/.github/workflows/add-pr-labels.yml similarity index 50% rename from .github/workflows/pull-request-labels.yml rename to .github/workflows/add-pr-labels.yml index 43856beaa1f..78d619f4a14 100644 --- a/.github/workflows/pull-request-labels.yml +++ b/.github/workflows/add-pr-labels.yml @@ -11,10 +11,5 @@ on: jobs: add-pr-label: - name: Add PR Labels - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - steps: - - uses: actions/labeler@v5 + uses: vyos/.github/.github/workflows/add-pr-labels.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/auto-author-assign.yml b/.github/workflows/auto-author-assign.yml index 0bfe972c08e..1f69f480732 100644 --- a/.github/workflows/auto-author-assign.yml +++ b/.github/workflows/auto-author-assign.yml @@ -7,11 +7,6 @@ permissions: pull-requests: write jobs: - # https://github.com/marketplace/actions/auto-author-assign assign-author: - runs-on: ubuntu-latest - steps: - - name: "Assign Author to PR" - uses: toshimaru/auto-author-assign@v1.6.2 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} + uses: vyos/.github/.github/workflows/assign-author.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/chceck-pr-message.yml b/.github/workflows/chceck-pr-message.yml new file mode 100644 index 00000000000..95c5b69ce9c --- /dev/null +++ b/.github/workflows/chceck-pr-message.yml @@ -0,0 +1,14 @@ +--- +name: Check pull request message format + +on: + pull_request: + branches: + - current + - crux + - equuleus + +jobs: + check-pr-title: + uses: vyos/.github/.github/workflows/check-pr-message.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/check-pr-conflicts.yml b/.github/workflows/check-pr-conflicts.yml new file mode 100644 index 00000000000..62a37a7faaf --- /dev/null +++ b/.github/workflows/check-pr-conflicts.yml @@ -0,0 +1,13 @@ + +name: "PR Conflicts checker" +on: + pull_request_target: + types: [synchronize] + +permissions: + pull-requests: write + +jobs: + check-pr-conflict-call: + uses: vyos/.github/.github/workflows/check-pr-merge-conflict.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/check-stale.yml b/.github/workflows/check-stale.yml new file mode 100644 index 00000000000..0b88acdb7e2 --- /dev/null +++ b/.github/workflows/check-stale.yml @@ -0,0 +1,9 @@ +name: "Issue and PR stale management" +on: + schedule: + - cron: "0 0 * * *" + +jobs: + stale: + uses: vyos/.github/.github/workflows/check-stale.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml new file mode 100644 index 00000000000..468543d6e5c --- /dev/null +++ b/.github/workflows/check-unused-imports.yml @@ -0,0 +1,11 @@ +name: Check for unused imports using Pylint +on: + pull_request_target: + branches: + - current + - sagitta + +jobs: + Check-Unused-Imports: + uses: vyos/.github/.github/workflows/check-unused-imports.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9e2e4bf0f07..f6472784d6b 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -1,14 +1,3 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# name: "Perform CodeQL Analysis" on: @@ -27,7 +16,7 @@ permissions: jobs: codeql-analysis-call: - uses: vyos/vyos-github-actions/.github/workflows/codeql-analysis.yml@current + uses: vyos/.github/.github/workflows/codeql-analysis.yml@feature/T6349-reusable-workflows secrets: inherit with: languages: "['python']" diff --git a/.github/workflows/label-backport.yml b/.github/workflows/label-backport.yml new file mode 100644 index 00000000000..581363eb138 --- /dev/null +++ b/.github/workflows/label-backport.yml @@ -0,0 +1,8 @@ +name: Mergifyio backport + +on: [issue_comment] + +jobs: + mergifyio_backport: + uses: vyos/.github/.github/workflows/label-backport.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/linit-j2.yml b/.github/workflows/linit-j2.yml new file mode 100644 index 00000000000..093fe7ffed5 --- /dev/null +++ b/.github/workflows/linit-j2.yml @@ -0,0 +1,14 @@ +--- +name: J2 Lint + +on: + pull_request: + branches: + - current + - crux + - equuleus + +jobs: + j2lint: + uses: vyos/.github/.github/workflows/lint-j2.yml@feature/T6349-reusable-workflows + secrets: inherit diff --git a/.github/workflows/mergifyio_backport.yml b/.github/workflows/mergifyio_backport.yml deleted file mode 100644 index d9f863d9ad3..00000000000 --- a/.github/workflows/mergifyio_backport.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Mergifyio backport - -on: [issue_comment] - -jobs: - mergifyio_backport: - if: github.repository == 'vyos/vyos-1x' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: actions-ecosystem/action-regex-match@v2 - id: regex-match - with: - text: ${{ github.event.comment.body }} - regex: '@[Mm][Ee][Rr][Gg][Ii][Ff][Yy][Ii][Oo] backport ' - - - uses: actions-ecosystem/action-add-labels@v1 - if: ${{ steps.regex-match.outputs.match != '' }} - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - labels: backport diff --git a/.github/workflows/pr-conflicts.yml b/.github/workflows/pr-conflicts.yml deleted file mode 100644 index 2fd0bb42d1a..00000000000 --- a/.github/workflows/pr-conflicts.yml +++ /dev/null @@ -1,18 +0,0 @@ -name: "PR Conflicts checker" -on: - pull_request_target: - types: [synchronize] - -jobs: - Conflict_Check: - name: 'Check PR status: conflicts and resolution' - runs-on: ubuntu-latest - steps: - - name: check if PRs are dirty - uses: eps1lon/actions-label-merge-conflict@v3 - with: - dirtyLabel: "state: conflict" - removeOnDirtyLabel: "state: conflict resolved" - repoToken: "${{ secrets.GITHUB_TOKEN }}" - commentOnDirty: "This pull request has conflicts, please resolve those before we can evaluate the pull request." - commentOnClean: "Conflicts have been resolved. A maintainer will review the pull request shortly." diff --git a/.github/workflows/pull-request-management.yml b/.github/workflows/pull-request-management.yml deleted file mode 100644 index 3a855c10738..00000000000 --- a/.github/workflows/pull-request-management.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -name: Build Pull Request Package - -on: - pull_request: - branches: - - current - - crux - - equuleus - -jobs: - j2lint: - name: Validate j2 files - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - timeout-minutes: 2 - - name: Setup J2Lint - timeout-minutes: 2 - run: | - sudo pip install git+https://github.com/aristanetworks/j2lint.git@341b5d5db86e095b622f09770cb6367a1583620e - - name: Run J2lint - timeout-minutes: 2 - run: | - j2lint $GITHUB_WORKSPACE/data diff --git a/.github/workflows/pull-request-message-check.yml b/.github/workflows/pull-request-message-check.yml deleted file mode 100644 index 8c206a5abdf..00000000000 --- a/.github/workflows/pull-request-message-check.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -name: Check pull request message format - -on: - pull_request: - branches: - - current - - crux - - equuleus - -jobs: - check-pr-title: - name: Check pull request title - runs-on: ubuntu-20.04 - steps: - - uses: actions/checkout@v2 - timeout-minutes: 2 - - name: Install the requests library - run: pip3 install requests - - name: Check the PR title - timeout-minutes: 2 - run: | - ./scripts/check-pr-title-and-commit-messages.py '${{ github.event.pull_request.url }}' diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index d21d151f7a7..00000000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Issue and PR stale management" -on: - schedule: - - cron: "0 0 * * *" - -jobs: - stale: - runs-on: ubuntu-latest - if: github.repository == 'vyos/vyos-1x' - steps: - # Issue stale management - - uses: actions/stale@v6 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - days-before-stale: 90 - days-before-close: -1 - stale-issue-message: 'This issue is stale because it has been open 90 days with no activity. The issue will be reviewed by a maintainer and may be closed' - stale-issue-label: 'state: stale' - exempt-issue-labels: 'state: accepted, state: in-progress' - stale-pr-message: 'This PR is stale because it has been open 30 days with no activity. The PR will be reviewed by a maintainer and may be closed' - stale-pr-label: 'state: stale' - exempt-pr-labels: 'state: accepted, state: in-progress' diff --git a/.github/workflows/unused-imports.yml b/.github/workflows/unused-imports.yml deleted file mode 100644 index da57bd2702d..00000000000 --- a/.github/workflows/unused-imports.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Check for unused imports using Pylint -on: - pull_request_target: - branches: - - current - - sagitta - -jobs: - Check-Unused-Imports: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: 3.11 - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install pylint - - name: Analysing the code with pylint - run: make unused-imports diff --git a/scripts/check-pr-title-and-commit-messages.py b/scripts/check-pr-title-and-commit-messages.py deleted file mode 100755 index 001f6cf828a..00000000000 --- a/scripts/check-pr-title-and-commit-messages.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import re -import sys -import time - -import requests - -# Use the same regex for PR title and commit messages for now -title_regex = r'^(([a-zA-Z\-_.]+:\s)?)T\d+:\s+[^\s]+.*' -commit_regex = title_regex - -def check_pr_title(title): - if not re.match(title_regex, title): - print("PR title '{}' does not match the required format!".format(title)) - print("Valid title example: T99999: make IPsec secure") - sys.exit(1) - -def check_commit_message(title): - if not re.match(commit_regex, title): - print("Commit title '{}' does not match the required format!".format(title)) - print("Valid title example: T99999: make IPsec secure") - sys.exit(1) - -if __name__ == '__main__': - if len(sys.argv) < 2: - print("Please specify pull request URL!") - sys.exit(1) - - # There seems to be a race condition that causes this scripts to receive - # an incomplete PR object that is missing certain fields, - # which causes temporary CI failures that require re-running the script - # - # It's probably better to add a small delay to prevent that - time.sleep(5) - - # Get the pull request object - pr = requests.get(sys.argv[1]).json() - if "title" not in pr: - print("The PR object does not have a title field!") - print("Did not receive a valid pull request object, please check the URL!") - sys.exit(1) - - check_pr_title(pr["title"]) - - # Get the list of commits - commits = requests.get(pr["commits_url"]).json() - for c in commits: - # Retrieve every individual commit and check its title - co = requests.get(c["url"]).json() - check_commit_message(co["commit"]["message"]) From a9e461edfc645f07ddfdad7b0d6ab9c9548c73a7 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sat, 18 May 2024 14:38:03 +0000 Subject: [PATCH 113/188] T6364: CGNAT drop hard limit that allows only one translation rule As PoC for CGNAT had a hard limit of using only one translation rule for one internal pool. Drop this limit and extend the usage number of the rules. ``` set nat cgnat rule 100 source pool 'int-01' set nat cgnat rule 100 translation pool 'ext-01' set nat cgnat rule 120 source pool 'vyos-int-02' set nat cgnat rule 120 translation pool 'vyos-ext-02' ``` --- src/conf_mode/nat_cgnat.py | 110 ++++++++++++++++++++++--------------- 1 file changed, 66 insertions(+), 44 deletions(-) diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index 9a20a3c543c..fbfb431f668 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -189,11 +189,6 @@ def verify(config): if 'rule' not in config: raise ConfigError(f'Rule must be defined!') - # As PoC allow only one rule for CGNAT translations - # one internal pool and one external pool - if len(config['rule']) > 1: - raise ConfigError(f'Only one rule is allowed for translations!') - for pool in ('external', 'internal'): if pool not in config['pool']: raise ConfigError(f'{pool} pool must be defined!') @@ -208,6 +203,8 @@ def verify(config): internal_pools_query = "keys(pool.internal)" internal_pools: list = jmespath.search(internal_pools_query, config) + used_external_pools = {} + used_internal_pools = {} for rule, rule_config in config['rule'].items(): if 'source' not in rule_config: raise ConfigError(f'Rule "{rule}" source pool must be defined!') @@ -217,57 +214,82 @@ def verify(config): if 'translation' not in rule_config: raise ConfigError(f'Rule "{rule}" translation pool must be defined!') + # Check if pool exists internal_pool = rule_config['source']['pool'] if internal_pool not in internal_pools: raise ConfigError(f'Internal pool "{internal_pool}" does not exist!') - external_pool = rule_config['translation']['pool'] if external_pool not in external_pools: raise ConfigError(f'External pool "{external_pool}" does not exist!') + # Check pool duplication in different rules + if external_pool in used_external_pools: + raise ConfigError( + f'External pool "{external_pool}" is already used in rule ' + f'{used_external_pools[external_pool]} and cannot be used in ' + f'rule {rule}!' + ) + + if internal_pool in used_internal_pools: + raise ConfigError( + f'Internal pool "{internal_pool}" is already used in rule ' + f'{used_internal_pools[internal_pool]} and cannot be used in ' + f'rule {rule}!' + ) + + used_external_pools[external_pool] = rule + used_internal_pools[internal_pool] = rule + def generate(config): if not config: return None - # first external pool as we allow only one as PoC - ext_pool_name = jmespath.search("rule.*.translation | [0]", config).get('pool') - int_pool_name = jmespath.search("rule.*.source | [0]", config).get('pool') - ext_query = f'pool.external."{ext_pool_name}".range | keys(@)' - int_query = f'pool.internal."{int_pool_name}".range' - external_ranges = jmespath.search(ext_query, config) - internal_ranges = [jmespath.search(int_query, config)] - - external_list_count = [] - external_list_hosts = [] - internal_list_count = [] - internal_list_hosts = [] - for ext_range in external_ranges: - # External hosts count - e_count = IPOperations(ext_range).get_ips_count() - external_list_count.append(e_count) - # External hosts list - e_hosts = IPOperations(ext_range).convert_prefix_to_list_ips() - external_list_hosts.extend(e_hosts) - for int_range in internal_ranges: - # Internal hosts count - i_count = IPOperations(int_range).get_ips_count() - internal_list_count.append(i_count) - # Internal hosts list - i_hosts = IPOperations(int_range).convert_prefix_to_list_ips() - internal_list_hosts.extend(i_hosts) - - external_host_count = sum(external_list_count) - internal_host_count = sum(internal_list_count) - ports_per_user = int( - jmespath.search(f'pool.external."{ext_pool_name}".per_user_limit.port', config) - ) - external_port_range: str = jmespath.search( - f'pool.external."{ext_pool_name}".external_port_range', config - ) - proto_maps, other_maps = generate_port_rules( - external_list_hosts, internal_list_hosts, ports_per_user, external_port_range - ) + proto_maps = [] + other_maps = [] + + for rule, rule_config in config['rule'].items(): + ext_pool_name: str = rule_config['translation']['pool'] + int_pool_name: str = rule_config['source']['pool'] + + external_ranges: list = [range for range in config['pool']['external'][ext_pool_name]['range']] + internal_ranges: list = [config['pool']['internal'][int_pool_name]['range']] + external_list_hosts_count = [] + external_list_hosts = [] + internal_list_hosts_count = [] + internal_list_hosts = [] + + for ext_range in external_ranges: + # External hosts count + e_count = IPOperations(ext_range).get_ips_count() + external_list_hosts_count.append(e_count) + # External hosts list + e_hosts = IPOperations(ext_range).convert_prefix_to_list_ips() + external_list_hosts.extend(e_hosts) + + for int_range in internal_ranges: + # Internal hosts count + i_count = IPOperations(int_range).get_ips_count() + internal_list_hosts_count.append(i_count) + # Internal hosts list + i_hosts = IPOperations(int_range).convert_prefix_to_list_ips() + internal_list_hosts.extend(i_hosts) + + external_host_count = sum(external_list_hosts_count) + internal_host_count = sum(internal_list_hosts_count) + ports_per_user = int( + jmespath.search(f'pool.external."{ext_pool_name}".per_user_limit.port', config) + ) + external_port_range: str = jmespath.search( + f'pool.external."{ext_pool_name}".external_port_range', config + ) + + rule_proto_maps, rule_other_maps = generate_port_rules( + external_list_hosts, internal_list_hosts, ports_per_user, external_port_range + ) + + proto_maps.extend(rule_proto_maps) + other_maps.extend(rule_other_maps) config['proto_map_elements'] = ', '.join(proto_maps) config['other_map_elements'] = ', '.join(other_maps) From 2371c26b55db133b3864e54f460a046898e55abf Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Sat, 18 May 2024 15:13:44 +0000 Subject: [PATCH 114/188] T5169: Allow to set CGNAT multiple internal pools Allow to set multiple CGNAT internal pools ``` set nat cgnat pool internal int-01 range '100.64.0.0/28' set nat cgnat pool internal int-01 range '100.64.222.11-100.64.222.14' ``` --- interface-definitions/nat_cgnat.xml.in | 1 + src/conf_mode/nat_cgnat.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/interface-definitions/nat_cgnat.xml.in b/interface-definitions/nat_cgnat.xml.in index caa26b4d911..fce5e655db1 100644 --- a/interface-definitions/nat_cgnat.xml.in +++ b/interface-definitions/nat_cgnat.xml.in @@ -123,6 +123,7 @@ + diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index fbfb431f668..5ad65de80ab 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -253,7 +253,7 @@ def generate(config): int_pool_name: str = rule_config['source']['pool'] external_ranges: list = [range for range in config['pool']['external'][ext_pool_name]['range']] - internal_ranges: list = [config['pool']['internal'][int_pool_name]['range']] + internal_ranges: list = [range for range in config['pool']['internal'][int_pool_name]['range']] external_list_hosts_count = [] external_list_hosts = [] internal_list_hosts_count = [] From 65c8e9e24aec4e90a52319956e062ae7640fec34 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Sun, 19 May 2024 00:27:03 +0530 Subject: [PATCH 115/188] T6349: updated pr-labels workflow permission (#3485) --- .github/workflows/add-pr-labels.yml | 4 ++++ .github/workflows/auto-author-assign.yml | 2 ++ .github/workflows/chceck-pr-message.yml | 4 ++++ .github/workflows/check-pr-conflicts.yml | 1 + .github/workflows/check-stale.yml | 4 ++++ .github/workflows/check-unused-imports.yml | 8 ++++++-- .github/workflows/label-backport.yml | 6 +++++- .github/workflows/linit-j2.yml | 4 ++++ src/op_mode/cgnat.py | 1 - 9 files changed, 30 insertions(+), 4 deletions(-) diff --git a/.github/workflows/add-pr-labels.yml b/.github/workflows/add-pr-labels.yml index 78d619f4a14..1723cceb01d 100644 --- a/.github/workflows/add-pr-labels.yml +++ b/.github/workflows/add-pr-labels.yml @@ -9,6 +9,10 @@ on: - equuleus - sagitta +permissions: + pull-requests: write + contents: read + jobs: add-pr-label: uses: vyos/.github/.github/workflows/add-pr-labels.yml@feature/T6349-reusable-workflows diff --git a/.github/workflows/auto-author-assign.yml b/.github/workflows/auto-author-assign.yml index 1f69f480732..c3696ea476d 100644 --- a/.github/workflows/auto-author-assign.yml +++ b/.github/workflows/auto-author-assign.yml @@ -3,8 +3,10 @@ on: pull_request_target: types: [opened, reopened, ready_for_review, locked] + permissions: pull-requests: write + contents: read jobs: assign-author: diff --git a/.github/workflows/chceck-pr-message.yml b/.github/workflows/chceck-pr-message.yml index 95c5b69ce9c..e7e45696173 100644 --- a/.github/workflows/chceck-pr-message.yml +++ b/.github/workflows/chceck-pr-message.yml @@ -8,6 +8,10 @@ on: - crux - equuleus +permissions: + pull-requests: write + contents: read + jobs: check-pr-title: uses: vyos/.github/.github/workflows/check-pr-message.yml@feature/T6349-reusable-workflows diff --git a/.github/workflows/check-pr-conflicts.yml b/.github/workflows/check-pr-conflicts.yml index 62a37a7faaf..0c659e6ed97 100644 --- a/.github/workflows/check-pr-conflicts.yml +++ b/.github/workflows/check-pr-conflicts.yml @@ -6,6 +6,7 @@ on: permissions: pull-requests: write + contents: read jobs: check-pr-conflict-call: diff --git a/.github/workflows/check-stale.yml b/.github/workflows/check-stale.yml index 0b88acdb7e2..b5ec533f184 100644 --- a/.github/workflows/check-stale.yml +++ b/.github/workflows/check-stale.yml @@ -3,6 +3,10 @@ on: schedule: - cron: "0 0 * * *" +permissions: + pull-requests: write + contents: read + jobs: stale: uses: vyos/.github/.github/workflows/check-stale.yml@feature/T6349-reusable-workflows diff --git a/.github/workflows/check-unused-imports.yml b/.github/workflows/check-unused-imports.yml index 468543d6e5c..aada264f77c 100644 --- a/.github/workflows/check-unused-imports.yml +++ b/.github/workflows/check-unused-imports.yml @@ -1,11 +1,15 @@ name: Check for unused imports using Pylint on: - pull_request_target: + pull_request: branches: - current - sagitta + workflow_dispatch: + +permissions: + contents: read jobs: - Check-Unused-Imports: + check-unused-imports: uses: vyos/.github/.github/workflows/check-unused-imports.yml@feature/T6349-reusable-workflows secrets: inherit diff --git a/.github/workflows/label-backport.yml b/.github/workflows/label-backport.yml index 581363eb138..9192b81843b 100644 --- a/.github/workflows/label-backport.yml +++ b/.github/workflows/label-backport.yml @@ -2,7 +2,11 @@ name: Mergifyio backport on: [issue_comment] +permissions: + pull-requests: write + contents: read + jobs: - mergifyio_backport: + mergifyio-backport: uses: vyos/.github/.github/workflows/label-backport.yml@feature/T6349-reusable-workflows secrets: inherit diff --git a/.github/workflows/linit-j2.yml b/.github/workflows/linit-j2.yml index 093fe7ffed5..364a65a14e8 100644 --- a/.github/workflows/linit-j2.yml +++ b/.github/workflows/linit-j2.yml @@ -8,6 +8,10 @@ on: - crux - equuleus +permissions: + pull-requests: write + contents: read + jobs: j2lint: uses: vyos/.github/.github/workflows/lint-j2.yml@feature/T6349-reusable-workflows diff --git a/src/op_mode/cgnat.py b/src/op_mode/cgnat.py index a98269a15f3..e58b15809b2 100755 --- a/src/op_mode/cgnat.py +++ b/src/op_mode/cgnat.py @@ -16,7 +16,6 @@ import json import sys -import typing from tabulate import tabulate From 17a94c2d64058e1574f45824253d9772265728dc Mon Sep 17 00:00:00 2001 From: kumvijaya Date: Mon, 20 May 2024 12:12:00 +0530 Subject: [PATCH 116/188] T6372: added codeowners --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000000..19139429851 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @vyos/reviewers \ No newline at end of file From 0d6e44179bae5f73d37502884194656b34b1c4f9 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Mon, 20 May 2024 20:26:35 +0200 Subject: [PATCH 117/188] op-mode: T6367: fix "force commit-archive" TypeError /usr/bin/config-mgmt requires an argument OR to be symbolically linked to *commit-revision or *commit-archive, for which it interprets argv[0] through the useful trickery: https://github.com/vyos/vyos-1x/blob/current/python/vyos/config_mgmt.py#L693-L700 Traceback (most recent call last): File "/usr/bin/config-mgmt", line 33, in sys.exit(load_entry_point('vyos==1.3.0', 'console_scripts', 'config-mgmt')()) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/vyos/config_mgmt.py", line 746, in run func = getattr(config_mgmt, args['subcommand']) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ TypeError: attribute name must be string, not 'NoneType' --- op-mode-definitions/force-commit-archive.xml.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/op-mode-definitions/force-commit-archive.xml.in b/op-mode-definitions/force-commit-archive.xml.in index 162323c20a0..46836f96733 100644 --- a/op-mode-definitions/force-commit-archive.xml.in +++ b/op-mode-definitions/force-commit-archive.xml.in @@ -6,7 +6,7 @@ Manually archive configuration - /usr/bin/config-mgmt + /etc/commit/post-hooks.d/02vyos-commit-archive; printf "\n" From c554c483817bfc6ef4f0175298d23355696f8665 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 21 May 2024 08:06:35 +0000 Subject: [PATCH 118/188] T6366: CGNAT add ability to get external and internal allocations Add the ability to show port allocation per external or internal address With huge entries, it is necessary to filter it by specific external/internal IP address --- op-mode-definitions/nat.xml.in | 20 ++++++++++++++++ src/op_mode/cgnat.py | 44 +++++++++++++++++++++++++--------- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/op-mode-definitions/nat.xml.in b/op-mode-definitions/nat.xml.in index 6398c0e0789..13e7fd81d93 100644 --- a/op-mode-definitions/nat.xml.in +++ b/op-mode-definitions/nat.xml.in @@ -16,6 +16,26 @@ Show allocated CGNAT parameters + + + + Show CGNAT allocations for an external IP address + + <x.x.x.x> + + + sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation --external-address "$6" + + + + Show CGNAT allocations for an internal IP address + + <x.x.x.x> + + + sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation --internal-address "$6" + + sudo ${vyos_op_scripts_dir}/cgnat.py show_allocation diff --git a/src/op_mode/cgnat.py b/src/op_mode/cgnat.py index e58b15809b2..9ad8f92f9ae 100755 --- a/src/op_mode/cgnat.py +++ b/src/op_mode/cgnat.py @@ -16,6 +16,7 @@ import json import sys +import typing from tabulate import tabulate @@ -27,15 +28,11 @@ CGNAT_TABLE = 'cgnat' -def _get_raw_data(): - """ Get CGNAT dictionary - """ +def _get_raw_data(external_address: str = '', internal_address: str = '') -> list[dict]: + """Get CGNAT dictionary and filter by external or internal address if provided.""" cmd_output = cmd(f'nft --json list table ip {CGNAT_TABLE}') data = json.loads(cmd_output) - return data - -def _get_formatted_output(data): elements = data['nftables'][2]['map']['elem'] allocations = [] for elem in elements: @@ -44,23 +41,48 @@ def _get_formatted_output(data): start_port = elem[1]['concat'][1]['range'][0] end_port = elem[1]['concat'][1]['range'][1] port_range = f'{start_port}-{end_port}' - allocations.append((internal, external, port_range)) + if (internal_address and internal != internal_address) or ( + external_address and external != external_address + ): + continue + + allocations.append( + { + 'internal_address': internal, + 'external_address': external, + 'port_range': port_range, + } + ) + + return allocations + + +def _get_formatted_output(allocations: list[dict]) -> str: + # Convert the list of dictionaries to a list of tuples for tabulate headers = ['Internal IP', 'External IP', 'Port range'] - output = tabulate(allocations, headers, numalign="left") + data = [ + (alloc['internal_address'], alloc['external_address'], alloc['port_range']) + for alloc in allocations + ] + output = tabulate(data, headers, numalign="left") return output -def show_allocation(raw: bool): +def show_allocation( + raw: bool, + external_address: typing.Optional[str], + internal_address: typing.Optional[str], +) -> str: config = ConfigTreeQuery() if not config.exists('nat cgnat'): raise vyos.opmode.UnconfiguredSubsystem('CGNAT is not configured') if raw: - return _get_raw_data() + return _get_raw_data(external_address, internal_address) else: - raw_data = _get_raw_data() + raw_data = _get_raw_data(external_address, internal_address) return _get_formatted_output(raw_data) From 5cb9b84bd9ce909460d8da7f039d9371143ede6c Mon Sep 17 00:00:00 2001 From: l0crian1 Date: Tue, 21 May 2024 09:35:29 -0400 Subject: [PATCH 119/188] T6375: Fix/Update NAT logging Fixed broken logging for "show log nat" Added the following commands: show log nat source show log nat source rule show log nat destination nat show log nat destination nat rule show log nat static show log nat static rule --- interface-definitions/nat.xml.in | 1 + op-mode-definitions/show-log.xml.in | 50 +++++++++++++++++++++++++++-- python/vyos/nat.py | 6 ++-- 3 files changed, 51 insertions(+), 6 deletions(-) diff --git a/interface-definitions/nat.xml.in b/interface-definitions/nat.xml.in index 0a639bd8089..73a74813798 100644 --- a/interface-definitions/nat.xml.in +++ b/interface-definitions/nat.xml.in @@ -141,6 +141,7 @@ #include + #include Translation address or prefix diff --git a/op-mode-definitions/show-log.xml.in b/op-mode-definitions/show-log.xml.in index e132703641e..c3aa324baf1 100644 --- a/op-mode-definitions/show-log.xml.in +++ b/op-mode-definitions/show-log.xml.in @@ -464,12 +464,56 @@ journalctl --no-hostname --boot --unit lldpd.service - + Show log for Network Address Translation (NAT) - egrep -i "kernel:.*\[NAT-[A-Z]{3,}-[0-9]+(-MASQ)?\]" $(find /var/log -maxdepth 1 -type f -name messages\* | sort -t. -k2nr) - + + + + Show NAT destination log + + journalctl --no-hostname --boot -k | egrep "\[DST-NAT-[0-9]+\]" + + + + Show NAT destination log for specified rule + + journalctl --no-hostname --boot -k | egrep "\[DST-NAT-$6\]" + + + + + + Show NAT source log + + journalctl --no-hostname --boot -k | egrep "\[SRC-NAT-[0-9]+(-MASQ)?\]""" + + + + Show NAT source log for specified rule + + journalctl --no-hostname --boot -k | egrep "\[SRC-NAT-$6(-MASQ)?\]" + + + + + + Show NAT static log + + journalctl --no-hostname --boot -k | egrep "\[STATIC-(SRC|DST)-NAT-[0-9]+\]" + + + + Show NAT static log for specified rule + + journalctl --no-hostname --boot -k | egrep "\[STATIC-(SRC|DST)-NAT-$6\]" + + + + + journalctl --no-hostname --boot -k | egrep "\[(STATIC-)?(DST|SRC)-NAT-[0-9]+(-MASQ)?\]" + Show log for Neighbor Discovery Protocol (NDP) Proxy diff --git a/python/vyos/nat.py b/python/vyos/nat.py index 2ada29add11..e5454878877 100644 --- a/python/vyos/nat.py +++ b/python/vyos/nat.py @@ -300,12 +300,12 @@ def parse_nat_static_rule(rule_conf, rule_id, nat_type): output.append('counter') - if translation_str: - output.append(translation_str) - if 'log' in rule_conf: output.append(f'log prefix "[{log_prefix}{log_suffix}]"') + if translation_str: + output.append(translation_str) + output.append(f'comment "{log_prefix}"') return " ".join(output) From e50b7afc9d5b727d04933116ccf364a2b9a48c30 Mon Sep 17 00:00:00 2001 From: khramshinr Date: Tue, 21 May 2024 20:02:31 +0600 Subject: [PATCH 120/188] T6373: QoS Policy Limiter - classes for marked traffic do not work --- python/vyos/qos/base.py | 11 ++++++++--- smoketest/scripts/cli/test_qos.py | 21 +++++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/python/vyos/qos/base.py b/python/vyos/qos/base.py index 87927ba9dfc..98e486e42b7 100644 --- a/python/vyos/qos/base.py +++ b/python/vyos/qos/base.py @@ -247,9 +247,15 @@ def update(self, config, direction, priority=None): filter_cmd_base += ' protocol all' if 'match' in cls_config: - is_filtered = False + has_filter = False for index, (match, match_config) in enumerate(cls_config['match'].items(), start=1): filter_cmd = filter_cmd_base + if not has_filter: + for key in ['mark', 'vif', 'ip', 'ipv6']: + if key in match_config: + has_filter = True + break + if self.qostype == 'shaper' and 'prio ' not in filter_cmd: filter_cmd += f' prio {index}' if 'mark' in match_config: @@ -332,13 +338,12 @@ def update(self, config, direction, priority=None): cls = int(cls) filter_cmd += f' flowid {self._parent:x}:{cls:x}' self._cmd(filter_cmd) - is_filtered = True vlan_expression = "match.*.vif" match_vlan = jmespath.search(vlan_expression, cls_config) if any(tmp in ['exceed', 'bandwidth', 'burst'] for tmp in cls_config) \ - and is_filtered: + and has_filter: # For "vif" "basic match" is used instead of "action police" T5961 if not match_vlan: filter_cmd += f' action police' diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index bcf5139c71b..5977b2f414d 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -738,6 +738,27 @@ def test_13_shaper_delete_only_rule(self): self.cli_commit() self.assertEqual('', cmd(f'tc filter show dev {interface}')) + def test_14_policy_limiter_marked_traffic(self): + policy_name = 'smoke_test' + base_policy_path = ['qos', 'policy', 'limiter', policy_name] + + self.cli_set(['qos', 'interface', self._interfaces[0], 'ingress', policy_name]) + self.cli_set(base_policy_path + ['class', '100', 'bandwidth', '20gbit']) + self.cli_set(base_policy_path + ['class', '100', 'burst', '3760k']) + self.cli_set(base_policy_path + ['class', '100', 'match', 'INTERNAL', 'mark', '100']) + self.cli_set(base_policy_path + ['class', '100', 'priority', '20']) + self.cli_set(base_policy_path + ['default', 'bandwidth', '1gbit']) + self.cli_set(base_policy_path + ['default', 'burst', '125000000b']) + self.cli_commit() + + tc_filters = cmd(f'tc filter show dev {self._interfaces[0]} ingress') + # class 100 + self.assertIn('filter parent ffff: protocol all pref 20 fw chain 0', tc_filters) + self.assertIn('action order 1: police 0x1 rate 20Gbit burst 3847500b mtu 2Kb action drop overhead 0b', tc_filters) + # default + self.assertIn('filter parent ffff: protocol all pref 255 basic chain 0', tc_filters) + self.assertIn('action order 1: police 0x2 rate 1Gbit burst 125000000b mtu 2Kb action drop overhead 0b', tc_filters) + if __name__ == '__main__': unittest.main(verbosity=2) From e1450096b4c667a4c33a3fcd8f67ebf6a39d441d Mon Sep 17 00:00:00 2001 From: Alex W Date: Tue, 21 May 2024 23:25:06 +0100 Subject: [PATCH 121/188] reverse-proxy: T6370: Set custom HTTP headers in reverse-proxy responses --- data/templates/load-balancing/haproxy.cfg.j2 | 10 +++++++ .../haproxy/http-response-headers.xml.i | 29 +++++++++++++++++++ .../load-balancing_reverse-proxy.xml.in | 2 ++ .../cli/test_load-balancing_reverse-proxy.py | 21 ++++++++++++++ src/conf_mode/load-balancing_reverse-proxy.py | 6 ++++ 5 files changed, 68 insertions(+) create mode 100644 interface-definitions/include/haproxy/http-response-headers.xml.i diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index 7917c82570e..797bf17e7c1 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -81,6 +81,11 @@ frontend {{ front }} {% endif %} {% endfor %} {% endif %} +{% if front_config.http_response_headers is vyos_defined %} +{% for header, header_config in front_config.http_response_headers.items() %} + http-response set-header {{ header }} '{{ header_config['value'] }}' +{% endfor %} +{% endif %} {% endif %} {% if front_config.rule is vyos_defined %} {% for rule, rule_config in front_config.rule.items() %} @@ -158,6 +163,11 @@ backend {{ back }} {% endif %} {% if back_config.mode is vyos_defined %} mode {{ back_config.mode }} +{% if back_config.http_response_headers is vyos_defined %} +{% for header, header_config in back_config.http_response_headers.items() %} + http-response set-header {{ header }} '{{ header_config['value'] }}' +{% endfor %} +{% endif %} {% endif %} {% if back_config.rule is vyos_defined %} {% for rule, rule_config in back_config.rule.items() %} diff --git a/interface-definitions/include/haproxy/http-response-headers.xml.i b/interface-definitions/include/haproxy/http-response-headers.xml.i new file mode 100644 index 00000000000..9e7ddfd28e4 --- /dev/null +++ b/interface-definitions/include/haproxy/http-response-headers.xml.i @@ -0,0 +1,29 @@ + + + + Headers to include in HTTP response + + txt + HTTP header name + + + [-a-zA-Z]+ + + Header names must only include alphabetical characters and hyphens + + + + + HTTP header value + + txt + HTTP header value + + + [[:ascii:]]{1,256} + + + + + + diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index 6a3b3cef1ce..011e1b53c69 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -39,6 +39,7 @@ #include #include #include + #include Redirect HTTP to HTTPS @@ -90,6 +91,7 @@ #include #include + #include Backend parameters diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index c8b17316f95..370a9276af8 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -385,5 +385,26 @@ def test_06_lb_reverse_proxy_tcp_mode(self): self.assertIn(f'mode {mode}', config) self.assertIn(f'server {bk_name} {bk_server}:{bk_server_port}', config) + def test_07_lb_reverse_proxy_http_response_headers(self): + # Setup base + self.configure_pki() + self.base_config() + + # Set example headers in both frontend and backend + self.cli_set(base_path + ['service', 'https_front', 'http-response-headers', 'Cache-Control', 'value', 'max-age=604800']) + self.cli_set(base_path + ['backend', 'bk-01', 'http-response-headers', 'Proxy-Backend-ID', 'value', 'bk-01']) + self.cli_commit() + + # Test headers are present in generated configuration file + config = read_file(HAPROXY_CONF) + self.assertIn('http-response set-header Cache-Control \'max-age=604800\'', config) + self.assertIn('http-response set-header Proxy-Backend-ID \'bk-01\'', config) + + # Test setting alongside modes other than http is blocked by validation conditions + self.cli_set(base_path + ['service', 'https_front', 'mode', 'tcp']) + with self.assertRaises(ConfigSessionError) as e: + self.cli_commit() + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index 1569d8d713e..a4efb1cd8c0 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -88,6 +88,12 @@ def verify(lb): if {'send_proxy', 'send_proxy_v2'} <= set(bk_server_conf): raise ConfigError(f'Cannot use both "send-proxy" and "send-proxy-v2" for server "{bk_server}"') + # Check if http-response-headers are configured in any frontend/backend where mode != http + for group in ['service', 'backend']: + for config_name, config in lb[group].items(): + if 'http_response_headers' in config and ('mode' not in config or config['mode'] != 'http'): + raise ConfigError(f'{group} {config_name} must be set to http mode to use http_response_headers!') + if 'ssl' in back_config: if {'no_verify', 'ca_certificate'} <= set(back_config['ssl']): raise ConfigError(f'backend {back} cannot have both ssl options no-verify and ca-certificate set!') From a37dbf7de235db2de43c4645afccc7f428308d9f Mon Sep 17 00:00:00 2001 From: kumvijaya Date: Wed, 22 May 2024 15:11:46 +0530 Subject: [PATCH 122/188] T6378: remove labler yml as it is kept in reusable workflow repo --- .github/labeler.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 .github/labeler.yml diff --git a/.github/labeler.yml b/.github/labeler.yml deleted file mode 100644 index e0b9ee43097..00000000000 --- a/.github/labeler.yml +++ /dev/null @@ -1,12 +0,0 @@ -equuleus: - - any: - - base-branch: 'equuleus' -current: - - any: - - base-branch: 'current' -crux: - - any: - - base-branch: 'crux' -sagitta: - - any: - - base-branch: 'sagitta' From 7bba95c8052af5b0cc5908cb9e740caa01b44161 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 22 May 2024 17:52:42 +0100 Subject: [PATCH 123/188] rollback-soft: T6384: tell the user to compare or commit after applying the diff --- python/vyos/config_mgmt.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/vyos/config_mgmt.py b/python/vyos/config_mgmt.py index fc51d781c7f..70b6ea20311 100644 --- a/python/vyos/config_mgmt.py +++ b/python/vyos/config_mgmt.py @@ -283,6 +283,8 @@ def rollback_soft(self, rev: int): rollback_ct = self._get_config_tree_revision(rev) try: load(rollback_ct, switch='explicit') + print('Rollback diff has been applied.') + print('Use "compare" to review the changes or "commit" to apply them.') except LoadConfigError as e: raise ConfigMgmtError(e) from e From 5a5dda14fd3d472680568f1792e9fbdb030f3995 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 22 May 2024 18:10:57 +0100 Subject: [PATCH 124/188] vyos.utils.io: T6385: handle keyboard interrupts in ask_yes_no and return False if the user interrupts the prompt with Ctrl-C --- python/vyos/utils/io.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/vyos/utils/io.py b/python/vyos/utils/io.py index a8c430f28f7..205210b6622 100644 --- a/python/vyos/utils/io.py +++ b/python/vyos/utils/io.py @@ -72,6 +72,8 @@ def ask_yes_no(question, default=False) -> bool: stdout.write("Please respond with yes/y or no/n\n") except EOFError: stdout.write("\nPlease respond with yes/y or no/n\n") + except KeyboardInterrupt: + return False def is_interactive(): """Try to determine if the routine was called from an interactive shell.""" From 03eae30b27433055ddc10f09fc134b83e9bd6cec Mon Sep 17 00:00:00 2001 From: Ryazanov Alexander Mihailovich Date: Sat, 18 May 2024 16:50:55 +0300 Subject: [PATCH 125/188] nat: T6365: remove warnings for negated interface selections by name --- src/conf_mode/nat.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 4cd9b570d5b..0563519869f 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -149,8 +149,12 @@ def verify(nat): if 'name' in config['outbound_interface'] and 'group' in config['outbound_interface']: raise ConfigError(f'{err_msg} cannot specify both interface group and interface name for nat source rule "{rule}"') elif 'name' in config['outbound_interface']: - if config['outbound_interface']['name'] not in 'any' and config['outbound_interface']['name'] not in interfaces(): - Warning(f'NAT interface "{config["outbound_interface"]["name"]}" for source NAT rule "{rule}" does not exist!') + interface_name = config['outbound_interface']['name'] + if interface_name not in 'any': + if interface_name[0] == '!': + interface_name = interface_name[1:] + if interface_name not in interfaces(): + Warning(f'NAT interface "{interface_name}" for source NAT rule "{rule}" does not exist!') else: group_name = config['outbound_interface']['group'] if group_name[0] == '!': @@ -182,8 +186,12 @@ def verify(nat): if 'name' in config['inbound_interface'] and 'group' in config['inbound_interface']: raise ConfigError(f'{err_msg} cannot specify both interface group and interface name for destination nat rule "{rule}"') elif 'name' in config['inbound_interface']: - if config['inbound_interface']['name'] not in 'any' and config['inbound_interface']['name'] not in interfaces(): - Warning(f'NAT interface "{config["inbound_interface"]["name"]}" for destination NAT rule "{rule}" does not exist!') + interface_name = config['inbound_interface']['name'] + if interface_name not in 'any': + if interface_name[0] == '!': + interface_name = interface_name[1:] + if interface_name not in interfaces(): + Warning(f'NAT interface "{interface_name}" for destination NAT rule "{rule}" does not exist!') else: group_name = config['inbound_interface']['group'] if group_name[0] == '!': From 3870247517741ce23e2fcee8aaa1d194f0ad621b Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 22 May 2024 19:46:46 +0200 Subject: [PATCH 126/188] nat: T6365: use string startswith() over [0] index access --- src/conf_mode/nat.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index 0563519869f..db02ca66fbd 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -151,7 +151,7 @@ def verify(nat): elif 'name' in config['outbound_interface']: interface_name = config['outbound_interface']['name'] if interface_name not in 'any': - if interface_name[0] == '!': + if interface_name.startswith('!'): interface_name = interface_name[1:] if interface_name not in interfaces(): Warning(f'NAT interface "{interface_name}" for source NAT rule "{rule}" does not exist!') @@ -188,7 +188,7 @@ def verify(nat): elif 'name' in config['inbound_interface']: interface_name = config['inbound_interface']['name'] if interface_name not in 'any': - if interface_name[0] == '!': + if interface_name.startswith('!'): interface_name = interface_name[1:] if interface_name not in interfaces(): Warning(f'NAT interface "{interface_name}" for destination NAT rule "{rule}" does not exist!') From 645c43ba60d29ca676a4323ccc5ca16c6bd8127a Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 22 May 2024 19:48:32 +0200 Subject: [PATCH 127/188] nat: T6365: use interface_exists() over netifaces.interfaces() --- src/conf_mode/nat.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/conf_mode/nat.py b/src/conf_mode/nat.py index db02ca66fbd..f74bb217e50 100755 --- a/src/conf_mode/nat.py +++ b/src/conf_mode/nat.py @@ -17,7 +17,6 @@ import os from sys import exit -from netifaces import interfaces from vyos.base import Warning from vyos.config import Config @@ -30,6 +29,7 @@ from vyos.utils.process import cmd from vyos.utils.process import run from vyos.utils.network import is_addr_assigned +from vyos.utils.network import interface_exists from vyos import ConfigError from vyos import airbag @@ -153,8 +153,8 @@ def verify(nat): if interface_name not in 'any': if interface_name.startswith('!'): interface_name = interface_name[1:] - if interface_name not in interfaces(): - Warning(f'NAT interface "{interface_name}" for source NAT rule "{rule}" does not exist!') + if not interface_exists(interface_name): + Warning(f'Interface "{interface_name}" for source NAT rule "{rule}" does not exist!') else: group_name = config['outbound_interface']['group'] if group_name[0] == '!': @@ -190,8 +190,8 @@ def verify(nat): if interface_name not in 'any': if interface_name.startswith('!'): interface_name = interface_name[1:] - if interface_name not in interfaces(): - Warning(f'NAT interface "{interface_name}" for destination NAT rule "{rule}" does not exist!') + if not interface_exists(interface_name): + Warning(f'Interface "{interface_name}" for destination NAT rule "{rule}" does not exist!') else: group_name = config['inbound_interface']['group'] if group_name[0] == '!': From 59781ff365a5e1b15ef6c4c2481f3d3815548b9d Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 22 May 2024 20:09:40 +0200 Subject: [PATCH 128/188] nat66: T6365: remove warnings for negated interface selections by name --- src/conf_mode/nat66.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/conf_mode/nat66.py b/src/conf_mode/nat66.py index fe017527d9c..075738dad22 100755 --- a/src/conf_mode/nat66.py +++ b/src/conf_mode/nat66.py @@ -17,15 +17,15 @@ import os from sys import exit -from netifaces import interfaces from vyos.base import Warning from vyos.config import Config from vyos.configdep import set_dependents, call_dependents from vyos.template import render -from vyos.utils.process import cmd -from vyos.utils.kernel import check_kmod from vyos.utils.dict import dict_search +from vyos.utils.kernel import check_kmod +from vyos.utils.network import interface_exists +from vyos.utils.process import cmd from vyos.template import is_ipv6 from vyos import ConfigError from vyos import airbag @@ -64,8 +64,12 @@ def verify(nat): if 'name' in config['outbound_interface'] and 'group' in config['outbound_interface']: raise ConfigError(f'{err_msg} cannot specify both interface group and interface name for nat source rule "{rule}"') elif 'name' in config['outbound_interface']: - if config['outbound_interface']['name'] not in 'any' and config['outbound_interface']['name'] not in interfaces(): - Warning(f'NAT66 interface "{config["outbound_interface"]["name"]}" for source NAT66 rule "{rule}" does not exist!') + interface_name = config['outbound_interface']['name'] + if interface_name not in 'any': + if interface_name.startswith('!'): + interface_name = interface_name[1:] + if not interface_exists(interface_name): + Warning(f'Interface "{interface_name}" for source NAT66 rule "{rule}" does not exist!') addr = dict_search('translation.address', config) if addr != None: @@ -88,8 +92,12 @@ def verify(nat): if 'name' in config['inbound_interface'] and 'group' in config['inbound_interface']: raise ConfigError(f'{err_msg} cannot specify both interface group and interface name for destination nat rule "{rule}"') elif 'name' in config['inbound_interface']: - if config['inbound_interface']['name'] not in 'any' and config['inbound_interface']['name'] not in interfaces(): - Warning(f'NAT66 interface "{config["inbound_interface"]["name"]}" for destination NAT66 rule "{rule}" does not exist!') + interface_name = config['inbound_interface']['name'] + if interface_name not in 'any': + if interface_name.startswith('!'): + interface_name = interface_name[1:] + if not interface_exists(interface_name): + Warning(f'Interface "{interface_name}" for destination NAT66 rule "{rule}" does not exist!') return None From 7fe568ca1672f1dfbd2b56ee3ef7a6ab48b03070 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 22 May 2024 21:31:32 +0200 Subject: [PATCH 129/188] nat: T6345: source NAT port mapping "fully-random" is superfluous in Kernel >=5.0 random - In kernel 5.0 and newer this is the same as fully-random. In earlier kernels the port mapping will be randomized using a seeded MD5 hash mix using source and destination address and destination port. https://git.netfilter.org/nftables/commit/?id=fbe27464dee4588d906492749251454 --- .../include/nat-translation-options.xml.i | 8 +- .../include/version/nat-version.xml.i | 2 +- smoketest/config-tests/nat-basic | 85 ++++++ smoketest/configs/nat-basic | 256 ++++++++++++++++++ src/migration-scripts/nat/7-to-8 | 62 +++++ 5 files changed, 406 insertions(+), 7 deletions(-) create mode 100644 smoketest/config-tests/nat-basic create mode 100644 smoketest/configs/nat-basic create mode 100755 src/migration-scripts/nat/7-to-8 diff --git a/interface-definitions/include/nat-translation-options.xml.i b/interface-definitions/include/nat-translation-options.xml.i index 6b95de0459c..c8900590f0f 100644 --- a/interface-definitions/include/nat-translation-options.xml.i +++ b/interface-definitions/include/nat-translation-options.xml.i @@ -28,22 +28,18 @@ Port mapping options - random fully-random none + random none random Randomize source port mapping - - fully-random - Full port randomization - none Do not apply port randomization - (random|fully-random|none) + (random|none) none diff --git a/interface-definitions/include/version/nat-version.xml.i b/interface-definitions/include/version/nat-version.xml.i index 656da6e14c9..173e91ed3ea 100644 --- a/interface-definitions/include/version/nat-version.xml.i +++ b/interface-definitions/include/version/nat-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/smoketest/config-tests/nat-basic b/smoketest/config-tests/nat-basic new file mode 100644 index 00000000000..9fea08b02a3 --- /dev/null +++ b/smoketest/config-tests/nat-basic @@ -0,0 +1,85 @@ +set interfaces ethernet eth0 offload rps +set interfaces ethernet eth0 disable +set interfaces ethernet eth1 offload gro +set interfaces ethernet eth1 offload gso +set interfaces ethernet eth1 offload rps +set interfaces ethernet eth1 offload sg +set interfaces ethernet eth1 offload tso +set interfaces ethernet eth2 offload gro +set interfaces ethernet eth2 offload gso +set interfaces ethernet eth2 offload rps +set interfaces ethernet eth2 offload sg +set interfaces ethernet eth2 offload tso +set interfaces ethernet eth3 offload gro +set interfaces ethernet eth3 offload gso +set interfaces ethernet eth3 offload rps +set interfaces ethernet eth3 offload sg +set interfaces ethernet eth3 offload tso +set interfaces bonding bond10 hash-policy 'layer3+4' +set interfaces bonding bond10 member interface 'eth2' +set interfaces bonding bond10 member interface 'eth3' +set interfaces bonding bond10 mode '802.3ad' +set interfaces bonding bond10 vif 50 address '192.168.189.1/24' +set interfaces loopback lo +set interfaces pppoe pppoe7 authentication password 'vyos' +set interfaces pppoe pppoe7 authentication username 'vyos' +set interfaces pppoe pppoe7 dhcpv6-options pd 0 interface bond10.50 address '1' +set interfaces pppoe pppoe7 dhcpv6-options pd 0 length '56' +set interfaces pppoe pppoe7 ip adjust-mss '1452' +set interfaces pppoe pppoe7 ipv6 address autoconf +set interfaces pppoe pppoe7 ipv6 adjust-mss '1432' +set interfaces pppoe pppoe7 mtu '1492' +set interfaces pppoe pppoe7 no-peer-dns +set interfaces pppoe pppoe7 source-interface 'eth1' +set service lldp interface eth1 disable +set service ntp allow-client address '192.168.189.0/24' +set service ntp server time1.vyos.net +set service ntp server time2.vyos.net +set service ntp listen-address '192.168.189.1' +set service ssh dynamic-protection +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 lease '604800' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 option default-router '192.168.189.1' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 option domain-name 'vyos.net' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 option name-server '1.1.1.1' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 option name-server '9.9.9.9' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 range 0 start '192.168.189.20' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 range 0 stop '192.168.189.254' +set service dhcp-server shared-network-name LAN subnet 192.168.189.0/24 subnet-id '1' +set service router-advert interface bond10.50 prefix ::/64 preferred-lifetime '2700' +set service router-advert interface bond10.50 prefix ::/64 valid-lifetime '5400' +set system config-management commit-revisions '100' +set system domain-name 'vyos.net' +set system host-name 'R1' +set system login user vyos authentication encrypted-password '$6$2Ta6TWHd/U$NmrX0x9kexCimeOcYK1MfhMpITF9ELxHcaBU/znBq.X2ukQOj61fVI2UYP/xBzP4QtiTcdkgs7WOQMHWsRymO/' +set system login user vyos authentication plaintext-password '' +set system name-server '1.1.1.1' +set system name-server '9.9.9.9' +set system console device ttyS0 speed '115200' +set nat destination rule 1000 destination port '3389' +set nat destination rule 1000 inbound-interface name 'pppoe7' +set nat destination rule 1000 protocol 'tcp' +set nat destination rule 1000 translation address '192.168.189.5' +set nat destination rule 1000 translation port '3389' +set nat destination rule 10022 destination port '10022' +set nat destination rule 10022 inbound-interface name 'pppoe7' +set nat destination rule 10022 protocol 'tcp' +set nat destination rule 10022 translation address '192.168.189.2' +set nat destination rule 10022 translation port '22' +set nat destination rule 10300 destination port '10300' +set nat destination rule 10300 inbound-interface name 'pppoe7' +set nat destination rule 10300 protocol 'udp' +set nat destination rule 10300 translation address '192.168.189.2' +set nat destination rule 10300 translation port '10300' +set nat source rule 10 outbound-interface name 'eth1' +set nat source rule 10 source address '192.168.189.0/24' +set nat source rule 10 translation address 'masquerade' +set nat source rule 10 translation options port-mapping 'random' +set nat source rule 50 outbound-interface name 'pppoe7' +set nat source rule 50 protocol 'udp' +set nat source rule 50 source address '192.168.189.2' +set nat source rule 50 source port '10300' +set nat source rule 50 translation address 'masquerade' +set nat source rule 50 translation port '10300' +set nat source rule 100 outbound-interface name 'pppoe7' +set nat source rule 100 source address '192.168.189.0/24' +set nat source rule 100 translation address 'masquerade' diff --git a/smoketest/configs/nat-basic b/smoketest/configs/nat-basic new file mode 100644 index 00000000000..52f369f3409 --- /dev/null +++ b/smoketest/configs/nat-basic @@ -0,0 +1,256 @@ +interfaces { + bonding bond10 { + hash-policy "layer3+4" + member { + interface "eth2" + interface "eth3" + } + mode "802.3ad" + vif 50 { + address "192.168.189.1/24" + } + } + ethernet eth0 { + disable + offload { + gro + gso + rps + sg + tso + } + } + ethernet eth1 { + offload { + gro + gso + rps + sg + tso + } + } + ethernet eth2 { + offload { + gro + gso + rps + sg + tso + } + } + ethernet eth3 { + offload { + gro + gso + rps + sg + tso + } + } + loopback lo { + } + pppoe pppoe7 { + authentication { + password "vyos" + username "vyos" + } + dhcpv6-options { + pd 0 { + interface bond10.50 { + address "1" + } + length "56" + } + } + ip { + adjust-mss "1452" + } + ipv6 { + address { + autoconf + } + adjust-mss "1432" + } + mtu "1492" + no-peer-dns + source-interface "eth1" + } +} +nat { + destination { + rule 1000 { + destination { + port "3389" + } + inbound-interface { + name "pppoe7" + } + protocol "tcp" + translation { + address "192.168.189.5" + port "3389" + } + } + rule 10022 { + destination { + port "10022" + } + inbound-interface { + name "pppoe7" + } + protocol "tcp" + translation { + address "192.168.189.2" + port "22" + } + } + rule 10300 { + destination { + port "10300" + } + inbound-interface { + name "pppoe7" + } + protocol "udp" + translation { + address "192.168.189.2" + port "10300" + } + } + } + source { + rule 10 { + outbound-interface { + name "eth1" + } + source { + address "192.168.189.0/24" + } + translation { + address "masquerade" + options { + port-mapping fully-random + } + } + } + rule 50 { + outbound-interface { + name "pppoe7" + } + protocol "udp" + source { + address "192.168.189.2" + port "10300" + } + translation { + address "masquerade" + port "10300" + } + } + rule 100 { + outbound-interface { + name "pppoe7" + } + source { + address "192.168.189.0/24" + } + translation { + address "masquerade" + } + } + } +} +service { + dhcp-server { + shared-network-name LAN { + subnet 192.168.189.0/24 { + default-router "192.168.189.1" + domain-name "vyos.net" + lease "604800" + name-server "1.1.1.1" + name-server "9.9.9.9" + range 0 { + start "192.168.189.20" + stop "192.168.189.254" + } + } + } + } + lldp { + interface all { + } + interface eth1 { + disable + } + } + ntp { + allow-client { + address "192.168.189.0/24" + } + listen-address "192.168.189.1" + server time1.vyos.net { + } + server time2.vyos.net { + } + } + router-advert { + interface bond10.50 { + prefix ::/64 { + preferred-lifetime "2700" + valid-lifetime "5400" + } + } + } + ssh { + disable-host-validation + dynamic-protection { + } + } +} +system { + config-management { + commit-revisions "100" + } + conntrack { + modules { + ftp + h323 + nfs + pptp + sip + sqlnet + tftp + } + } + console { + device ttyS0 { + speed "115200" + } + } + domain-name "vyos.net" + host-name "R1" + login { + user vyos { + authentication { + encrypted-password $6$2Ta6TWHd/U$NmrX0x9kexCimeOcYK1MfhMpITF9ELxHcaBU/znBq.X2ukQOj61fVI2UYP/xBzP4QtiTcdkgs7WOQMHWsRymO/ + plaintext-password "" + } + } + } + name-server "1.1.1.1" + name-server "9.9.9.9" + syslog { + global { + facility all { + level "info" + } + facility local7 { + level "debug" + } + } + } +} + +// Warning: Do not remove the following line. +// vyos-config-version: "bgp@5:broadcast-relay@1:cluster@2:config-management@1:conntrack@5:conntrack-sync@2:container@2:dhcp-relay@2:dhcp-server@8:dhcpv6-server@1:dns-dynamic@4:dns-forwarding@4:firewall@15:flow-accounting@1:https@6:ids@1:interfaces@32:ipoe-server@3:ipsec@13:isis@3:l2tp@9:lldp@2:mdns@1:monitoring@1:nat@7:nat66@3:ntp@3:openconnect@3:ospf@2:pim@1:policy@8:pppoe-server@10:pptp@5:qos@2:quagga@11:rip@1:rpki@2:salt@1:snmp@3:ssh@2:sstp@6:system@27:vrf@3:vrrp@4:vyos-accel-ppp@2:wanloadbalance@3:webproxy@2" +// Release version: 1.4.0-epa3 diff --git a/src/migration-scripts/nat/7-to-8 b/src/migration-scripts/nat/7-to-8 new file mode 100755 index 00000000000..ab2ffa6d3a7 --- /dev/null +++ b/src/migration-scripts/nat/7-to-8 @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T6345: random - In kernel 5.0 and newer this is the same as fully-random. +# In earlier kernels the port mapping will be randomized using a seeded +# MD5 hash mix using source and destination address and destination port. +# drop fully-random from CLI + +from sys import argv,exit +from vyos.configtree import ConfigTree + +if len(argv) < 2: + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) + +if not config.exists(['nat']): + # Nothing to do + exit(0) + +for direction in ['source', 'destination']: + # If a node doesn't exist, we obviously have nothing to do. + if not config.exists(['nat', direction]): + continue + + # However, we also need to handle the case when a 'source' or 'destination' sub-node does exist, + # but there are no rules under it. + if not config.list_nodes(['nat', direction]): + continue + + for rule in config.list_nodes(['nat', direction, 'rule']): + port_mapping = ['nat', direction, 'rule', rule, 'translation', 'options', 'port-mapping'] + if config.exists(port_mapping): + tmp = config.return_value(port_mapping) + if tmp == 'fully-random': + config.set(port_mapping, value='random') + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print(f'Failed to save the modified config: {e}') + exit(1) From 55f2681bc5ecf490a695dff17c59171427e87aee Mon Sep 17 00:00:00 2001 From: Ginko <152240782+Giggum@users.noreply.github.com> Date: Thu, 23 May 2024 01:09:58 -0400 Subject: [PATCH 130/188] dhcpv6-server: T6381: fix typos in select ConfigError messages in VyOS current (#3508) --- src/conf_mode/service_dhcpv6-server.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/conf_mode/service_dhcpv6-server.py b/src/conf_mode/service_dhcpv6-server.py index c7333dd3aa1..7af88007c48 100755 --- a/src/conf_mode/service_dhcpv6-server.py +++ b/src/conf_mode/service_dhcpv6-server.py @@ -106,14 +106,14 @@ def verify(dhcpv6): # Stop address must be greater or equal to start address if not ip_address(stop) >= ip_address(start): - raise ConfigError(f'Range stop address "{stop}" must be greater then or equal ' \ + raise ConfigError(f'Range stop address "{stop}" must be greater than or equal ' \ f'to the range start address "{start}"!') # DHCPv6 range start address must be unique - two ranges can't # start with the same address - makes no sense if start in range6_start: raise ConfigError(f'Conflicting DHCPv6 lease range: '\ - f'Pool start address "{start}" defined multipe times!') + f'Pool start address "{start}" defined multiple times!') range6_start.append(start) @@ -121,7 +121,7 @@ def verify(dhcpv6): # end with the same address - makes no sense if stop in range6_stop: raise ConfigError(f'Conflicting DHCPv6 lease range: '\ - f'Pool stop address "{stop}" defined multipe times!') + f'Pool stop address "{stop}" defined multiple times!') range6_stop.append(stop) @@ -180,7 +180,7 @@ def verify(dhcpv6): if 'option' in subnet_config: if 'vendor_option' in subnet_config['option']: if len(dict_search('option.vendor_option.cisco.tftp_server', subnet_config)) > 2: - raise ConfigError(f'No more then two Cisco tftp-servers should be defined for subnet "{subnet}"!') + raise ConfigError(f'No more than two Cisco tftp-servers should be defined for subnet "{subnet}"!') # Subnets must be unique if subnet in subnets: From cd32928e1856b0a7e9781709cfc3b0db16b6abcb Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 23 May 2024 21:36:04 +0200 Subject: [PATCH 131/188] suricata: T751: move CLI from "service ids suricata" -> "service suricata" --- .../service_ids_suricata.xml.in | 250 ------------------ interface-definitions/service_suricata.xml.in | 246 +++++++++++++++++ ...ce_ids_suricata.py => service_suricata.py} | 2 +- 3 files changed, 247 insertions(+), 251 deletions(-) delete mode 100644 interface-definitions/service_ids_suricata.xml.in create mode 100644 interface-definitions/service_suricata.xml.in rename src/conf_mode/{service_ids_suricata.py => service_suricata.py} (99%) diff --git a/interface-definitions/service_ids_suricata.xml.in b/interface-definitions/service_ids_suricata.xml.in deleted file mode 100644 index 8c1973567c1..00000000000 --- a/interface-definitions/service_ids_suricata.xml.in +++ /dev/null @@ -1,250 +0,0 @@ - - - - - - - - - Network IDS, IPS and Network Security Monitoring - 740 - - - #include - - - Address group name - - home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server - - - [a-z0-9-]+ - - - - - - IP address or subnet - - ipv4 - IPv4 address to match - - - ipv6 - IPv6 address to match - - - ipv4net - IPv4 prefix to match - - - ipv6net - IPv6 prefix to match - - - !ipv4 - Exclude the specified IPv4 address from matches - - - !ipv6 - Exclude the specified IPv6 address from matches - - - !ipv4net - Exclude the specified IPv6 prefix from matches - - - !ipv6net - Exclude the specified IPv6 prefix from matches - - - - - - - - - - - - - - - - - Address group - - service ids suricata address-group - home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server - - - string - Address group to match - - - !string - Exclude the specified address group from matches - - - !?[a-z0-9-]+ - - - - - - - - - Port group name - - http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports - - - [a-z0-9-]+ - - - - - - Port number - - u32:1-65535 - Numeric port to match - - - !u32:1-65535 - Numeric port to exclude from matches - - - start-end - Numbered port range (e.g. 1001-1005) to match - - - !start-end - Numbered port range (e.g. !1001-1005) to exclude from matches - - - - - - - - - - - Port group - - service ids suricata port-group - http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports - - - string - Port group to match - - - !string - Exclude the specified port group from matches - - - !?[a-z0-9-]+ - - - - - - - - - Suricata log outputs - - - - - Extensible Event Format (EVE) - - - - - EVE logging destination - - regular syslog - - - regular - Log to filename - - - syslog - Log to syslog - - - (regular|syslog) - - - regular - - - - Log file - - filename - File name in default Suricata log directory - - - /path - Absolute file path - - - eve.json - - - - Log types - - alert anomaly drop files http dns tls smtp dnp3 ftp rdp nfs smb tftp ikev2 dcerpc krb5 snmp rfb sip dhcp ssh mqtt http2 flow netflow - - - alert - Record events for rule matches - - - anomaly - Record unexpected conditions such as truncated packets, packets with invalid IP/UDP/TCP length values, and other events that render the packet invalid for further processing or describe unexpected behavior on an established stream - - - drop - Record events for dropped packets - - - file - Record file details (e.g., MD5) for files extracted from application protocols (e.g., HTTP) - - - application (http, dns, tls, ...) - Record application-level transactions - - - flow - Record bi-directional flows - - - netflow - Record uni-directional flows - - - (alert|anomaly|http|dns|tls|files|drop|smtp|dnp3|ftp|rdp|nfs|smb|tftp|ikev2|dcerpc|krb5|snmp|rfb|sip|dhcp|ssh|mqtt|http2|flow|netflow) - - - - - - - - - - - - - - - diff --git a/interface-definitions/service_suricata.xml.in b/interface-definitions/service_suricata.xml.in new file mode 100644 index 00000000000..e21320bfee9 --- /dev/null +++ b/interface-definitions/service_suricata.xml.in @@ -0,0 +1,246 @@ + + + + + + + Network IDS, IPS and Security Monitoring + 740 + + + #include + + + Address group name + + home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server + + + [a-z0-9-]+ + + + + + + IP address or subnet + + ipv4 + IPv4 address to match + + + ipv6 + IPv6 address to match + + + ipv4net + IPv4 prefix to match + + + ipv6net + IPv6 prefix to match + + + !ipv4 + Exclude the specified IPv4 address from matches + + + !ipv6 + Exclude the specified IPv6 address from matches + + + !ipv4net + Exclude the specified IPv6 prefix from matches + + + !ipv6net + Exclude the specified IPv6 prefix from matches + + + + + + + + + + + + + + + + + Address group + + service ids suricata address-group + home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server + + + string + Address group to match + + + !string + Exclude the specified address group from matches + + + !?[a-z0-9-]+ + + + + + + + + + Port group name + + http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports + + + [a-z0-9-]+ + + + + + + Port number + + u32:1-65535 + Numeric port to match + + + !u32:1-65535 + Numeric port to exclude from matches + + + start-end + Numbered port range (e.g. 1001-1005) to match + + + !start-end + Numbered port range (e.g. !1001-1005) to exclude from matches + + + + + + + + + + + Port group + + service ids suricata port-group + http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports + + + string + Port group to match + + + !string + Exclude the specified port group from matches + + + !?[a-z0-9-]+ + + + + + + + + + Suricata log outputs + + + + + Extensible Event Format (EVE) + + + + + EVE logging destination + + regular syslog + + + regular + Log to filename + + + syslog + Log to syslog + + + (regular|syslog) + + + regular + + + + Log file + + filename + File name in default Suricata log directory + + + /path + Absolute file path + + + eve.json + + + + Log types + + alert anomaly drop files http dns tls smtp dnp3 ftp rdp nfs smb tftp ikev2 dcerpc krb5 snmp rfb sip dhcp ssh mqtt http2 flow netflow + + + alert + Record events for rule matches + + + anomaly + Record unexpected conditions such as truncated packets, packets with invalid IP/UDP/TCP length values, and other events that render the packet invalid for further processing or describe unexpected behavior on an established stream + + + drop + Record events for dropped packets + + + file + Record file details (e.g., MD5) for files extracted from application protocols (e.g., HTTP) + + + application (http, dns, tls, ...) + Record application-level transactions + + + flow + Record bi-directional flows + + + netflow + Record uni-directional flows + + + (alert|anomaly|http|dns|tls|files|drop|smtp|dnp3|ftp|rdp|nfs|smb|tftp|ikev2|dcerpc|krb5|snmp|rfb|sip|dhcp|ssh|mqtt|http2|flow|netflow) + + + + + + + + + + + + + diff --git a/src/conf_mode/service_ids_suricata.py b/src/conf_mode/service_suricata.py similarity index 99% rename from src/conf_mode/service_ids_suricata.py rename to src/conf_mode/service_suricata.py index 49fbce244c4..cce4de6e3e9 100755 --- a/src/conf_mode/service_ids_suricata.py +++ b/src/conf_mode/service_suricata.py @@ -65,7 +65,7 @@ def get_config(config=None): conf = config else: conf = Config() - base = ['service', 'ids', 'suricata'] + base = ['service', 'suricata'] if not conf.exists(base): return None From 2af04a53a4c1aa30118f4c799eb7d1c4cd63be66 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 23 May 2024 21:54:26 +0200 Subject: [PATCH 132/188] suricata: T751: remove implicit default dictionary --- interface-definitions/service_suricata.xml.in | 16 ++----- src/conf_mode/service_suricata.py | 47 ++++--------------- 2 files changed, 13 insertions(+), 50 deletions(-) diff --git a/interface-definitions/service_suricata.xml.in b/interface-definitions/service_suricata.xml.in index e21320bfee9..e0159e2bad7 100644 --- a/interface-definitions/service_suricata.xml.in +++ b/interface-definitions/service_suricata.xml.in @@ -12,9 +12,6 @@ Address group name - - home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server - [a-z0-9-]+ @@ -73,14 +70,13 @@ Address group service ids suricata address-group - home-net external-net http-servers smtp-servers sql-servers dns-servers telnet-servers aim-servers dc-servers dnp3-server dnp3-client modbus-client modbus-server enip-client enip-server - string + txt Address group to match - !string + !txt Exclude the specified address group from matches @@ -94,9 +90,6 @@ Port group name - - http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports - [a-z0-9-]+ @@ -133,14 +126,13 @@ Port group service ids suricata port-group - http-ports shellcode-ports oracle-ports ssh-ports dnp3-ports modbus-ports file-data-ports ftp-ports geneve-ports vxlan-ports teredo-ports - string + txt Port group to match - !string + !txt Exclude the specified port group from matches diff --git a/src/conf_mode/service_suricata.py b/src/conf_mode/service_suricata.py index cce4de6e3e9..06d68a6379e 100755 --- a/src/conf_mode/service_suricata.py +++ b/src/conf_mode/service_suricata.py @@ -29,53 +29,18 @@ config_file = '/run/suricata/suricata.yaml' rotate_file = '/etc/logrotate.d/suricata' -address_group_defaults = { - 'home-net': {'address': ['192.168.0.0/16','10.0.0.0/8','172.16.0.0/12']}, - 'external-net': {'group': ['!home-net']}, - 'http-servers': {'group': ['home-net']}, - 'smtp-servers': {'group': ['home-net']}, - 'sql-servers': {'group': ['home-net']}, - 'dns-servers': {'group': ['home-net']}, - 'telnet-servers': {'group': ['home-net']}, - 'aim-servers': {'group': ['external-net']}, - 'dc-servers': {'group': ['home-net']}, - 'dnp3-server': {'group': ['home-net']}, - 'modbus-client': {'group': ['home-net']}, - 'modbus-server': {'group': ['home-net']}, - 'enip-client': {'group': ['home-net']}, - 'enip-server': {'group': ['home-net']}, -} - -port_group_defaults = { - 'http-ports': {'port': ['80']}, - 'shellcode-ports': {'port': ['!80']}, - 'oracle-ports': {'port': ['1521']}, - 'ssh-ports': {'port': ['22']}, - 'dnp3-ports': {'port': ['20000']}, - 'modbus-ports': {'port': ['502']}, - 'file-data-ports': {'port': ['110', '143'], 'group': ['http-ports']}, - 'ftp-ports': {'port': ['21']}, - 'geneve-ports': {'port': ['6081']}, - 'vxlan-ports': {'port': ['4789']}, - 'teredo-ports': {'port': ['3544']}, -} - def get_config(config=None): if config: conf = config else: conf = Config() base = ['service', 'suricata'] + if not conf.exists(base): return None suricata = conf.get_config_dict(base, - get_first_key=True, - with_recursive_defaults=True) - - # Ensure minimal defaults are present - suricata['address-group'] = address_group_defaults | suricata.get('address-group', {}) - suricata['port-group'] = port_group_defaults | suricata.get('port-group', {}) + get_first_key=True, with_recursive_defaults=True) return suricata @@ -114,7 +79,13 @@ def verify(suricata): return None if 'interface' not in suricata: - raise ConfigError('No interfaces configured') + raise ConfigError('No interfaces configured!') + + if 'address-group' not in suricata: + raise ConfigError('No address-group configured!') + + if 'port-group' not in suricata: + raise ConfigError('No port-group configured!') try: topological_sort(suricata['address-group']) From 01464a6069fa7cf02d5545c02ed03fa69679c32c Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 23 May 2024 21:54:41 +0200 Subject: [PATCH 133/188] suricata: T751: use key_mangling in get_config_dict() --- data/templates/ids/suricata.j2 | 4 ++-- src/conf_mode/service_suricata.py | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/data/templates/ids/suricata.j2 b/data/templates/ids/suricata.j2 index 1bd90b67f21..585db93ebf3 100644 --- a/data/templates/ids/suricata.j2 +++ b/data/templates/ids/suricata.j2 @@ -15,12 +15,12 @@ vars: # more specific is better for alert accuracy and performance address-groups: -{% for (name, value) in suricata['address-group'] %} +{% for (name, value) in suricata['address_group'] %} {{ name }}: "[{{ value | join(',') }}]" {% endfor %} port-groups: -{% for (name, value) in suricata['port-group'] %} +{% for (name, value) in suricata['port_group'] %} {{ name }}: "[{{ value | join(',') }}]" {% endfor %} diff --git a/src/conf_mode/service_suricata.py b/src/conf_mode/service_suricata.py index 06d68a6379e..69b369e0b70 100755 --- a/src/conf_mode/service_suricata.py +++ b/src/conf_mode/service_suricata.py @@ -39,7 +39,7 @@ def get_config(config=None): if not conf.exists(base): return None - suricata = conf.get_config_dict(base, + suricata = conf.get_config_dict(base, key_mangling=('-', '_'), get_first_key=True, with_recursive_defaults=True) return suricata @@ -81,19 +81,19 @@ def verify(suricata): if 'interface' not in suricata: raise ConfigError('No interfaces configured!') - if 'address-group' not in suricata: + if 'address_group' not in suricata: raise ConfigError('No address-group configured!') - if 'port-group' not in suricata: + if 'port_group' not in suricata: raise ConfigError('No port-group configured!') try: - topological_sort(suricata['address-group']) + topological_sort(suricata['address_group']) except (ConfigError,StopIteration) as e: raise ConfigError(f'Invalid address-group: {e}') try: - topological_sort(suricata['port-group']) + topological_sort(suricata['port_group']) except (ConfigError,StopIteration) as e: raise ConfigError(f'Invalid port-group: {e}') @@ -126,12 +126,12 @@ def format_group(group): return format_group # Format the address group - suricata['address-group'] = map(to_config('address'), - topological_sort(suricata['address-group'])) + suricata['address_group'] = map(to_config('address'), + topological_sort(suricata['address_group'])) # Format the port group - suricata['port-group'] = map(to_config('port'), - topological_sort(suricata['port-group'])) + suricata['port_group'] = map(to_config('port'), + topological_sort(suricata['port_group'])) render(config_file, 'ids/suricata.j2', {'suricata': suricata}) render(rotate_file, 'ids/suricata_logrotate.j2', suricata) From 609563d6acfeafbed46b1ac5e6bd497ce097e3bc Mon Sep 17 00:00:00 2001 From: Gregor Michels Date: Fri, 24 May 2024 09:54:52 +0200 Subject: [PATCH 134/188] load-balancing haproxy: T6391: fix typo in timeout help (#3513) Co-authored-by: Gregor Michels --- interface-definitions/include/haproxy/timeout.xml.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/interface-definitions/include/haproxy/timeout.xml.i b/interface-definitions/include/haproxy/timeout.xml.i index 250b3568308..79e7303b11c 100644 --- a/interface-definitions/include/haproxy/timeout.xml.i +++ b/interface-definitions/include/haproxy/timeout.xml.i @@ -1,7 +1,7 @@ - Tiemout options + Timeout options From 65fba1cd27af67c543e120effc12882bd0191f03 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 25 May 2024 21:26:08 +0200 Subject: [PATCH 135/188] op-mode: T6377: must call pki.py helper as root to work with ACME certificates This fixes the error: vyos@vyos:~$ show pki certificate Traceback (most recent call last): File "/usr/lib/python3/dist-packages/vyos/config.py", line 111, in config_dict_mangle_acme tmp = read_file(f'{vyos_certbot_dir}/live/{name}/cert.pem') ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/vyos/utils/file.py", line 44, in read_file raise e File "/usr/lib/python3/dist-packages/vyos/utils/file.py", line 38, in read_file with open(fname, 'r') as f: ^^^^^^^^^^^^^^^^ PermissionError: [Errno 13] Permission denied: '/config/auth/letsencrypt/live/vyos/cert.pem' --- op-mode-definitions/pki.xml.in | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/op-mode-definitions/pki.xml.in b/op-mode-definitions/pki.xml.in index a5e01bade9c..f76b4f4e149 100644 --- a/op-mode-definitions/pki.xml.in +++ b/op-mode-definitions/pki.xml.in @@ -495,7 +495,7 @@ Show x509 CA certificates - ${vyos_op_scripts_dir}/pki.py --action show --ca "all" + sudo ${vyos_op_scripts_dir}/pki.py --action show --ca "all" @@ -504,13 +504,13 @@ pki ca - ${vyos_op_scripts_dir}/pki.py --action show --ca "$4" + sudo ${vyos_op_scripts_dir}/pki.py --action show --ca "$4" Show x509 CA certificate in PEM format - ${vyos_op_scripts_dir}/pki.py --action show --ca "$4" --pem + sudo ${vyos_op_scripts_dir}/pki.py --action show --ca "$4" --pem @@ -518,7 +518,7 @@ Show x509 certificates - ${vyos_op_scripts_dir}/pki.py --action show --certificate "all" + sudo ${vyos_op_scripts_dir}/pki.py --action show --certificate "all" @@ -527,7 +527,7 @@ pki certificate - ${vyos_op_scripts_dir}/pki.py --action show --certificate "$4" + sudo ${vyos_op_scripts_dir}/pki.py --action show --certificate "$4" @@ -542,7 +542,7 @@ sha256 sha384 sha512 - ${vyos_op_scripts_dir}/pki.py --action show --certificate "$4" --fingerprint "$6" + sudo ${vyos_op_scripts_dir}/pki.py --action show --certificate "$4" --fingerprint "$6" From b6ee07c7efbb818787deba20116f4289853fb5c9 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sat, 25 May 2024 21:53:19 +0200 Subject: [PATCH 136/188] op-mode: T6400: pki: unable to generate fingerprint for ACME issued certificates This fixes (for and ACME generated certificate) vyos@vyos:~$ show pki certificate vyos fingerprint sha512 Traceback (most recent call last): File "/usr/libexec/vyos/op_mode/pki.py", line 1081, in show_certificate_fingerprint(args.certificate, args.fingerprint) File "/usr/libexec/vyos/op_mode/pki.py", line 934, in show_certificate_fingerprint print(get_certificate_fingerprint(cert, hash)) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ File "/usr/lib/python3/dist-packages/vyos/pki.py", line 76, in get_certificate_fingerprint fp = cert.fingerprint(hash_algorithm) ^^^^^^^^^^^^^^^^ AttributeError: 'bool' object has no attribute 'fingerprint' After the fix: vyos@vyos# run show pki certificate vyos fingerprint sha256 10:2C:EF:2C:DA:7A:EE:C6:D7:8E:53:12:F0:F5:DE:B9:E9:D0:6C:B4:49:1C:8B:70:2B:D9:AF:FC:9B:75:A3:D2 --- src/op_mode/pki.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/op_mode/pki.py b/src/op_mode/pki.py index b1ca6ee296d..361b60e0e8f 100755 --- a/src/op_mode/pki.py +++ b/src/op_mode/pki.py @@ -876,7 +876,7 @@ def show_certificate_authority(name=None, pem=False): print("Certificate Authorities:") print(tabulate.tabulate(data, headers)) -def show_certificate(name=None, pem=False): +def show_certificate(name=None, pem=False, fingerprint_hash=None): headers = ['Name', 'Type', 'Subject CN', 'Issuer CN', 'Issued', 'Expiry', 'Revoked', 'Private Key', 'CA Present'] data = [] certs = get_config_certificate() @@ -897,6 +897,9 @@ def show_certificate(name=None, pem=False): if name and pem: print(encode_certificate(cert)) return + elif name and fingerprint_hash: + print(get_certificate_fingerprint(cert, fingerprint_hash)) + return ca_name = get_certificate_ca(cert, ca_certs) cert_subject_cn = cert.subject.rfc4514_string().split(",")[0] @@ -923,12 +926,6 @@ def show_certificate(name=None, pem=False): print("Certificates:") print(tabulate.tabulate(data, headers)) -def show_certificate_fingerprint(name, hash): - cert = get_config_certificate(name=name) - cert = load_certificate(cert['certificate']) - - print(get_certificate_fingerprint(cert, hash)) - def show_crl(name=None, pem=False): headers = ['CA Name', 'Updated', 'Revokes'] data = [] @@ -1074,7 +1071,7 @@ def show_crl(name=None, pem=False): if args.fingerprint is None: show_certificate(None if args.certificate == 'all' else args.certificate, args.pem) else: - show_certificate_fingerprint(args.certificate, args.fingerprint) + show_certificate(args.certificate, fingerprint_hash=args.fingerprint) elif args.crl: show_crl(None if args.crl == 'all' else args.crl, args.pem) else: From f7b0bc68b7950a6c6e68b9e6708ef8a4b7b9b423 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Sun, 26 May 2024 14:53:40 +0200 Subject: [PATCH 137/188] smoketest: T6395: check for VFIO options to be present --- smoketest/scripts/system/test_kernel_options.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/smoketest/scripts/system/test_kernel_options.py b/smoketest/scripts/system/test_kernel_options.py index 18922d93df6..bc7f658b03f 100755 --- a/smoketest/scripts/system/test_kernel_options.py +++ b/smoketest/scripts/system/test_kernel_options.py @@ -111,5 +111,14 @@ def test_ip_routing_support(self): tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) + def test_vfio(self): + options_to_check = [ + 'CONFIG_VFIO', 'CONFIG_VFIO_GROUP', 'CONFIG_VFIO_CONTAINER', + 'CONFIG_VFIO_IOMMU_TYPE1', 'CONFIG_VFIO_NOIOMMU', 'CONFIG_VFIO_VIRQFD' + ] + for option in options_to_check: + tmp = re.findall(f'{option}=(y|m)', self._config_data) + self.assertTrue(tmp) + if __name__ == '__main__': unittest.main(verbosity=2) From d4d70929a81b2ee1f66a9412a3545911b3874a62 Mon Sep 17 00:00:00 2001 From: Alex W Date: Sun, 26 May 2024 21:50:01 +0100 Subject: [PATCH 138/188] reverse-proxy: T6402: Fix invalid checks in validation script --- src/conf_mode/load-balancing_reverse-proxy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index a4efb1cd8c0..b6db110ae3b 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -88,22 +88,22 @@ def verify(lb): if {'send_proxy', 'send_proxy_v2'} <= set(bk_server_conf): raise ConfigError(f'Cannot use both "send-proxy" and "send-proxy-v2" for server "{bk_server}"') + if 'ssl' in back_config: + if {'no_verify', 'ca_certificate'} <= set(back_config['ssl']): + raise ConfigError(f'backend {back} cannot have both ssl options no-verify and ca-certificate set!') + # Check if http-response-headers are configured in any frontend/backend where mode != http for group in ['service', 'backend']: for config_name, config in lb[group].items(): if 'http_response_headers' in config and ('mode' not in config or config['mode'] != 'http'): raise ConfigError(f'{group} {config_name} must be set to http mode to use http_response_headers!') - if 'ssl' in back_config: - if {'no_verify', 'ca_certificate'} <= set(back_config['ssl']): - raise ConfigError(f'backend {back} cannot have both ssl options no-verify and ca-certificate set!') - for front, front_config in lb['service'].items(): for cert in dict_search('ssl.certificate', front_config) or []: verify_pki_certificate(lb, cert) for back, back_config in lb['backend'].items(): - tmp = dict_search('ssl.ca_certificate', front_config) + tmp = dict_search('ssl.ca_certificate', back_config) if tmp: verify_pki_ca_certificate(lb, tmp) From 4d84f786f64d2b80046100ead5d0e8c1eef7418c Mon Sep 17 00:00:00 2001 From: khramshinr Date: Thu, 23 May 2024 17:39:56 +0600 Subject: [PATCH 139/188] T4576: Accel-ppp logging level configuration add ability to change logging level config for: * VPN L2TP * VPN PPTP * VPN SSTP * IPoE Server * PPPoE Serve --- data/templates/accel-ppp/ipoe.config.j2 | 4 +- data/templates/accel-ppp/l2tp.config.j2 | 4 +- data/templates/accel-ppp/pppoe.config.j2 | 4 +- data/templates/accel-ppp/pptp.config.j2 | 4 +- data/templates/accel-ppp/sstp.config.j2 | 4 +- .../include/accel-ppp/log.xml.i | 42 +++++++++++++++++++ .../service_ipoe-server.xml.in | 1 + .../service_pppoe-server.xml.in | 1 + interface-definitions/vpn_l2tp.xml.in | 1 + interface-definitions/vpn_pptp.xml.in | 1 + interface-definitions/vpn_sstp.xml.in | 1 + smoketest/scripts/cli/base_accel_ppp_test.py | 18 ++++++++ 12 files changed, 80 insertions(+), 5 deletions(-) create mode 100644 interface-definitions/include/accel-ppp/log.xml.i diff --git a/data/templates/accel-ppp/ipoe.config.j2 b/data/templates/accel-ppp/ipoe.config.j2 index c898129852a..d87b90473d2 100644 --- a/data/templates/accel-ppp/ipoe.config.j2 +++ b/data/templates/accel-ppp/ipoe.config.j2 @@ -29,7 +29,9 @@ max-starting={{ max_concurrent_sessions }} [log] syslog=accel-ipoe,daemon copy=1 -level=5 +{% if log.level is vyos_defined %} +level={{ log.level }} +{% endif %} [ipoe] verbose=1 diff --git a/data/templates/accel-ppp/l2tp.config.j2 b/data/templates/accel-ppp/l2tp.config.j2 index 4ce9042c2a4..db4db66a769 100644 --- a/data/templates/accel-ppp/l2tp.config.j2 +++ b/data/templates/accel-ppp/l2tp.config.j2 @@ -28,7 +28,9 @@ max-starting={{ max_concurrent_sessions }} [log] syslog=accel-l2tp,daemon copy=1 -level=5 +{% if log.level is vyos_defined %} +level={{ log.level }} +{% endif %} [client-ip-range] 0.0.0.0/0 diff --git a/data/templates/accel-ppp/pppoe.config.j2 b/data/templates/accel-ppp/pppoe.config.j2 index 42bc8440c99..6711f2ec9c4 100644 --- a/data/templates/accel-ppp/pppoe.config.j2 +++ b/data/templates/accel-ppp/pppoe.config.j2 @@ -27,7 +27,9 @@ thread-count={{ thread_count }} [log] syslog=accel-pppoe,daemon copy=1 -level=5 +{% if log.level is vyos_defined %} +level={{ log.level }} +{% endif %} {% if authentication.mode is vyos_defined("noauth") %} [auth] diff --git a/data/templates/accel-ppp/pptp.config.j2 b/data/templates/accel-ppp/pptp.config.j2 index a04bd40c00f..44f35998bda 100644 --- a/data/templates/accel-ppp/pptp.config.j2 +++ b/data/templates/accel-ppp/pptp.config.j2 @@ -28,7 +28,9 @@ max-starting={{ max_concurrent_sessions }} [log] syslog=accel-pptp,daemon copy=1 -level=5 +{% if log.level is vyos_defined %} +level={{ log.level }} +{% endif %} [client-ip-range] 0.0.0.0/0 diff --git a/data/templates/accel-ppp/sstp.config.j2 b/data/templates/accel-ppp/sstp.config.j2 index 22fb5550637..38da829f366 100644 --- a/data/templates/accel-ppp/sstp.config.j2 +++ b/data/templates/accel-ppp/sstp.config.j2 @@ -29,7 +29,9 @@ max-starting={{ max_concurrent_sessions }} [log] syslog=accel-sstp,daemon copy=1 -level=5 +{% if log.level is vyos_defined %} +level={{ log.level }} +{% endif %} [client-ip-range] 0.0.0.0/0 diff --git a/interface-definitions/include/accel-ppp/log.xml.i b/interface-definitions/include/accel-ppp/log.xml.i new file mode 100644 index 00000000000..96ce93ff9d5 --- /dev/null +++ b/interface-definitions/include/accel-ppp/log.xml.i @@ -0,0 +1,42 @@ + + + + Server logging + + + + + Specifies log level + + 0 + Turn off logging + + + 1 + Log only error messages + + + 2 + Log error and warning messages + + + 3 + Log error, warning and minimum information messages + + + 4 + Log error, warning and full information messages + + + 5 + Log all messages including debug messages + + + + + + 3 + + + + diff --git a/interface-definitions/service_ipoe-server.xml.in b/interface-definitions/service_ipoe-server.xml.in index 414c9a73156..c7542f0d05c 100644 --- a/interface-definitions/service_ipoe-server.xml.in +++ b/interface-definitions/service_ipoe-server.xml.in @@ -189,6 +189,7 @@ #include #include #include + #include diff --git a/interface-definitions/service_pppoe-server.xml.in b/interface-definitions/service_pppoe-server.xml.in index 5d357c2f9af..81228938fa6 100644 --- a/interface-definitions/service_pppoe-server.xml.in +++ b/interface-definitions/service_pppoe-server.xml.in @@ -153,6 +153,7 @@ #include #include #include + #include diff --git a/interface-definitions/vpn_l2tp.xml.in b/interface-definitions/vpn_l2tp.xml.in index 85a375db435..c00e825344a 100644 --- a/interface-definitions/vpn_l2tp.xml.in +++ b/interface-definitions/vpn_l2tp.xml.in @@ -140,6 +140,7 @@ #include #include #include + #include diff --git a/interface-definitions/vpn_pptp.xml.in b/interface-definitions/vpn_pptp.xml.in index a63633f5720..8aec0cb1c3c 100644 --- a/interface-definitions/vpn_pptp.xml.in +++ b/interface-definitions/vpn_pptp.xml.in @@ -56,6 +56,7 @@ #include #include #include + #include diff --git a/interface-definitions/vpn_sstp.xml.in b/interface-definitions/vpn_sstp.xml.in index d9ed1c04051..5fd5c95ca22 100644 --- a/interface-definitions/vpn_sstp.xml.in +++ b/interface-definitions/vpn_sstp.xml.in @@ -62,6 +62,7 @@ Host-name must be alphanumeric and can contain hyphens + #include diff --git a/smoketest/scripts/cli/base_accel_ppp_test.py b/smoketest/scripts/cli/base_accel_ppp_test.py index ab723e707f4..212dc58ab69 100644 --- a/smoketest/scripts/cli/base_accel_ppp_test.py +++ b/smoketest/scripts/cli/base_accel_ppp_test.py @@ -628,3 +628,21 @@ def test_accel_limits(self): self.assertEqual(conf['connlimit']['limit'], limits) self.assertEqual(conf['connlimit']['burst'], burst) self.assertEqual(conf['connlimit']['timeout'], timeout) + + def test_accel_log_level(self): + self.basic_config() + self.cli_commit() + + # check default value + conf = ConfigParser(allow_no_value=True) + conf.read(self._config_file) + self.assertEqual(conf['log']['level'], '3') + + for log_level in range(0, 5): + self.set(['log', 'level', str(log_level)]) + self.cli_commit() + # Validate configuration values + conf = ConfigParser(allow_no_value=True) + conf.read(self._config_file) + + self.assertEqual(conf['log']['level'], str(log_level)) From 380e998b10341b6dd42bb94d00a9d7a462ada27a Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Mon, 27 May 2024 15:23:53 +0100 Subject: [PATCH 140/188] openvpn: T6374: ensure that TLS role is configured for site-to-site with TLS --- src/conf_mode/interfaces_openvpn.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index 0ecffd3beb0..5b9c217575d 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -260,6 +260,11 @@ def verify(openvpn): # OpenVPN site-to-site - VERIFY # elif openvpn['mode'] == 'site-to-site': + # XXX: site-to-site is the only mode that still can work without TLS, + # so we need to make sure that if TLS is used, then TLS role is also specified + if 'shared_secret_key' not in openvpn['tls'] and 'role' not in openvpn['tls']: + raise ConfigError('"tls role" is required for site-to-site OpenVPN with TLS') + if 'local_address' not in openvpn and 'is_bridge_member' not in openvpn: raise ConfigError('Must specify "local-address" or add interface to bridge') From cbb61faed494381b0c655d811920413b31fd294d Mon Sep 17 00:00:00 2001 From: khramshinr Date: Mon, 27 May 2024 20:34:52 +0600 Subject: [PATCH 141/188] T5786: Add set/show system image to /image endpoint --- python/vyos/configsession.py | 6 +++ smoketest/scripts/cli/test_service_https.py | 41 ++++++++++++++++++ src/services/vyos-http-api-server | 46 ++++++++++++++------- 3 files changed, 77 insertions(+), 16 deletions(-) diff --git a/python/vyos/configsession.py b/python/vyos/configsession.py index ab7a631bb01..beec6010b26 100644 --- a/python/vyos/configsession.py +++ b/python/vyos/configsession.py @@ -34,6 +34,8 @@ '--action', 'add', '--no-prompt', '--image-path'] REMOVE_IMAGE = ['/usr/libexec/vyos/op_mode/image_manager.py', '--action', 'delete', '--no-prompt', '--image-name'] +SET_DEFAULT_IMAGE = ['/usr/libexec/vyos/op_mode/image_manager.py', + '--action', 'set', '--no-prompt', '--image-name'] GENERATE = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'generate'] SHOW = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'show'] RESET = ['/opt/vyatta/bin/vyatta-op-cmd-wrapper', 'reset'] @@ -235,6 +237,10 @@ def remove_image(self, name): out = self.__run_command(REMOVE_IMAGE + [name]) return out + def set_default_image(self, name): + out = self.__run_command(SET_DEFAULT_IMAGE + [name]) + return out + def generate(self, path): out = self.__run_command(GENERATE + path) return out diff --git a/smoketest/scripts/cli/test_service_https.py b/smoketest/scripts/cli/test_service_https.py index f2a64627f2f..8a6386e4f91 100755 --- a/smoketest/scripts/cli/test_service_https.py +++ b/smoketest/scripts/cli/test_service_https.py @@ -411,6 +411,47 @@ def test_api_reset(self): r = request('POST', url, verify=False, headers=headers, data=payload) self.assertEqual(r.status_code, 200) + @ignore_warning(InsecureRequestWarning) + def test_api_image(self): + address = '127.0.0.1' + key = 'VyOS-key' + url = f'https://{address}/image' + headers = {} + + self.cli_set(base_path + ['api', 'keys', 'id', 'key-01', 'key', key]) + self.cli_commit() + + payload = { + 'data': '{"op": "add"}', + 'key': f'{key}', + } + r = request('POST', url, verify=False, headers=headers, data=payload) + self.assertEqual(r.status_code, 400) + self.assertIn('Missing required field "url"', r.json().get('error')) + + payload = { + 'data': '{"op": "delete"}', + 'key': f'{key}', + } + r = request('POST', url, verify=False, headers=headers, data=payload) + self.assertEqual(r.status_code, 400) + self.assertIn('Missing required field "name"', r.json().get('error')) + + payload = { + 'data': '{"op": "set_default"}', + 'key': f'{key}', + } + r = request('POST', url, verify=False, headers=headers, data=payload) + self.assertEqual(r.status_code, 400) + self.assertIn('Missing required field "name"', r.json().get('error')) + + payload = { + 'data': '{"op": "show"}', + 'key': f'{key}', + } + r = request('POST', url, verify=False, headers=headers, data=payload) + self.assertEqual(r.status_code, 200) + @ignore_warning(InsecureRequestWarning) def test_api_config_file_load_http(self): # Test load config from HTTP URL diff --git a/src/services/vyos-http-api-server b/src/services/vyos-http-api-server index ecbf6fcf9f0..7f5233c6b54 100755 --- a/src/services/vyos-http-api-server +++ b/src/services/vyos-http-api-server @@ -23,16 +23,17 @@ import logging import signal import traceback import threading +from enum import Enum from time import sleep -from typing import List, Union, Callable, Dict +from typing import List, Union, Callable, Dict, Self from fastapi import FastAPI, Depends, Request, Response, HTTPException from fastapi import BackgroundTasks from fastapi.responses import HTMLResponse from fastapi.exceptions import RequestValidationError from fastapi.routing import APIRoute -from pydantic import BaseModel, StrictStr, validator +from pydantic import BaseModel, StrictStr, validator, model_validator from starlette.middleware.cors import CORSMiddleware from starlette.datastructures import FormData from starlette.formparsers import FormParser, MultiPartParser @@ -177,16 +178,35 @@ class ConfigFileModel(ApiModel): } } + +class ImageOp(str, Enum): + add = "add" + delete = "delete" + show = "show" + set_default = "set_default" + + class ImageModel(ApiModel): - op: StrictStr + op: ImageOp url: StrictStr = None name: StrictStr = None + @model_validator(mode='after') + def check_data(self) -> Self: + if self.op == 'add': + if not self.url: + raise ValueError("Missing required field \"url\"") + elif self.op in ['delete', 'set_default']: + if not self.name: + raise ValueError("Missing required field \"name\"") + + return self + class Config: schema_extra = { "example": { "key": "id_key", - "op": "add | delete", + "op": "add | delete | show | set_default", "url": "imagelocation", "name": "imagename", } @@ -668,19 +688,13 @@ def image_op(data: ImageModel): try: if op == 'add': - if data.url: - url = data.url - else: - return error(400, "Missing required field \"url\"") - res = session.install_image(url) + res = session.install_image(data.url) elif op == 'delete': - if data.name: - name = data.name - else: - return error(400, "Missing required field \"name\"") - res = session.remove_image(name) - else: - return error(400, f"'{op}' is not a valid operation") + res = session.remove_image(data.name) + elif op == 'show': + res = session.show(["system", "image"]) + elif op == 'set_default': + res = session.set_default_image(data.name) except ConfigSessionError as e: return error(400, str(e)) except Exception as e: From 5146cb23fff56e5a84db8c84120b836ceeae47f2 Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Mon, 27 May 2024 12:13:39 +0200 Subject: [PATCH 142/188] T6406: check for required kernel config --- smoketest/scripts/system/test_kernel_options.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/smoketest/scripts/system/test_kernel_options.py b/smoketest/scripts/system/test_kernel_options.py index bc7f658b03f..4666e98e71b 100755 --- a/smoketest/scripts/system/test_kernel_options.py +++ b/smoketest/scripts/system/test_kernel_options.py @@ -120,5 +120,13 @@ def test_vfio(self): tmp = re.findall(f'{option}=(y|m)', self._config_data) self.assertTrue(tmp) + def test_container_cpu(self): + options_to_check = [ + 'CONFIG_CGROUP_SCHED', 'CONFIG_CPUSETS', 'CONFIG_CGROUP_CPUACCT', 'CONFIG_CFS_BANDWIDTH' + ] + for option in options_to_check: + tmp = re.findall(f'{option}=(y|m)', self._config_data) + self.assertTrue(tmp) + if __name__ == '__main__': unittest.main(verbosity=2) From fb6602f431f5595b97ea3726467ec782fa50ceb8 Mon Sep 17 00:00:00 2001 From: Alex W Date: Mon, 27 May 2024 21:47:24 +0100 Subject: [PATCH 143/188] reverse-proxy: T6409: Remove unused backend parameters --- .../version/reverseproxy-version.xml.i | 3 ++ .../load-balancing_reverse-proxy.xml.in | 13 ----- .../xml-component-version.xml.in | 1 + src/migration-scripts/reverse-proxy/0-to-1 | 48 +++++++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 interface-definitions/include/version/reverseproxy-version.xml.i create mode 100755 src/migration-scripts/reverse-proxy/0-to-1 diff --git a/interface-definitions/include/version/reverseproxy-version.xml.i b/interface-definitions/include/version/reverseproxy-version.xml.i new file mode 100644 index 00000000000..907ea1e5ee8 --- /dev/null +++ b/interface-definitions/include/version/reverseproxy-version.xml.i @@ -0,0 +1,3 @@ + + + diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index 011e1b53c69..e50e6e579c4 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -92,19 +92,6 @@ #include #include #include - - - Backend parameters - - - - - HTTP health check - - - - - HTTP check configuration diff --git a/interface-definitions/xml-component-version.xml.in b/interface-definitions/xml-component-version.xml.in index 10a1be242c5..67d86a1d020 100644 --- a/interface-definitions/xml-component-version.xml.in +++ b/interface-definitions/xml-component-version.xml.in @@ -48,4 +48,5 @@ #include #include #include + #include diff --git a/src/migration-scripts/reverse-proxy/0-to-1 b/src/migration-scripts/reverse-proxy/0-to-1 new file mode 100755 index 00000000000..62be52ea6c5 --- /dev/null +++ b/src/migration-scripts/reverse-proxy/0-to-1 @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T6409: Remove unused 'backend bk-example parameters' node + +from sys import argv, exit +from vyos.configtree import ConfigTree + +if len(argv) < 2: + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +config = ConfigTree(config_file) +base = ['load-balancing', 'reverse-proxy', 'backend'] +if not config.exists(base): + # Nothing to do + exit(0) +else: + # we need to run this for every configured network + for backend in config.list_nodes(base): + param_node = base + [backend, 'parameters'] + if config.exists(param_node): + config.delete(param_node) + + try: + with open(file_name, 'w') as f: + f.write(config.to_string()) + except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) From 81dea053e7178b8fea836a85aacde2a38ffb9e09 Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Mon, 27 May 2024 13:12:54 +0200 Subject: [PATCH 144/188] T6406: add container cpu limit option --- interface-definitions/container.xml.in | 18 ++++++++++++++++++ smoketest/scripts/cli/test_container.py | 16 ++++++++++++++++ src/conf_mode/container.py | 9 ++++++++- 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/interface-definitions/container.xml.in b/interface-definitions/container.xml.in index 2296a3e9e8f..4370f5888e5 100644 --- a/interface-definitions/container.xml.in +++ b/interface-definitions/container.xml.in @@ -192,6 +192,24 @@ + + + This limits the number of CPU resources the container can use + + u32:0 + Unlimited + + + txt + Amount of CPU time the container can use in amount of cores (up to three decimals) + + + (0|[1-9]\d*)(\.\d{1,3})? + + Container CPU limit must be a (decimal) number in range 0 to number of threads + + 0 + Memory (RAM) available to this container diff --git a/smoketest/scripts/cli/test_container.py b/smoketest/scripts/cli/test_container.py index 3201883b8ca..ef31d0f6fc1 100755 --- a/smoketest/scripts/cli/test_container.py +++ b/smoketest/scripts/cli/test_container.py @@ -91,6 +91,22 @@ def test_basic(self): # Check for running process self.assertEqual(process_named_running(PROCESS_NAME), pid) + def test_cpu_limit(self): + cont_name = 'c2' + + self.cli_set(base_path + ['name', cont_name, 'allow-host-networks']) + self.cli_set(base_path + ['name', cont_name, 'image', cont_image]) + self.cli_set(base_path + ['name', cont_name, 'cpus', '1.25']) + + self.cli_commit() + + pid = 0 + with open(PROCESS_PIDFILE.format(cont_name), 'r') as f: + pid = int(f.read()) + + # Check for running process + self.assertEqual(process_named_running(PROCESS_NAME), pid) + def test_ipv4_network(self): prefix = '192.0.2.0/24' base_name = 'ipv4' diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 91a10e89138..2294b8e575c 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -16,6 +16,7 @@ import os +from decimal import Decimal from hashlib import sha256 from ipaddress import ip_address from ipaddress import ip_network @@ -127,6 +128,11 @@ def verify(container): f'locally. Please use "add container image {image}" to add it '\ f'to the system! Container "{name}" will not be started!') + if 'cpus' in container_config: + cores = os.cpu_count() + if Decimal(container_config['cpus']) > cores: + raise ConfigError(f'Cannot set limit to more cores than available "{name}"!') + if 'network' in container_config: if len(container_config['network']) > 1: raise ConfigError(f'Only one network can be specified for container "{name}"!') @@ -257,6 +263,7 @@ def verify(container): def generate_run_arguments(name, container_config): image = container_config['image'] + cpus = container_config['cpus'] memory = container_config['memory'] shared_memory = container_config['shared_memory'] restart = container_config['restart'] @@ -333,7 +340,7 @@ def generate_run_arguments(name, container_config): if 'allow_host_pid' in container_config: host_pid = '--pid host' - container_base_cmd = f'--detach --interactive --tty --replace {capabilities} ' \ + container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpus} ' \ f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \ f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label} {uid} {host_pid}' From 74910564f82e2837cd7eb35ea21f07601e5f8f0d Mon Sep 17 00:00:00 2001 From: Nicolas Vollmar Date: Tue, 28 May 2024 08:17:15 +0200 Subject: [PATCH 145/188] T6406: rename cpus to cpu --- interface-definitions/container.xml.in | 2 +- smoketest/scripts/cli/test_container.py | 2 +- src/conf_mode/container.py | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/interface-definitions/container.xml.in b/interface-definitions/container.xml.in index 4370f5888e5..1ad7215e5d1 100644 --- a/interface-definitions/container.xml.in +++ b/interface-definitions/container.xml.in @@ -192,7 +192,7 @@ - + This limits the number of CPU resources the container can use diff --git a/smoketest/scripts/cli/test_container.py b/smoketest/scripts/cli/test_container.py index ef31d0f6fc1..90f821c6072 100755 --- a/smoketest/scripts/cli/test_container.py +++ b/smoketest/scripts/cli/test_container.py @@ -96,7 +96,7 @@ def test_cpu_limit(self): self.cli_set(base_path + ['name', cont_name, 'allow-host-networks']) self.cli_set(base_path + ['name', cont_name, 'image', cont_image]) - self.cli_set(base_path + ['name', cont_name, 'cpus', '1.25']) + self.cli_set(base_path + ['name', cont_name, 'cpu-quota', '1.25']) self.cli_commit() diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index 2294b8e575c..ca09dff9f0d 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -128,9 +128,9 @@ def verify(container): f'locally. Please use "add container image {image}" to add it '\ f'to the system! Container "{name}" will not be started!') - if 'cpus' in container_config: - cores = os.cpu_count() - if Decimal(container_config['cpus']) > cores: + if 'cpu_quota' in container_config: + cores = vyos.cpu.get_core_count() + if Decimal(container_config['cpu_quota']) > cores: raise ConfigError(f'Cannot set limit to more cores than available "{name}"!') if 'network' in container_config: @@ -263,7 +263,7 @@ def verify(container): def generate_run_arguments(name, container_config): image = container_config['image'] - cpus = container_config['cpus'] + cpu_quota = container_config['cpu_quota'] memory = container_config['memory'] shared_memory = container_config['shared_memory'] restart = container_config['restart'] @@ -340,7 +340,7 @@ def generate_run_arguments(name, container_config): if 'allow_host_pid' in container_config: host_pid = '--pid host' - container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpus} ' \ + container_base_cmd = f'--detach --interactive --tty --replace {capabilities} --cpus {cpu_quota} ' \ f'--memory {memory}m --shm-size {shared_memory}m --memory-swap 0 --restart {restart} ' \ f'--name {name} {hostname} {device} {port} {volume} {env_opt} {label} {uid} {host_pid}' From 24d259e57b52eda16f357c5de7c3cb23ac69a18c Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Tue, 28 May 2024 14:43:56 +0300 Subject: [PATCH 146/188] op mode: T6389: Check architecture and flavor compatibility on upgrade attempts --- src/op_mode/image_installer.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/src/op_mode/image_installer.py b/src/op_mode/image_installer.py index 0d2d7076c50..bdc16de1562 100755 --- a/src/op_mode/image_installer.py +++ b/src/op_mode/image_installer.py @@ -40,13 +40,14 @@ from vyos.utils.io import ask_input, ask_yes_no, select_entry from vyos.utils.file import chmod_2775 from vyos.utils.process import cmd, run -from vyos.version import get_remote_version +from vyos.version import get_remote_version, get_version_data # define text messages MSG_ERR_NOT_LIVE: str = 'The system is already installed. Please use "add system image" instead.' MSG_ERR_LIVE: str = 'The system is in live-boot mode. Please use "install image" instead.' MSG_ERR_NO_DISK: str = 'No suitable disk was found. There must be at least one disk of 2GB or greater size.' MSG_ERR_IMPROPER_IMAGE: str = 'Missing sha256sum.txt.\nEither this image is corrupted, or of era 1.2.x (md5sum) and would downgrade image tools;\ndisallowed in either case.' +MSG_ERR_ARCHITECTURE_MISMATCH: str = 'Upgrading to a different image architecture will break your system.' MSG_INFO_INSTALL_WELCOME: str = 'Welcome to VyOS installation!\nThis command will install VyOS to your permanent storage.' MSG_INFO_INSTALL_EXIT: str = 'Exiting from VyOS installation' MSG_INFO_INSTALL_SUCCESS: str = 'The image installed successfully; please reboot now.' @@ -79,6 +80,9 @@ MSG_WARN_IMAGE_NAME_WRONG: str = 'The suggested name is unsupported!\n'\ 'It must be between 1 and 64 characters long and contains only the next characters: .+-_ a-z A-Z 0-9' MSG_WARN_PASSWORD_CONFIRM: str = 'The entered values did not match. Try again' +MSG_WARN_FLAVOR_MISMATCH: str = 'The running image flavor is "{0}". The new image flavor is "{1}".\n' \ +'Installing a different image flavor may cause functionality degradation or break your system.\n' \ +'Do you want to continue with installation?' CONST_MIN_DISK_SIZE: int = 2147483648 # 2 GB CONST_MIN_ROOT_SIZE: int = 1610612736 # 1.5 GB # a reserved space: 2MB for header, 1 MB for BIOS partition, 256 MB for EFI @@ -693,6 +697,31 @@ def is_raid_install(install_object: Union[disk.DiskDetails, raid.RaidDetails]) - return False +def validate_compatibility(iso_path: str) -> None: + """Check architecture and flavor compatibility with the running image + + Args: + iso_path (str): a path to the mounted ISO image + """ + old_data = get_version_data() + old_flavor = old_data.get('flavor', '') + old_architecture = old_data.get('architecture') or cmd('dpkg --print-architecture') + + new_data = get_version_data(f'{iso_path}/version.json') + new_flavor = new_data.get('flavor', '') + new_architecture = new_data.get('architecture', '') + + if not old_architecture == new_architecture: + print(MSG_ERR_ARCHITECTURE_MISMATCH) + cleanup() + exit(MSG_INFO_INSTALL_EXIT) + + if not old_flavor == new_flavor: + if not ask_yes_no(MSG_WARN_FLAVOR_MISMATCH.format(old_flavor, new_flavor), default=False): + cleanup() + exit(MSG_INFO_INSTALL_EXIT) + + def install_image() -> None: """Install an image to a disk """ @@ -876,6 +905,9 @@ def add_image(image_path: str, vrf: str = None, username: str = '', Path(DIR_ISO_MOUNT).mkdir(mode=0o755, parents=True) disk.partition_mount(iso_path, DIR_ISO_MOUNT, 'iso9660') + print('Validating image compatibility') + validate_compatibility(DIR_ISO_MOUNT) + # check sums print('Validating image checksums') if not Path(DIR_ISO_MOUNT).joinpath('sha256sum.txt').exists(): From 55e02bef4f3aeff5e33cddc79f22b93fc671e491 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Tue, 28 May 2024 13:06:51 +0000 Subject: [PATCH 147/188] T6411: CGNAT fix sequences for external address ranges Fix the bug where address external alocation was not rely on sequences of the external IP addresses (if set) --- smoketest/scripts/cli/test_cgnat.py | 39 +++++++++++++++++++++++++++++ src/conf_mode/nat_cgnat.py | 6 ++++- 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/smoketest/scripts/cli/test_cgnat.py b/smoketest/scripts/cli/test_cgnat.py index c65c58820fc..02dad3de572 100755 --- a/smoketest/scripts/cli/test_cgnat.py +++ b/smoketest/scripts/cli/test_cgnat.py @@ -95,5 +95,44 @@ def test_cgnat(self): self.verify_nftables(nftables_search, 'ip cgnat', inverse=False, args='-s') + def test_cgnat_sequence(self): + internal_name = 'earth' + external_name = 'milky_way' + internal_net = '100.64.0.0/28' + + ext_addr_alpha_proxima = '192.0.2.121/32' + ext_addr_beta_cygni = '198.51.100.23/32' + ext_addr_gamma_leonis = '203.0.113.102/32' + + ext_seq_beta_cygni = '3' + ext_seq_gamma_leonis = '10' + + external_ports = '1024-65535' + ports_per_subscriber = '10000' + rule = '100' + + nftables_search = [ + ['100.64.0.0 : 198.51.100.23 . 1024-11023, 100.64.0.1 : 198.51.100.23 . 11024-21023'], + ['100.64.0.4 : 198.51.100.23 . 41024-51023, 100.64.0.5 : 198.51.100.23 . 51024-61023'], + ['100.64.0.6 : 203.0.113.102 . 1024-11023, 100.64.0.7 : 203.0.113.102 . 11024-21023'], + ['100.64.0.8 : 203.0.113.102 . 21024-31023, 100.64.0.9 : 203.0.113.102 . 31024-41023'], + ['100.64.0.10 : 203.0.113.102 . 41024-51023, 100.64.0.11 : 203.0.113.102 . 51024-61023'], + ['100.64.0.12 : 192.0.2.121 . 1024-11023, 100.64.0.13 : 192.0.2.121 . 11024-21023'], + ['100.64.0.14 : 192.0.2.121 . 21024-31023, 100.64.0.15 : 192.0.2.121 . 31024-41023'], + ] + + self.cli_set(base_path + ['pool', 'external', external_name, 'external-port-range', external_ports]) + self.cli_set(base_path + ['pool', 'external', external_name, 'per-user-limit', 'port', ports_per_subscriber]) + self.cli_set(base_path + ['pool', 'external', external_name, 'range', ext_addr_alpha_proxima]) + self.cli_set(base_path + ['pool', 'external', external_name, 'range', ext_addr_beta_cygni, 'seq', ext_seq_beta_cygni]) + self.cli_set(base_path + ['pool', 'external', external_name, 'range', ext_addr_gamma_leonis, 'seq', ext_seq_gamma_leonis]) + self.cli_set(base_path + ['pool', 'internal', internal_name, 'range', internal_net]) + self.cli_set(base_path + ['rule', rule, 'source', 'pool', internal_name]) + self.cli_set(base_path + ['rule', rule, 'translation', 'pool', external_name]) + self.cli_commit() + + self.verify_nftables(nftables_search, 'ip cgnat', inverse=False, args='-s') + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index 5ad65de80ab..957b12c28d3 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -252,7 +252,11 @@ def generate(config): ext_pool_name: str = rule_config['translation']['pool'] int_pool_name: str = rule_config['source']['pool'] - external_ranges: list = [range for range in config['pool']['external'][ext_pool_name]['range']] + # Sort the external ranges by sequence + external_ranges: list = sorted( + config['pool']['external'][ext_pool_name]['range'], + key=lambda r: int(config['pool']['external'][ext_pool_name]['range'][r].get('seq', 999999)) + ) internal_ranges: list = [range for range in config['pool']['internal'][int_pool_name]['range']] external_list_hosts_count = [] external_list_hosts = [] From 03fd368ed263ca28c9b1b5e29f486217784d15ef Mon Sep 17 00:00:00 2001 From: fett0 Date: Wed, 29 May 2024 01:06:00 +0000 Subject: [PATCH 148/188] ISIS: T6332: Fix isis not working only ipv6 --- data/templates/frr/isisd.frr.j2 | 5 ++- .../include/isis/protocol-common-config.xml.i | 35 +++++++++++++++++++ smoketest/scripts/cli/test_protocols_isis.py | 15 ++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/data/templates/frr/isisd.frr.j2 b/data/templates/frr/isisd.frr.j2 index 1e1cc3c271e..5570caaa7df 100644 --- a/data/templates/frr/isisd.frr.j2 +++ b/data/templates/frr/isisd.frr.j2 @@ -178,7 +178,7 @@ advertise-passive-only {% for priority, priority_limit_options in fast_reroute.lfa.local.priority_limit.items() %} {% for level in priority_limit_options %} fast-reroute priority-limit {{ priority }} {{ level | replace('_', '-') }} -{% endfor %} +{% endfor %} {% endfor %} {% endif %} {% if fast_reroute.lfa.local.tiebreaker is vyos_defined %} @@ -233,6 +233,9 @@ fast-reroute remote-lfa prefix-list {{ prefix_list }} {% endfor %} {% endfor %} {% endif %} +{% if topology is vyos_defined %} +topology {{ topology }} +{% endif %} {% if level is vyos_defined('level-2') %} is-type level-2-only {% elif level is vyos_defined %} diff --git a/interface-definitions/include/isis/protocol-common-config.xml.i b/interface-definitions/include/isis/protocol-common-config.xml.i index 404f03cb590..0e79ca5f258 100644 --- a/interface-definitions/include/isis/protocol-common-config.xml.i +++ b/interface-definitions/include/isis/protocol-common-config.xml.i @@ -165,6 +165,41 @@ #include + + + Configure IS-IS topologies + + ipv4-multicast ipv4-mgmt ipv6-unicast ipv6-multicast ipv6-mgmt ipv6-dstsrc + + + ipv4-multicast + Use IPv4 multicast topology + + + ipv4-mgmt + Use IPv4 management topology + + + ipv6-unicast + Use IPv6 unicast topology + + + ipv6-multicast + Use IPv6 multicast topology + + + ipv6-mgmt + Use IPv6 management topology + + + ipv6-dstsrc + Use IPv6 dst-src topology + + + (ipv4-multicast|ipv4-mgmt|ipv6-unicast|ipv6-multicast|ipv6-mgmt|ipv6-dstsrc) + + + IS-IS fast reroute configuration diff --git a/smoketest/scripts/cli/test_protocols_isis.py b/smoketest/scripts/cli/test_protocols_isis.py index 0fd18a6da4c..9c57f202014 100755 --- a/smoketest/scripts/cli/test_protocols_isis.py +++ b/smoketest/scripts/cli/test_protocols_isis.py @@ -395,5 +395,20 @@ def test_isis_09_lfa(self): self.cli_delete(['policy', 'prefix-list', prefix_list]) self.cli_commit() + def test_isis_10_topology(self): + topologies = ['ipv4-multicast', 'ipv4-mgmt', 'ipv6-unicast', 'ipv6-multicast', 'ipv6-mgmt'] + interface = 'lo' + + # Set a basic IS-IS config + self.cli_set(base_path + ['net', net]) + + self.cli_set(base_path + ['interface', interface]) + for topology in topologies: + self.cli_set(base_path + ['topology', topology]) + self.cli_commit() + tmp = self.getFRRconfig(f'router isis {domain}', daemon='isisd') + self.assertIn(f' net {net}', tmp) + self.assertIn(f' topology {topology}', tmp) + if __name__ == '__main__': unittest.main(verbosity=2) From 8c67e6a317cce2e419ac16f2e5cf1def8a364afc Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 29 May 2024 11:40:49 +0000 Subject: [PATCH 149/188] T6349: Reuse repo sync --- .github/workflows/repo-sync.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/workflows/repo-sync.yml diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml new file mode 100644 index 00000000000..90cb2dbea1b --- /dev/null +++ b/.github/workflows/repo-sync.yml @@ -0,0 +1,13 @@ +name: Repo-sync + +on: + pull_request: + types: + - closed + branches: + - current + +jobs: + trigger-sync: + uses: vyos/.github/.github/workflows/trigger-repo-sync.yml@feature/T6349-reusable-workflows + secrets: inherit From f4069582273e1ee9916dea7de1e6ec176db81bc6 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 29 May 2024 13:58:17 +0100 Subject: [PATCH 150/188] openvpn: T6374: only check TLS role for s2s if TLS is configured --- src/conf_mode/interfaces_openvpn.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/conf_mode/interfaces_openvpn.py b/src/conf_mode/interfaces_openvpn.py index 5b9c217575d..627cc90ba7c 100755 --- a/src/conf_mode/interfaces_openvpn.py +++ b/src/conf_mode/interfaces_openvpn.py @@ -168,6 +168,14 @@ def verify_pki(openvpn): 'verification, consult the documentation for details.') if tls: + if mode == 'site-to-site': + # XXX: site-to-site with PSKs is the only mode that can work without TLS, + # so 'tls role' is not mandatory for it, + # but we need to check that if it uses peer certificate fingerprints rather than PSKs, + # then the TLS role is set + if ('shared_secret_key' not in tls) and ('role' not in tls): + raise ConfigError('"tls role" is required for site-to-site OpenVPN with TLS') + if (mode in ['server', 'client']) and ('ca_certificate' not in tls): raise ConfigError(f'Must specify "tls ca-certificate" on openvpn interface {interface},\ it is required in server and client modes') @@ -260,11 +268,6 @@ def verify(openvpn): # OpenVPN site-to-site - VERIFY # elif openvpn['mode'] == 'site-to-site': - # XXX: site-to-site is the only mode that still can work without TLS, - # so we need to make sure that if TLS is used, then TLS role is also specified - if 'shared_secret_key' not in openvpn['tls'] and 'role' not in openvpn['tls']: - raise ConfigError('"tls role" is required for site-to-site OpenVPN with TLS') - if 'local_address' not in openvpn and 'is_bridge_member' not in openvpn: raise ConfigError('Must specify "local-address" or add interface to bridge') From f3c14280a6259b6877e341cc0f47d290325abedc Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Wed, 29 May 2024 13:18:28 +0000 Subject: [PATCH 151/188] T6415: Fix variables for repo sync --- .github/workflows/repo-sync.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 90cb2dbea1b..84fd0d7dac5 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -10,4 +10,7 @@ on: jobs: trigger-sync: uses: vyos/.github/.github/workflows/trigger-repo-sync.yml@feature/T6349-reusable-workflows - secrets: inherit + secrets: + REMOTE_REPO: ${{ secrets.REMOTE_REPO }} + REMOTE_OWNER: ${{ secrets.REMOTE_OWNER }} + PAT: ${{ secrets.PAT }} From b7595ee9d328778105c70e3d4399ac45f555b304 Mon Sep 17 00:00:00 2001 From: Ginko <152240782+Giggum@users.noreply.github.com> Date: Wed, 29 May 2024 14:27:22 -0400 Subject: [PATCH 152/188] nat: T6371: fix op mode display of configured ports when comma separated list of ports/ranges exists Before: Issuing the op mode command "show nat source rules" will throw an exception if the user has configured NAT rules using a list of ports as a comma-separated list (e.g. '!22,telnet,http,123,1001-1005'). Also there was no handling for the "!" rule and so '!53' would display as '53'. With this PR: Introduced iteration to capture all configured ports and append to the appropriate string for display to the user as well as handling of '!' if present in user's configuration. --- src/op_mode/nat.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/src/op_mode/nat.py b/src/op_mode/nat.py index 4ab524fb7ac..16a545cda41 100755 --- a/src/op_mode/nat.py +++ b/src/op_mode/nat.py @@ -99,6 +99,23 @@ def _get_raw_translation(direction, family, address=None): def _get_formatted_output_rules(data, direction, family): + def _get_ports_for_output(my_dict): + # Get and insert all configured ports or port ranges into output string + for index, port in enumerate(my_dict['set']): + if 'range' in str(my_dict['set'][index]): + output = my_dict['set'][index]['range'] + output = '-'.join(map(str, output)) + else: + output = str(port) + if index == 0: + output = str(output) + else: + output = ','.join([output,output]) + # Handle case where configured ports are a negated list + if my_dict['op'] == '!=': + output = '!' + output + return(output) + # Add default values before loop sport, dport, proto = 'any', 'any', 'any' saddr = '::/0' if family == 'inet6' else '0.0.0.0/0' @@ -126,21 +143,9 @@ def _get_formatted_output_rules(data, direction, family): elif my_dict['field'] == 'daddr': daddr = f'{op}{my_dict["prefix"]["addr"]}/{my_dict["prefix"]["len"]}' elif my_dict['field'] == 'sport': - # Port range or single port - if jmespath.search('set[*].range', my_dict): - sport = my_dict['set'][0]['range'] - sport = '-'.join(map(str, sport)) - else: - sport = my_dict.get('set') - sport = ','.join(map(str, sport)) + sport = _get_ports_for_output(my_dict) elif my_dict['field'] == 'dport': - # Port range or single port - if jmespath.search('set[*].range', my_dict): - dport = my_dict["set"][0]["range"] - dport = '-'.join(map(str, dport)) - else: - dport = my_dict.get('set') - dport = ','.join(map(str, dport)) + dport = _get_ports_for_output(my_dict) else: field = jmespath.search('left.payload.field', match) if field == 'saddr': From 2980eb0ad527f0ef0f1527c0ea97842ca2a8ede5 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 29 May 2024 22:47:22 +0200 Subject: [PATCH 153/188] op-mode: T5231: add command to restart reverse-proxy --- op-mode-definitions/reverse-proxy.xml.in | 23 +++++++++++++++++++ op-mode-definitions/show-reverse-proxy.xml.in | 13 ----------- 2 files changed, 23 insertions(+), 13 deletions(-) create mode 100644 op-mode-definitions/reverse-proxy.xml.in delete mode 100644 op-mode-definitions/show-reverse-proxy.xml.in diff --git a/op-mode-definitions/reverse-proxy.xml.in b/op-mode-definitions/reverse-proxy.xml.in new file mode 100644 index 00000000000..4af24880b86 --- /dev/null +++ b/op-mode-definitions/reverse-proxy.xml.in @@ -0,0 +1,23 @@ + + + + + + + Restart reverse-proxy service + + if cli-shell-api existsActive load-balancing reverse-proxy; then sudo systemctl restart haproxy.service; else echo "Reverse-Proxy not configured"; fi + + + + + + + + Show load-balancing reverse-proxy + + sudo ${vyos_op_scripts_dir}/reverseproxy.py show + + + + diff --git a/op-mode-definitions/show-reverse-proxy.xml.in b/op-mode-definitions/show-reverse-proxy.xml.in deleted file mode 100644 index ed0fee84358..00000000000 --- a/op-mode-definitions/show-reverse-proxy.xml.in +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - Show load-balancing reverse-proxy - - sudo ${vyos_op_scripts_dir}/reverseproxy.py show - - - - From a2f0b25452c67528077f343d75de09d038e97fee Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 29 May 2024 22:49:00 +0200 Subject: [PATCH 154/188] reverse-proxy: T5231: better mark v4v6 listen any address haproxy supports both ":::80 v4v6" and "[::]:80 v4v6" as listen statement, where the later one is more humand readable. Both act in the same way. --- data/templates/load-balancing/haproxy.cfg.j2 | 2 +- smoketest/scripts/cli/test_load-balancing_reverse-proxy.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index 797bf17e7c1..b786a58f879 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -62,7 +62,7 @@ frontend {{ front }} bind {{ address | bracketize_ipv6 }}:{{ front_config.port }} {{ ssl_directive }} {{ ssl_front | join(' ') }} {% endfor %} {% else %} - bind :::{{ front_config.port }} v4v6 {{ ssl_directive }} {{ ssl_front | join(' ') }} + bind [::]:{{ front_config.port }} v4v6 {{ ssl_directive }} {{ ssl_front | join(' ') }} {% endif %} {% if front_config.redirect_http_to_https is vyos_defined %} http-request redirect scheme https unless { ssl_fc } diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index 370a9276af8..2b2f93cdf76 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -218,7 +218,7 @@ def test_01_lb_reverse_proxy_domain(self): # Frontend self.assertIn(f'frontend {frontend}', config) - self.assertIn(f'bind :::{front_port} v4v6', config) + self.assertIn(f'bind [::]:{front_port} v4v6', config) self.assertIn(f'mode {mode}', config) for domain in domains_bk_first: self.assertIn(f'acl {rule_ten} hdr(host) -i {domain}', config) @@ -371,7 +371,7 @@ def test_06_lb_reverse_proxy_tcp_mode(self): # Frontend self.assertIn(f'frontend {frontend}', config) - self.assertIn(f'bind :::{front_port} v4v6', config) + self.assertIn(f'bind [::]:{front_port} v4v6', config) self.assertIn(f'mode {mode}', config) self.assertIn(f'tcp-request inspect-delay {tcp_request_delay}', config) From 6000c47f068503522b0ccfe57c51f34ad9892e87 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 29 May 2024 22:51:00 +0200 Subject: [PATCH 155/188] reverse-proxy: T5231: remove frontend ca-certificate code path The code path to handle the ca certificate used for the frontend service is removed, as there is no way on the XLI to define the CA certificate used for the frontend service. --- src/conf_mode/load-balancing_reverse-proxy.py | 33 ++++++++----------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index b6db110ae3b..16dcba85220 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -126,30 +126,23 @@ def generate(lb): # SSL Certificates for frontend for front, front_config in lb['service'].items(): - if 'ssl' in front_config: + if 'ssl' not in front_config: + continue - if 'certificate' in front_config['ssl']: - cert_names = front_config['ssl']['certificate'] + if 'certificate' in front_config['ssl']: + cert_names = front_config['ssl']['certificate'] - for cert_name in cert_names: - pki_cert = lb['pki']['certificate'][cert_name] - cert_file_path = os.path.join(load_balancing_dir, f'{cert_name}.pem') - cert_key_path = os.path.join(load_balancing_dir, f'{cert_name}.pem.key') + for cert_name in cert_names: + pki_cert = lb['pki']['certificate'][cert_name] + cert_file_path = os.path.join(load_balancing_dir, f'{cert_name}.pem') + cert_key_path = os.path.join(load_balancing_dir, f'{cert_name}.pem.key') - with open(cert_file_path, 'w') as f: - f.write(wrap_certificate(pki_cert['certificate'])) + with open(cert_file_path, 'w') as f: + f.write(wrap_certificate(pki_cert['certificate'])) - if 'private' in pki_cert and 'key' in pki_cert['private']: - with open(cert_key_path, 'w') as f: - f.write(wrap_private_key(pki_cert['private']['key'])) - - if 'ca_certificate' in front_config['ssl']: - ca_name = front_config['ssl']['ca_certificate'] - pki_ca_cert = lb['pki']['ca'][ca_name] - ca_cert_file_path = os.path.join(load_balancing_dir, f'{ca_name}.pem') - - with open(ca_cert_file_path, 'w') as f: - f.write(wrap_certificate(pki_ca_cert['certificate'])) + if 'private' in pki_cert and 'key' in pki_cert['private']: + with open(cert_key_path, 'w') as f: + f.write(wrap_private_key(pki_cert['private']['key'])) # SSL Certificates for backend for back, back_config in lb['backend'].items(): From d83a6e5c5dc7e97e773f08bec7ba377530baafc9 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 29 May 2024 22:52:08 +0200 Subject: [PATCH 156/188] reverse-proxy: T6419: build full CA chain when verifying backend server --- src/conf_mode/load-balancing_reverse-proxy.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index 16dcba85220..a6fc407b03c 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -28,7 +28,11 @@ from vyos.utils.network import is_listen_port_bind_service from vyos.pki import wrap_certificate from vyos.pki import wrap_private_key +from vyos.pki import find_chain +from vyos.pki import load_certificate +from vyos.pki import encode_certificate from vyos.template import render +from vyos.utils.file import write_file from vyos import ConfigError from vyos import airbag airbag.enable() @@ -146,15 +150,22 @@ def generate(lb): # SSL Certificates for backend for back, back_config in lb['backend'].items(): - if 'ssl' in back_config: + if 'ssl' not in back_config: + continue + + if 'ca_certificate' in back_config['ssl']: + ca_name = back_config['ssl']['ca_certificate'] + ca_cert_file_path = os.path.join(load_balancing_dir, f'{ca_name}.pem') + ca_chains = [] - if 'ca_certificate' in back_config['ssl']: - ca_name = back_config['ssl']['ca_certificate'] - pki_ca_cert = lb['pki']['ca'][ca_name] - ca_cert_file_path = os.path.join(load_balancing_dir, f'{ca_name}.pem') + loaded_ca_certs = {load_certificate(c['certificate']) + for c in lb['pki']['ca'].values()} if 'ca' in lb['pki'] else {} - with open(ca_cert_file_path, 'w') as f: - f.write(wrap_certificate(pki_ca_cert['certificate'])) + pki_ca_cert = lb['pki']['ca'][ca_name] + loaded_ca_cert = load_certificate(pki_ca_cert['certificate']) + ca_full_chain = find_chain(loaded_ca_cert, loaded_ca_certs) + ca_chains.append('\n'.join(encode_certificate(c) for c in ca_full_chain)) + write_file(ca_cert_file_path, '\n'.join(ca_chains)) render(load_balancing_conf_file, 'load-balancing/haproxy.cfg.j2', lb) render(systemd_override, 'load-balancing/override_haproxy.conf.j2', lb) From 4b189a76c0a9a28504aab6715658840b929fc243 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 29 May 2024 23:25:15 +0200 Subject: [PATCH 157/188] reverse-proxy: T6419: build full CA chain for frontend SSL certificate --- src/conf_mode/load-balancing_reverse-proxy.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index a6fc407b03c..1c1252df0f8 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -26,11 +26,11 @@ from vyos.utils.process import call from vyos.utils.network import check_port_availability from vyos.utils.network import is_listen_port_bind_service -from vyos.pki import wrap_certificate -from vyos.pki import wrap_private_key from vyos.pki import find_chain from vyos.pki import load_certificate +from vyos.pki import load_private_key from vyos.pki import encode_certificate +from vyos.pki import encode_private_key from vyos.template import render from vyos.utils.file import write_file from vyos import ConfigError @@ -128,6 +128,9 @@ def generate(lb): if not os.path.isdir(load_balancing_dir): os.mkdir(load_balancing_dir) + loaded_ca_certs = {load_certificate(c['certificate']) + for c in lb['pki']['ca'].values()} if 'ca' in lb['pki'] else {} + # SSL Certificates for frontend for front, front_config in lb['service'].items(): if 'ssl' not in front_config: @@ -141,12 +144,16 @@ def generate(lb): cert_file_path = os.path.join(load_balancing_dir, f'{cert_name}.pem') cert_key_path = os.path.join(load_balancing_dir, f'{cert_name}.pem.key') - with open(cert_file_path, 'w') as f: - f.write(wrap_certificate(pki_cert['certificate'])) + loaded_pki_cert = load_certificate(pki_cert['certificate']) + cert_full_chain = find_chain(loaded_pki_cert, loaded_ca_certs) + + write_file(cert_file_path, + '\n'.join(encode_certificate(c) for c in cert_full_chain)) if 'private' in pki_cert and 'key' in pki_cert['private']: - with open(cert_key_path, 'w') as f: - f.write(wrap_private_key(pki_cert['private']['key'])) + loaded_key = load_private_key(pki_cert['private']['key'], passphrase=None, wrap_tags=True) + key_pem = encode_private_key(loaded_key, passphrase=None) + write_file(cert_key_path, key_pem) # SSL Certificates for backend for back, back_config in lb['backend'].items(): @@ -158,9 +165,6 @@ def generate(lb): ca_cert_file_path = os.path.join(load_balancing_dir, f'{ca_name}.pem') ca_chains = [] - loaded_ca_certs = {load_certificate(c['certificate']) - for c in lb['pki']['ca'].values()} if 'ca' in lb['pki'] else {} - pki_ca_cert = lb['pki']['ca'][ca_name] loaded_ca_cert = load_certificate(pki_ca_cert['certificate']) ca_full_chain = find_chain(loaded_ca_cert, loaded_ca_certs) @@ -172,7 +176,6 @@ def generate(lb): return None - def apply(lb): call('systemctl daemon-reload') if not lb: From 8439f8a43e93c0560f1abfc2aa60990f521b4d4d Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Wed, 29 May 2024 23:43:29 +0200 Subject: [PATCH 158/188] container: T6406: fix NameError: name 'vyos' is not defined Commit 74910564f ("T6406: rename cpus to cpu") did not import the function from the Python module. --- src/conf_mode/container.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/conf_mode/container.py b/src/conf_mode/container.py index ca09dff9f0d..3efeb9b40fe 100755 --- a/src/conf_mode/container.py +++ b/src/conf_mode/container.py @@ -29,6 +29,7 @@ from vyos.configdict import is_node_changed from vyos.configverify import verify_vrf from vyos.ifconfig import Interface +from vyos.cpu import get_core_count from vyos.utils.file import write_file from vyos.utils.process import call from vyos.utils.process import cmd @@ -129,7 +130,7 @@ def verify(container): f'to the system! Container "{name}" will not be started!') if 'cpu_quota' in container_config: - cores = vyos.cpu.get_core_count() + cores = get_core_count() if Decimal(container_config['cpu_quota']) > cores: raise ConfigError(f'Cannot set limit to more cores than available "{name}"!') From dcaeb33ffb6db5640090dd9fa94e45b0e04da940 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 30 May 2024 06:24:32 +0000 Subject: [PATCH 159/188] T6415: Enable repo-sync workflow to be triggered manually --- .github/workflows/repo-sync.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index 84fd0d7dac5..a2bf54f745f 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -6,6 +6,7 @@ on: - closed branches: - current + workflow_dispatch: jobs: trigger-sync: From 96d0e23a32a0e1b990ce022546ed7225956a0494 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 30 May 2024 09:38:55 +0200 Subject: [PATCH 160/188] hostname: T6421: enforce explicit CLI priority for host-name and domain-name To prevent any possible races in the future the host-name and domain-name nodes should be set with explicit priorities! --- interface-definitions/system_domain-name.xml.in | 1 + interface-definitions/system_host-name.xml.in | 1 + 2 files changed, 2 insertions(+) diff --git a/interface-definitions/system_domain-name.xml.in b/interface-definitions/system_domain-name.xml.in index bfca9b8ce40..695af29d9ea 100644 --- a/interface-definitions/system_domain-name.xml.in +++ b/interface-definitions/system_domain-name.xml.in @@ -5,6 +5,7 @@ System domain name + 6 diff --git a/interface-definitions/system_host-name.xml.in b/interface-definitions/system_host-name.xml.in index 423531a68a7..f74baab48d6 100644 --- a/interface-definitions/system_host-name.xml.in +++ b/interface-definitions/system_host-name.xml.in @@ -6,6 +6,7 @@ System host name (default: vyos) + 5 #include From cf07a55d183be1f4d28b8b50a0784513d91d6fe2 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 30 May 2024 09:44:55 +0200 Subject: [PATCH 161/188] vyos.ifconfig: T6421: verify /etc/hostname exists before reading Inspired-By: Brandon Zhi --- python/vyos/ifconfig/interface.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/python/vyos/ifconfig/interface.py b/python/vyos/ifconfig/interface.py index f0897bc216d..117479adee6 100644 --- a/python/vyos/ifconfig/interface.py +++ b/python/vyos/ifconfig/interface.py @@ -42,6 +42,7 @@ from vyos.utils.process import run from vyos.template import is_ipv4 from vyos.template import is_ipv6 +from vyos.utils.file import read_file from vyos.utils.network import is_intf_addr_assigned from vyos.utils.network import is_ipv6_link_local from vyos.utils.assertion import assert_boolean @@ -1356,12 +1357,13 @@ def set_dhcp(self, enable): if enable and 'disable' not in self.config: if dict_search('dhcp_options.host_name', self.config) == None: # read configured system hostname. - # maybe change to vyos hostd client ??? + # maybe change to vyos-hostsd client ??? hostname = 'vyos' - with open('/etc/hostname', 'r') as f: - hostname = f.read().rstrip('\n') - tmp = {'dhcp_options' : { 'host_name' : hostname}} - self.config = dict_merge(tmp, self.config) + hostname_file = '/etc/hostname' + if os.path.isfile(hostname_file): + hostname = read_file(hostname_file) + tmp = {'dhcp_options' : { 'host_name' : hostname}} + self.config = dict_merge(tmp, self.config) render(systemd_override_file, 'dhcp-client/override.conf.j2', self.config) render(dhclient_config_file, 'dhcp-client/ipv4.j2', self.config) From e6fe6e50a5c817e18c453e7bc42bb2e1c4b17671 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 30 May 2024 11:20:56 +0200 Subject: [PATCH 162/188] op-mode: ipsec: T6407: fix profile generation Commit 952b1656f51 ("ipsec: T5606: T5871: Use multi node for CA certificates") added support for multiple CA certificates which broke the OP mode command to generate the IPSec profiles as it did not expect a list and was rather working on a string. Now multiple CAs can be rendered into the Apple IOS profile. --- data/templates/ipsec/ios_profile.j2 | 11 ++++++++--- src/op_mode/ikev2_profile_generator.py | 19 +++++++++++++------ 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/data/templates/ipsec/ios_profile.j2 b/data/templates/ipsec/ios_profile.j2 index eb74924b8f7..a9ae1c7a938 100644 --- a/data/templates/ipsec/ios_profile.j2 +++ b/data/templates/ipsec/ios_profile.j2 @@ -83,12 +83,15 @@ +{% if certs is vyos_defined %} +{% for cert in certs %} + PayloadIdentifier - org.example.ca + org.{{ cert.ca_cn | lower | replace(' ', '.') | replace('_', '.') }} PayloadUUID - {{ '' | get_uuid }} + {{ cert.ca_cn | generate_uuid4 }} PayloadType com.apple.security.root PayloadVersion @@ -96,9 +99,11 @@ PayloadContent - {{ ca_cert }} + {{ cert.ca_cert }} +{% endfor %} +{% endif %} diff --git a/src/op_mode/ikev2_profile_generator.py b/src/op_mode/ikev2_profile_generator.py index 2b29f94bf40..4ac4fb14aac 100755 --- a/src/op_mode/ikev2_profile_generator.py +++ b/src/op_mode/ikev2_profile_generator.py @@ -144,15 +144,22 @@ data['rfqdn'] = '.'.join(tmp) pki = conf.get_config_dict(pki_base, get_first_key=True) -ca_name = data['authentication']['x509']['ca_certificate'] cert_name = data['authentication']['x509']['certificate'] -ca_cert = load_certificate(pki['ca'][ca_name]['certificate']) -cert = load_certificate(pki['certificate'][cert_name]['certificate']) +data['certs'] = [] + +for ca_name in data['authentication']['x509']['ca_certificate']: + tmp = {} + ca_cert = load_certificate(pki['ca'][ca_name]['certificate']) + cert = load_certificate(pki['certificate'][cert_name]['certificate']) + + + tmp['ca_cn'] = ca_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + tmp['cert_cn'] = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value + tmp['ca_cert'] = conf.value(pki_base + ['ca', ca_name, 'certificate']) + + data['certs'].append(tmp) -data['ca_cn'] = ca_cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value -data['cert_cn'] = cert.subject.get_attributes_for_oid(NameOID.COMMON_NAME)[0].value -data['ca_cert'] = conf.value(pki_base + ['ca', ca_name, 'certificate']) esp_proposals = conf.get_config_dict(ipsec_base + ['esp-group', data['esp_group'], 'proposal'], key_mangling=('-', '_'), get_first_key=True) From 19d8415512dcf87dc3a87feabf128652ffc74594 Mon Sep 17 00:00:00 2001 From: Haim Gelfenbeyn Date: Thu, 30 May 2024 09:30:27 -0400 Subject: [PATCH 163/188] dns: T6422: allow multiple redundant NS records NS is unlike CNAME or PTR, multiple NS records are perfectly valid and is a common use case: multiple redundant DNS servers is a common configuration and should be supported. --- .../service_dns_forwarding.xml.in | 1 + src/conf_mode/service_dns_forwarding.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/interface-definitions/service_dns_forwarding.xml.in b/interface-definitions/service_dns_forwarding.xml.in index b52b4bda3d1..5667028b746 100644 --- a/interface-definitions/service_dns_forwarding.xml.in +++ b/interface-definitions/service_dns_forwarding.xml.in @@ -311,6 +311,7 @@ [-_a-zA-Z0-9.]{1,63}(?<!\.) + #include diff --git a/src/conf_mode/service_dns_forwarding.py b/src/conf_mode/service_dns_forwarding.py index 7e863073ac4..e8318a83ec5 100755 --- a/src/conf_mode/service_dns_forwarding.py +++ b/src/conf_mode/service_dns_forwarding.py @@ -102,7 +102,7 @@ def get_config(config=None): 'ttl': rdata['ttl'], 'value': address }) - elif rtype in ['cname', 'ptr', 'ns']: + elif rtype in ['cname', 'ptr']: if not 'target' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: target is required') continue @@ -113,6 +113,19 @@ def get_config(config=None): 'ttl': rdata['ttl'], 'value': '{}.'.format(rdata['target']) }) + elif rtype == 'ns': + if not 'target' in rdata: + dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at leaast one target is required') + continue + + for target in rdata['target']: + zone['records'].append({ + 'name': subnode, + 'type': rtype.upper(), + 'ttl': rdata['ttl'], + 'value': '{}.'.format(target) + }) + elif rtype == 'mx': if not 'server' in rdata: dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one server is required') From dd2516904527c74e01e0ced5166afe72a479ee00 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 30 May 2024 17:25:27 +0200 Subject: [PATCH 164/188] reverse-proxy: T6409: unindent migration script code path --- src/migration-scripts/reverse-proxy/0-to-1 | 26 +++++++++++----------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/migration-scripts/reverse-proxy/0-to-1 b/src/migration-scripts/reverse-proxy/0-to-1 index 62be52ea6c5..d614938156f 100755 --- a/src/migration-scripts/reverse-proxy/0-to-1 +++ b/src/migration-scripts/reverse-proxy/0-to-1 @@ -33,16 +33,16 @@ base = ['load-balancing', 'reverse-proxy', 'backend'] if not config.exists(base): # Nothing to do exit(0) -else: - # we need to run this for every configured network - for backend in config.list_nodes(base): - param_node = base + [backend, 'parameters'] - if config.exists(param_node): - config.delete(param_node) - - try: - with open(file_name, 'w') as f: - f.write(config.to_string()) - except OSError as e: - print("Failed to save the modified config: {}".format(e)) - exit(1) + +# we need to run this for every configured network +for backend in config.list_nodes(base): + param_node = base + [backend, 'parameters'] + if config.exists(param_node): + config.delete(param_node) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) From f2d0701f50061374b5a4f55d33201629b3293248 Mon Sep 17 00:00:00 2001 From: Haim Gelfenbeyn Date: Thu, 30 May 2024 12:13:42 -0400 Subject: [PATCH 165/188] T6422: Smoke test for NS record configration in authoritative DNS, typo & style fixes --- smoketest/scripts/cli/test_service_dns_forwarding.py | 10 ++++++++++ src/conf_mode/service_dns_forwarding.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/smoketest/scripts/cli/test_service_dns_forwarding.py b/smoketest/scripts/cli/test_service_dns_forwarding.py index 079c584ba56..4db1d749594 100755 --- a/smoketest/scripts/cli/test_service_dns_forwarding.py +++ b/smoketest/scripts/cli/test_service_dns_forwarding.py @@ -291,5 +291,15 @@ def test_edns_subnet_allow_list(self): tmp = get_config_value('edns-subnet-allow-list') self.assertEqual(tmp, ','.join(options)) + def test_multiple_ns_records(self): + test_zone = 'example.com' + self.cli_set(base_path + ['authoritative-domain', test_zone, 'records', 'ns', 'test', 'target', f'ns1.{test_zone}']) + self.cli_set(base_path + ['authoritative-domain', test_zone, 'records', 'ns', 'test', 'target', f'ns2.{test_zone}']) + self.cli_commit() + zone_config = read_file(f'{PDNS_REC_RUN_DIR}/zone.{test_zone}.conf') + self.assertRegex(zone_config, fr'test\s+\d+\s+NS\s+ns1\.{test_zone}\.') + self.assertRegex(zone_config, fr'test\s+\d+\s+NS\s+ns2\.{test_zone}\.') + + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/service_dns_forwarding.py b/src/conf_mode/service_dns_forwarding.py index e8318a83ec5..70686534ff3 100755 --- a/src/conf_mode/service_dns_forwarding.py +++ b/src/conf_mode/service_dns_forwarding.py @@ -115,7 +115,7 @@ def get_config(config=None): }) elif rtype == 'ns': if not 'target' in rdata: - dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at leaast one target is required') + dns['authoritative_zone_errors'].append(f'{subnode}.{node}: at least one target is required') continue for target in rdata['target']: @@ -123,7 +123,7 @@ def get_config(config=None): 'name': subnode, 'type': rtype.upper(), 'ttl': rdata['ttl'], - 'value': '{}.'.format(target) + 'value': f'{target}.' }) elif rtype == 'mx': From 0c75e2470f8db900ffcac4e3c84669b6aa4580dd Mon Sep 17 00:00:00 2001 From: Giggum <152240782+Giggum@users.noreply.github.com> Date: Thu, 30 May 2024 22:37:24 -0400 Subject: [PATCH 166/188] conntrack: T6396: correction to helper message for custom timeout rule --- interface-definitions/system_conntrack.xml.in | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interface-definitions/system_conntrack.xml.in b/interface-definitions/system_conntrack.xml.in index 219c6e28e4f..66f3d4e057e 100644 --- a/interface-definitions/system_conntrack.xml.in +++ b/interface-definitions/system_conntrack.xml.in @@ -406,7 +406,7 @@ - Ignore rule number must be between 1 and 999999 + Timeout rule number must be between 1 and 999999 #include @@ -421,7 +421,7 @@ - Interface to ignore connections tracking on + Interface to apply custom connection timers on any @@ -464,7 +464,7 @@ - Ignore rule number must be between 1 and 999999 + Timeout rule number must be between 1 and 999999 #include @@ -479,7 +479,7 @@ - Interface to ignore connections tracking on + Interface to apply custom connection timers on any From d5271e084cca8af54f425816916a821b0eab1a5a Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Fri, 31 May 2024 09:48:52 +0200 Subject: [PATCH 167/188] op-mode: T683: remove superfluous debug print in snmpv3 display code This was a leftover from the early days. --- src/op_mode/snmp_v3.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/op_mode/snmp_v3.py b/src/op_mode/snmp_v3.py index a1f76f0bce1..abeb524dd97 100755 --- a/src/op_mode/snmp_v3.py +++ b/src/op_mode/snmp_v3.py @@ -85,7 +85,7 @@ 'user': [], 'view': [] } - + if c.exists_effective('service snmp v3 group'): for g in c.list_effective_nodes('service snmp v3 group'): group = { @@ -146,7 +146,6 @@ data['trap'].append(trap) - print(data) if args.all: # Special case, print all templates ! tmpl = jinja2.Template(GROUP_OUTP_TMPL_SRC) From 76c366dff8408e0dc176ccbbeb068d4b15775b83 Mon Sep 17 00:00:00 2001 From: Christian Breunig Date: Thu, 30 May 2024 17:10:37 +0200 Subject: [PATCH 168/188] GitHub: add action to build package on PR --- .github/workflows/build-package.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/build-package.yml diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml new file mode 100644 index 00000000000..0200aceb471 --- /dev/null +++ b/.github/workflows/build-package.yml @@ -0,0 +1,17 @@ +name: Debian Package Build +on: + pull_request: + branches: + - current + +jobs: + package-build: + runs-on: ubuntu-latest + container: + image: vyos/vyos-build:current + options: --sysctl net.ipv6.conf.lo.disable_ipv6=0 + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build Debian package + run: dpkg-buildpackage -uc -us -tc -b From 5563bcd9eaef5d5d8a020b92d43a86028e3c26c1 Mon Sep 17 00:00:00 2001 From: Roman Khramshin Date: Fri, 31 May 2024 16:04:42 +0700 Subject: [PATCH 169/188] T5307: QoS - traffic-class-map services (#3492) added new syntax to work with class match filters in QoS policy --- .../include/qos/class-match-group.xml.i | 15 +++ .../include/qos/class-match-ipv4.xml.i | 31 ++++++ .../include/qos/class-match-ipv6.xml.i | 31 ++++++ .../include/qos/class-match-mark.xml.i | 14 +++ .../include/qos/class-match-vif.xml.i | 15 +++ .../include/qos/class-match.xml.i | 89 +---------------- interface-definitions/qos.xml.in | 39 ++++++++ smoketest/scripts/cli/test_qos.py | 95 +++++++++++++++++++ .../qos/list_traffic_match_group.py | 35 +++++++ src/conf_mode/qos.py | 77 ++++++++++++++- 10 files changed, 352 insertions(+), 89 deletions(-) create mode 100644 interface-definitions/include/qos/class-match-group.xml.i create mode 100644 interface-definitions/include/qos/class-match-ipv4.xml.i create mode 100644 interface-definitions/include/qos/class-match-ipv6.xml.i create mode 100644 interface-definitions/include/qos/class-match-mark.xml.i create mode 100644 interface-definitions/include/qos/class-match-vif.xml.i create mode 100644 src/completion/qos/list_traffic_match_group.py diff --git a/interface-definitions/include/qos/class-match-group.xml.i b/interface-definitions/include/qos/class-match-group.xml.i new file mode 100644 index 00000000000..40e3b72592b --- /dev/null +++ b/interface-definitions/include/qos/class-match-group.xml.i @@ -0,0 +1,15 @@ + + + + Filter group for QoS policy + + txt + Match group name + + + + + + + + diff --git a/interface-definitions/include/qos/class-match-ipv4.xml.i b/interface-definitions/include/qos/class-match-ipv4.xml.i new file mode 100644 index 00000000000..dc44d32d56f --- /dev/null +++ b/interface-definitions/include/qos/class-match-ipv4.xml.i @@ -0,0 +1,31 @@ + + + + Match IP protocol header + + + + + Match on destination port or address + + + #include + #include + + + #include + #include + #include + + + Match on source port or address + + + #include + #include + + + #include + + + diff --git a/interface-definitions/include/qos/class-match-ipv6.xml.i b/interface-definitions/include/qos/class-match-ipv6.xml.i new file mode 100644 index 00000000000..ed7aceff960 --- /dev/null +++ b/interface-definitions/include/qos/class-match-ipv6.xml.i @@ -0,0 +1,31 @@ + + + + Match IPv6 protocol header + + + + + Match on destination port or address + + + #include + #include + + + #include + #include + #include + + + Match on source port or address + + + #include + #include + + + #include + + + diff --git a/interface-definitions/include/qos/class-match-mark.xml.i b/interface-definitions/include/qos/class-match-mark.xml.i new file mode 100644 index 00000000000..a7481c6aac0 --- /dev/null +++ b/interface-definitions/include/qos/class-match-mark.xml.i @@ -0,0 +1,14 @@ + + + + Match on mark applied by firewall + + u32 + FW mark to match + + + + + + + diff --git a/interface-definitions/include/qos/class-match-vif.xml.i b/interface-definitions/include/qos/class-match-vif.xml.i new file mode 100644 index 00000000000..ec58db60659 --- /dev/null +++ b/interface-definitions/include/qos/class-match-vif.xml.i @@ -0,0 +1,15 @@ + + + + Virtual Local Area Network (VLAN) ID for this match + + u32:0-4095 + Virtual Local Area Network (VLAN) tag + + + + + VLAN ID must be between 0 and 4095 + + + diff --git a/interface-definitions/include/qos/class-match.xml.i b/interface-definitions/include/qos/class-match.xml.i index 4ba12f8f734..77d1933a3f2 100644 --- a/interface-definitions/include/qos/class-match.xml.i +++ b/interface-definitions/include/qos/class-match.xml.i @@ -5,7 +5,7 @@ [^-].* - Match queue name cannot start with hyphen (-) + Match queue name cannot start with hyphen #include @@ -89,89 +89,10 @@ #include - - - Match IP protocol header - - - - - Match on destination port or address - - - #include - #include - - - #include - #include - #include - - - Match on source port or address - - - #include - #include - - - #include - - - - - Match IPv6 protocol header - - - - - Match on destination port or address - - - #include - #include - - - #include - #include - #include - - - Match on source port or address - - - #include - #include - - - #include - - - - - Match on mark applied by firewall - - u32 - FW mark to match - - - - - - - - - Virtual Local Area Network (VLAN) ID for this match - - u32:0-4095 - Virtual Local Area Network (VLAN) tag - - - - - VLAN ID must be between 0 and 4095 - - + #include + #include + #include + #include diff --git a/interface-definitions/qos.xml.in b/interface-definitions/qos.xml.in index 8f9ae3fa626..927594c11ea 100644 --- a/interface-definitions/qos.xml.in +++ b/interface-definitions/qos.xml.in @@ -281,6 +281,7 @@ #include #include #include + #include #include 20 @@ -415,6 +416,7 @@ #include #include #include + #include #include #include @@ -542,6 +544,8 @@ #include #include #include + #include + Packet scheduling quantum @@ -645,6 +649,7 @@ #include #include #include + #include #include #include #include @@ -767,6 +772,7 @@ #include + #include Realtime class settings @@ -830,6 +836,39 @@ + + + Filter group for QoS policy + + txt + Match group name + + + [^-].* + + Match group name cannot start with hyphen + + + #include + + + Class matching rule name + + [^-].* + + Match queue name cannot start with hyphen + + + #include + #include + #include + #include + #include + + + #include + + diff --git a/smoketest/scripts/cli/test_qos.py b/smoketest/scripts/cli/test_qos.py index 5977b2f414d..b98c0e9b797 100755 --- a/smoketest/scripts/cli/test_qos.py +++ b/smoketest/scripts/cli/test_qos.py @@ -759,6 +759,101 @@ def test_14_policy_limiter_marked_traffic(self): self.assertIn('filter parent ffff: protocol all pref 255 basic chain 0', tc_filters) self.assertIn('action order 1: police 0x2 rate 1Gbit burst 125000000b mtu 2Kb action drop overhead 0b', tc_filters) + def test_15_traffic_match_group(self): + interface = self._interfaces[0] + self.cli_set(['qos', 'interface', interface, 'egress', 'VyOS-HTB']) + base_policy_path = ['qos', 'policy', 'shaper', 'VyOS-HTB'] + + #old syntax + self.cli_set(base_policy_path + ['bandwidth', '100mbit']) + self.cli_set(base_policy_path + ['class', '10', 'bandwidth', '40%']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF11', 'ip', 'dscp', 'AF11']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF41', 'ip', 'dscp', 'AF41']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF43', 'ip', 'dscp', 'AF43']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'CS4', 'ip', 'dscp', 'CS4']) + self.cli_set(base_policy_path + ['class', '10', 'priority', '1']) + self.cli_set(base_policy_path + ['class', '10', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['class', '20', 'bandwidth', '30%']) + self.cli_set(base_policy_path + ['class', '20', 'match', 'EF', 'ip', 'dscp', 'EF']) + self.cli_set(base_policy_path + ['class', '20', 'match', 'CS5', 'ip', 'dscp', 'CS5']) + self.cli_set(base_policy_path + ['class', '20', 'priority', '2']) + self.cli_set(base_policy_path + ['class', '20', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['default', 'bandwidth', '20%']) + self.cli_set(base_policy_path + ['default', 'queue-type', 'fair-queue']) + self.cli_commit() + + tc_filters_old = cmd(f'tc -details filter show dev {interface}') + self.assertIn('match 00280000/00ff0000', tc_filters_old) + self.assertIn('match 00880000/00ff0000', tc_filters_old) + self.assertIn('match 00980000/00ff0000', tc_filters_old) + self.assertIn('match 00800000/00ff0000', tc_filters_old) + self.assertIn('match 00a00000/00ff0000', tc_filters_old) + self.assertIn('match 00b80000/00ff0000', tc_filters_old) + # delete config by old syntax + self.cli_delete(base_policy_path) + self.cli_delete(['qos', 'interface', interface, 'egress', 'VyOS-HTB']) + self.cli_commit() + self.assertEqual('', cmd(f'tc -s filter show dev {interface}')) + + self.cli_set(['qos', 'interface', interface, 'egress', 'VyOS-HTB']) + # prepare traffic match group + self.cli_set(['qos', 'traffic-match-group', 'VOICE', 'description', 'voice shaper']) + self.cli_set(['qos', 'traffic-match-group', 'VOICE', 'match', 'EF', 'ip', 'dscp', 'EF']) + self.cli_set(['qos', 'traffic-match-group', 'VOICE', 'match', 'CS5', 'ip', 'dscp', 'CS5']) + + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME_COMMON', 'description', 'real time common filters']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME_COMMON', 'match', 'AF43', 'ip', 'dscp', 'AF43']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME_COMMON', 'match', 'CS4', 'ip', 'dscp', 'CS4']) + + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME', 'description', 'real time shaper']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME', 'match', 'AF41', 'ip', 'dscp', 'AF41']) + self.cli_set(['qos', 'traffic-match-group', 'REAL_TIME', 'match-group', 'REAL_TIME_COMMON']) + + # new syntax + self.cli_set(base_policy_path + ['bandwidth', '100mbit']) + self.cli_set(base_policy_path + ['class', '10', 'bandwidth', '40%']) + self.cli_set(base_policy_path + ['class', '10', 'match', 'AF11', 'ip', 'dscp', 'AF11']) + self.cli_set(base_policy_path + ['class', '10', 'match-group', 'REAL_TIME']) + self.cli_set(base_policy_path + ['class', '10', 'priority', '1']) + self.cli_set(base_policy_path + ['class', '10', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['class', '20', 'bandwidth', '30%']) + self.cli_set(base_policy_path + ['class', '20', 'match-group', 'VOICE']) + self.cli_set(base_policy_path + ['class', '20', 'priority', '2']) + self.cli_set(base_policy_path + ['class', '20', 'queue-type', 'fair-queue']) + self.cli_set(base_policy_path + ['default', 'bandwidth', '20%']) + self.cli_set(base_policy_path + ['default', 'queue-type', 'fair-queue']) + self.cli_commit() + + self.assertEqual(tc_filters_old, cmd(f'tc -details filter show dev {interface}')) + + def test_16_wrong_traffic_match_group(self): + interface = self._interfaces[0] + self.cli_set(['qos', 'interface', interface]) + + # Can not use both IPv6 and IPv4 in one match + self.cli_set(['qos', 'traffic-match-group', '1', 'match', 'one', 'ip', 'dscp', 'EF']) + self.cli_set(['qos', 'traffic-match-group', '1', 'match', 'one', 'ipv6', 'dscp', 'EF']) + with self.assertRaises(ConfigSessionError) as e: + self.cli_commit() + + # check contain itself, should commit success + self.cli_delete(['qos', 'traffic-match-group', '1', 'match', 'one', 'ipv6']) + self.cli_set(['qos', 'traffic-match-group', '1', 'match-group', '1']) + self.cli_commit() + + # check cycle dependency, should commit success + self.cli_set(['qos', 'traffic-match-group', '1', 'match-group', '3']) + self.cli_set(['qos', 'traffic-match-group', '2', 'match', 'one', 'ip', 'dscp', 'CS4']) + self.cli_set(['qos', 'traffic-match-group', '2', 'match-group', '1']) + + self.cli_set(['qos', 'traffic-match-group', '3', 'match', 'one', 'ipv6', 'dscp', 'CS4']) + self.cli_set(['qos', 'traffic-match-group', '3', 'match-group', '2']) + self.cli_commit() + + # inherit from non exist group, should commit success with warning + self.cli_set(['qos', 'traffic-match-group', '3', 'match-group', 'unexpected']) + self.cli_commit() + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/completion/qos/list_traffic_match_group.py b/src/completion/qos/list_traffic_match_group.py new file mode 100644 index 00000000000..015d7ada91b --- /dev/null +++ b/src/completion/qos/list_traffic_match_group.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from vyos.config import Config + + +def get_qos_traffic_match_group(): + config = Config() + base = ['qos', 'traffic-match-group'] + conf = config.get_config_dict(base, key_mangling=('-', '_')) + groups = [] + + for group in conf.get('traffic_match_group', []): + groups.append(group) + + return groups + + +if __name__ == "__main__": + groups = get_qos_traffic_match_group() + print(" ".join(groups)) + diff --git a/src/conf_mode/qos.py b/src/conf_mode/qos.py index 8a590cbc65a..45248fb4a93 100755 --- a/src/conf_mode/qos.py +++ b/src/conf_mode/qos.py @@ -17,6 +17,7 @@ from sys import exit from netifaces import interfaces +from vyos.base import Warning from vyos.config import Config from vyos.configdep import set_dependents from vyos.configdep import call_dependents @@ -89,6 +90,36 @@ def _clean_conf_dict(conf): return conf +def _get_group_filters(config: dict, group_name: str, visited=None) -> dict: + filters = dict() + if not visited: + visited = [group_name, ] + else: + if group_name in visited: + return filters + visited.append(group_name) + + for filter, filter_config in config.get(group_name, {}).items(): + if filter == 'match': + for match, match_config in filter_config.items(): + filters[f'{group_name}-{match}'] = match_config + elif filter == 'match_group': + for group in filter_config: + filters.update(_get_group_filters(config, group, visited)) + + return filters + + +def _get_group_match(config:dict, group_name:str) -> dict: + match = dict() + for key, val in _get_group_filters(config, group_name).items(): + # delete duplicate matches + if val not in match.values(): + match[key] = val + + return match + + def get_config(config=None): if config: conf = config @@ -135,11 +166,27 @@ def get_config(config=None): qos = conf.merge_defaults(qos, recursive=True) + if 'traffic_match_group' in qos: + for group, group_config in qos['traffic_match_group'].items(): + if 'match_group' in group_config: + qos['traffic_match_group'][group]['match'] = _get_group_match(qos['traffic_match_group'], group) + for policy in qos.get('policy', []): for p_name, p_config in qos['policy'][policy].items(): # cleanup empty match config if 'class' in p_config: for cls, cls_config in p_config['class'].items(): + if 'match_group' in cls_config: + # merge group match to match + for group in cls_config['match_group']: + for match, match_conf in qos['traffic_match_group'].get(group, {'match': {}})['match'].items(): + if 'match' not in cls_config: + cls_config['match'] = dict() + if match in cls_config['match']: + cls_config['match'][f'{group}-{match}'] = match_conf + else: + cls_config['match'][match] = match_conf + if 'match' in cls_config: cls_config['match'] = _clean_conf_dict(cls_config['match']) if cls_config['match'] == {}: @@ -147,6 +194,22 @@ def get_config(config=None): return qos + +def _verify_match(cls_config: dict) -> None: + if 'match' in cls_config: + for match, match_config in cls_config['match'].items(): + if {'ip', 'ipv6'} <= set(match_config): + raise ConfigError( + f'Can not use both IPv6 and IPv4 in one match ({match})!') + + +def _verify_match_group_exist(cls_config, qos): + if 'match_group' in cls_config: + for group in cls_config['match_group']: + if 'traffic_match_group' not in qos or group not in qos['traffic_match_group']: + Warning(f'Match group "{group}" does not exist!') + + def verify(qos): if not qos or 'interface' not in qos: return None @@ -174,11 +237,8 @@ def verify(qos): # bandwidth is not mandatory for priority-queue - that is why this is on the exception list if 'bandwidth' not in cls_config and policy_type not in ['priority_queue', 'round_robin', 'shaper_hfsc']: raise ConfigError(f'Bandwidth must be defined for policy "{policy}" class "{cls}"!') - if 'match' in cls_config: - for match, match_config in cls_config['match'].items(): - if {'ip', 'ipv6'} <= set(match_config): - raise ConfigError(f'Can not use both IPv6 and IPv4 in one match ({match})!') - + _verify_match(cls_config) + _verify_match_group_exist(cls_config, qos) if policy_type in ['random_detect']: if 'precedence' in policy_config: for precedence, precedence_config in policy_config['precedence'].items(): @@ -216,8 +276,14 @@ def verify(qos): if direction not in tmp: raise ConfigError(f'Selected QoS policy on interface "{interface}" only supports "{tmp}"!') + if 'traffic_match_group' in qos: + for group, group_config in qos['traffic_match_group'].items(): + _verify_match(group_config) + _verify_match_group_exist(group_config, qos) + return None + def generate(qos): if not qos or 'interface' not in qos: return None @@ -254,6 +320,7 @@ def apply(qos): return None + if __name__ == '__main__': try: c = get_config() From f37dbb732a66f618df54dcd5db5735cbf0d33c86 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Fri, 31 May 2024 15:55:30 +0530 Subject: [PATCH 170/188] T6415: repo sync using pull_request_target --- .github/workflows/repo-sync.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/repo-sync.yml b/.github/workflows/repo-sync.yml index a2bf54f745f..36f323cdd01 100644 --- a/.github/workflows/repo-sync.yml +++ b/.github/workflows/repo-sync.yml @@ -1,7 +1,7 @@ name: Repo-sync on: - pull_request: + pull_request_target: types: - closed branches: From 34024e630ec70ac5dc59fc57f818f6cd9dc15cd2 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Fri, 31 May 2024 21:51:25 +1000 Subject: [PATCH 171/188] tunnel: T6157: fixing GRE tunnel uniqueness checks Unset params would mistakenly match when None and trigger a validation error even when used params were unique. Updated check to ensure unique source-addresses if not None, and that (source-interfaces, source-addresses) are unique together appropriately. --- src/conf_mode/interfaces_tunnel.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/conf_mode/interfaces_tunnel.py b/src/conf_mode/interfaces_tunnel.py index 43ba7285736..98ef98d1253 100755 --- a/src/conf_mode/interfaces_tunnel.py +++ b/src/conf_mode/interfaces_tunnel.py @@ -145,11 +145,20 @@ def verify(tunnel): # If no IP GRE key is defined we can not have more then one GRE tunnel # bound to any one interface/IP address and the same remote. This will # result in a OS PermissionError: add tunnel "gre0" failed: File exists - if (their_address == our_address or our_source_if == their_source_if) and \ - our_remote == their_remote: - raise ConfigError(f'Missing required "ip key" parameter when '\ - 'running more then one GRE based tunnel on the '\ - 'same source-interface/source-address') + if our_remote == their_remote: + if our_address is not None and their_address == our_address: + # If set to the same values, this is always a fail + raise ConfigError(f'Missing required "ip key" parameter when '\ + 'running more then one GRE based tunnel on the '\ + 'same source-address') + + if their_source_if == our_source_if and their_address == our_address: + # Note that lack of None check on these is deliberate. + # source-if and source-ip matching while unset (all None) is a fail + # source-ifs set and matching with unset source-ips is a fail + raise ConfigError(f'Missing required "ip key" parameter when '\ + 'running more then one GRE based tunnel on the '\ + 'same source-interface') # Keys are not allowed with ipip and sit tunnels if tunnel['encapsulation'] in ['ipip', 'sit']: From 39004c453fb8f71171ba3433ee559b5ff745bebe Mon Sep 17 00:00:00 2001 From: fett0 Date: Fri, 31 May 2024 19:42:00 +0000 Subject: [PATCH 172/188] isis: T6429: fix isis metric-style configuration missing --- data/templates/frr/isisd.frr.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/templates/frr/isisd.frr.j2 b/data/templates/frr/isisd.frr.j2 index 5570caaa7df..eb14aade690 100644 --- a/data/templates/frr/isisd.frr.j2 +++ b/data/templates/frr/isisd.frr.j2 @@ -76,6 +76,9 @@ advertise-passive-only {% if set_overload_bit is vyos_defined %} set-overload-bit {% endif %} +{% if metric_style is vyos_defined %} + metric-style {{ metric_style }} +{% endif %} {% if domain_password.md5 is vyos_defined %} domain-password md5 {{ domain_password.plaintext_password }} {% elif domain_password.plaintext_password is vyos_defined %} From 3ad333fc62807f5816f826d7bc0c4c8e0ac96167 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Sat, 1 Jun 2024 20:42:36 +1000 Subject: [PATCH 173/188] nat64: T6403: validate source prefix for RFC compliance Simplest fix is to comply with RFC6052. The code change is just masking out the relevant bits and ensuring they're zeroed. --- src/conf_mode/nat64.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/conf_mode/nat64.py b/src/conf_mode/nat64.py index c1e7ebf85d5..32a1c47d197 100755 --- a/src/conf_mode/nat64.py +++ b/src/conf_mode/nat64.py @@ -20,7 +20,7 @@ import os import re -from ipaddress import IPv6Network +from ipaddress import IPv6Network, IPv6Address from json import dumps as json_write from vyos import ConfigError @@ -103,8 +103,14 @@ def verify(nat64) -> None: # Verify that source.prefix is set and is a /96 if not dict_search("source.prefix", instance): raise ConfigError(f"Source NAT64 rule {rule} missing source prefix") - if IPv6Network(instance["source"]["prefix"]).prefixlen != 96: + src_prefix = IPv6Network(instance["source"]["prefix"]) + if src_prefix.prefixlen != 96: raise ConfigError(f"Source NAT64 rule {rule} source prefix must be /96") + if (int(src_prefix[0]) & int(IPv6Address('0:0:0:0:ff00::'))) != 0: + raise ConfigError( + f'Source NAT64 rule {rule} source prefix is not RFC6052-compliant: ' + 'bits 64 to 71 (9th octet) must be zeroed' + ) pools = dict_search("translation.pool", instance) if pools: From ac7ee2b36df23c3a4dd2be393132631556b6ef40 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Sat, 1 Jun 2024 23:42:57 +1000 Subject: [PATCH 174/188] vxlan: T6401: Avoid calling get_vxlan_vni_filter() unless we need it `bridge vni show dev vxlanX` will exit with an error if no VNI filters are installed, but the getter is used even when we haven't installed any. This fix avoids fetching a list of VNI filters unless we know we've created some. --- python/vyos/ifconfig/vxlan.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/vyos/ifconfig/vxlan.py b/python/vyos/ifconfig/vxlan.py index bdb48e30361..918aea20296 100644 --- a/python/vyos/ifconfig/vxlan.py +++ b/python/vyos/ifconfig/vxlan.py @@ -138,10 +138,13 @@ def set_vlan_vni_mapping(self, state): raise ValueError('Value out of range') if 'vlan_to_vni_removed' in self.config: - cur_vni_filter = get_vxlan_vni_filter(self.ifname) + cur_vni_filter = None + if dict_search('parameters.vni_filter', self.config) != None: + cur_vni_filter = get_vxlan_vni_filter(self.ifname) + for vlan, vlan_config in self.config['vlan_to_vni_removed'].items(): # If VNI filtering is enabled, remove matching VNI filter - if dict_search('parameters.vni_filter', self.config) != None: + if cur_vni_filter != None: vni = vlan_config['vni'] if vni in cur_vni_filter: self._cmd(f'bridge vni delete dev {self.ifname} vni {vni}') From 3e5cc0b7fb8ae4a0f8b7c9270d9db0a0f252c448 Mon Sep 17 00:00:00 2001 From: Alex W Date: Mon, 3 Jun 2024 05:54:07 +0100 Subject: [PATCH 175/188] reverse-proxy: T6434: Support additional healthcheck options (#3574) --- data/templates/load-balancing/haproxy.cfg.j2 | 7 +++ .../load-balancing_reverse-proxy.xml.in | 31 +++++++++++ .../cli/test_load-balancing_reverse-proxy.py | 53 +++++++++++++++++++ src/conf_mode/load-balancing_reverse-proxy.py | 13 ++++- 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index b786a58f879..c6027e09b15 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -131,6 +131,13 @@ frontend {{ front }} {% if backend is vyos_defined %} {% for back, back_config in backend.items() %} backend {{ back }} +{% if back_config.health_check is vyos_defined %} +{% if back_config.health_check == 'smtp' %} + option smtpchk +{% else %} + option {{ back_config.health_check }}-check +{% endif %} +{% endif %} {% if back_config.http_check is vyos_defined %} option httpchk {% endif %} diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index e50e6e579c4..ce757a5d67a 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -151,6 +151,37 @@ + + + Non HTTP health check options + + ldap mysql pgsql redis smtp + + + ldap + LDAP protocol check + + + mysql + MySQL protocol check + + + pgsql + PostgreSQL protocol check + + + redis + Redis protocol check + + + smtp + SMTP protocol check + + + (ldap|mysql|redis|pgsql|smtp) + + + #include diff --git a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py index 2b2f93cdf76..aa796f59f13 100755 --- a/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py +++ b/smoketest/scripts/cli/test_load-balancing_reverse-proxy.py @@ -338,6 +338,11 @@ def test_05_lb_reverse_proxy_backend_http_check(self): self.assertIn('http-check send meth GET uri /health', config) self.assertIn('http-check expect string success', config) + # Test configuring both http-check & health-check fails validation script + self.cli_set(base_path + ['backend', 'bk-01', 'health-check', 'ldap']) + with self.assertRaises(ConfigSessionError) as e: + self.cli_commit() + def test_06_lb_reverse_proxy_tcp_mode(self): frontend = 'tcp_8443' mode = 'tcp' @@ -405,6 +410,54 @@ def test_07_lb_reverse_proxy_http_response_headers(self): with self.assertRaises(ConfigSessionError) as e: self.cli_commit() + def test_08_lb_reverse_proxy_tcp_health_checks(self): + # Setup PKI + self.configure_pki() + + # Define variables + frontend = 'fe_ldaps' + mode = 'tcp' + health_check = 'ldap' + front_port = '636' + bk_name = 'bk_ldap' + bk_servers = ['192.0.2.11', '192.0.2.12'] + bk_server_port = '389' + + # Configure frontend + self.cli_set(base_path + ['service', frontend, 'mode', mode]) + self.cli_set(base_path + ['service', frontend, 'port', front_port]) + self.cli_set(base_path + ['service', frontend, 'ssl', 'certificate', 'smoketest']) + + # Configure backend + self.cli_set(base_path + ['backend', bk_name, 'mode', mode]) + self.cli_set(base_path + ['backend', bk_name, 'health-check', health_check]) + for index, bk_server in enumerate(bk_servers): + self.cli_set(base_path + ['backend', bk_name, 'server', f'srv-{index}', 'address', bk_server]) + self.cli_set(base_path + ['backend', bk_name, 'server', f'srv-{index}', 'port', bk_server_port]) + + # Commit & read config + self.cli_commit() + config = read_file(HAPROXY_CONF) + + # Validate Frontend + self.assertIn(f'frontend {frontend}', config) + self.assertIn(f'bind [::]:{front_port} v4v6 ssl crt /run/haproxy/smoketest.pem', config) + self.assertIn(f'mode {mode}', config) + self.assertIn(f'backend {bk_name}', config) + + # Validate Backend + self.assertIn(f'backend {bk_name}', config) + self.assertIn(f'option {health_check}-check', config) + self.assertIn(f'mode {mode}', config) + for index, bk_server in enumerate(bk_servers): + self.assertIn(f'server srv-{index} {bk_server}:{bk_server_port}', config) + + # Validate SMTP option renders correctly + self.cli_set(base_path + ['backend', bk_name, 'health-check', 'smtp']) + self.cli_commit() + config = read_file(HAPROXY_CONF) + self.assertIn(f'option smtpchk', config) + if __name__ == '__main__': unittest.main(verbosity=2) diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index 1c1252df0f8..09c68dadd40 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -79,12 +79,21 @@ def verify(lb): raise ConfigError(f'"TCP" port "{tmp_port}" is used by another service') for back, back_config in lb['backend'].items(): - if 'http-check' in back_config: - http_check = back_config['http-check'] + if 'http_check' in back_config: + http_check = back_config['http_check'] if 'expect' in http_check and 'status' in http_check['expect'] and 'string' in http_check['expect']: raise ConfigError(f'"expect status" and "expect string" can not be configured together!') + + if 'health_check' in back_config: + if 'mode' not in back_config or back_config['mode'] != 'tcp': + raise ConfigError(f'backend "{back}" can only be configured with {back_config["health_check"]} ' + + f'health-check whilst in TCP mode!') + if 'http_check' in back_config: + raise ConfigError(f'backend "{back}" cannot be configured with both http-check and health-check!') + if 'server' not in back_config: raise ConfigError(f'"{back} server" must be configured!') + for bk_server, bk_server_conf in back_config['server'].items(): if 'address' not in bk_server_conf or 'port' not in bk_server_conf: raise ConfigError(f'"backend {back} server {bk_server} address and port" must be configured!') From 5490c76f9b9f53751fc527f455090f0a3820e8fe Mon Sep 17 00:00:00 2001 From: Hannes Tamme Date: Mon, 3 Jun 2024 14:21:06 +0300 Subject: [PATCH 176/188] bfd: T6440: BFD peer length typo --- src/conf_mode/protocols_bfd.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/conf_mode/protocols_bfd.py b/src/conf_mode/protocols_bfd.py index 1c01a901347..1361bb1a95b 100755 --- a/src/conf_mode/protocols_bfd.py +++ b/src/conf_mode/protocols_bfd.py @@ -49,7 +49,7 @@ def verify(bfd): for peer, peer_config in bfd['peer'].items(): # IPv6 link local peers require an explicit local address/interface if is_ipv6_link_local(peer): - if 'source' not in peer_config or len(peer_config['source'] < 2): + if 'source' not in peer_config or len(peer_config['source']) < 2: raise ConfigError('BFD IPv6 link-local peers require explicit local address and interface setting') # IPv6 peers require an explicit local address From de1479b06cb9b292fe4919c5949f3d3599ea11c7 Mon Sep 17 00:00:00 2001 From: Andrew Topp Date: Tue, 4 Jun 2024 20:22:57 +1000 Subject: [PATCH 177/188] T6431: op-mode command monitor traceroute missing recursive symlink Likely this was copied from mtr in the past but the symlink wasn't added to the Makefile. I've also swapped the completion help text around to match the commands. --- Makefile | 1 + op-mode-definitions/mtr.xml.in | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 44a6e35edfa..3b26273d639 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,7 @@ op_mode_definitions: $(op_xml_obj) ln -s ../node.tag $(OP_TMPL_DIR)/ping/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/traceroute/node.tag/node.tag/ ln -s ../node.tag $(OP_TMPL_DIR)/mtr/node.tag/node.tag/ + ln -s ../node.tag $(OP_TMPL_DIR)/monitor/traceroute/node.tag/node.tag/ # XXX: test if there are empty node.def files - this is not allowed as these # could mask help strings or mandatory priority statements diff --git a/op-mode-definitions/mtr.xml.in b/op-mode-definitions/mtr.xml.in index 8239aec4c9b..66729e2bc5c 100644 --- a/op-mode-definitions/mtr.xml.in +++ b/op-mode-definitions/mtr.xml.in @@ -13,7 +13,7 @@ - mtr options + Traceroute options @@ -35,7 +35,7 @@ - Traceroute options + mtr options From 770edf016838523c248e3c8a36c5f327a0b98415 Mon Sep 17 00:00:00 2001 From: Nicolas Fort Date: Fri, 24 May 2024 16:44:41 +0000 Subject: [PATCH 178/188] T3900: T6394: extend functionalities in firewall; move netfilter sysctl timeout parameters defined in conntrack to firewall global-opton section. --- data/templates/conntrack/sysctl.conf.j2 | 20 +--- .../firewall/sysctl-firewall.conf.j2 | 28 ++++++ .../include/firewall/action-and-notrack.xml.i | 2 +- .../include/firewall/global-options.xml.i | 8 ++ .../timeout-common-protocols.xml.i | 3 +- .../include/version/firewall-version.xml.i | 2 +- interface-definitions/system_conntrack.xml.in | 1 - smoketest/scripts/cli/test_firewall.py | 96 ++++++++++++++++++- .../scripts/cli/test_system_conntrack.py | 60 ------------ src/conf_mode/firewall.py | 39 +------- src/conf_mode/system_conntrack.py | 2 +- src/migration-scripts/firewall/15-to-16 | 55 +++++++++++ 12 files changed, 193 insertions(+), 123 deletions(-) create mode 100644 data/templates/firewall/sysctl-firewall.conf.j2 rename interface-definitions/include/{conntrack => firewall}/timeout-common-protocols.xml.i (98%) create mode 100755 src/migration-scripts/firewall/15-to-16 diff --git a/data/templates/conntrack/sysctl.conf.j2 b/data/templates/conntrack/sysctl.conf.j2 index 986f75c6162..554512f4d10 100644 --- a/data/templates/conntrack/sysctl.conf.j2 +++ b/data/templates/conntrack/sysctl.conf.j2 @@ -3,25 +3,7 @@ net.netfilter.nf_conntrack_expect_max = {{ expect_table_size }} net.netfilter.nf_conntrack_max = {{ table_size }} - net.ipv4.tcp_max_syn_backlog = {{ tcp.half_open_connections }} - net.netfilter.nf_conntrack_tcp_loose = {{ '1' if tcp.loose is vyos_defined('enable') else '0' }} net.netfilter.nf_conntrack_tcp_max_retrans = {{ tcp.max_retrans }} - -net.netfilter.nf_conntrack_icmp_timeout = {{ timeout.icmp }} -net.netfilter.nf_conntrack_generic_timeout = {{ timeout.other }} - -net.netfilter.nf_conntrack_tcp_timeout_close_wait = {{ timeout.tcp.close_wait }} -net.netfilter.nf_conntrack_tcp_timeout_close = {{ timeout.tcp.close }} -net.netfilter.nf_conntrack_tcp_timeout_established = {{ timeout.tcp.established }} -net.netfilter.nf_conntrack_tcp_timeout_fin_wait = {{ timeout.tcp.fin_wait }} -net.netfilter.nf_conntrack_tcp_timeout_last_ack = {{ timeout.tcp.last_ack }} -net.netfilter.nf_conntrack_tcp_timeout_syn_recv = {{ timeout.tcp.syn_recv }} -net.netfilter.nf_conntrack_tcp_timeout_syn_sent = {{ timeout.tcp.syn_sent }} -net.netfilter.nf_conntrack_tcp_timeout_time_wait = {{ timeout.tcp.time_wait }} - -net.netfilter.nf_conntrack_udp_timeout = {{ timeout.udp.other }} -net.netfilter.nf_conntrack_udp_timeout_stream = {{ timeout.udp.stream }} - -net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }} +net.netfilter.nf_conntrack_acct = {{ '1' if flow_accounting is vyos_defined else '0' }} \ No newline at end of file diff --git a/data/templates/firewall/sysctl-firewall.conf.j2 b/data/templates/firewall/sysctl-firewall.conf.j2 new file mode 100644 index 00000000000..b9c3311e27b --- /dev/null +++ b/data/templates/firewall/sysctl-firewall.conf.j2 @@ -0,0 +1,28 @@ +# Autogenerated by firewall.py + +# gloabl options +net.ipv4.icmp_echo_ignore_all = {{ 0 if global_options.all_ping == 'enable' else 1 }} +net.ipv4.icmp_echo_ignore_broadcasts = {{ 0 if global_options.broadcast_ping == 'enable' else 1 }} +net.ipv4.conf.all.bc_forwarding = {{ 1 if global_options.directed_broadcast == 'enable' else 0 }} +net.ipv4.conf.*.accept_source_route = {{ 1 if global_options.ip_src_route == 'enable' else 0 }} +net.ipv6.conf.*.accept_redirects = {{ 1 if global_options.ipv6_receive_redirects == 'enable' else 0 }} +net.ipv6.conf.*.accept_source_route = {{ 0 if global_options.ipv6_src_route == 'enable' else -1 }} +net.ipv4.conf.all.log_martians = {{ 1 if global_options.log_martians == 'enable' else 0 }} +net.ipv4.conf.*.accept_redirects = {{ 1 if global_options.receive_redirects == 'enable' else 0 }} +net.ipv4.conf.*.send_redirects = {{ 1 if global_options.send_redirects == 'enable' else 0 }} +net.ipv4.tcp_syncookies = {{ 1 if global_options.syn_cookies == 'enable' else 0 }} +net.ipv4.tcp_rfc1337 = {{ 1 if global_options.twa_hazards_protection == 'enable' else 0 }} + +## Timeout values: +net.netfilter.nf_conntrack_icmp_timeout = {{ global_options.timeout.icmp }} +net.netfilter.nf_conntrack_generic_timeout = {{ global_options.timeout.other }} +net.netfilter.nf_conntrack_tcp_timeout_close_wait = {{ global_options.timeout.tcp.close_wait }} +net.netfilter.nf_conntrack_tcp_timeout_close = {{ global_options.timeout.tcp.close }} +net.netfilter.nf_conntrack_tcp_timeout_established = {{ global_options.timeout.tcp.established }} +net.netfilter.nf_conntrack_tcp_timeout_fin_wait = {{ global_options.timeout.tcp.fin_wait }} +net.netfilter.nf_conntrack_tcp_timeout_last_ack = {{ global_options.timeout.tcp.last_ack }} +net.netfilter.nf_conntrack_tcp_timeout_syn_recv = {{ global_options.timeout.tcp.syn_recv }} +net.netfilter.nf_conntrack_tcp_timeout_syn_sent = {{ global_options.timeout.tcp.syn_sent }} +net.netfilter.nf_conntrack_tcp_timeout_time_wait = {{ global_options.timeout.tcp.time_wait }} +net.netfilter.nf_conntrack_udp_timeout = {{ global_options.timeout.udp.other }} +net.netfilter.nf_conntrack_udp_timeout_stream = {{ global_options.timeout.udp.stream }} diff --git a/interface-definitions/include/firewall/action-and-notrack.xml.i b/interface-definitions/include/firewall/action-and-notrack.xml.i index e063c58d546..de11f7dd50a 100644 --- a/interface-definitions/include/firewall/action-and-notrack.xml.i +++ b/interface-definitions/include/firewall/action-and-notrack.xml.i @@ -35,7 +35,7 @@ notrack - Igone connection tracking + Ignore connection tracking (accept|continue|jump|notrack|reject|return|drop|queue) diff --git a/interface-definitions/include/firewall/global-options.xml.i b/interface-definitions/include/firewall/global-options.xml.i index 9cd0b3239f7..9039b76fd63 100644 --- a/interface-definitions/include/firewall/global-options.xml.i +++ b/interface-definitions/include/firewall/global-options.xml.i @@ -244,6 +244,14 @@ enable + + + Connection timeout options + + + #include + + RFC1337 TCP TIME-WAIT assasination hazards protection diff --git a/interface-definitions/include/conntrack/timeout-common-protocols.xml.i b/interface-definitions/include/firewall/timeout-common-protocols.xml.i similarity index 98% rename from interface-definitions/include/conntrack/timeout-common-protocols.xml.i rename to interface-definitions/include/firewall/timeout-common-protocols.xml.i index 2676d846e57..037d7d2b169 100644 --- a/interface-definitions/include/conntrack/timeout-common-protocols.xml.i +++ b/interface-definitions/include/firewall/timeout-common-protocols.xml.i @@ -1,4 +1,4 @@ - + ICMP timeout in seconds @@ -169,4 +169,3 @@ - diff --git a/interface-definitions/include/version/firewall-version.xml.i b/interface-definitions/include/version/firewall-version.xml.i index fa8e26f78b2..560ed9e5f59 100644 --- a/interface-definitions/include/version/firewall-version.xml.i +++ b/interface-definitions/include/version/firewall-version.xml.i @@ -1,3 +1,3 @@ - + diff --git a/interface-definitions/system_conntrack.xml.in b/interface-definitions/system_conntrack.xml.in index 219c6e28e4f..33aa832a8f8 100644 --- a/interface-definitions/system_conntrack.xml.in +++ b/interface-definitions/system_conntrack.xml.in @@ -509,7 +509,6 @@ - #include diff --git a/smoketest/scripts/cli/test_firewall.py b/smoketest/scripts/cli/test_firewall.py index 5cf2b214675..0943d8e24df 100755 --- a/smoketest/scripts/cli/test_firewall.py +++ b/smoketest/scripts/cli/test_firewall.py @@ -23,6 +23,7 @@ from vyos.configsession import ConfigSessionError from vyos.utils.process import run +from vyos.utils.file import read_file sysfs_config = { 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'default': '0', 'test_value': 'disable'}, @@ -38,6 +39,10 @@ 'twa_hazards_protection': {'sysfs': '/proc/sys/net/ipv4/tcp_rfc1337', 'default': '0', 'test_value': 'enable'} } +def get_sysctl(parameter): + tmp = parameter.replace(r'.', r'/') + return read_file(f'/proc/sys/{tmp}') + class TestFirewall(VyOSUnitTestSHIM.TestCase): @classmethod def setUpClass(cls): @@ -240,7 +245,7 @@ def test_ipv4_basic_rules(self): self.cli_set(['firewall', 'ipv4', 'output', 'raw', 'rule', '1', 'action', 'accept']) self.cli_set(['firewall', 'ipv4', 'output', 'raw', 'rule', '1', 'protocol', 'udp']) - self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'action', 'drop']) + self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'action', 'notrack']) self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'protocol', 'tcp']) self.cli_set(['firewall', 'ipv4', 'prerouting', 'raw', 'rule', '1', 'destination', 'port', '23']) @@ -270,7 +275,7 @@ def test_ipv4_basic_rules(self): ['OUT-raw default-action drop', 'drop'], ['chain VYOS_PREROUTING_raw'], ['type filter hook prerouting priority raw; policy accept;'], - ['tcp dport 23', 'drop'], + ['tcp dport 23', 'notrack'], ['PRE-raw default-action accept', 'accept'], ['chain NAME_smoketest'], ['saddr 172.16.20.10', 'daddr 172.16.10.10', 'log prefix "[ipv4-NAM-smoketest-1-A]" log level debug', 'ip ttl 15', 'accept'], @@ -474,7 +479,7 @@ def test_ipv6_basic_rules(self): self.cli_set(['firewall', 'ipv6', 'output', 'filter', 'rule', '3', 'outbound-interface', 'name', interface]) self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'default-action', 'drop']) - self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'rule', '1', 'action', 'accept']) + self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'rule', '1', 'action', 'notrack']) self.cli_set(['firewall', 'ipv6', 'output', 'raw', 'rule', '1', 'protocol', 'udp']) self.cli_set(['firewall', 'ipv6', 'prerouting', 'raw', 'rule', '1', 'action', 'drop']) @@ -498,7 +503,7 @@ def test_ipv6_basic_rules(self): ['log prefix "[ipv6-OUT-filter-default-D]"','OUT-filter default-action drop', 'drop'], ['chain VYOS_IPV6_OUTPUT_raw'], ['type filter hook output priority raw; policy accept;'], - ['udp', 'accept'], + ['udp', 'notrack'], ['OUT-raw default-action drop', 'drop'], ['chain VYOS_IPV6_PREROUTING_raw'], ['type filter hook prerouting priority raw; policy accept;'], @@ -770,6 +775,89 @@ def test_sysfs(self): with open(path, 'r') as f: self.assertNotEqual(f.read().strip(), conf['default'], msg=path) + def test_timeout_sysctl(self): + timeout_config = { + 'net.netfilter.nf_conntrack_icmp_timeout' :{ + 'cli' : ['global-options', 'timeout', 'icmp'], + 'test_value' : '180', + 'default_value' : '30', + }, + 'net.netfilter.nf_conntrack_generic_timeout' :{ + 'cli' : ['global-options', 'timeout', 'other'], + 'test_value' : '1200', + 'default_value' : '600', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_close_wait' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'close-wait'], + 'test_value' : '30', + 'default_value' : '60', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_close' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'close'], + 'test_value' : '20', + 'default_value' : '10', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_established' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'established'], + 'test_value' : '1000', + 'default_value' : '432000', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_fin_wait' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'fin-wait'], + 'test_value' : '240', + 'default_value' : '120', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_last_ack' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'last-ack'], + 'test_value' : '300', + 'default_value' : '30', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_syn_recv' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'syn-recv'], + 'test_value' : '100', + 'default_value' : '60', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_syn_sent' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'syn-sent'], + 'test_value' : '300', + 'default_value' : '120', + }, + 'net.netfilter.nf_conntrack_tcp_timeout_time_wait' :{ + 'cli' : ['global-options', 'timeout', 'tcp', 'time-wait'], + 'test_value' : '303', + 'default_value' : '120', + }, + 'net.netfilter.nf_conntrack_udp_timeout' :{ + 'cli' : ['global-options', 'timeout', 'udp', 'other'], + 'test_value' : '90', + 'default_value' : '30', + }, + 'net.netfilter.nf_conntrack_udp_timeout_stream' :{ + 'cli' : ['global-options', 'timeout', 'udp', 'stream'], + 'test_value' : '200', + 'default_value' : '180', + }, + } + + for parameter, parameter_config in timeout_config.items(): + self.cli_set(['firewall'] + parameter_config['cli'] + [parameter_config['test_value']]) + + # commit changes + self.cli_commit() + + # validate configuration + for parameter, parameter_config in timeout_config.items(): + tmp = parameter_config['test_value'] + self.assertEqual(get_sysctl(f'{parameter}'), tmp) + + # delete all configuration options and revert back to defaults + self.cli_delete(['firewall', 'global-options', 'timeout']) + self.cli_commit() + + # validate configuration + for parameter, parameter_config in timeout_config.items(): + self.assertEqual(get_sysctl(f'{parameter}'), parameter_config['default_value']) + ### Zone def test_zone_basic(self): self.cli_set(['firewall', 'ipv4', 'name', 'smoketest', 'default-action', 'drop']) diff --git a/smoketest/scripts/cli/test_system_conntrack.py b/smoketest/scripts/cli/test_system_conntrack.py index c6d8a5436af..3ae7b6217a3 100755 --- a/smoketest/scripts/cli/test_system_conntrack.py +++ b/smoketest/scripts/cli/test_system_conntrack.py @@ -68,66 +68,6 @@ def test_conntrack_options(self): 'test_value' : '128', 'default_value' : '3', }, - 'net.netfilter.nf_conntrack_icmp_timeout' :{ - 'cli' : ['timeout', 'icmp'], - 'test_value' : '180', - 'default_value' : '30', - }, - 'net.netfilter.nf_conntrack_generic_timeout' :{ - 'cli' : ['timeout', 'other'], - 'test_value' : '1200', - 'default_value' : '600', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_close_wait' :{ - 'cli' : ['timeout', 'tcp', 'close-wait'], - 'test_value' : '30', - 'default_value' : '60', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_close' :{ - 'cli' : ['timeout', 'tcp', 'close'], - 'test_value' : '20', - 'default_value' : '10', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_established' :{ - 'cli' : ['timeout', 'tcp', 'established'], - 'test_value' : '1000', - 'default_value' : '432000', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_fin_wait' :{ - 'cli' : ['timeout', 'tcp', 'fin-wait'], - 'test_value' : '240', - 'default_value' : '120', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_last_ack' :{ - 'cli' : ['timeout', 'tcp', 'last-ack'], - 'test_value' : '300', - 'default_value' : '30', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_syn_recv' :{ - 'cli' : ['timeout', 'tcp', 'syn-recv'], - 'test_value' : '100', - 'default_value' : '60', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_syn_sent' :{ - 'cli' : ['timeout', 'tcp', 'syn-sent'], - 'test_value' : '300', - 'default_value' : '120', - }, - 'net.netfilter.nf_conntrack_tcp_timeout_time_wait' :{ - 'cli' : ['timeout', 'tcp', 'time-wait'], - 'test_value' : '303', - 'default_value' : '120', - }, - 'net.netfilter.nf_conntrack_udp_timeout' :{ - 'cli' : ['timeout', 'udp', 'other'], - 'test_value' : '90', - 'default_value' : '30', - }, - 'net.netfilter.nf_conntrack_udp_timeout_stream' :{ - 'cli' : ['timeout', 'udp', 'stream'], - 'test_value' : '200', - 'default_value' : '180', - }, } for parameter, parameter_config in conntrack_config.items(): diff --git a/src/conf_mode/firewall.py b/src/conf_mode/firewall.py index acf3805d2cb..4c289b921ca 100755 --- a/src/conf_mode/firewall.py +++ b/src/conf_mode/firewall.py @@ -33,6 +33,7 @@ from vyos.utils.dict import dict_search_args from vyos.utils.dict import dict_search_recursive from vyos.utils.process import call +from vyos.utils.process import cmd from vyos.utils.process import rc_cmd from vyos import ConfigError from vyos import airbag @@ -40,20 +41,7 @@ airbag.enable() nftables_conf = '/run/nftables.conf' - -sysfs_config = { - 'all_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_all', 'enable': '0', 'disable': '1'}, - 'broadcast_ping': {'sysfs': '/proc/sys/net/ipv4/icmp_echo_ignore_broadcasts', 'enable': '0', 'disable': '1'}, - 'directed_broadcast' : {'sysfs': '/proc/sys/net/ipv4/conf/all/bc_forwarding', 'enable': '1', 'disable': '0'}, - 'ip_src_route': {'sysfs': '/proc/sys/net/ipv4/conf/*/accept_source_route'}, - 'ipv6_receive_redirects': {'sysfs': '/proc/sys/net/ipv6/conf/*/accept_redirects'}, - 'ipv6_src_route': {'sysfs': '/proc/sys/net/ipv6/conf/*/accept_source_route', 'enable': '0', 'disable': '-1'}, - 'log_martians': {'sysfs': '/proc/sys/net/ipv4/conf/all/log_martians'}, - 'receive_redirects': {'sysfs': '/proc/sys/net/ipv4/conf/*/accept_redirects'}, - 'send_redirects': {'sysfs': '/proc/sys/net/ipv4/conf/*/send_redirects'}, - 'syn_cookies': {'sysfs': '/proc/sys/net/ipv4/tcp_syncookies'}, - 'twa_hazards_protection': {'sysfs': '/proc/sys/net/ipv4/tcp_rfc1337'} -} +sysctl_file = r'/run/sysctl/10-vyos-firewall.conf' valid_groups = [ 'address_group', @@ -467,33 +455,16 @@ def generate(firewall): local_zone_conf['from_local'][zone] = zone_conf['from'][local_zone] render(nftables_conf, 'firewall/nftables.j2', firewall) + render(sysctl_file, 'firewall/sysctl-firewall.conf.j2', firewall) return None -def apply_sysfs(firewall): - for name, conf in sysfs_config.items(): - paths = glob(conf['sysfs']) - value = None - - if name in firewall['global_options']: - conf_value = firewall['global_options'][name] - if conf_value in conf: - value = conf[conf_value] - elif conf_value == 'enable': - value = '1' - elif conf_value == 'disable': - value = '0' - - if value: - for path in paths: - with open(path, 'w') as f: - f.write(value) - def apply(firewall): install_result, output = rc_cmd(f'nft --file {nftables_conf}') if install_result == 1: raise ConfigError(f'Failed to apply firewall: {output}') - apply_sysfs(firewall) + # Apply firewall global-options sysctl settings + cmd(f'sysctl -f {sysctl_file}') call_dependents() diff --git a/src/conf_mode/system_conntrack.py b/src/conf_mode/system_conntrack.py index d9c38fd956d..aa290788cb3 100755 --- a/src/conf_mode/system_conntrack.py +++ b/src/conf_mode/system_conntrack.py @@ -166,7 +166,7 @@ def verify(conntrack): if not group_obj: Warning(f'{error_group} "{group_name}" has no members!') - Warning(f'It is prefered to defined {inet} conntrack ignore rules in the section') + Warning(f'It is prefered to define {inet} conntrack ignore rules in section') if dict_search_args(conntrack, 'timeout', 'custom', inet, 'rule') != None: for rule, rule_config in conntrack['timeout']['custom'][inet]['rule'].items(): diff --git a/src/migration-scripts/firewall/15-to-16 b/src/migration-scripts/firewall/15-to-16 new file mode 100755 index 00000000000..7c8d38fe6da --- /dev/null +++ b/src/migration-scripts/firewall/15-to-16 @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2022-2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +# T6394: Migrate conntrack timeout options to firewall global-options + # from: set system conntrack timeout .. + # to: set firewall global-options timeout ... + +from sys import argv +from sys import exit + +from vyos.configtree import ConfigTree + +if len(argv) < 2: + print("Must specify file name!") + exit(1) + +file_name = argv[1] + +with open(file_name, 'r') as f: + config_file = f.read() + +firewall_base = ['firewall', 'global-options'] +conntrack_base = ['system', 'conntrack', 'timeout'] +config = ConfigTree(config_file) + +if not config.exists(conntrack_base): + # Nothing to do + exit(0) + +for protocol in ['icmp', 'tcp', 'udp', 'other']: + if config.exists(conntrack_base + [protocol]): + if not config.exists(firewall_base): + config.set(firewall_base + ['timeout']) + config.copy(conntrack_base + [protocol], firewall_base + ['timeout', protocol]) + config.delete(conntrack_base + [protocol]) + +try: + with open(file_name, 'w') as f: + f.write(config.to_string()) +except OSError as e: + print("Failed to save the modified config: {}".format(e)) + exit(1) \ No newline at end of file From 3d14676bf9b6dcef77ec5587447015b3d58e194a Mon Sep 17 00:00:00 2001 From: fett0 Date: Tue, 4 Jun 2024 15:14:33 +0000 Subject: [PATCH 179/188] ISIS: T6332: add smoketest option --- smoketest/scripts/cli/test_protocols_isis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/smoketest/scripts/cli/test_protocols_isis.py b/smoketest/scripts/cli/test_protocols_isis.py index 9c57f202014..769f3dd33a4 100755 --- a/smoketest/scripts/cli/test_protocols_isis.py +++ b/smoketest/scripts/cli/test_protocols_isis.py @@ -60,6 +60,7 @@ def test_isis_01_redistribute(self): prefix_list = 'EXPORT-ISIS' route_map = 'EXPORT-ISIS' rule = '10' + metric_style = 'transition' self.cli_set(['policy', 'prefix-list', prefix_list, 'rule', rule, 'action', 'permit']) self.cli_set(['policy', 'prefix-list', prefix_list, 'rule', rule, 'prefix', '203.0.113.0/24']) @@ -80,6 +81,7 @@ def test_isis_01_redistribute(self): self.cli_commit() self.cli_set(base_path + ['redistribute', 'ipv4', 'connected', 'level-2', 'route-map', route_map]) + self.cli_set(base_path + ['metric-style', metric_style]) self.cli_set(base_path + ['log-adjacency-changes']) # Commit all changes @@ -88,6 +90,7 @@ def test_isis_01_redistribute(self): # Verify all changes tmp = self.getFRRconfig(f'router isis {domain}', daemon='isisd') self.assertIn(f' net {net}', tmp) + self.assertIn(f' metric-style {metric_style}', tmp) self.assertIn(f' log-adjacency-changes', tmp) self.assertIn(f' redistribute ipv4 connected level-2 route-map {route_map}', tmp) @@ -401,7 +404,6 @@ def test_isis_10_topology(self): # Set a basic IS-IS config self.cli_set(base_path + ['net', net]) - self.cli_set(base_path + ['interface', interface]) for topology in topologies: self.cli_set(base_path + ['topology', topology]) From 02658cc2b466cb76835ad79b477c76fadc0c0cc2 Mon Sep 17 00:00:00 2001 From: Daniil Baturin Date: Wed, 5 Jun 2024 15:55:22 +0100 Subject: [PATCH 180/188] show version: T6446: display the support URL for LTS builds --- data/templates/login/default_motd.j2 | 3 +++ 1 file changed, 3 insertions(+) diff --git a/data/templates/login/default_motd.j2 b/data/templates/login/default_motd.j2 index 543c6f8e031..0d52092f8e4 100644 --- a/data/templates/login/default_motd.j2 +++ b/data/templates/login/default_motd.j2 @@ -4,6 +4,9 @@ Welcome to VyOS! . VyOS {{ version_data.version }} └ ──┘ {{ version_data.release_train }} +{% if version_data.lts_build %} + * Support portal: {{ version_data.support_url }} +{% endif %} * Documentation: {{ version_data.documentation_url }} * Project news: {{ version_data.project_news_url }} * Bug reports: {{ version_data.bugtracker_url }} From c71f6cb19de3400462fde1b745221f9355c393c0 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 5 Jun 2024 12:36:21 -0500 Subject: [PATCH 181/188] migration: T6006: update config.boot.default and move to vyos-1x --- data/config.boot.default | 53 ++++++++++++++++++++++++++++++++++++++++ python/vyos/defaults.py | 2 ++ src/init/vyos-router | 12 +++++++-- 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 data/config.boot.default diff --git a/data/config.boot.default b/data/config.boot.default new file mode 100644 index 00000000000..93369d9b7d3 --- /dev/null +++ b/data/config.boot.default @@ -0,0 +1,53 @@ +interfaces { + loopback lo { + } +} +service { + ntp { + allow-client { + address "127.0.0.0/8" + address "169.254.0.0/16" + address "10.0.0.0/8" + address "172.16.0.0/12" + address "192.168.0.0/16" + address "::1/128" + address "fe80::/10" + address "fc00::/7" + } + server time1.vyos.net { + } + server time2.vyos.net { + } + server time3.vyos.net { + } + } +} +system { + config-management { + commit-revisions "100" + } + console { + device ttyS0 { + speed "115200" + } + } + host-name "vyos" + login { + user vyos { + authentication { + encrypted-password "$6$QxPS.uk6mfo$9QBSo8u1FkH16gMyAVhus6fU3LOzvLR9Z9.82m3tiHFAxTtIkhaZSWssSgzt4v4dGAL8rhVQxTg0oAG9/q11h/" + plaintext-password "" + } + } + } + syslog { + global { + facility all { + level "info" + } + facility local7 { + level "debug" + } + } + } +} diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 64145a42ec2..7b0f7d9d906 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -46,3 +46,5 @@ commit_lock = '/opt/vyatta/config/.lock' component_version_json = os.path.join(directories['data'], 'component-versions.json') + +config_default = os.path.join(directories['data'], 'config.boot.default') diff --git a/src/init/vyos-router b/src/init/vyos-router index 15e37df07cb..a88810f084c 100755 --- a/src/init/vyos-router +++ b/src/init/vyos-router @@ -22,6 +22,7 @@ declare progname=${0##*/} declare action=$1; shift declare -x BOOTFILE=$vyatta_sysconfdir/config/config.boot +declare -x DEFAULT_BOOTFILE=$vyatta_sysconfdir/config.boot.default # If vyos-config= boot option is present, use that file instead for x in $(cat /proc/cmdline); do @@ -129,9 +130,16 @@ unmount_encrypted_config() { # if necessary, provide initial config init_bootfile () { + # define and version default boot config if not present + if [ ! -r $DEFAULT_BOOTFILE ]; then + if [ -f $vyos_data_dir/config.boot.default ]; then + cp $vyos_data_dir/config.boot.default $DEFAULT_BOOTFILE + $vyos_libexec_dir/system-versions-foot.py >> $DEFAULT_BOOTFILE + fi + fi if [ ! -r $BOOTFILE ] ; then - if [ -f $vyatta_sysconfdir/config.boot.default ]; then - cp $vyatta_sysconfdir/config.boot.default $BOOTFILE + if [ -f $DEFAULT_BOOTFILE ]; then + cp $DEFAULT_BOOTFILE $BOOTFILE else $vyos_libexec_dir/system-versions-foot.py > $BOOTFILE fi From 4f18517c9b0a912582933ddbc5424759c676c290 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 5 Jun 2024 19:53:11 -0500 Subject: [PATCH 182/188] migration: T6447: add module compose_config --- python/vyos/compose_config.py | 84 +++++++++++++++++++++++++++++++++++ python/vyos/configtree.py | 10 ++++- 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 python/vyos/compose_config.py diff --git a/python/vyos/compose_config.py b/python/vyos/compose_config.py new file mode 100644 index 00000000000..efa28babe32 --- /dev/null +++ b/python/vyos/compose_config.py @@ -0,0 +1,84 @@ +# Copyright 2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +"""This module allows iterating over function calls to modify an existing +config. +""" + +from pathlib import Path +from typing import TypeAlias, Union, Callable + +from vyos.configtree import ConfigTree +from vyos.configtree import deep_copy as ct_deep_copy +from vyos.utils.system import load_as_module + +ConfigObj: TypeAlias = Union[str, ConfigTree] + +class ComposeConfigError(Exception): + """Raised when an error occurs modifying a config object. + """ + +class ComposeConfig: + """Apply function to config tree: for iteration over functions or files. + """ + def __init__(self, config_obj: ConfigObj, checkpoint_file=None): + if isinstance(config_obj, ConfigTree): + self.config_tree = config_obj + else: + self.config_tree = ConfigTree(config_obj) + + self.checkpoint = self.config_tree + self.checkpoint_file = checkpoint_file + + def apply_func(self, func: Callable): + """Apply the function to the config tree. + """ + if not callable(func): + raise ComposeConfigError(f'{func.__name__} is not callable') + + if self.checkpoint_file is not None: + self.checkpoint = ct_deep_copy(self.config_tree) + + try: + func(self.config_tree) + except Exception as e: + self.config_tree = self.checkpoint + raise ComposeConfigError(e) from e + + def apply_file(self, func_file: str, func_name: str): + """Apply named function from file. + """ + try: + mod_name = Path(func_file).stem.replace('-', '_') + mod = load_as_module(mod_name, func_file) + func = getattr(mod, func_name) + except Exception as e: + raise ComposeConfigError(f'Error with {func_file}: {e}') from e + + try: + self.apply_func(func) + except ComposeConfigError as e: + raise ComposeConfigError(f'Error in {func_file}: {e}') from e + + def to_string(self, with_version=False) -> str: + """Return the rendered config tree. + """ + return self.config_tree.to_string(no_version=not with_version) + + def write(self, config_file: str, with_version=False): + """Write the config tree to a file. + """ + config_str = self.to_string(with_version=with_version) + Path(config_file).write_text(config_str) diff --git a/python/vyos/configtree.py b/python/vyos/configtree.py index e4b282d72e4..afd6e030bbb 100644 --- a/python/vyos/configtree.py +++ b/python/vyos/configtree.py @@ -175,9 +175,11 @@ def _get_config(self): def get_version_string(self): return self.__version - def to_string(self, ordered_values=False): + def to_string(self, ordered_values=False, no_version=False): config_string = self.__to_string(self.__config, ordered_values).decode() config_string = unescape_backslash(config_string) + if no_version: + return config_string config_string = "{0}\n{1}".format(config_string, self.__version) return config_string @@ -482,3 +484,9 @@ def to_commands(self): add = self.add.to_commands() delete = self.delete.to_commands(op="delete") return delete + "\n" + add + +def deep_copy(config_tree: ConfigTree) -> ConfigTree: + """An inelegant, but reasonably fast, copy; replace with backend copy + """ + D = DiffTree(None, config_tree) + return D.add From d34edc5c15f939f87648b0f8588f99474c53b459 Mon Sep 17 00:00:00 2001 From: John Estabrook Date: Wed, 5 Jun 2024 19:57:18 -0500 Subject: [PATCH 183/188] migration: T6006: add activation script dir and helper function --- debian/rules | 5 + debian/vyos-1x.install | 1 + python/vyos/defaults.py | 1 + src/activation-scripts/20-ethernet_offload.py | 103 ++++++++++++++++++ src/helpers/run-config-activation.py | 83 ++++++++++++++ src/init/vyos-router | 11 ++ 6 files changed, 204 insertions(+) create mode 100755 src/activation-scripts/20-ethernet_offload.py create mode 100755 src/helpers/run-config-activation.py diff --git a/debian/rules b/debian/rules index d007089a42e..9da40465f11 100755 --- a/debian/rules +++ b/debian/rules @@ -11,6 +11,7 @@ VYOS_MIBS_DIR := usr/share/snmp/mibs VYOS_LOCALUI_DIR := srv/localui MIGRATION_SCRIPTS_DIR := opt/vyatta/etc/config-migrate/migrate +ACTIVATION_SCRIPTS_DIR := usr/libexec/vyos/activate SYSTEM_SCRIPTS_DIR := usr/libexec/vyos/system SERVICES_DIR := usr/libexec/vyos/services @@ -67,6 +68,10 @@ override_dh_auto_install: mkdir -p $(DIR)/$(MIGRATION_SCRIPTS_DIR) cp -r src/migration-scripts/* $(DIR)/$(MIGRATION_SCRIPTS_DIR) + # Install activation scripts + mkdir -p $(DIR)/$(ACTIVATION_SCRIPTS_DIR) + cp -r src/activation-scripts/* $(DIR)/$(ACTIVATION_SCRIPTS_DIR) + # Install system scripts mkdir -p $(DIR)/$(SYSTEM_SCRIPTS_DIR) cp -r src/system/* $(DIR)/$(SYSTEM_SCRIPTS_DIR) diff --git a/debian/vyos-1x.install b/debian/vyos-1x.install index 9e43669bed2..b3978d38aff 100644 --- a/debian/vyos-1x.install +++ b/debian/vyos-1x.install @@ -28,6 +28,7 @@ usr/bin/vyos-config-to-commands usr/bin/vyos-config-to-json usr/bin/vyos-hostsd-client usr/lib +usr/libexec/vyos/activate usr/libexec/vyos/completion usr/libexec/vyos/conf_mode usr/libexec/vyos/init diff --git a/python/vyos/defaults.py b/python/vyos/defaults.py index 7b0f7d9d906..e7cd69a8b96 100644 --- a/python/vyos/defaults.py +++ b/python/vyos/defaults.py @@ -25,6 +25,7 @@ 'services' : f'{base_dir}/services', 'config' : '/opt/vyatta/etc/config', 'migrate' : '/opt/vyatta/etc/config-migrate/migrate', + 'activate' : f'{base_dir}/activate', 'log' : '/var/log/vyatta', 'templates' : '/usr/share/vyos/templates/', 'certbot' : '/config/auth/letsencrypt', diff --git a/src/activation-scripts/20-ethernet_offload.py b/src/activation-scripts/20-ethernet_offload.py new file mode 100755 index 00000000000..33b0ea469c3 --- /dev/null +++ b/src/activation-scripts/20-ethernet_offload.py @@ -0,0 +1,103 @@ +# Copyright 2021-2024 VyOS maintainers and contributors +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this library. If not, see . + +# T3619: mirror Linux Kernel defaults for ethernet offloading options into VyOS +# CLI. See https://vyos.dev/T3619#102254 for all the details. +# T3787: Remove deprecated UDP fragmentation offloading option +# T6006: add to activation-scripts: migration-scripts/interfaces/20-to-21 + +from vyos.ethtool import Ethtool +from vyos.configtree import ConfigTree + +def activate(config: ConfigTree): + base = ['interfaces', 'ethernet'] + + if not config.exists(base): + return + + for ifname in config.list_nodes(base): + eth = Ethtool(ifname) + + # If GRO is enabled by the Kernel - we reflect this on the CLI. If GRO is + # enabled via CLI but not supported by the NIC - we remove it from the CLI + configured = config.exists(base + [ifname, 'offload', 'gro']) + enabled, fixed = eth.get_generic_receive_offload() + if configured and fixed: + config.delete(base + [ifname, 'offload', 'gro']) + elif enabled and not fixed: + config.set(base + [ifname, 'offload', 'gro']) + + # If GSO is enabled by the Kernel - we reflect this on the CLI. If GSO is + # enabled via CLI but not supported by the NIC - we remove it from the CLI + configured = config.exists(base + [ifname, 'offload', 'gso']) + enabled, fixed = eth.get_generic_segmentation_offload() + if configured and fixed: + config.delete(base + [ifname, 'offload', 'gso']) + elif enabled and not fixed: + config.set(base + [ifname, 'offload', 'gso']) + + # If LRO is enabled by the Kernel - we reflect this on the CLI. If LRO is + # enabled via CLI but not supported by the NIC - we remove it from the CLI + configured = config.exists(base + [ifname, 'offload', 'lro']) + enabled, fixed = eth.get_large_receive_offload() + if configured and fixed: + config.delete(base + [ifname, 'offload', 'lro']) + elif enabled and not fixed: + config.set(base + [ifname, 'offload', 'lro']) + + # If SG is enabled by the Kernel - we reflect this on the CLI. If SG is + # enabled via CLI but not supported by the NIC - we remove it from the CLI + configured = config.exists(base + [ifname, 'offload', 'sg']) + enabled, fixed = eth.get_scatter_gather() + if configured and fixed: + config.delete(base + [ifname, 'offload', 'sg']) + elif enabled and not fixed: + config.set(base + [ifname, 'offload', 'sg']) + + # If TSO is enabled by the Kernel - we reflect this on the CLI. If TSO is + # enabled via CLI but not supported by the NIC - we remove it from the CLI + configured = config.exists(base + [ifname, 'offload', 'tso']) + enabled, fixed = eth.get_tcp_segmentation_offload() + if configured and fixed: + config.delete(base + [ifname, 'offload', 'tso']) + elif enabled and not fixed: + config.set(base + [ifname, 'offload', 'tso']) + + # Remove deprecated UDP fragmentation offloading option + if config.exists(base + [ifname, 'offload', 'ufo']): + config.delete(base + [ifname, 'offload', 'ufo']) + + # Also while processing the interface configuration, not all adapters support + # changing the speed and duplex settings. If the desired speed and duplex + # values do not work for the NIC driver, we change them back to the default + # value of "auto" - which will be applied if the CLI node is deleted. + speed_path = base + [ifname, 'speed'] + duplex_path = base + [ifname, 'duplex'] + # speed and duplex must always be set at the same time if not set to "auto" + if config.exists(speed_path) and config.exists(duplex_path): + speed = config.return_value(speed_path) + duplex = config.return_value(duplex_path) + if speed != 'auto' and duplex != 'auto': + if not eth.check_speed_duplex(speed, duplex): + config.delete(speed_path) + config.delete(duplex_path) + + # Also while processing the interface configuration, not all adapters support + # changing disabling flow-control - or change this setting. If disabling + # flow-control is not supported by the NIC, we remove the setting from CLI + flow_control_path = base + [ifname, 'disable-flow-control'] + if config.exists(flow_control_path): + if not eth.check_flow_control(): + config.delete(flow_control_path) diff --git a/src/helpers/run-config-activation.py b/src/helpers/run-config-activation.py new file mode 100755 index 00000000000..58293702add --- /dev/null +++ b/src/helpers/run-config-activation.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2024 VyOS maintainers and contributors +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License version 2 or later as +# published by the Free Software Foundation. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import re +import logging +from pathlib import Path +from argparse import ArgumentParser + +from vyos.compose_config import ComposeConfig +from vyos.compose_config import ComposeConfigError +from vyos.defaults import directories + +parser = ArgumentParser() +parser.add_argument('config_file', type=str, + help="configuration file to modify with system-specific settings") +parser.add_argument('--test-script', type=str, + help="test effect of named script") + +args = parser.parse_args() + +checkpoint_file = '/run/vyos-activate-checkpoint' +log_file = Path(directories['config']).joinpath('vyos-activate.log') + +logger = logging.getLogger(__name__) +fh = logging.FileHandler(log_file) +formatter = logging.Formatter('%(message)s') +fh.setFormatter(formatter) +logger.addHandler(fh) + +if 'vyos-activate-debug' in Path('/proc/cmdline').read_text(): + print(f'\nactivate-debug enabled: file {checkpoint_file}_* on error') + debug = checkpoint_file + logger.setLevel(logging.DEBUG) +else: + debug = None + logger.setLevel(logging.INFO) + +def sort_key(s: Path): + s = s.stem + pre, rem = re.match(r'(\d*)(?:-)?(.+)', s).groups() + return int(pre or 0), rem + +def file_ext(file_name: str) -> str: + """Return an identifier from file name for checkpoint file extension. + """ + return Path(file_name).stem + +script_dir = Path(directories['activate']) + +if args.test_script: + script_list = [script_dir.joinpath(args.test_script)] +else: + script_list = sorted(script_dir.glob('*.py'), key=sort_key) + +config_file = args.config_file +config_str = Path(config_file).read_text() + +compose = ComposeConfig(config_str, checkpoint_file=debug) + +for file in script_list: + file = file.as_posix() + logger.info(f'calling {file}') + try: + compose.apply_file(file, func_name='activate') + except ComposeConfigError as e: + if debug: + compose.write(f'{compose.checkpoint_file}_{file_ext(file)}') + logger.error(f'config-activation error in {file}: {e}') + +compose.write(config_file, with_version=True) diff --git a/src/init/vyos-router b/src/init/vyos-router index a88810f084c..59004fdc1da 100755 --- a/src/init/vyos-router +++ b/src/init/vyos-router @@ -157,6 +157,15 @@ migrate_bootfile () fi } +# configure system-specific settings +system_config () +{ + if [ -x $vyos_libexec_dir/run-config-activation.py ]; then + log_progress_msg system + sg ${GROUP} -c "$vyos_libexec_dir/run-config-activation.py $BOOTFILE" + fi +} + # load the initial config load_bootfile () { @@ -501,6 +510,8 @@ start () update_interface_config + disabled system_config || system_config + for s in ${subinit[@]} ; do if ! disabled $s; then log_progress_msg $s From 7a0d80e11326622021e1b282df0d3aac0b233ba1 Mon Sep 17 00:00:00 2001 From: Viacheslav Hletenko Date: Thu, 6 Jun 2024 18:24:01 +0300 Subject: [PATCH 184/188] T6412: CGNAT fix allocation calcluation for verify (#3585) Fix external address/port allocation for CGN. It fixes some cases where external address/ports can be allocated again to another user. --- src/conf_mode/nat_cgnat.py | 71 ++++++++++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 15 deletions(-) diff --git a/src/conf_mode/nat_cgnat.py b/src/conf_mode/nat_cgnat.py index 957b12c28d3..d429f6e2142 100755 --- a/src/conf_mode/nat_cgnat.py +++ b/src/conf_mode/nat_cgnat.py @@ -111,7 +111,19 @@ def generate_port_rules( port_count: int, global_port_range: str = '1024-65535', ) -> list: - """Generates list of nftables rules for the batch file.""" + """Generates a list of nftables option rules for the batch file. + + Args: + external_hosts (list): A list of external host IPs. + internal_hosts (list): A list of internal host IPs. + port_count (int): The number of ports required per host. + global_port_range (str): The global port range to be used. Default is '1024-65535'. + + Returns: + list: A list containing two elements: + - proto_map_elements (list): A list of proto map elements. + - other_map_elements (list): A list of other map elements. + """ rules = [] proto_map_elements = [] other_map_elements = [] @@ -120,13 +132,6 @@ def generate_port_rules( # Calculate the required number of ports per host required_ports_per_host = port_count - - # Check if there are enough external addresses for all internal hosts - if required_ports_per_host * len(internal_hosts) > total_possible_ports * len( - external_hosts - ): - raise ConfigError("Not enough ports available for the specified parameters!") - current_port = start_port current_external_index = 0 @@ -141,13 +146,6 @@ def generate_port_rules( current_port = start_port next_end_port = current_port + required_ports_per_host - 1 - # Ensure the same port is not assigned to the same external host - if any( - rule.endswith(f'{external_host}:{current_port}-{next_end_port}') - for rule in rules - ): - raise ConfigError("Not enough ports available for the specified parameters") - proto_map_elements.append( f'{internal_host} : {external_host} . {current_port}-{next_end_port}' ) @@ -240,6 +238,49 @@ def verify(config): used_external_pools[external_pool] = rule used_internal_pools[internal_pool] = rule + # Check calculation for allocation + external_port_range: str = config['pool']['external'][external_pool]['external_port_range'] + + external_ip_ranges: list = list( + config['pool']['external'][external_pool]['range'] + ) + internal_ip_ranges: list = config['pool']['internal'][internal_pool]['range'] + start_port, end_port = map(int, external_port_range.split('-')) + ports_per_range_count: int = (end_port - start_port) + 1 + + external_list_hosts_count = [] + external_list_hosts = [] + internal_list_hosts_count = [] + internal_list_hosts = [] + for ext_range in external_ip_ranges: + # External hosts count + e_count = IPOperations(ext_range).get_ips_count() + external_list_hosts_count.append(e_count) + # External hosts list + e_hosts = IPOperations(ext_range).convert_prefix_to_list_ips() + external_list_hosts.extend(e_hosts) + for int_range in internal_ip_ranges: + # Internal hosts count + i_count = IPOperations(int_range).get_ips_count() + internal_list_hosts_count.append(i_count) + # Internal hosts list + i_hosts = IPOperations(int_range).convert_prefix_to_list_ips() + internal_list_hosts.extend(i_hosts) + + external_host_count = sum(external_list_hosts_count) + internal_host_count = sum(internal_list_hosts_count) + ports_per_user: int = int( + config['pool']['external'][external_pool]['per_user_limit']['port'] + ) + users_per_extip = ports_per_range_count // ports_per_user + max_users = users_per_extip * external_host_count + + if internal_host_count > max_users: + raise ConfigError( + f'Rule "{rule}" does not have enough ports available for the ' + f'specified parameters' + ) + def generate(config): if not config: From 61f8250184e927de9ab6bddc207b917bef7da42b Mon Sep 17 00:00:00 2001 From: Nataliia Solomko Date: Thu, 6 Jun 2024 13:55:25 +0300 Subject: [PATCH 185/188] xml: T6423: enforce priority on nodes having an owner --- .../load-balancing_reverse-proxy.xml.in | 1 + interface-definitions/load-balancing_wan.xml.in | 2 +- interface-definitions/protocols_static_arp.xml.in | 1 + .../protocols_static_multicast.xml.in | 1 + .../protocols_static_neighbor-proxy.xml.in | 1 + interface-definitions/service_aws_glb.xml.in | 2 +- interface-definitions/service_config-sync.xml.in | 1 + interface-definitions/service_console-server.xml.in | 1 + interface-definitions/service_event-handler.xml.in | 1 + .../service_monitoring_telegraf.xml.in | 2 +- .../service_monitoring_zabbix-agent.xml.in | 1 + interface-definitions/service_sla.xml.in | 1 + interface-definitions/system_login_banner.xml.in | 1 + interface-definitions/system_proxy.xml.in | 1 + scripts/build-command-templates | 12 +++++++++++- 15 files changed, 25 insertions(+), 4 deletions(-) diff --git a/interface-definitions/load-balancing_reverse-proxy.xml.in b/interface-definitions/load-balancing_reverse-proxy.xml.in index e50e6e579c4..09fa9eb2964 100644 --- a/interface-definitions/load-balancing_reverse-proxy.xml.in +++ b/interface-definitions/load-balancing_reverse-proxy.xml.in @@ -5,6 +5,7 @@ Configure reverse-proxy + 900 diff --git a/interface-definitions/load-balancing_wan.xml.in b/interface-definitions/load-balancing_wan.xml.in index e117fd1b277..310aa034381 100644 --- a/interface-definitions/load-balancing_wan.xml.in +++ b/interface-definitions/load-balancing_wan.xml.in @@ -3,12 +3,12 @@ Configure load-balancing - 900 Configure Wide Area Network (WAN) load-balancing + 900 diff --git a/interface-definitions/protocols_static_arp.xml.in b/interface-definitions/protocols_static_arp.xml.in index 05c69f1ed42..0c5d6e4ed8b 100644 --- a/interface-definitions/protocols_static_arp.xml.in +++ b/interface-definitions/protocols_static_arp.xml.in @@ -7,6 +7,7 @@ Static ARP translation + 481 diff --git a/interface-definitions/protocols_static_multicast.xml.in b/interface-definitions/protocols_static_multicast.xml.in index c8e28ed352f..caf95ed7c21 100644 --- a/interface-definitions/protocols_static_multicast.xml.in +++ b/interface-definitions/protocols_static_multicast.xml.in @@ -7,6 +7,7 @@ Multicast static route + 481 diff --git a/interface-definitions/protocols_static_neighbor-proxy.xml.in b/interface-definitions/protocols_static_neighbor-proxy.xml.in index 1c8433a39a5..7347976f99f 100644 --- a/interface-definitions/protocols_static_neighbor-proxy.xml.in +++ b/interface-definitions/protocols_static_neighbor-proxy.xml.in @@ -7,6 +7,7 @@ Neighbor proxy parameters + 481 diff --git a/interface-definitions/service_aws_glb.xml.in b/interface-definitions/service_aws_glb.xml.in index c749fd04eb3..71de1f03ac5 100644 --- a/interface-definitions/service_aws_glb.xml.in +++ b/interface-definitions/service_aws_glb.xml.in @@ -5,12 +5,12 @@ Amazon Web Service - 1280 Gateway load-balancer tunnel handler + 1280 diff --git a/interface-definitions/service_config-sync.xml.in b/interface-definitions/service_config-sync.xml.in index 648c14aeebd..af4e8ed51ca 100644 --- a/interface-definitions/service_config-sync.xml.in +++ b/interface-definitions/service_config-sync.xml.in @@ -5,6 +5,7 @@ Configuration synchronization + 10000 diff --git a/interface-definitions/service_console-server.xml.in b/interface-definitions/service_console-server.xml.in index fc6dbe954d4..68835dafd48 100644 --- a/interface-definitions/service_console-server.xml.in +++ b/interface-definitions/service_console-server.xml.in @@ -5,6 +5,7 @@ Serial Console Server + 2 diff --git a/interface-definitions/service_event-handler.xml.in b/interface-definitions/service_event-handler.xml.in index 2cee4f59541..41540816bec 100644 --- a/interface-definitions/service_event-handler.xml.in +++ b/interface-definitions/service_event-handler.xml.in @@ -5,6 +5,7 @@ Service event handler + 2 diff --git a/interface-definitions/service_monitoring_telegraf.xml.in b/interface-definitions/service_monitoring_telegraf.xml.in index 4d694114a1a..2624023ea64 100644 --- a/interface-definitions/service_monitoring_telegraf.xml.in +++ b/interface-definitions/service_monitoring_telegraf.xml.in @@ -5,12 +5,12 @@ Monitoring services - 1280 Telegraf metric collector + 1280 diff --git a/interface-definitions/service_monitoring_zabbix-agent.xml.in b/interface-definitions/service_monitoring_zabbix-agent.xml.in index 40f2df642c4..3754e914578 100644 --- a/interface-definitions/service_monitoring_zabbix-agent.xml.in +++ b/interface-definitions/service_monitoring_zabbix-agent.xml.in @@ -7,6 +7,7 @@ Zabbix-agent settings + 1280 diff --git a/interface-definitions/service_sla.xml.in b/interface-definitions/service_sla.xml.in index 0c4f8a591a1..2cd68195aea 100644 --- a/interface-definitions/service_sla.xml.in +++ b/interface-definitions/service_sla.xml.in @@ -5,6 +5,7 @@ Service level agreement (SLA) + 2 diff --git a/interface-definitions/system_login_banner.xml.in b/interface-definitions/system_login_banner.xml.in index 211505ae441..c90e38c3c5f 100644 --- a/interface-definitions/system_login_banner.xml.in +++ b/interface-definitions/system_login_banner.xml.in @@ -11,6 +11,7 @@ System login banners + 410 diff --git a/interface-definitions/system_proxy.xml.in b/interface-definitions/system_proxy.xml.in index 214534dbbea..5b0df5c7062 100644 --- a/interface-definitions/system_proxy.xml.in +++ b/interface-definitions/system_proxy.xml.in @@ -5,6 +5,7 @@ Sets a proxy for system wide use + 100 diff --git a/scripts/build-command-templates b/scripts/build-command-templates index 2e7f8b99454..36929abb236 100755 --- a/scripts/build-command-templates +++ b/scripts/build-command-templates @@ -287,6 +287,12 @@ def process_node(n, tmpl_dir): props = get_properties(props_elem, n.find("defaultValue")) if owner: props["owner"] = owner + # tag is mandatory if the parent node has an owner + if "priority" not in props: + raise ValueError( + f" tag should be set for the node <{name}> path '{' '.join(my_tmpl_dir[1:])}'" + ) + # Type should not be set for non-tag, non-leaf nodes # For non-valueless leaf nodes, set the type to txt: to make them have some type, # actual value validation is handled by constraints translated to syntax:expression: @@ -335,4 +341,8 @@ nodes = root.iterfind("*") for n in nodes: if n.tag == "syntaxVersion": continue - process_node(n, [output_dir]) + try: + process_node(n, [output_dir]) + except ValueError as e: + print(e) + sys.exit(1) From d3acecdf129cd940f8b2d1b229a6e2a343cab74b Mon Sep 17 00:00:00 2001 From: zsdc Date: Thu, 6 Jun 2024 23:04:51 +0300 Subject: [PATCH 186/188] grub: T6453: Fixed GRUB variables parsing To parse variables with `=` a variable name should be limited by alphanumerical characters only. --- python/vyos/system/grub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/vyos/system/grub.py b/python/vyos/system/grub.py index faf68c2d17b..daddb799a01 100644 --- a/python/vyos/system/grub.py +++ b/python/vyos/system/grub.py @@ -49,7 +49,7 @@ BOOT_OPTS_STEM: str = 'boot=live rootdelay=5 noautologin net.ifnames=0 biosdevname=0 vyos-union=/boot/' # prepare regexes -REGEX_GRUB_VARS: str = r'^set (?P.+)=[\'"]?(?P.*)(?\w+)=[\'"]?(?P.*)(?.+)$' REGEX_KERNEL_CMDLINE: str = r'^BOOT_IMAGE=/(?Pboot|live)/((?P.+)/)?vmlinuz.*$' REGEX_GRUB_BOOT_OPTS: str = r'^\s*set boot_opts="(?P[^$]+)"$' From 60d7c0ecaff49ec62f4600a460f5fbe7b26a0d9c Mon Sep 17 00:00:00 2001 From: Alex W Date: Fri, 7 Jun 2024 11:32:41 +0100 Subject: [PATCH 187/188] reverse-proxy: T6454: Set default value of http for haproxy mode --- data/templates/load-balancing/haproxy.cfg.j2 | 40 +++++++++---------- .../include/haproxy/mode.xml.i | 1 + src/conf_mode/load-balancing_reverse-proxy.py | 4 +- 3 files changed, 21 insertions(+), 24 deletions(-) diff --git a/data/templates/load-balancing/haproxy.cfg.j2 b/data/templates/load-balancing/haproxy.cfg.j2 index c6027e09b15..c18a998b896 100644 --- a/data/templates/load-balancing/haproxy.cfg.j2 +++ b/data/templates/load-balancing/haproxy.cfg.j2 @@ -67,25 +67,23 @@ frontend {{ front }} {% if front_config.redirect_http_to_https is vyos_defined %} http-request redirect scheme https unless { ssl_fc } {% endif %} -{% if front_config.mode is vyos_defined %} mode {{ front_config.mode }} -{% if front_config.tcp_request.inspect_delay is vyos_defined %} +{% if front_config.tcp_request.inspect_delay is vyos_defined %} tcp-request inspect-delay {{ front_config.tcp_request.inspect_delay }} -{% endif %} -{# add tcp-request related directive if ssl is configed #} -{% if front_config.mode is vyos_defined('tcp') and front_config.rule is vyos_defined %} -{% for rule, rule_config in front_config.rule.items() %} -{% if rule_config.ssl is vyos_defined %} +{% endif %} +{# add tcp-request related directive if ssl is configured #} +{% if front_config.mode == 'tcp' and front_config.rule is vyos_defined %} +{% for rule, rule_config in front_config.rule.items() %} +{% if rule_config.ssl is vyos_defined %} tcp-request content accept if { req_ssl_hello_type 1 } -{% break %} -{% endif %} -{% endfor %} -{% endif %} -{% if front_config.http_response_headers is vyos_defined %} -{% for header, header_config in front_config.http_response_headers.items() %} +{% break %} +{% endif %} +{% endfor %} +{% endif %} +{% if front_config.http_response_headers is vyos_defined %} +{% for header, header_config in front_config.http_response_headers.items() %} http-response set-header {{ header }} '{{ header_config['value'] }}' -{% endfor %} -{% endif %} +{% endfor %} {% endif %} {% if front_config.rule is vyos_defined %} {% for rule, rule_config in front_config.rule.items() %} @@ -162,19 +160,17 @@ backend {{ back }} {% set balance_translate = {'least-connection': 'leastconn', 'round-robin': 'roundrobin', 'source-address': 'source'} %} balance {{ balance_translate[back_config.balance] }} {% endif %} -{# If mode is not TCP skip Forwarded #} -{% if back_config.mode is not vyos_defined('tcp') %} +{# If mode is HTTP add X-Forwarded headers #} +{% if back_config.mode == 'http' %} option forwardfor http-request set-header X-Forwarded-Port %[dst_port] http-request add-header X-Forwarded-Proto https if { ssl_fc } {% endif %} -{% if back_config.mode is vyos_defined %} mode {{ back_config.mode }} -{% if back_config.http_response_headers is vyos_defined %} -{% for header, header_config in back_config.http_response_headers.items() %} +{% if back_config.http_response_headers is vyos_defined %} +{% for header, header_config in back_config.http_response_headers.items() %} http-response set-header {{ header }} '{{ header_config['value'] }}' -{% endfor %} -{% endif %} +{% endfor %} {% endif %} {% if back_config.rule is vyos_defined %} {% for rule, rule_config in back_config.rule.items() %} diff --git a/interface-definitions/include/haproxy/mode.xml.i b/interface-definitions/include/haproxy/mode.xml.i index 672ea65b4d2..d013e027d60 100644 --- a/interface-definitions/include/haproxy/mode.xml.i +++ b/interface-definitions/include/haproxy/mode.xml.i @@ -18,5 +18,6 @@ (http|tcp) + http diff --git a/src/conf_mode/load-balancing_reverse-proxy.py b/src/conf_mode/load-balancing_reverse-proxy.py index 09c68dadd40..17226efe9a4 100755 --- a/src/conf_mode/load-balancing_reverse-proxy.py +++ b/src/conf_mode/load-balancing_reverse-proxy.py @@ -85,7 +85,7 @@ def verify(lb): raise ConfigError(f'"expect status" and "expect string" can not be configured together!') if 'health_check' in back_config: - if 'mode' not in back_config or back_config['mode'] != 'tcp': + if back_config['mode'] != 'tcp': raise ConfigError(f'backend "{back}" can only be configured with {back_config["health_check"]} ' + f'health-check whilst in TCP mode!') if 'http_check' in back_config: @@ -108,7 +108,7 @@ def verify(lb): # Check if http-response-headers are configured in any frontend/backend where mode != http for group in ['service', 'backend']: for config_name, config in lb[group].items(): - if 'http_response_headers' in config and ('mode' not in config or config['mode'] != 'http'): + if 'http_response_headers' in config and config['mode'] != 'http': raise ConfigError(f'{group} {config_name} must be set to http mode to use http_response_headers!') for front, front_config in lb['service'].items(): From 395bd4eb850ff5763a82f29b1ff398c41e200f09 Mon Sep 17 00:00:00 2001 From: Vijayakumar A <36878324+kumvijaya@users.noreply.github.com> Date: Sun, 9 Jun 2024 14:54:24 +0530 Subject: [PATCH 188/188] T6449: added pr update trigger (#3596) --- .github/workflows/chceck-pr-message.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/chceck-pr-message.yml b/.github/workflows/chceck-pr-message.yml index e7e45696173..4606620147a 100644 --- a/.github/workflows/chceck-pr-message.yml +++ b/.github/workflows/chceck-pr-message.yml @@ -2,11 +2,12 @@ name: Check pull request message format on: - pull_request: + pull_request_target: branches: - current - crux - equuleus + types: [opened, synchronize, edited] permissions: pull-requests: write