WIP: Python pillar library

This imports the Python based pillar logic used in our Salt
infrastructures in the form of reusable libraries, helping us to
deduplicate code and making it easier to maintain changes.
The respective "#!py" pillars will then be adjusted to reference library
calls, with only the infrastructure specific data needing to be passed
as function parameters.

Signed-off-by: Georg Pfuetzenreuter <[email protected]>

infrastructure-formula/python/.gitignore

@@ -0,0 +1,4 @@
+*.egg-info
+__pycache__
+dist
+venv

infrastructure-formula/python/README.md

@@ -0,0 +1,3 @@
+# Python pillar helpers
+
+A Python library to be used in `#!py` pillar SLS files. It allows for rendering of formula pillars based off data in YAML datasetes.

infrastructure-formula/python/main/__init__.py

@@ -0,0 +1,18 @@
+"""
+Copyright (C) 2024 Georg Pfuetzenreuter <[email protected]>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+from .__version__ import __version__ as __version__

infrastructure-formula/python/main/__version__.py

@@ -0,0 +1,18 @@
+"""
+Copyright (C) 2024 Georg Pfuetzenreuter <[email protected]>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+__version__ = '1.0'

infrastructure-formula/python/pillar/__init__.py

@@ -0,0 +1,18 @@
+"""
+Copyright (C) 2024 Georg Pfuetzenreuter <[email protected]>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+from opensuse_infrastructure_formula.__version__ import __version__ as __version__

infrastructure-formula/python/pillar/infrastructure.py

@@ -0,0 +1,150 @@
+"""
+Copyright (C) 2024 Georg Pfuetzenreuter <[email protected]>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+from yaml import safe_load
+
+root = '/srv/salt-git/pillar'
+
+def generate_infrastructure_pillar(enabled_domains, log):
+    pillar = {
+        'infrastructure': {
+            'domains': {},
+        }
+    }
+
+    for domain in enabled_domains:
+        pillar['infrastructure']['domains'][domain] = {
+            'clusters': {},
+            'machines': {},
+        }
+
+        domainpillar = pillar['infrastructure']['domains'][domain]
+        domaindir = f'{root}/domain/'
+        mydomaindir = f'{domaindir}{domain.replace(".", "_")}'
+
+        msg = f'Parsing domain {domain}'
+        log(f'{msg} ...')
+
+        domaindata = {
+                'clusters': {},
+                'hosts': {},
+                'inherited_clusters': {},
+        }
+
+        for file in ['clusters', 'hosts']:
+            with open(f'{mydomaindir}/{file}.yaml') as fh:
+                domaindata[file] = safe_load(fh)
+
+        for cluster, clusterconfig in domaindata['clusters'].items():
+            log(f'{msg} => cluster {cluster} ...')
+
+            if 'delegate_to' in clusterconfig:
+                delegated_domain = clusterconfig['delegate_to']
+
+                with open(f'{domaindir}/{delegated_domain}/clusters.yaml') as fh:
+                    domaindata['inherited_clusters'].update({
+                        delegated_domain: safe_load(fh),
+                    })
+
+                if cluster in domaindata['inherited_clusters']:
+                    clusterconfig = domaindata['inherited_clusters'][cluster]
+                else:
+                    log(f'Delegation of cluster {cluster} to {delegated_domain} is not possible!')
+
+            clusterpillar = {
+                    'storage': clusterconfig['storage'],
+            }
+
+            if 'primary_node' in clusterconfig:
+                clusterpillar['primary'] = clusterconfig['primary_node']
+
+            if 'netapp' in clusterconfig:
+                clusterpillar.update({
+                    'netapp': clusterconfig['netapp'],
+                })
+
+            log(clusterpillar)
+            domainpillar['clusters'][cluster] = clusterpillar
+
+        for host, hostconfig in domaindata['hosts'].items():
+            log(f'{msg} => host {host} ...')
+
+            hostpillar = {
+                    'cluster': hostconfig['cluster'],
+                    'disks': hostconfig.get('disks', {}),
+                    'extra': {
+                        'legacy': hostconfig.get('legacy_boot', False),
+                    },
+                    'image': hostconfig.get('image', 'admin-minimal-latest'),
+                    'interfaces': {},
+                    'ram': hostconfig['ram'],
+                    'vcpu': hostconfig['vcpu'],
+            }
+
+            if 'node' in hostconfig:
+                node = hostconfig['node']
+
+                # the node key is compared against the hypervisor minion ID, which is always a FQDN in our infrastructure
+                if '.' in node:
+                    hostpillar['node'] = node
+                else:
+                    hostpillar['node'] = f'{node}.{domain}'
+
+            hostinterfaces = hostconfig.get('interfaces', {})
+
+            ip4 = hostconfig.get('ip4')
+            ip6 = hostconfig.get('ip6')
+
+            if not ip4 and not ip6 and hostinterfaces:
+                if 'primary_interface' in hostconfig:
+                    interface = hostconfig['primary_interface']
+                elif len(hostinterfaces) == 1:
+                    interface = next(iter(hostinterfaces))
+                else:
+                    interface = 'eth0'
+
+                if interface in hostinterfaces:
+                    ip4 = hostinterfaces[interface].get('ip4')
+                    ip6 = hostinterfaces[interface].get('ip6')
+
+            hostpillar['ip4'] = ip4
+            hostpillar['ip6'] = ip6
+
+            for interface, ifconfig in hostinterfaces.items():
+                iftype = ifconfig.get('type', 'direct')
+
+                ifpillar = {
+                        'mac': ifconfig['mac'],
+                        'type': iftype,
+                        'source': ifconfig['source'] if 'source' in ifconfig else f'x-{interface}',
+                }
+
+                if iftype == 'direct':
+                    ifpillar['mode'] = ifconfig.get('mode', 'bridge')
+
+                for i in [4, 6]:
+                    ipf = f'ip{i}'
+
+                    if ipf in ifconfig:
+                        ifpillar[ipf] = ifconfig[ipf]
+
+                hostpillar['interfaces'][interface] = ifpillar
+
+            log(hostpillar)
+            domainpillar['machines'][host] = hostpillar
+
+    return pillar

infrastructure-formula/python/pillar/juniper_junos.py

@@ -0,0 +1,175 @@
+"""
+Copyright (C) 2024 SUSE LLC <[email protected]>
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""
+
+from bisect import insort
+from ipaddress import ip_network
+from pathlib import PosixPath
+import yaml
+
+root = '/srv/salt-git/pillar'
+
+def generate_juniper_junos_pillar(enabled_domains, minion_id, spacemap, log_debug, log_error, log_warning):
+    minion = minion_id.replace('LAB-', '')
+    data = {'networks': {}, 'switching': {}}
+    config = {}
+    log_debug(f'Starting juniper_junos pillar construction ...')
+
+    minion_s = minion.split('-')
+    if len(minion_s) != 4:
+        log_error(f'Cannot parse minion ID')
+        return {}
+    space = minion_s[1].lower()
+    log_debug(f'Minion space set to "{space}"')
+
+    for domain in enabled_domains:
+        domain_space = domain.split('.')[0]
+        if domain_space in spacemap:
+            domain_space = spacemap[domain_space]
+        log_debug(f'Domain space set to "{domain_space}"')
+        domain = domain.replace('.', '_')
+
+        for dataset in data.keys():
+            log_debug(f'Scanning domain {domain}, dataset {dataset} ...')
+            file = f'{root}/domain/{domain}/{dataset}.yaml'
+
+            if PosixPath(file).is_file():
+                with open(file) as fh:
+
+                    if dataset == 'switching':
+                        log_debug('Updating data ...')
+                        data[dataset].update(yaml.safe_load(fh))
+
+                    elif dataset == 'networks':
+                        log_debug('Not updating data, scanning networks ...')
+                        for network, nwconfig in yaml.safe_load(fh).items():
+                            done = False
+
+                            for existing_network, existing_nwconfig in data[dataset].items():
+                                if network == existing_network or nwconfig.get('id') == existing_nwconfig.get('id'):
+                                    mynetwork = existing_network
+                                    log_debug(f'Mapping network {network} to existing network {mynetwork}')
+
+                                    if nwconfig.get('description') != data[dataset][mynetwork].get('description'):
+                                        log_warning(f'Conflicting descriptions in network {mynetwork}')
+                                    if nwconfig.get('id') != data[dataset][mynetwork].get('id'):
+                                        log_error(f'Conflicting ID: {network} != {mynetwork}, refusing to continue!')
+                                        return {}
+
+                                    if 'groups' not in data[dataset][mynetwork]:
+                                        data[dataset][mynetwork]['groups'] = []
+                                    for group in nwconfig.get('groups', []):
+                                        insort(data[dataset][mynetwork]['groups'], group)
+
+                                    done = True
+                                    break
+
+                            if not done:
+                                if space == domain_space:
+                                    log_debug(f'Creating new network {network}')
+                                    data[dataset][network] = nwconfig
+                                else:
+                                    log_debug(f'Ignoring network {network}')
+
+            else:
+                log_warning(f'File {file} does not exist.')
+
+    if minion in data['switching']:
+        config.update(data['switching'][minion])
+    else:
+        return {}
+
+    log_debug(f'Constructing juniper_junos pillar for {minion}')
+
+    vlids = []
+    groups = {}
+    for interface, ifconfig in config.get('interfaces', {}).items():
+        log_debug(f'Parsing interface {interface} ...')
+        for vlid in ifconfig.get('vlan', {}).get('ids', []):
+            if vlid not in vlids:
+                vlids.append(vlid)
+
+        group = None
+        if 'group' in ifconfig:
+            group = ifconfig['group']
+        elif 'addresses' in ifconfig:
+            group = '__lonely'
+        elif 'vlan' in ifconfig and 'all' in ifconfig['vlan'].get('ids', []):
+            group = '__all'
+        if group:
+            if group not in groups:
+                groups.update({group: {'interfaces': [], 'networks': []}})  # noqa 206
+            log_debug(f'Appending interface {interface} to group {group}')
+            groups[group]['interfaces'].append(interface)
+
+    group_names = groups.keys()
+
+    for network, nwconfig in data['networks'].items():
+        matching_groups = [group for group in nwconfig.get('groups', []) if group in group_names]
+
+        if nwconfig['id'] in vlids or any(matching_groups) or network.startswith(('ICCL_', 'ICCP_')):
+            log_debug(f'Adding network {network} to config ...')
+            if 'vlans' not in config:
+                config.update({'vlans': {}})
+            if network not in config['vlans']:
+                config['vlans'].update({network: {}})
+            config['vlans'][network].update({'id': nwconfig['id']})
+            if 'description' in nwconfig:
+                config['vlans'][network].update({'description': nwconfig['description']})
+            for group in matching_groups:
+                groups[group]['networks'].append(network)
+
+    for group, members in groups.items():
+        for interface in members['interfaces']:
+            ifconfig = config['interfaces'][interface]
+
+            unit = 0
+            if '.' in interface:
+                ifsplit = interface.split('.')
+                ifname = ifsplit[0]
+                ifsuffix = ifsplit[1]
+                if ifsuffix.isdigit():
+                    unit = int(ifsuffix)
+
+            if 'units' not in ifconfig:
+                ifconfig.update({'units': {}})
+            if unit not in ifconfig['units']:
+                ifconfig['units'].update({unit: {}})
+            if members['networks']:
+                ifconfig['units'][unit].update({'vlan': {'ids': [], 'type': ifconfig.get('vlan', {}).get('type', 'access')}})  # noqa 206
+            for network in members['networks']:
+                if config['vlans'][network]['id'] not in ifconfig['units'][unit]['vlan']['ids']:
+                    insort(ifconfig['units'][unit]['vlan']['ids'], config['vlans'][network]['id'])
+            if 'addresses' in ifconfig:
+                for address in ifconfig['addresses']:
+                    address_version = ip_network(address, False).version
+                    if address_version == 4:
+                        family = 'inet'
+                    elif address_version == 6:
+                        family = 'inet6'
+                    else:
+                        log_error(f'Illegal address: {address}')
+                    if family not in ifconfig['units'][unit]:
+                        ifconfig['units'][unit].update({family: {'addresses': []}})  # noqa 206
+                    ifconfig['units'][unit][family]['addresses'].append(address)
+                del ifconfig['addresses']
+            elif group == '__all':
+                ifconfig['units'][unit].update({'vlan': {'ids': ['all'], 'type': ifconfig.get('vlan', {}).get('type', 'trunk')}})  # noqa 206
+            if unit > 0:
+                config['interfaces'][ifname] = config['interfaces'].pop(interface)
+
+    log_debug(f'Returning juniper_junos pillar for {minion}: {config}')
+    return {'juniper_junos': config}