Configure bond0 interface

We want to use bonded interface pairs on these system. The nodes
aren't yet wired for it, but setting this up now will allow us to
refer to the `bond0` interface in e.g. VLAN configurations (and means
we won't have to re-work those later).

Because we're using OVNKubernetes, we can't use nmstate [1] to enact
the configuration. The recommendation is to apply the configuration
using a MachineConfig [2] resource, but this is complicated by the
fact that our nodes don't all have the same interface names, and it's
not possible to apply node-specific machineconfigs [3].

We work around this solution by:

1. Copying nmconnection files for *all hosts* to *every host*, but
  placing them in `/etc/mco` (just because that's a convenient
  available directory, it seems relatively topical, and it's not
  possible to create new directories using the `directories` section
  of an ignition config).

2. Installing a systemd unit that runs a shell script at boot that
  copies the host-specific configs from `/etc/mco` into
  `/etc/NetworkManager/system-connections`.

[1]: https://docs.openshift.com/container-platform/4.10/networking/k8s_nmstate/k8s-nmstate-about-the-k8s-nmstate-operator.html
[2]: https://docs.openshift.com/container-platform/4.10/post_installation_configuration/machine-configuration-tasks.html
[3]: openshift/machine-config-operator#1720

x-branch: feature/bond0

cluster-scope/overlays/nerc-ocp-infra/kustomization.yaml

@@ -8,6 +8,7 @@ resources:
 - ../../bundles/acm
 - ../../bundles/odf
 - clusterversion.yaml
+- machineconfigs/configure-bond0
 
 patches:
   - path: oauths/cluster_patch.yaml

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/Makefile

@@ -0,0 +1,10 @@
+MACHINECONFIGS = \
+	configure-bond0.yaml
+
+%.yaml: src/%.in.yaml
+	yq -y --argjson config "$$(yq .spec.config $< | butane -d src)" '.spec.config |= $$config' $< > $@ || { rm -f $@; exit 1; }
+
+all: $(MACHINECONFIGS)
+
+clean:
+	rm -f $(MACHINECONFIGS)

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/README.md

@@ -0,0 +1,11 @@
+Including file content in ignition configs is a pain, because it has to be base64 encoded. The `Makefile` in this directory uses [Butane][] to transpile `MachineConfig` resources, automatically including and encoding content from files.
+
+To run the `Makefile`, you will need:
+
+- [Butane][]
+- [yq][]
+- [jq][]
+
+[yq]: https://kislyuk.github.io/yq/
+[jq]: https://stedolan.github.io/jq/
+[butane]: https://coreos.github.io/butane/

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/configure-bond0.yaml

@@ -0,0 +1,66 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  name: configure-bond0
+  labels:
+    machineconfiguration.openshift.io/role: master
+spec:
+  config:
+    ignition:
+      version: 3.2.0
+    storage:
+      files:
+        - path: /etc/mco/ctl-0-nic1.nmconnection
+          contents:
+            source: data:,%5Bconnection%5D%0Aid%3Dens6f0%0Atype%3Dethernet%0Ainterface-name%3Dens6f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+        - path: /etc/mco/ctl-0-nic2.nmconnection
+          contents:
+            source: data:,%5Bconnection%5D%0Aid%3Dens6f1%0Atype%3Dethernet%0Ainterface-name%3Dens6f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+        - path: /etc/mco/ctl-1-nic1.nmconnection
+          contents:
+            source: data:,%5Bconnection%5D%0Aid%3Dens5f0%0Atype%3Dethernet%0Ainterface-name%3Dens5f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+        - path: /etc/mco/ctl-1-nic2.nmconnection
+          contents:
+            source: data:,%5Bconnection%5D%0Aid%3Dens5f1%0Atype%3Dethernet%0Ainterface-name%3Dens5f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+        - path: /etc/mco/ctl-2-nic1.nmconnection
+          contents:
+            source: data:,%5Bconnection%5D%0Aid%3Dens5f0%0Atype%3Dethernet%0Ainterface-name%3Dens5f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+        - path: /etc/mco/ctl-2-nic2.nmconnection
+          contents:
+            source: data:,%5Bconnection%5D%0Aid%3Dens5f1%0Atype%3Dethernet%0Ainterface-name%3Dens5f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A
+        - path: /etc/NetworkConfiguration/system-connections/bond0.nmconnection
+          contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/0zMwQqDMAzG8XvepaOCjO2QJ5EeavsNCzaRmgp7++EQtlvgn+83JRVBsqISqGSeVbIne2/4nlTE0F4xwUmsuHLspteOrXXQD7n9Nbev8cDOA9EEW9AEFqha56f3nmg6tUBVM3iOa5QE1xrVUqoKD+P5UrZjDFRhi2Y+bcpL2pyVCu3Gj/voPX0CAAD//04u+Q/FAAAA
+        - path: /etc/mco/configure-bond0-ports.sh
+          contents:
+            compression: gzip
+            source: data:;base64,H4sIAAAAAAAC/5SQMU8DMQyFd/+KR9sJcURl7cRQiaVlgK3qkEt81OrFOcUpFFH+O+KEhNgOj3563yd7fuVaUdd6OxClY5SCZoDjGtyW61sux41X/8LF2btVTk3IqhyqZLUmivm250jpdWLFsdr1rabf1X9djqjLBYOvB8hPO4XsFg+PT8/b+826UQkfy5u7zz+aFWImAJAOux2aDosRsd+vUA+sY/Y9YcBsjGZTT8LlAj5LxXKEdEIxKxPNsVY7FUbIpXCoMO5FT2f0vuXeqLDVXDhM/gF9BQAA///ZzNpjrQEAAA==
+          mode: 493
+    systemd:
+      units:
+        - contents: '[Unit]
+
+            Description = Apply node-specific network configuration for bond0
+
+            RequiredBy = NetworkManager.service
+
+            After = network-pre.target
+
+            Before = NetworkManager.service
+
+
+            [Service]
+
+            Type = oneshot
+
+            ExecStart = /etc/mco/configure-bond0-ports.sh
+
+
+            [Install]
+
+            WantedBy = multi-user.service
+
+            '
+          enabled: true
+          name: configure-bond0-ports.service

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/kustomization.yaml

@@ -0,0 +1,4 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- configure-bond0.yaml

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/bond0.nmconnection

@@ -0,0 +1,17 @@
+[connection]
+id=bond0
+type=bond
+interface-name=bond0
+autoconnect=true
+connection.autoconnect-slaves=1
+
+[ethernet]
+mtu=9000
+
+[bond]
+mode=balance-rr
+miimon=140
+
+[ipv4]
+method=auto
+dhcp-timeout=86400

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0-ports.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+mkdir -p /etc/NetworkManager/system-connections-disabled
+mv /etc/NetworkManager/system-connections/ens*.nmconnection /etc/NetworkManager/system-connections-disabled/
+
+for path in /etc/mco/$HOSTNAME-nic{1,2}.nmconnection; do
+    if [[ -f $path ]]; then
+        cp "$path" /etc/NetworkManager/system-connections/ || exit 1
+    fi
+done
+
+# Ensure correct selinux labels
+restorecon /etc/NetworkManager/system-connections

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/configure-bond0.in.yaml

@@ -0,0 +1,57 @@
+apiVersion: machineconfiguration.openshift.io/v1
+kind: MachineConfig
+metadata:
+  name: configure-bond0
+  labels:
+    machineconfiguration.openshift.io/role: master
+spec:
+  config:
+    variant: fcos
+
+    # butane version 1.3.0 generates ignition 3.2.0 configs
+    version: 1.3.0
+
+    storage:
+      files:
+        - path: /etc/mco/ctl-0-nic1.nmconnection
+          contents:
+            local: ctl-0-nic1.nmconnection
+        - path: /etc/mco/ctl-0-nic2.nmconnection
+          contents:
+            local: ctl-0-nic2.nmconnection
+        - path: /etc/mco/ctl-1-nic1.nmconnection
+          contents:
+            local: ctl-1-nic1.nmconnection
+        - path: /etc/mco/ctl-1-nic2.nmconnection
+          contents:
+            local: ctl-1-nic2.nmconnection
+        - path: /etc/mco/ctl-2-nic1.nmconnection
+          contents:
+            local: ctl-2-nic1.nmconnection
+        - path: /etc/mco/ctl-2-nic2.nmconnection
+          contents:
+            local: ctl-2-nic2.nmconnection
+        - path: /etc/NetworkConfiguration/system-connections/bond0.nmconnection
+          contents:
+            local: bond0.nmconnection
+        - path: /etc/mco/configure-bond0-ports.sh
+          contents:
+            local: configure-bond0-ports.sh
+          mode: 0755
+    systemd:
+      units:
+        - name: configure-bond0-ports.service
+          enabled: true
+          contents: |
+            [Unit]
+            Description = Apply node-specific network configuration for bond0
+            RequiredBy = NetworkManager.service
+            After = network-pre.target
+            Before = NetworkManager.service
+
+            [Service]
+            Type = oneshot
+            ExecStart = /etc/mco/configure-bond0-ports.sh
+
+            [Install]
+            WantedBy = multi-user.service

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic1.nmconnection

@@ -0,0 +1,10 @@
+[connection]
+id=ens6f0
+type=ethernet
+interface-name=ens6f0
+master=bond0
+slave-type=bond
+autoconnect=true
+
+[ethernet]
+mtu=9000

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-0-nic2.nmconnection

@@ -0,0 +1,10 @@
+[connection]
+id=ens6f1
+type=ethernet
+interface-name=ens6f1
+master=bond0
+slave-type=bond
+autoconnect=true
+
+[ethernet]
+mtu=9000

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic1.nmconnection

@@ -0,0 +1,10 @@
+[connection]
+id=ens5f0
+type=ethernet
+interface-name=ens5f0
+master=bond0
+slave-type=bond
+autoconnect=true
+
+[ethernet]
+mtu=9000

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-1-nic2.nmconnection

@@ -0,0 +1,10 @@
+[connection]
+id=ens5f1
+type=ethernet
+interface-name=ens5f1
+master=bond0
+slave-type=bond
+autoconnect=true
+
+[ethernet]
+mtu=9000

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic1.nmconnection

@@ -0,0 +1,10 @@
+[connection]
+id=ens5f0
+type=ethernet
+interface-name=ens5f0
+master=bond0
+slave-type=bond
+autoconnect=true
+
+[ethernet]
+mtu=9000

cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0/src/ctl-2-nic2.nmconnection

@@ -0,0 +1,10 @@
+[connection]
+id=ens5f1
+type=ethernet
+interface-name=ens5f1
+master=bond0
+slave-type=bond
+autoconnect=true
+
+[ethernet]
+mtu=9000