From 3d790edbab5fa58c6f37115705cdcedd54cd137f Mon Sep 17 00:00:00 2001 From: Lars Kellogg-Stedman Date: Thu, 23 Jun 2022 18:28:34 -0400 Subject: [PATCH] Configure bond0 interface We want to use bonded interface pairs on these system. The nodes aren't yet wired for it, but setting this up now will allow us to refer to the `bond0` interface in e.g. VLAN configurations (and means we won't have to re-work those later). Because we're using OVNKubernetes, we can't use nmstate [1] to enact the configuration. The recommendation is to apply the configuration using a MachineConfig [2] resource, but this is complicated by the fact that our nodes don't all have the same interface names, and it's not possible to apply node-specific machineconfigs [3]. We work around this solution by: 1. Copying nmconnection files for *all hosts* to *every host*, but placing them in `/etc/mco` (just because that's a convenient available directory, it seems relatively topical, and it's not possible to create new directories using the `directories` section of an ignition config). 2. Installing a systemd unit that runs a shell script at boot that copies the host-specific configs from `/etc/mco` into `/etc/NetworkManager/system-connections`. [1]: https://docs.openshift.com/container-platform/4.10/networking/k8s_nmstate/k8s-nmstate-about-the-k8s-nmstate-operator.html [2]: https://docs.openshift.com/container-platform/4.10/post_installation_configuration/machine-configuration-tasks.html [3]: https://github.com/openshift/machine-config-operator/issues/1720 x-branch: feature/bond0 --- .../nerc-ocp-infra/machineconfigs/Makefile | 10 +++ .../nerc-ocp-infra/machineconfigs/README.md | 11 +++ .../machineconfigs/configure-bond0.in.yaml | 40 +++++++++++ .../machineconfigs/configure-bond0.yaml | 72 +++++++++++++++++++ .../machineconfigs/kustomization.yaml | 6 ++ .../src/apply-network-config.sh | 10 +++ .../machineconfigs/src/bond0.nmconnection | 17 +++++ .../src/ctl-0-nic1.nmconnection | 10 +++ .../src/ctl-0-nic2.nmconnection | 10 +++ .../src/ctl-1-nic1.nmconnection | 10 +++ .../src/ctl-1-nic2.nmconnection | 10 +++ .../src/ctl-2-nic1.nmconnection | 10 +++ .../src/ctl-2-nic2.nmconnection | 10 +++ 13 files changed, 226 insertions(+) create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/Makefile create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/README.md create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.in.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/kustomization.yaml create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/apply-network-config.sh create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/bond0.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic1.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic2.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic1.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic2.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic1.nmconnection create mode 100644 cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic2.nmconnection diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/Makefile b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/Makefile new file mode 100644 index 00000000..8ab57ab9 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/Makefile @@ -0,0 +1,10 @@ +MACHINECONFIGS = \ + configure-bond0.yaml + +%.yaml: %.in.yaml + yq --argjson config "$$(yq .spec.config $< | butane -d src)" '.spec.config |= $$config' $< > $@ || { rm -f $@; exit 1; } + +all: $(MACHINECONFIGS) + +clean: + rm -f $(MACHINECONFIGS) diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/README.md b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/README.md new file mode 100644 index 00000000..5a36c0de --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/README.md @@ -0,0 +1,11 @@ +Including file content in ignition configs is a pain, because it has to be base64 encoded. The `Makefile` in this directory uses [Butane][] to transpile `MachineConfig` resources, automatically including and encoding content from files. + +To run the `Makefile`, you will need: + +- [Butane][] +- [yq][] +- [jq][] + +[yq]: https://kislyuk.github.io/yq/ +[jq]: https://stedolan.github.io/jq/ +[butane]: https://coreos.github.io/butane/ diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.in.yaml b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.in.yaml new file mode 100644 index 00000000..f55bc98f --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.in.yaml @@ -0,0 +1,40 @@ +apiVersion: machineconfiguration.openshift.io/v1 +kind: MachineConfig +metadata: + name: configure-bond0 + labels: + machineconfiguration.openshift.io/role: master +spec: + config: + variant: fcos + + # butane version 1.3.0 generates ignition 3.2.0 configs + version: 1.3.0 + + storage: + files: + - path: /etc/mco/ctl-0-nic1.nmconnection + contents: + local: ctl-0-nic1.nmconnection + - path: /etc/mco/ctl-0-nic2.nmconnection + contents: + local: ctl-0-nic2.nmconnection + - path: /etc/mco/ctl-1-nic1.nmconnection + contents: + local: ctl-1-nic1.nmconnection + - path: /etc/mco/ctl-1-nic2.nmconnection + contents: + local: ctl-1-nic2.nmconnection + - path: /etc/mco/ctl-2-nic1.nmconnection + contents: + local: ctl-2-nic1.nmconnection + - path: /etc/mco/ctl-2-nic2.nmconnection + contents: + local: ctl-2-nic2.nmconnection + - path: /etc/NetworkConfiguration/system-connections/bond0.nmconnection + contents: + local: bond0.nmconnection + - path: /etc/mco/apply-network-config.sh + contents: + local: apply-network-config.sh + mode: 0755 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.yaml b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.yaml new file mode 100644 index 00000000..5a767728 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/configure-bond0.yaml @@ -0,0 +1,72 @@ +{ + "apiVersion": "machineconfiguration.openshift.io/v1", + "kind": "MachineConfig", + "metadata": { + "name": "configure-bond0", + "labels": { + "machineconfiguration.openshift.io/role": "master" + } + }, + "spec": { + "config": { + "ignition": { + "version": "3.2.0" + }, + "storage": { + "files": [ + { + "path": "/etc/mco/ctl-0-nic1.nmconnection", + "contents": { + "source": "data:,%5Bconnection%5D%0Aid%3Dens6f0%0Atype%3Dethernet%0Ainterface-name%3Dens6f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A" + } + }, + { + "path": "/etc/mco/ctl-0-nic2.nmconnection", + "contents": { + "source": "data:,%5Bconnection%5D%0Aid%3Dens6f1%0Atype%3Dethernet%0Ainterface-name%3Dens6f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A" + } + }, + { + "path": "/etc/mco/ctl-1-nic1.nmconnection", + "contents": { + "source": "data:,%5Bconnection%5D%0Aid%3Dens5f0%0Atype%3Dethernet%0Ainterface-name%3Dens5f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A" + } + }, + { + "path": "/etc/mco/ctl-1-nic2.nmconnection", + "contents": { + "source": "data:,%5Bconnection%5D%0Aid%3Dens5f1%0Atype%3Dethernet%0Ainterface-name%3Dens5f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A" + } + }, + { + "path": "/etc/mco/ctl-2-nic1.nmconnection", + "contents": { + "source": "data:,%5Bconnection%5D%0Aid%3Dens5f0%0Atype%3Dethernet%0Ainterface-name%3Dens5f0%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A" + } + }, + { + "path": "/etc/mco/ctl-2-nic2.nmconnection", + "contents": { + "source": "data:,%5Bconnection%5D%0Aid%3Dens5f1%0Atype%3Dethernet%0Ainterface-name%3Dens5f1%0Amaster%3Dbond0%0Aslave-type%3Dbond%0Aautoconnect%3Dtrue%0A%0A%5Bethernet%5D%0Amtu%3D9000%0A" + } + }, + { + "path": "/etc/NetworkConfiguration/system-connections/bond0.nmconnection", + "contents": { + "compression": "gzip", + "source": "data:;base64,H4sIAAAAAAAC/0zMwQqDMAzG8XvepaOCjO2QJ5EeavsNCzaRmgp7++EQtlvgn+83JRVBsqISqGSeVbIne2/4nlTE0F4xwUmsuHLspteOrXXQD7n9Nbev8cDOA9EEW9AEFqha56f3nmg6tUBVM3iOa5QE1xrVUqoKD+P5UrZjDFRhi2Y+bcpL2pyVCu3Gj/voPX0CAAD//04u+Q/FAAAA" + } + }, + { + "path": "/etc/mco/apply-network-config.sh", + "contents": { + "compression": "gzip", + "source": "data:;base64,H4sIAAAAAAAC/5TPPUsEMRDG8TrzKcbzKjEOZ3uVhWBzZ6HdsUU2mbjDmsmSBF9w/e7CIojdXvvAjz/P5QX1otS7OgCkMUhBOyFx83Tk9p7LeHDqXrhQ/ayNk/VZlX2TrNUGqa5/5QDpbSUh1np1o+lvOrdFADEXnFwbUH518pm2D49Pz8e7w71V8V+769vvf5k9hgxGIp5OaCNuF991e2wDKxjjJ9ws42btE5xn5A9puAMTBUJWhp8AAAD///CXzntOAQAA" + }, + "mode": 493 + } + ] + } + } + } +} diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/kustomization.yaml b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/kustomization.yaml new file mode 100644 index 00000000..48c9ab8e --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- ctl-0-bonding.yaml +- ctl-1-bonding.yaml +- ctl-2-bonding.yaml diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/apply-network-config.sh b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/apply-network-config.sh new file mode 100644 index 00000000..7198a737 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/apply-network-config.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +mkdir -p /etc/NetworkManager/system-connections-disabled +mv /etc/NetworkManager/system-connections/ens*.nmconnection /etc/NetworkManager/system-connections-disabled/ + +for path in /etc/mco/$HOSTNAME-nic{1,2}.nmconnection; do + if [[ -f $path ]]; then + cp "$path" /etc/NetworkManager/system-connections/ || exit 1 + fi +done diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/bond0.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/bond0.nmconnection new file mode 100644 index 00000000..d7cef358 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/bond0.nmconnection @@ -0,0 +1,17 @@ +[connection] +id=bond0 +type=bond +interface-name=bond0 +autoconnect=true +connection.autoconnect-slaves=1 + +[ethernet] +mtu=9000 + +[bond] +mode=balance-rr +miimon=140 + +[ipv4] +method=auto +dhcp-timeout=86400 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic1.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic1.nmconnection new file mode 100644 index 00000000..8ef6fb2a --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic1.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens6f0 +type=ethernet +interface-name=ens6f0 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic2.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic2.nmconnection new file mode 100644 index 00000000..45bbb655 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-0-nic2.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens6f1 +type=ethernet +interface-name=ens6f1 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic1.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic1.nmconnection new file mode 100644 index 00000000..744a7729 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic1.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f0 +type=ethernet +interface-name=ens5f0 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic2.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic2.nmconnection new file mode 100644 index 00000000..6c017d79 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-1-nic2.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f1 +type=ethernet +interface-name=ens5f1 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic1.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic1.nmconnection new file mode 100644 index 00000000..744a7729 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic1.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f0 +type=ethernet +interface-name=ens5f0 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000 diff --git a/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic2.nmconnection b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic2.nmconnection new file mode 100644 index 00000000..6c017d79 --- /dev/null +++ b/cluster-scope/overlays/nerc-ocp-infra/machineconfigs/src/ctl-2-nic2.nmconnection @@ -0,0 +1,10 @@ +[connection] +id=ens5f1 +type=ethernet +interface-name=ens5f1 +master=bond0 +slave-type=bond +autoconnect=true + +[ethernet] +mtu=9000