diff --git a/Makefile b/Makefile index f0ed853..d6a0363 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ SHELL := /bin/bash +PULL_SECRET ?= openshift_pull.json OUT_DIR = $(CURDIR)/build/_output/ all: build-manifests test @@ -11,21 +12,13 @@ build-manifests: ./hack/build-manifests.sh ${OUT_DIR} kcli-run-ocp: - ./kcli/run.sh ./kcli/ocp.yaml openshift-pull.json - -kcli-add-slb: - ./kcli/add-slb.sh - -kcli-del-slb: - ./kcli/del-slb.sh + ./kcli/run.sh ./kcli/ocp.yaml ${PULL_SECRET} kcli-run-rhcos: - ./kcli/run.sh ./kcli/rhcos.yaml openshift-pull.json + ./kcli/run.sh ./kcli/rhcos.yaml ${PULL_SECRET} .PHONY: \ test \ build-manifests \ kcli-run-ocp \ - kcli-add-slb \ - kcli-del-slb \ kcli-run-rhcos diff --git a/add-slb-nncp.yaml b/add-slb-nncp.yaml deleted file mode 100644 index 9307133..0000000 --- a/add-slb-nncp.yaml +++ /dev/null @@ -1,33 +0,0 @@ -apiVersion: nmstate.io/v1 -kind: NodeNetworkConfigurationPolicy -metadata: - name: slb -spec: - capture: - primary-nic: interfaces.description == "primary" - secondary-nic: interfaces.description == "secondary" - desiredState: - interfaces: - - name: brcnv - type: ovs-interface - state: up - mac-address: "{{ capture.primary-nic.interfaces.0.mac-address }}" - ipv4: - enabled: true - dhcp: true - - name: brcnv - type: ovs-bridge - state: up - bridge: - options: - stp: false - mcast-snooping-enable: false - rstp: false - port: - - name: bond0 - link-aggregation: - mode: balance-slb - port: - - name: "{{ capture.primary-nic.interfaces.0.name }}" - - name: "{{ capture.secondary-nic.interfaces.0.name }}" - - name: brcnv diff --git a/cnvnet-nad.yaml b/cnvnet-nad.yaml new file mode 100644 index 0000000..ea5b7d7 --- /dev/null +++ b/cnvnet-nad.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: k8s.cni.cncf.io/v1 +kind: NetworkAttachmentDefinition +metadata: + name: cnvnet +spec: + config: |2 + { + "cniVersion": "0.3.1", + "name": "cnvnet", + "type": "ovn-k8s-cni-overlay", + "topology": "localnet", + "vlan": 20, + "netAttachDefName": "default/cnvnet" + } + diff --git a/custom-config.fcc.tmpl b/custom-config.fcc.tmpl index 000b248..f2f535c 100644 --- a/custom-config.fcc.tmpl +++ b/custom-config.fcc.tmpl @@ -16,7 +16,6 @@ storage: inline: | [main] no-auto-default=* - dhcp=dhclient - path: /usr/local/bin/capture-macs mode: 0755 contents: diff --git a/del-slb-nncp.yaml b/del-slb-nncp.yaml deleted file mode 100644 index 2b22814..0000000 --- a/del-slb-nncp.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: nmstate.io/v1 -kind: NodeNetworkConfigurationPolicy -metadata: - name: slb -spec: - capture: - primary-nic: interfaces.description == "primary" - secondary-nic: interfaces.description == "secondary" - desiredState: - interfaces: - - name: brcnv - type: ovs-interface - state: absent - - name: brcnv - type: ovs-bridge - state: absent - - name: "{{ capture.primary-nic.interfaces.0.name }}" - type: ethernet - state: up - ipv4: - enabled: true - dhcp: true - - name: "{{ capture.secondary-nic.interfaces.0.name }}" - type: ethernet - state: up - ipv4: - enabled: false diff --git a/hack/build-manifests.sh b/hack/build-manifests.sh index 181efdc..c32eee5 100755 --- a/hack/build-manifests.sh +++ b/hack/build-manifests.sh @@ -4,12 +4,6 @@ set -ex OUT_DIR=$1 -copy_nncp_manifests() { - local nncp_glob="*-slb-nncp.yaml" - - cp ${nncp_glob} ${OUT_DIR} -} - build_custom_config() { local output_fcc=${OUT_DIR}/custom-config.fcc local output_ign=${OUT_DIR}/custom-config.ign @@ -26,23 +20,21 @@ build_custom_config() { } build_mco() { - local output_worker_mco=${OUT_DIR}/mco_ovs_workers.yml - local output_supervisor_mco=${OUT_DIR}/mco_ovs_supervisor.yml + local output_worker_mco=${OUT_DIR}/mco_ovs_workers.yml + local output_supervisor_mco=${OUT_DIR}/mco_ovs_supervisor.yml - # Base64 encode the `init-interfaces.sh` file - base64_script_content=$(base64 -w 0 < init-interfaces.sh) && export base64_script_content + # Base64 encode the `init-interfaces.sh` file + export base64_script_content=$(base64 -w 0 < init-interfaces.sh) # Paste the content into each MCO file - envsubst \$base64_script_content < mco_ovs_workers.yml.tmpl > "${output_worker_mco}" - envsubst \$base64_script_content < mco_ovs_supervisor.yml.tmpl > "${output_supervisor_mco}" + envsubst < mco_ovs_workers.yml.tmpl > "${output_worker_mco}" + envsubst < mco_ovs_supervisor.yml.tmpl > "${output_supervisor_mco}" } if [[ ! -d "${OUT_DIR}" ]]; then mkdir -p "${OUT_DIR}" fi -copy_nncp_manifests - build_custom_config build_mco diff --git a/init-interfaces.sh b/init-interfaces.sh index 815db00..a74930f 100644 --- a/init-interfaces.sh +++ b/init-interfaces.sh @@ -3,134 +3,69 @@ set -ex is_configured() { - [[ $(grep primary /etc/NetworkManager/system-connections/* | wc -l) -ge 1 && $(grep secondary /etc/NetworkManager/system-connections/* | wc -l) -ge 1 ]] + [[ $(nmstatectl show --json bondcnv |jq '.interfaces |length') -eq 1 ]] } -is_con_exists() { - local con_name=$1 - if nmcli -t -g NAME con show | grep -w -q "$con_name"; then - return 0 # true - fi - return 1 # false +read_mac() { + local field=$1 + awk -F= "/$field/ {print \$2}" < /boot/mac_addresses | tr '[:lower:]' '[:upper:]' } -is_con_active() { - local con_name=$1 - if nmcli -t -g NAME con show --active | grep -w -q "$con_name"; then - return 0 # true - fi - return 1 # false +find_interface_by_mac() { + local mac=$1 + nmstatectl show --json |jq -r ".interfaces[] | select(.\"mac-address\"==\"$mac\").name" } -get_con_name_by_mac_or_device() { - local mac=$(echo $1 | sed -e 's/\\\|://g') - local dev_name=$2 - while read -r con; do - if [[ "$(nmcli -g 802-3-ethernet.mac-address c show "${con}" | tr '[A-Z]' '[a-z]' | sed -e 's/\\\|://g')" == "$mac" || $(nmcli -g connection.interface-name c show "${con}") == "${dev_name}" ]]; then - echo "${con}" - break - fi - done <<< "$(nmcli -g NAME c show)" +create_cnvnet() { + nmstatectl apply << EOF +ovn: + bridge-mappings: + - localnet: cnvnet + bridge: br-ex +EOF } -generate_new_con_name() { - local device_name=$1 - printf "ethernet-%s-%s" "$device_name" "$RANDOM" -} +create_bondcnv() { + if [[ ! -f /boot/mac_addresses ]] ; then + echo "no mac address configuration file found .. exiting" + exit 1 + fi + + if is_configured; then + echo "interfaces already configured" + exit 0 + fi + + local default_device=$(find_interface_by_mac $(read_mac PRIMARY_MAC)) + local secondary_device=$(find_interface_by_mac $(read_mac SECONDARY_MAC)) + + echo -e "default dev: $default_device \nsecondary dev: $secondary_device" + if [[ -z "$default_device" ]] || [[ -z "$secondary_device" ]]; then + echo "error: primary/secondary device name not found" + exit 1 + fi -set_description() { - local mac=$1 - local nic=$2 - local description=$3 - local connections=$(grep -REl "type=ethernet" /etc/NetworkManager/system-connections | xargs -I{} -- grep -El -i "mac-address=${mac}|interface-name=${nic}" "{}") - IFS=$'\n' - for connection in ${connections}; do - if ! grep nmstate.interface.description "${connection}"; then - echo "" >> "${connection}" - echo "[user]" >> "${connection}" - echo "nmstate.interface.description=${description}" >> "${connection}" - else - sed -i "s/nmstate\.interface\.description=.*/nmstate.interface.description=$description/" "${connection}" - fi - done - unset IFS +# We cannot use nmpolicy [1] or /etc/nmstate yet [2] +# [1] https://issues.redhat.com/browse/RHEL-26617 +# [2] https://github.com/openshift/machine-config-operator/pull/4212 + nmstatectl apply << EOF +interfaces: +- name: bondcnv + type: bond + state: up + ipv4: + enabled: true + dhcp: true + copy-mac-from: $default_device + link-aggregation: + mode: balance-xor + options: + xmit_hash_policy: vlan+srcmac + balance-slb: 1 + port: + - $default_device + - $secondary_device +EOF } -if [[ ! -f /boot/mac_addresses ]] ; then - echo "no mac address configuration file found .. exiting" - exit 1 -fi - -if is_configured; then - echo "interfaces already configured" - exit 0 -fi - -primary_mac="$(awk -F= '/PRIMARY_MAC/ {print $2}' < /boot/mac_addresses | tr '[:upper:]' '[:lower:]')" -secondary_mac="$(awk -F= '/SECONDARY_MAC/ {print $2}' < /boot/mac_addresses | tr '[:upper:]' '[:lower:]')" - -default_device="" -secondary_device="" -default_connection_name="" -secondary_connection_name="" - -for dev in $(nmcli device status | awk '/ethernet/ {print $1}'); do - dev_mac=$(nmcli -g GENERAL.HWADDR dev show "$dev" | sed -e 's/\\//g' | tr '[:upper:]' '[:lower:]') - case $dev_mac in - "${primary_mac}") - default_device="$dev" - default_connection_name=$(get_con_name_by_mac_or_device "$primary_mac" "$dev") - ;; - "${secondary_mac}") - secondary_device="$dev" - secondary_connection_name=$(get_con_name_by_mac_or_device "$secondary_mac" "$dev") - ;; - *) - ;; - esac -done - -echo -e "default dev: $default_device (CONNECTION.NAME $default_connection_name)\nsecondary dev: $secondary_device (CONNECTION.NAME $secondary_connection_name)" -if [[ -z "$default_device" ]] || [[ -z "$secondary_device" ]]; then - echo "error: primary/secondary device name not found" - exit 1 -fi - -if eval ! is_con_exists "\"$default_connection_name\""; then - default_connection_name="$(generate_new_con_name "${default_device}")" && export default_connection_name - nmcli con add type ethernet \ - conn.interface "${default_device}" \ - connection.autoconnect yes \ - ipv4.method auto \ - con-name "${default_connection_name}" \ - 802-3-ethernet.mac-address "${primary_mac}" -fi -if eval ! is_con_active "\"$default_connection_name\""; then - nmcli con up "${default_connection_name}" -fi - -if eval ! is_con_exists "\"$secondary_connection_name\""; then - secondary_connection_name="$(generate_new_con_name "${secondary_device}")" && export secondary_connection_name - nmcli con add type ethernet \ - conn.interface "${secondary_device}" \ - connection.autoconnect yes \ - ipv4.method disabled \ - ipv6.method disabled \ - con-name "${secondary_connection_name}" \ - 802-3-ethernet.mac-address "${secondary_mac}" -fi -if eval ! is_con_active "\"${secondary_connection_name}\""; then - nmcli con mod "${secondary_connection_name}" \ - connection.interface-name "${secondary_device}" \ - connection.autoconnect yes \ - ipv4.method disabled \ - ipv6.method disabled \ - 802-3-ethernet.mac-address "${secondary_mac}" - nmcli con up "${secondary_connection_name}" || /bin/true -fi - -set_description "${primary_mac}" "${default_device}" primary -set_description "${secondary_mac}" "${secondary_device}" secondary - -nmcli c reload - +$@ diff --git a/kcli/ocp.yaml b/kcli/ocp.yaml index 941cf99..444bfe2 100644 --- a/kcli/ocp.yaml +++ b/kcli/ocp.yaml @@ -1,6 +1,6 @@ parameters: vmrules: - - rhocs-slb-master-0: + - rhocs-slb-ctlplane-0: cmds: - echo redhat | passwd --stdin core cmdline: custom-config macAddressList=52:54:00:f6:80:01,52:54:00:f6:80:02 @@ -23,11 +23,8 @@ rhocs-slb: type: kube kubetype: openshift domain: redhat.com - version: nightly - tag: '4.10' - masters: 1 + tag: '4.14' + ctlplanes: 1 workers: 1 memory: 16384 numcpus: 16 - postscripts: - - kcli/deploy-knmstate.sh diff --git a/kcli/run.sh b/kcli/run.sh index 284cfdd..1a58802 100755 --- a/kcli/run.sh +++ b/kcli/run.sh @@ -1,4 +1,4 @@ -#!/bin/bash -e +#!/bin/bash -ex tmpdir=$(mktemp -d) @@ -20,11 +20,11 @@ patch -p1 < kcli/ignition.patch export base64_capture_macs_script_content=$(cat capture-macs.sh|base64 -w 0) envsubst < custom-config.fcc.tmpl > custom-config.fcc butane < custom-config.fcc > rhocs-slb-worker-0.ign -cp rhocs-slb-worker-0.ign rhocs-slb-master-0.ign +cp rhocs-slb-worker-0.ign rhocs-slb-ctlplane-0.ign mkdir -p manifests export base64_script_content=$(cat init-interfaces.sh|base64 -w 0) -envsubst < mco_ovs_workers.yml.tmpl > manifests/mco_ovs_workers.yml +envsubst < mco_ovs_workers.yml.tmpl > manifests/mco_ovs_workers.yml envsubst < mco_ovs_supervisor.yml.tmpl > manifests/mco_ovs_supervisor.yml if [[ $0 =~ run.sh ]]; then diff --git a/mco_ovs_supervisor.yml.tmpl b/mco_ovs_supervisor.yml.tmpl index b72fc7a..d372930 100644 --- a/mco_ovs_supervisor.yml.tmpl +++ b/mco_ovs_supervisor.yml.tmpl @@ -12,14 +12,24 @@ spec: units: - contents: | [Unit] - Before=kubelet.service - After=NetworkManager.service + Before=ovs-configuration.service + After=NetworkManager-wait-online.service [Service] Type=oneshot - ExecStart=/bin/sh /var/init-interfaces.sh + ExecStart=/bin/sh /var/init-interfaces.sh create_bondcnv [Install] WantedBy=multi-user.target - name: setup-ovs.service + name: create-bondcnv.service + enabled: true + - contents: | + [Unit] + After=ovs-configuration.service + [Service] + Type=oneshot + ExecStart=/bin/sh /var/init-interfaces.sh create_cnvnet + [Install] + WantedBy=multi-user.target + name: create-cnvnet.service enabled: true storage: files: diff --git a/mco_ovs_workers.yml.tmpl b/mco_ovs_workers.yml.tmpl index 049979e..2e62873 100644 --- a/mco_ovs_workers.yml.tmpl +++ b/mco_ovs_workers.yml.tmpl @@ -12,15 +12,26 @@ spec: units: - contents: | [Unit] - Before=kubelet.service - After=NetworkManager.service + Before=ovs-configuration.service + After=NetworkManager-wait-online.service [Service] Type=oneshot - ExecStart=/bin/sh /var/init-interfaces.sh + ExecStart=/bin/sh /var/init-interfaces.sh create_bondcnv [Install] WantedBy=multi-user.target - name: setup-ovs.service + name: create-bondcnv.service enabled: true + - contents: | + [Unit] + After=ovs-configuration.service + [Service] + Type=oneshot + ExecStart=/bin/sh /var/init-interfaces.sh create_cnvnet + [Install] + WantedBy=multi-user.target + name: create-cnvnet.service + enabled: true + storage: files: - contents: diff --git a/tests/test-coreos.sh b/tests/test-coreos.sh index 8bd77cf..666b7d0 100755 --- a/tests/test-coreos.sh +++ b/tests/test-coreos.sh @@ -3,7 +3,7 @@ set -ex COREOS_ASSEMBLER_REPO_URL=https://github.com/coreos/coreos-assembler.git -OPENSHIFT_VERSION=${OPENSHIFT_VERSION:-"4.13"} +OPENSHIFT_VERSION=${OPENSHIFT_VERSION:-"4.14"} COREOS_ASSEMBLER_REPO_BRANCH=${COREOS_ASSEMBLER_REPO_BRANCH:-"origin/rhcos-${OPENSHIFT_VERSION}"} IMAGE_INSTALLER_BRANCH=${IMAGE_INSTALLER_BRANCH:-"release-${OPENSHIFT_VERSION}"} RHCOS_SLB_TEST_PATH=mantle/kola/tests/misc/network.go