From 1e61f182dd14f3700eba9adf7e62a29493b49d72 Mon Sep 17 00:00:00 2001
From: Jonathan Lebon <jonathan@jlebon.com>
Date: Thu, 27 Jan 2022 16:54:08 -0500
Subject: [PATCH 1/3] overlay.d: add 35coreos-iptables

From `overlay.d/README.md`:

Contains systemd service and script for remaining on iptables-nft after
the migration to nft.

Split out because (1) it will roll out to next first, and (2) it can
more easily be deleted after the barrier release.

For more details, see:
https://github.com/coreos/fedora-coreos-tracker/issues/676
https://github.com/coreos/fedora-coreos-config/pull/1324
---
 .../coreos-enable-iptables-legacy.service     | 18 ++++
 .../coreos-enable-iptables-legacy.sh          | 82 +++++++++++++++++++
 .../35coreos-iptables/module-setup.sh         | 17 ++++
 overlay.d/README.md                           | 13 +++
 4 files changed, 130 insertions(+)
 create mode 100644 overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.service
 create mode 100755 overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.sh
 create mode 100644 overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/module-setup.sh

diff --git a/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.service b/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.service
new file mode 100644
index 0000000000..a59bc10531
--- /dev/null
+++ b/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.service
@@ -0,0 +1,18 @@
+[Unit]
+Description=CoreOS Enable iptables-legacy
+ConditionPathExists=/etc/initrd-release
+DefaultDependencies=false
+ConditionPathExists=/sysroot/etc/coreos/iptables-legacy.stamp
+
+# On first boot, allow Ignition config to install stamp file.
+After=ignition-files.service
+
+# On subsequent boots, just make sure the deployment is accessible.
+After=ostree-prepare-root.service
+
+Before=initrd.target
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/sbin/coreos-enable-iptables-legacy
diff --git a/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.sh b/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.sh
new file mode 100755
index 0000000000..4f364510bf
--- /dev/null
+++ b/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/coreos-enable-iptables-legacy.sh
@@ -0,0 +1,82 @@
+#!/bin/bash
+set -euo pipefail
+
+declare -A SYMLINKS=(
+    [ip6tables]=ip6tables-legacy
+    [ip6tables-restore]=ip6tables-legacy-restore
+    [ip6tables-save]=ip6tables-legacy-save
+    [iptables]=iptables-legacy
+    [iptables-restore]=iptables-legacy-restore
+    [iptables-save]=iptables-legacy-save
+)
+
+STAMP=/sysroot/etc/coreos/iptables-legacy.stamp
+IGNITION_RESULT=/sysroot/etc/.ignition-result.json
+
+# sanity-check the stamp file is present
+if [ ! -e "${STAMP}" ]; then
+    echo "File ${STAMP} not found; exiting."
+    exit 0
+fi
+
+# We only want to run once.
+rm "${STAMP}"
+
+# Ignore firstboot. We don't want the stamp file to be a long-term
+# provisioning-time API for moving to iptables-legacy, so explicitly check for
+# this and don't support it. We use the Ignition report file because it's less
+# hacky than parsing the kernel commandline for `ignition.firstboot`.
+if [ -e "${IGNITION_RESULT}" ]; then
+    ignition_boot=$(jq -r .provisioningBootID "${IGNITION_RESULT}")
+    if [ "$(cat /proc/sys/kernel/random/boot_id)" = "${ignition_boot}" ]; then
+        echo "First boot detected; exiting."
+        exit 0
+    fi
+fi
+
+# if legacy doesn't exist on the host anymore, do nothing
+for legacy in "${SYMLINKS[@]}"; do
+    path=/sysroot/usr/sbin/$legacy
+    if [ ! -e "$path" ]; then
+        echo "Executable $path no longer present; exiting."
+        exit 0
+    fi
+done
+
+symlink_is_default() {
+    local symlinkpath=$1; shift
+    # check that the deployment is still using the symlink (i.e. the user didn't
+    # do something funky), and that the OSTree default is still symlink-based
+    # (i.e. that we didn't change strategy and forgot to update this script)
+    if [ ! -L "/sysroot/$symlinkpath" ] || [ ! -L "/sysroot/usr/$symlinkpath" ]; then
+        return 1
+    fi
+    # compare symlink targets between deployment and OSTree default
+    if [ "$(readlink "/sysroot/$symlinkpath")" != "$(readlink "/sysroot/usr/$symlinkpath")" ]; then
+        return 1
+    fi
+    # it's the default
+    return 0
+}
+
+# If there are any modifications to the symlinks, do nothing. This is basically
+# like `ostree admin config-diff` but more focused and lighter/safer than doing
+# a bwrap call and grepping output.
+for symlink in "${!SYMLINKS[@]}"; do
+    symlinkpath=/etc/alternatives/$symlink
+    if ! symlink_is_default "$symlinkpath"; then
+        echo "Symlink $symlinkpath is not default; exiting without modifying."
+        exit 0
+    fi
+done
+
+# Update symlinks for legacy backend!
+for symlink in "${!SYMLINKS[@]}"; do
+    target=${SYMLINKS[$symlink]}
+    symlink=/etc/alternatives/$symlink
+    ln -vsf "/usr/sbin/$target" "/sysroot/$symlink"
+    # symlink labels don't matter, but relabel to appease unlabeled_t scanners
+    coreos-relabel "$symlink"
+done
+
+echo "Updated /sysroot to use iptables-legacy."
diff --git a/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/module-setup.sh b/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/module-setup.sh
new file mode 100644
index 0000000000..677f3f6188
--- /dev/null
+++ b/overlay.d/35coreos-iptables/usr/lib/dracut/modules.d/35coreos-iptables/module-setup.sh
@@ -0,0 +1,17 @@
+install_and_enable_unit() {
+    unit="$1"; shift
+    target="$1"; shift
+    inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
+    # note we `|| exit 1` here so we error out if e.g. the units are missing
+    # see https://github.com/coreos/fedora-coreos-config/issues/799
+    systemctl -q --root="$initdir" add-requires "$target" "$unit" || exit 1
+}
+
+install() {
+    inst_simple readlink
+
+    inst_simple "$moddir/coreos-enable-iptables-legacy.sh" \
+        "/usr/sbin/coreos-enable-iptables-legacy"
+    install_and_enable_unit "coreos-enable-iptables-legacy.service" \
+        "initrd.target"
+}
diff --git a/overlay.d/README.md b/overlay.d/README.md
index 4213f2c0b2..61a989718a 100644
--- a/overlay.d/README.md
+++ b/overlay.d/README.md
@@ -46,3 +46,16 @@ Add static chrony configuration for NTP servers provided on platforms
 such as `azure`, `aws`, `gcp`. The chrony config for these NTP servers
 should override other chrony configuration (e.g. DHCP-provided)
 configuration.
+
+35coreos-iptables
+-----------------
+
+Contains systemd service and script for remaining on iptables-nft after
+the migration to nft.
+
+Split out because (1) it will roll out to next first, and (2) it can
+more easily be deleted after the barrier release.
+
+For more details, see:
+https://github.com/coreos/fedora-coreos-tracker/issues/676
+https://github.com/coreos/fedora-coreos-config/pull/1324

From 63e2f81ec006546cd681b3039952aa0925e213d9 Mon Sep 17 00:00:00 2001
From: Jonathan Lebon <jonathan@jlebon.com>
Date: Thu, 27 Jan 2022 17:07:43 -0500
Subject: [PATCH 2/3] tests: add iptables-nft test

This is prep for enabling iptables-nft in `next`.

Because tests are shared between streams, this is a bit awkward. The way
this does it is:
- Make the iptables-legacy test exclusive and attach a Butane config
  that sets the legacy symlinks. On next, this will verify that this
  config can be used to boot into legacy. On !next, this will verify
  that the config can safely be used even before migration.
- Add an iptables non-exclusive test. On next, this will verify that the
  default backend is nft. On !next, it will verify that it is legacy.
  Once the migration is over on all streams, the latter check will be
  removed, so it'll purely check for nft.
---
 tests/kola/firewall/data/commonlib.sh         |  1 -
 tests/kola/firewall/iptables-legacy           | 18 ------------
 tests/kola/firewall/iptables-legacy/config.bu | 28 +++++++++++++++++++
 .../iptables-legacy/data/commonlib.sh         |  1 +
 tests/kola/firewall/iptables-legacy/test.sh   | 14 ++++++++++
 .../kola/firewall/iptables/data/commonlib.sh  |  1 +
 tests/kola/firewall/iptables/test.sh          | 26 +++++++++++++++++
 7 files changed, 70 insertions(+), 19 deletions(-)
 delete mode 120000 tests/kola/firewall/data/commonlib.sh
 delete mode 100755 tests/kola/firewall/iptables-legacy
 create mode 100644 tests/kola/firewall/iptables-legacy/config.bu
 create mode 120000 tests/kola/firewall/iptables-legacy/data/commonlib.sh
 create mode 100755 tests/kola/firewall/iptables-legacy/test.sh
 create mode 120000 tests/kola/firewall/iptables/data/commonlib.sh
 create mode 100755 tests/kola/firewall/iptables/test.sh

diff --git a/tests/kola/firewall/data/commonlib.sh b/tests/kola/firewall/data/commonlib.sh
deleted file mode 120000
index 1742d51e67..0000000000
--- a/tests/kola/firewall/data/commonlib.sh
+++ /dev/null
@@ -1 +0,0 @@
-../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/firewall/iptables-legacy b/tests/kola/firewall/iptables-legacy
deleted file mode 100755
index 594f691a5d..0000000000
--- a/tests/kola/firewall/iptables-legacy
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-# kola: { "distros": "fcos", "exclusive": false }
-# This test is currently scoped to only FCOS because the RHCOS version of `iptables`
-# is using the `nf_tables` backend.
-# TODO: modify this test to check for `nf_tables` backend when FCOS switches.
-#       See https://github.com/coreos/fedora-coreos-config/pull/1324
-
-set -xeuo pipefail
-
-. $KOLA_EXT_DATA/commonlib.sh
-
-# Make sure we're still on legacy iptables for now
-# https://github.com/coreos/fedora-coreos-tracker/issues/676#issuecomment-928028451
-if ! iptables --version | grep legacy; then
-    iptables --version # output for logs
-    fatal "iptables version is not legacy"
-fi
-ok "iptables still in legacy mode"
diff --git a/tests/kola/firewall/iptables-legacy/config.bu b/tests/kola/firewall/iptables-legacy/config.bu
new file mode 100644
index 0000000000..c7092c4022
--- /dev/null
+++ b/tests/kola/firewall/iptables-legacy/config.bu
@@ -0,0 +1,28 @@
+variant: fcos
+version: 1.4.0
+storage:
+  links:
+    - path: /etc/alternatives/iptables
+      target: /usr/sbin/iptables-legacy
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/iptables-restore
+      target: /usr/sbin/iptables-legacy-restore
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/iptables-save
+      target: /usr/sbin/iptables-legacy-save
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables
+      target: /usr/sbin/ip6tables-legacy
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables-restore
+      target: /usr/sbin/ip6tables-legacy-restore
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables-save
+      target: /usr/sbin/ip6tables-legacy-save
+      overwrite: true
+      hard: false
diff --git a/tests/kola/firewall/iptables-legacy/data/commonlib.sh b/tests/kola/firewall/iptables-legacy/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/firewall/iptables-legacy/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/firewall/iptables-legacy/test.sh b/tests/kola/firewall/iptables-legacy/test.sh
new file mode 100755
index 0000000000..8ef28366cd
--- /dev/null
+++ b/tests/kola/firewall/iptables-legacy/test.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+# kola: { "distros": "fcos", "exclusive": true }
+# This test verifies that one can configure a node to use the legacy iptables
+# backend. It is scoped to only FCOS because RHCOS only supports nft.
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+# Make sure we're on legacy iptables
+if ! iptables --version | grep legacy; then
+    iptables --version # output for logs
+    fatal "iptables version is not legacy"
+fi
+ok "iptables in legacy mode"
diff --git a/tests/kola/firewall/iptables/data/commonlib.sh b/tests/kola/firewall/iptables/data/commonlib.sh
new file mode 120000
index 0000000000..b8dcbdca1a
--- /dev/null
+++ b/tests/kola/firewall/iptables/data/commonlib.sh
@@ -0,0 +1 @@
+../../../data/commonlib.sh
\ No newline at end of file
diff --git a/tests/kola/firewall/iptables/test.sh b/tests/kola/firewall/iptables/test.sh
new file mode 100755
index 0000000000..3477f2f125
--- /dev/null
+++ b/tests/kola/firewall/iptables/test.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+# kola: { "exclusive": false }
+# Verifies that the expected iptables backend is configured.
+# https://github.com/coreos/fedora-coreos-tracker/issues/676
+set -xeuo pipefail
+
+. $KOLA_EXT_DATA/commonlib.sh
+
+# we're currently rolling out to next first
+case "$(get_fcos_stream)" in
+    "next-devel" | "next")
+        if ! iptables --version | grep nf_tables; then
+            iptables --version # output for logs
+            fatal "iptables version is not nft"
+        fi
+        ok "iptables in nft mode"
+        ;;
+    *)
+        # Make sure we're on legacy iptables
+        if ! iptables --version | grep legacy; then
+            iptables --version # output for logs
+            fatal "iptables version is not legacy"
+        fi
+        ok "iptables in legacy mode"
+        ;;
+esac

From 886869a52d29be0dcebad151f521758c97e3f092 Mon Sep 17 00:00:00 2001
From: Jonathan Lebon <jonathan@jlebon.com>
Date: Wed, 16 Feb 2022 16:04:57 -0500
Subject: [PATCH 3/3] tests/manual: add iptables-nft upgrade tests

These are manual upgrade tests that verify various upgrade paths for
iptables-nft.

It's manual in that you have to update the `OCIARCHIVE_URL` to point to
a URL of an ociarchive of a build with `35coreos-iptables`.

Long-term, I'd like to add external tests support directly in upgrade
tests so that we could have access to the artifacts vis e.g.
`KOLA_EXT_DATA` or a mount. But for now, this will do.

To run the tests, first update `OCIARCHIVE_URL`, and then:

```
    kola run -E /path/to/tests/manual/iptables-nft-migration ext.iptables-nft-migration.*
```
---
 .../tests/kola/already-migrated/config.bu     | 28 +++++++++++++++++++
 .../kola/already-migrated/data/common.sh      |  1 +
 .../tests/kola/already-migrated/test.sh       | 21 ++++++++++++++
 .../tests/kola/data/common.sh                 | 22 +++++++++++++++
 .../tests/kola/migrate-to-nft                 | 21 ++++++++++++++
 .../tests/kola/stay-on-legacy.day1/config.bu  |  6 ++++
 .../kola/stay-on-legacy.day1/data/common.sh   |  1 +
 .../tests/kola/stay-on-legacy.day1/test.sh    | 21 ++++++++++++++
 .../tests/kola/stay-on-legacy.day2            | 23 +++++++++++++++
 9 files changed, 144 insertions(+)
 create mode 100644 tests/manual/iptables-nft-migration/tests/kola/already-migrated/config.bu
 create mode 120000 tests/manual/iptables-nft-migration/tests/kola/already-migrated/data/common.sh
 create mode 100755 tests/manual/iptables-nft-migration/tests/kola/already-migrated/test.sh
 create mode 100644 tests/manual/iptables-nft-migration/tests/kola/data/common.sh
 create mode 100755 tests/manual/iptables-nft-migration/tests/kola/migrate-to-nft
 create mode 100644 tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/config.bu
 create mode 120000 tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/data/common.sh
 create mode 100755 tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/test.sh
 create mode 100755 tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day2

diff --git a/tests/manual/iptables-nft-migration/tests/kola/already-migrated/config.bu b/tests/manual/iptables-nft-migration/tests/kola/already-migrated/config.bu
new file mode 100644
index 0000000000..2692db6c75
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/already-migrated/config.bu
@@ -0,0 +1,28 @@
+variant: fcos
+version: 1.4.0
+storage:
+  links:
+    - path: /etc/alternatives/iptables
+      target: /usr/sbin/iptables-nft
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/iptables-restore
+      target: /usr/sbin/iptables-nft-restore
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/iptables-save
+      target: /usr/sbin/iptables-nft-save
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables
+      target: /usr/sbin/ip6tables-nft
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables-restore
+      target: /usr/sbin/ip6tables-nft-restore
+      overwrite: true
+      hard: false
+    - path: /etc/alternatives/ip6tables-save
+      target: /usr/sbin/ip6tables-nft-save
+      overwrite: true
+      hard: false
diff --git a/tests/manual/iptables-nft-migration/tests/kola/already-migrated/data/common.sh b/tests/manual/iptables-nft-migration/tests/kola/already-migrated/data/common.sh
new file mode 120000
index 0000000000..e232e75550
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/already-migrated/data/common.sh
@@ -0,0 +1 @@
+../../data/common.sh
\ No newline at end of file
diff --git a/tests/manual/iptables-nft-migration/tests/kola/already-migrated/test.sh b/tests/manual/iptables-nft-migration/tests/kola/already-migrated/test.sh
new file mode 100755
index 0000000000..76984b55e9
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/already-migrated/test.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -xeuo pipefail
+
+# kola: { "tags": "needs-internet" }
+
+. $KOLA_EXT_DATA/common.sh
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      assert_iptables_nft
+      assert_iptables_differs_from_default
+      upgrade
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      assert_iptables_nft
+      assert_iptables_matches_default
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/manual/iptables-nft-migration/tests/kola/data/common.sh b/tests/manual/iptables-nft-migration/tests/kola/data/common.sh
new file mode 100644
index 0000000000..7b9b98e516
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/data/common.sh
@@ -0,0 +1,22 @@
+OCIARCHIVE_URL=http://192.168.0.13:8000/fedora-coreos-35.20220210.dev.0-ostree.x86_64.ociarchive
+
+upgrade() {
+    curl -Lo /var/tmp/update.ociarchive "${OCIARCHIVE_URL}"
+    rpm-ostree rebase --experimental ostree-unverified-image:oci-archive:/var/tmp/update.ociarchive
+}
+
+assert_iptables_legacy() {
+    iptables --version | grep legacy
+}
+
+assert_iptables_nft() {
+    iptables --version | grep nf_tables
+}
+
+assert_iptables_differs_from_default() {
+    ostree admin config-diff | grep alternatives/iptables
+}
+
+assert_iptables_matches_default() {
+    ! ostree admin config-diff | grep alternatives/iptables
+}
diff --git a/tests/manual/iptables-nft-migration/tests/kola/migrate-to-nft b/tests/manual/iptables-nft-migration/tests/kola/migrate-to-nft
new file mode 100755
index 0000000000..919df443b1
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/migrate-to-nft
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -xeuo pipefail
+
+# kola: { "tags": "needs-internet" }
+
+. $KOLA_EXT_DATA/common.sh
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      assert_iptables_legacy
+      assert_iptables_matches_default
+      upgrade
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      assert_iptables_nft
+      assert_iptables_matches_default
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/config.bu b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/config.bu
new file mode 100644
index 0000000000..81e9d43319
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/config.bu
@@ -0,0 +1,6 @@
+variant: fcos
+version: 1.4.0
+storage:
+  files:
+    - path: /etc/coreos/iptables-legacy.stamp
+      mode: 0644
diff --git a/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/data/common.sh b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/data/common.sh
new file mode 120000
index 0000000000..e232e75550
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/data/common.sh
@@ -0,0 +1 @@
+../../data/common.sh
\ No newline at end of file
diff --git a/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/test.sh b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/test.sh
new file mode 100755
index 0000000000..2d263ea1a5
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day1/test.sh
@@ -0,0 +1,21 @@
+#!/bin/bash
+set -xeuo pipefail
+
+# kola: { "tags": "needs-internet" }
+
+. $KOLA_EXT_DATA/common.sh
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      assert_iptables_legacy
+      assert_iptables_matches_default
+      upgrade
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      assert_iptables_legacy
+      assert_iptables_differs_from_default
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac
diff --git a/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day2 b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day2
new file mode 100755
index 0000000000..85035164e3
--- /dev/null
+++ b/tests/manual/iptables-nft-migration/tests/kola/stay-on-legacy.day2
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -xeuo pipefail
+
+# kola: { "tags": "needs-internet" }
+
+. $KOLA_EXT_DATA/common.sh
+
+case "${AUTOPKGTEST_REBOOT_MARK:-}" in
+  "")
+      assert_iptables_legacy
+      assert_iptables_matches_default
+      mkdir -m 755 /etc/coreos/
+      touch /etc/coreos/iptables-legacy.stamp
+      upgrade
+      /tmp/autopkgtest-reboot rebooted
+      ;;
+
+  rebooted)
+      assert_iptables_legacy
+      assert_iptables_differs_from_default
+      ;;
+  *) fatal "unexpected mark: ${AUTOPKGTEST_REBOOT_MARK}";;
+esac