Merge pull request #5825 from jkonecny12/rhel-10-sync-master

RHEL-10: Sync infra from master

.github/workflows/kickstart-tests.yml.j2

@@ -21,6 +21,13 @@
 #
 # /kickstart-test --force rpm-ostree
 # ... run rpm-ostree test even if it is disabled
+#
+#
+# It is possible to run updated tests via a PR in kickstart-tests repository
+#
+# /kickstart-test --kstest-pr 1233 --testtype harddrive
+# ... run the tests of type harddrive with
+# https://github.com/rhinstaller/kickstart-tests/pull/1233 update of kickstart tests
 
 name: kickstart-tests
 on:
@@ -70,15 +77,26 @@ jobs:
         run: |
           # extract first line and cut out the "/kickstart-tests" first word
           ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//')
-          echo "workflow arguments are: $ARGS"
+          # parse --kstest-pr option
+          OPTS="$(getopt -q -o "" --long kstest-pr: -- $ARGS)" || true
+          PR=${OPTS#" --kstest-pr '"}
+          PR=$(echo $PR | cut -d " " -f1)
+          PR=${PR%"'"}
+          PR=${PR%"--"}
+          # remove --kstest-pr option
+          ARGS=$(echo "$ARGS" | sed -r -e "s#--kstest-pr( +|=)$PR ##" | sed 's/[[:space:]]*$//')
+          echo "test selection arguments are: $ARGS"
           echo "comment_args=${ARGS}" >> $GITHUB_OUTPUT
+          echo "kickstart-tests PR: $PR"
+          echo "kstest_pr=${PR}" >> $GITHUB_OUTPUT
 
     outputs:
       allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }}
       base_ref: ${{ fromJson(steps.pr_api.outputs.data).base.ref }}
       sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }}
       comment_args: ${{ steps.parse_comment_args.outputs.comment_args }}
       target_branch: ${{ fromJson(steps.pr_api.outputs.data).base.ref }}
+      kstest_pr: ${{ steps.parse_comment_args.outputs.kstest_pr }}
 
   run:
     needs: pr-info
@@ -134,8 +152,24 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: rhinstaller/kickstart-tests
+          fetch-depth: 0
           path: kickstart-tests
 
+      - name: Pull the request and rebase to the upstream target
+        working-directory: ./kickstart-tests
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [ -n "${{ needs.pr-info.outputs.kstest_pr }}" ]; then
+              gh pr checkout ${{ needs.pr-info.outputs.kstest_pr }}
+              BASE_REF=$(gh pr view --json baseRefName --template '{{.baseRefName}}' ${{ needs.pr-info.outputs.kstest_pr }})
+              git config user.name github-actions
+              git config user.email [email protected]
+              git log --oneline -1 $BASE_REF
+              git rebase $BASE_REF
+              echo "Rebasing PR ${{ needs.pr-info.outputs.kstest_pr }} to upstream $BASE_REF"
+          fi
+
       - name: Generate test cases
         working-directory: ./kickstart-tests
         run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases

.structure-config

@@ -8,14 +8,13 @@ INFRASTRUCTURE_FILES=(
 .coveragerc
 .shellcheckrc
 .github/
+.gitignore
 dockerfile/
 scripts/testing/
 scripts/jinja-render
 scripts/makebumpver
 scripts/rhel_version.py
 scripts/rhel_version.py.j2
-scripts/rebuild_boot_iso
-scripts/update_boot_iso
 dracut/.shellcheckrc
 docs/ci-status.rst
 CONTRIBUTING.rst

CONTRIBUTING.rst

@@ -10,8 +10,8 @@ Setting up development container
 --------------------------------
 
 The anaconda team uses a containerized development environment using toolbx.
-If you can install [toolbx](https://containertoolbx.org/) or
-[distrobox](https://distrobox.privatedns.org/) on your system, it is highly
+If you can install `toolbx <https://containertoolbx.org/>`_ or
+`distrobox <https://distrobox.privatedns.org/>`_ on your system, it is highly
 recommended to do that:
 
  - It is known to work and gives you reproducible results.
@@ -29,7 +29,7 @@ To create and enter a development toolbx for Anaconda just run these commands::
 Installing dependencies
 -----------------------
 
-If you are using [cockpit/tasks container](https://quay.io/repository/cockpit/tasks)
+If you are using `cockpit/tasks container <https://quay.io/repository/cockpit/tasks>`_
 for Web UI development only, you can skip this part.
 
 To get all the dependencies and prepare the environment in the container or
@@ -150,31 +150,33 @@ Local development workflow
 ^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 This workflow makes it possible to test changes to the Anaconda source code locally on your machine without any dependencies
-on external infrastructure. It uses two scripts, one called ``scripts/rebuild_boot_iso`` to build a fresh bootable installation image (boot.iso)
-from Anaconda source code on the given branch and corresponding Fedora/CentOS Stream packages. The second script, called ``scripts/update_boot_iso``
+on external infrastructure. It uses two scripts, one called ``scripts/testing/rebuild_iso`` to build a fresh bootable installation image
+from Anaconda source code on the given branch and corresponding Fedora/CentOS Stream packages. The second script, called ``scripts/testing/update_iso``
 uses the Anaconda updates image mechanism together with the ``mkksiso`` command provided by the Lorax project to very quickly
 create an updated version of the boot.iso when Anaconda code is changed. The updated boot.iso can then be booted on a VM or bare metal.
 
-The ``rebuild_boot_iso`` script
+The ``rebuild_iso`` script
 """""""""""""""""""""""""""""""
 
 This is just a simple script that rebuilds the boot.iso from Anaconda source code on the current branch & corresponding Fedora
-(on Fedora branches) or CentOS Stream (on RHEL branches) packages. The script makes sure to remove the old images first
+(on Fedora branches) or CentoOS Stream (on RHEL branches) packages. The script makes sure to remove the old images first
 and also records Anaconda Git revision that was used to build the image.
 
 This should take about 15 minutes on modern hardware.
 
-The ``update_boot_iso`` script
+See --help for further information.
+
+The ``update_iso`` script
 """"""""""""""""""""""""""""""
 
 This is the main script that enables local development by quickly updating a boot iso with local changes.
 This should take a couple seconds on modern hardware.
 
 For the most common use case ("I have changed the Anaconda source and want to see what it does.") just do this:
 
-1. run ``scripts/rebuild_boot_iso`` first, this creates ``result/iso/boot.iso``
+1. run ``scripts/testing/rebuild_iso`` first, this creates ``result/iso/boot.iso``
 2. change the Anaconda source code
-3. run ``scripts/update_boot_iso`` which creates the ``result/iso/updated_boot.iso``
+3. run ``scripts/testing/update_iso`` which creates the ``result/iso/updated_boot.iso``
 4. start the ``result/iso/updated_boot.iso`` in a VM or on bare metal
 
 The script also has a few command line options that might come handy:
@@ -183,6 +185,7 @@ The script also has a few command line options that might come handy:
 * ``-k, --ks-file`` add the specified kickstart file to the updated boot.iso and use it for installation
 * ``-v, --virt-install`` boot the updated iso in a temporary VM for super fast & simple debugging
 * ``-t, --tag`` use a specific Git revision when generating the updates image
+* You can specify custom ISO image (requirement for Live ISO usage) as optional positional parameter.
 
 Running the updated boot.iso
 """"""""""""""""""""""""""""

Makefile.am

@@ -374,7 +374,8 @@ container-iso-build:
 	sudo $(CONTAINER_ENGINE) run \
 	--rm \
 	--privileged \
-	--tmpfs /var/tmp:rw,mode=1777 \
+	$(shell test $(shell grep MemTotal /proc/meminfo | awk '{print $$2}') -gt 11000000 && \
+	  echo --tmpfs /var/tmp:rw,mode=1777) \
 	-v $(srcdir)/result/build/01-rpm-build:/anaconda-rpms:ro \
 	-v $(srcdir)/result/iso:/images:z \
 	$(CONTAINER_ADD_ARGS) \
@@ -404,7 +405,8 @@ container-live-iso-build:
 	--rm \
 	--tty \
 	--privileged \
-	--tmpfs /var/tmp:rw,mode=1777 \
+	$(shell test $(shell grep MemTotal /proc/meminfo | awk '{print $$2}') -gt 15000000 && \
+	  echo --tmpfs /var/tmp:rw,mode=1777) \
 	--device /dev/kvm \
 	-v $(srcdir)/result/build/01-rpm-build:/anaconda-rpms:ro \
 	-v $(srcdir)/result/iso:/images:z \
@@ -479,7 +481,11 @@ tests-pylint:
 # For a weird reason when pylint is not in TESTS included for automake (during configure)
 # it won't allow us to use makefile check to start the tests.
 # Unfortunately, we removed pylint from the TESTS because we don't want it to run as default
-	$(srcdir)/tests/pylint/runpylint | tee $(srcdir)/tests/pylint/runpylint.log
+	set -o pipefail ; \
+	$(srcdir)/tests/pylint/runpylint | tee $(srcdir)/tests/pylint/runpylint.log ; \
+	rc=$$? ; \
+	$(MAKE) grab-logs ; \
+	exit $$rc
 
 tests-unit-only:
 	@mkdir -p $(USER_SITE_PACKAGES)

dockerfile/anaconda-live-iso-creator/lmc-build

@@ -41,7 +41,7 @@ start_http_server() {
     echo "$!" > $HTTP_PID
 
     # extract container IP
-    IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+')
+    IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+' | head -n1)
     echo "http://$IP:8000/"
 }
 

dockerfile/anaconda-live-iso-creator/lmc-build.j2

@@ -36,7 +36,7 @@ start_http_server() {
     echo "$!" > $HTTP_PID
 
     # extract container IP
-    IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+')
+    IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+' | head -n1)
     echo "http://$IP:8000/"
 }
 

scripts/testing/install_dependencies.sh

@@ -39,9 +39,13 @@ sed 's/@PACKAGE_VERSION@/0/; s/@PACKAGE_RELEASE@/0/; s/%{__python3}/python3/' ./
 # get all build requires dependencies from the spec file and strip out version
 # version could be problematic because of fedora version you are running and
 # they are mostly not important for automake
-deps=$(rpmspec -q --buildrequires $TEMP | sed 's/>=.*$//')
+build_deps=$(rpmspec -q --buildrequires $TEMP | sed 's/>=.*$//')
+# add also runtime dependencies for the local development
+# remove anaconda packages and also '(glibc-langpack-en or glibc-all-langpacks)' which will fail otherwise
+requires_deps=$(rpmspec -q --requires $TEMP | grep -v -E "(anaconda-|-widgets| or )" | sed 's/>=.*$//')
+
 # shellcheck disable=SC2068
-dnf install $@ $deps  # do NOT quote the list or it falls apart
+dnf install $@ $build_deps $requires_deps  # do NOT quote the list or it falls apart
 
 # clean up the temp file
 rm $TEMP

scripts/testing/rebuild_iso

@@ -0,0 +1,78 @@
+#!/bin/bash
+#
+# rebuild_boot_iso
+#
+# This script is used to cleanly rebuild boot.iso from the current
+# checked out branch.
+#
+# ask for sudo now, so we have it when we get to the image build
+set -eu
+
+help() {
+    cat <<EOF
+This script will build the anaconda RPMs and then build boot.iso
+or Live ISO from these.
+
+Before starting the build it will remove all the previous build
+artifacts.
+
+Usage:
+  Start the build process:
+    $0 [options]
+
+Options:
+ -h, --help                         Print this help
+ -a, --add-rpms                     Add additional RPMs (for example anaconda-webui package)
+                                    these packages have to be already requested to be in ISO
+ -l, --live                         Build Live ISO instead of boot.iso
+EOF
+}
+
+BOOT_ISO="result/iso/boot.iso"
+PACKAGES_DIR="result/build/01-rpm-build/"
+BOOT_ISO_GIT_REVISION="result/iso/iso.git_rev"
+
+BUILD_TARGET="boot.iso"
+
+COPY_RPMS=""
+
+# parse arguments
+eval set -- "$(getopt -o ha:l --long help,add-rpms:,live -- "$@")"
+
+while true; do
+    case "${1:-}" in
+        -h|--help) help; exit 0 ;;
+        -a|--add-rpms) shift; COPY_RPMS="$COPY_RPMS $1" ;;
+        -l|--live) BUILD_TARGET="live" ;;
+        --) shift; break ;;
+    esac
+    shift
+done
+
+echo "warming up sudo!"
+sudo true
+# remove any previous package and relevant iso artifacts
+rm -rf result/build/
+rm -f ${BOOT_ISO}
+rm -f ${BOOT_ISO_GIT_REVISION}
+# make sure the iso folder actually exists
+mkdir -p result/iso/
+# note the Git revision from which we build the boot.iso
+git rev-parse HEAD > ${BOOT_ISO_GIT_REVISION}
+
+# build the anaconda rpms
+make -f ./Makefile.am container-rpms-scratch
+
+# copy additional web UI packages
+for i in $COPY_RPMS; do
+    cp "$i" "$PACKAGES_DIR"
+done
+
+# build the ISO
+if [ "$BUILD_TARGET" = "boot.iso" ]; then
+    make -f ./Makefile.am anaconda-iso-creator-build
+    make -f ./Makefile.am container-iso-build
+elif [ "$BUILD_TARGET" = "live" ]; then
+    make -f ./Makefile.am anaconda-live-iso-creator-build
+    make -f ./Makefile.am container-live-iso-build
+fi