From 13171c120e826f47d3151bd4cedb71d967488c75 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 13 Aug 2024 13:17:42 +0200 Subject: [PATCH 1/3] infra: Sync infra files from `master` More precisely sync: - Makefile.am - template files - install_dependencies.sh --- .github/workflows/kickstart-tests.yml.j2 | 36 ++++++++++++++++++- .structure-config | 3 +- Makefile.am | 12 +++++-- .../anaconda-live-iso-creator/lmc-build | 2 +- .../anaconda-live-iso-creator/lmc-build.j2 | 2 +- scripts/testing/install_dependencies.sh | 8 +++-- 6 files changed, 53 insertions(+), 10 deletions(-) diff --git a/.github/workflows/kickstart-tests.yml.j2 b/.github/workflows/kickstart-tests.yml.j2 index 30f0873d2a8..ec15c3b6413 100644 --- a/.github/workflows/kickstart-tests.yml.j2 +++ b/.github/workflows/kickstart-tests.yml.j2 @@ -21,6 +21,13 @@ # # /kickstart-test --force rpm-ostree # ... run rpm-ostree test even if it is disabled +# +# +# It is possible to run updated tests via a PR in kickstart-tests repository +# +# /kickstart-test --kstest-pr 1233 --testtype harddrive +# ... run the tests of type harddrive with +# https://github.com/rhinstaller/kickstart-tests/pull/1233 update of kickstart tests name: kickstart-tests on: @@ -70,8 +77,18 @@ jobs: run: | # extract first line and cut out the "/kickstart-tests" first word ARGS=$(echo "$BODY" | sed -n '1 s/^[^ ]* *//p' | sed 's/[[:space:]]*$//') - echo "workflow arguments are: $ARGS" + # parse --kstest-pr option + OPTS="$(getopt -q -o "" --long kstest-pr: -- $ARGS)" || true + PR=${OPTS#" --kstest-pr '"} + PR=$(echo $PR | cut -d " " -f1) + PR=${PR%"'"} + PR=${PR%"--"} + # remove --kstest-pr option + ARGS=$(echo "$ARGS" | sed -r -e "s#--kstest-pr( +|=)$PR ##" | sed 's/[[:space:]]*$//') + echo "test selection arguments are: $ARGS" echo "comment_args=${ARGS}" >> $GITHUB_OUTPUT + echo "kickstart-tests PR: $PR" + echo "kstest_pr=${PR}" >> $GITHUB_OUTPUT outputs: allowed_user: ${{ steps.check_user_perm.outputs.allowed_user }} @@ -79,6 +96,7 @@ jobs: sha: ${{ fromJson(steps.pr_api.outputs.data).head.sha }} comment_args: ${{ steps.parse_comment_args.outputs.comment_args }} target_branch: ${{ fromJson(steps.pr_api.outputs.data).base.ref }} + kstest_pr: ${{ steps.parse_comment_args.outputs.kstest_pr }} run: needs: pr-info @@ -134,8 +152,24 @@ jobs: uses: actions/checkout@v4 with: repository: rhinstaller/kickstart-tests + fetch-depth: 0 path: kickstart-tests + - name: Pull the request and rebase to the upstream target + working-directory: ./kickstart-tests + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + if [ -n "${{ needs.pr-info.outputs.kstest_pr }}" ]; then + gh pr checkout ${{ needs.pr-info.outputs.kstest_pr }} + BASE_REF=$(gh pr view --json baseRefName --template '{{.baseRefName}}' ${{ needs.pr-info.outputs.kstest_pr }}) + git config user.name github-actions + git config user.email github-actions@github.com + git log --oneline -1 $BASE_REF + git rebase $BASE_REF + echo "Rebasing PR ${{ needs.pr-info.outputs.kstest_pr }} to upstream $BASE_REF" + fi + - name: Generate test cases working-directory: ./kickstart-tests run: scripts/generate-testcases.py -t ./testlib/test_cases/kstest-template.tc.yaml.j2 . -o ./testlib/test_cases diff --git a/.structure-config b/.structure-config index 75f783b016e..7c54fc6accd 100644 --- a/.structure-config +++ b/.structure-config @@ -8,14 +8,13 @@ INFRASTRUCTURE_FILES=( .coveragerc .shellcheckrc .github/ +.gitignore dockerfile/ scripts/testing/ scripts/jinja-render scripts/makebumpver scripts/rhel_version.py scripts/rhel_version.py.j2 -scripts/rebuild_boot_iso -scripts/update_boot_iso dracut/.shellcheckrc docs/ci-status.rst CONTRIBUTING.rst diff --git a/Makefile.am b/Makefile.am index 8edadc28236..5e82aa4c59e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -374,7 +374,8 @@ container-iso-build: sudo $(CONTAINER_ENGINE) run \ --rm \ --privileged \ - --tmpfs /var/tmp:rw,mode=1777 \ + $(shell test $(shell grep MemTotal /proc/meminfo | awk '{print $$2}') -gt 11000000 && \ + echo --tmpfs /var/tmp:rw,mode=1777) \ -v $(srcdir)/result/build/01-rpm-build:/anaconda-rpms:ro \ -v $(srcdir)/result/iso:/images:z \ $(CONTAINER_ADD_ARGS) \ @@ -404,7 +405,8 @@ container-live-iso-build: --rm \ --tty \ --privileged \ - --tmpfs /var/tmp:rw,mode=1777 \ + $(shell test $(shell grep MemTotal /proc/meminfo | awk '{print $$2}') -gt 15000000 && \ + echo --tmpfs /var/tmp:rw,mode=1777) \ --device /dev/kvm \ -v $(srcdir)/result/build/01-rpm-build:/anaconda-rpms:ro \ -v $(srcdir)/result/iso:/images:z \ @@ -479,7 +481,11 @@ tests-pylint: # For a weird reason when pylint is not in TESTS included for automake (during configure) # it won't allow us to use makefile check to start the tests. # Unfortunately, we removed pylint from the TESTS because we don't want it to run as default - $(srcdir)/tests/pylint/runpylint | tee $(srcdir)/tests/pylint/runpylint.log + set -o pipefail ; \ + $(srcdir)/tests/pylint/runpylint | tee $(srcdir)/tests/pylint/runpylint.log ; \ + rc=$$? ; \ + $(MAKE) grab-logs ; \ + exit $$rc tests-unit-only: @mkdir -p $(USER_SITE_PACKAGES) diff --git a/dockerfile/anaconda-live-iso-creator/lmc-build b/dockerfile/anaconda-live-iso-creator/lmc-build index a4e6f792e46..d1143a30f51 100755 --- a/dockerfile/anaconda-live-iso-creator/lmc-build +++ b/dockerfile/anaconda-live-iso-creator/lmc-build @@ -41,7 +41,7 @@ start_http_server() { echo "$!" > $HTTP_PID # extract container IP - IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+') + IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+' | head -n1) echo "http://$IP:8000/" } diff --git a/dockerfile/anaconda-live-iso-creator/lmc-build.j2 b/dockerfile/anaconda-live-iso-creator/lmc-build.j2 index c3ee5d752fb..a119ae2c35c 100755 --- a/dockerfile/anaconda-live-iso-creator/lmc-build.j2 +++ b/dockerfile/anaconda-live-iso-creator/lmc-build.j2 @@ -36,7 +36,7 @@ start_http_server() { echo "$!" > $HTTP_PID # extract container IP - IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+') + IP=$(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+' | head -n1) echo "http://$IP:8000/" } diff --git a/scripts/testing/install_dependencies.sh b/scripts/testing/install_dependencies.sh index 61f6fa0dee8..3a00b545406 100755 --- a/scripts/testing/install_dependencies.sh +++ b/scripts/testing/install_dependencies.sh @@ -39,9 +39,13 @@ sed 's/@PACKAGE_VERSION@/0/; s/@PACKAGE_RELEASE@/0/; s/%{__python3}/python3/' ./ # get all build requires dependencies from the spec file and strip out version # version could be problematic because of fedora version you are running and # they are mostly not important for automake -deps=$(rpmspec -q --buildrequires $TEMP | sed 's/>=.*$//') +build_deps=$(rpmspec -q --buildrequires $TEMP | sed 's/>=.*$//') +# add also runtime dependencies for the local development +# remove anaconda packages and also '(glibc-langpack-en or glibc-all-langpacks)' which will fail otherwise +requires_deps=$(rpmspec -q --requires $TEMP | grep -v -E "(anaconda-|-widgets| or )" | sed 's/>=.*$//') + # shellcheck disable=SC2068 -dnf install $@ $deps # do NOT quote the list or it falls apart +dnf install $@ $build_deps $requires_deps # do NOT quote the list or it falls apart # clean up the temp file rm $TEMP From 31586d10cfc912768a140539417cc66bc02f8d60 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 13 Aug 2024 13:18:42 +0200 Subject: [PATCH 2/3] infra: Update ISO helper scripts from `master` These scripts were enhanced and moved on master. Let's make that consistent. --- scripts/rebuild_boot_iso | 24 ------ scripts/testing/rebuild_iso | 78 +++++++++++++++++ .../{update_boot_iso => testing/update_iso} | 85 ++++++++++++++----- 3 files changed, 142 insertions(+), 45 deletions(-) delete mode 100755 scripts/rebuild_boot_iso create mode 100755 scripts/testing/rebuild_iso rename scripts/{update_boot_iso => testing/update_iso} (76%) diff --git a/scripts/rebuild_boot_iso b/scripts/rebuild_boot_iso deleted file mode 100755 index 6f4454e832b..00000000000 --- a/scripts/rebuild_boot_iso +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# -# rebuild_boot_iso -# -# This script is used to cleanly rebuild boot.iso from the current -# checked out branch. -# -# ask for sudo now, so we have it when we get to the image build -sudo echo "warming up sudo!" -BOOT_ISO="result/iso/boot.iso" -UPDATED_BOOT_ISO="result/iso/boot.iso.git_rev" -BOOT_ISO_GIT_REVISION="result/iso/boot.iso.git_rev" -# remove any previous package and relevant iso artifacts -rm -rf result/build/ -rm -f ${BOOT_ISO} -rm -f ${UPDATED_BOOT_ISO} -rm -f ${BOOT_ISO_GIT_REVISION} -# make sure the iso folder actually exists -mkdir -p result/iso/ -# note the Git revision from which we build the boot.iso -git rev-parse HEAD > result/iso/boot.iso.git_rev -make -f ./Makefile.am container-rpms-scratch -make -f ./Makefile.am anaconda-iso-creator-build -make -f ./Makefile.am container-iso-build diff --git a/scripts/testing/rebuild_iso b/scripts/testing/rebuild_iso new file mode 100755 index 00000000000..1635f85bfe3 --- /dev/null +++ b/scripts/testing/rebuild_iso @@ -0,0 +1,78 @@ +#!/bin/bash +# +# rebuild_boot_iso +# +# This script is used to cleanly rebuild boot.iso from the current +# checked out branch. +# +# ask for sudo now, so we have it when we get to the image build +set -eu + +help() { + cat < ${BOOT_ISO_GIT_REVISION} + +# build the anaconda rpms +make -f ./Makefile.am container-rpms-scratch + +# copy additional web UI packages +for i in $COPY_RPMS; do + cp "$i" "$PACKAGES_DIR" +done + +# build the ISO +if [ "$BUILD_TARGET" = "boot.iso" ]; then + make -f ./Makefile.am anaconda-iso-creator-build + make -f ./Makefile.am container-iso-build +elif [ "$BUILD_TARGET" = "live" ]; then + make -f ./Makefile.am anaconda-live-iso-creator-build + make -f ./Makefile.am container-live-iso-build +fi diff --git a/scripts/update_boot_iso b/scripts/testing/update_iso similarity index 76% rename from scripts/update_boot_iso rename to scripts/testing/update_iso index 41990b13aaa..c786da07491 100755 --- a/scripts/update_boot_iso +++ b/scripts/testing/update_iso @@ -1,8 +1,8 @@ #!/usr/bin/python3 # -# update_boot_iso +# update_iso # -# This script is used to quickly update a boot.iso +# This script is used to quickly update an installer iso # via the mkksiso tool. See CONTRIBUTING.rst for more information # about how this works & --help for available boot options. @@ -13,15 +13,24 @@ import time import sys import subprocess +from pathlib import Path + # Absolute path to the main project directory -PROJECT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PROJECT_DIR = os.path.dirname( + os.path.dirname( + os.path.dirname(os.path.abspath(__file__)) + ) + ) # Relative path to the ISO folder within the project ISO_FOLDER = os.path.join(PROJECT_DIR, "result", "iso") +# Name of the file storing git revision of the anaconda repository +REVISION_FILE_NAME = "iso.git_rev" + # Initial boot ISO we will update INPUT_ISO = os.path.join(ISO_FOLDER, "boot.iso") -INPUT_ISO_REVISION_FILE = os.path.join(ISO_FOLDER, "boot.iso.git_rev") +INPUT_ISO_REVISION_FILE = os.path.join(ISO_FOLDER, REVISION_FILE_NAME) # Updated boot ISO (including Anaconda updates image and possibly other bits) UPDATED_ISO = os.path.join(ISO_FOLDER, "updated_boot.iso") @@ -58,10 +67,11 @@ def get_first_non_upstream_commit(): return None -def make_updates_image(git_id): +def make_updates_image(git_id, rpm_paths): """Build an updates image based on tag/hash and prepare it for inclusion in boot ISO. :param str git_id: git revision id (hash, tag, etc.) + :param [str] rpm_paths: list of paths to the RPM files we are adding to updates_image """ if git_id is None: print("** make updates:git_id is None, falling back to finding first non-upstream commit id") @@ -70,11 +80,17 @@ def make_updates_image(git_id): print("** error: could not determine a valid commit id") sys.exit(1) + rpm_args = "" + for rpm in rpm_paths: + rpm_args += " -a " + rpm + print("** preparing updates image via tag/hash: %s" % git_id) # Create the necessary folder structure os.makedirs(UPDATES_FOLDER, exist_ok=True) # Prepare updates image - os.system("./scripts/makeupdates -k -c -t %s" % git_id) + cmd = f"./scripts/makeupdates -c -t {git_id}{rpm_args}" + print("** Calling:", cmd) + os.system(cmd) # Move it next to the ISOs shutil.move(UPDATES_IMAGE, os.path.join(UPDATES_FOLDER, UPDATES_IMAGE)) print("** updates image is ready in: %s" % UPDATES_FOLDER) @@ -86,13 +102,13 @@ def check_input_iso_available(): If not, notify the user and exit. """ if os.path.exists(INPUT_ISO): - print("** using input boot ISO: %s" % INPUT_ISO) + print("** using input ISO: %s" % INPUT_ISO) else: - print("** error: input boot ISO (%s) not found" % INPUT_ISO) + print("** error: input ISO (%s) not found" % INPUT_ISO) sys.exit(1) def get_boot_iso_revision(): - """Check if we have a Git revision for the input boot.iso. + """Check if we have an Anaconda Git revision for the input ISO. :return: revision string or None if revision file was not found :rtype: bool @@ -100,7 +116,7 @@ def get_boot_iso_revision(): if os.path.exists(INPUT_ISO_REVISION_FILE): with open(INPUT_ISO_REVISION_FILE, "rt") as f: boot_iso_git_rev = f.read().strip() - print("** found Git revision for boot.iso:") + print("** found Anaconda Git revision for ISO:") print(boot_iso_git_rev) return boot_iso_git_rev else: @@ -210,10 +226,33 @@ def run_updated_iso_with_virt_install(): subprocess.run(cmd, check=True) print("** virt-install finished running") +def set_input_iso(args): + """Call checks and configure input ISO based on the user parameter + + This function will configure INPUT_ISO, UPDATED_ISO global variables. + """ + global INPUT_ISO + global UPDATED_ISO + global INPUT_ISO_REVISION_FILE + + INPUT_ISO = args.input_iso_path + # this code will take input ISO path and create updated ISO path by + # directory (parent) + file name without suffix (stem) + "-updated" + file suffix (suffix) + UPDATED_ISO = os.path.join(args.input_iso_path.parent, + f"{args.input_iso_path.stem}-updated{args.input_iso_path.suffix}") + INPUT_ISO_REVISION_FILE = os.path.join(args.input_iso_path.parent, REVISION_FILE_NAME) + + def main(): - parser = argparse.ArgumentParser(description="update Anaconda boot.iso") + parser = argparse.ArgumentParser(description="update Anaconda ISO image") + parser.add_argument('input_iso_path', metavar='INPUT_ISO', type=Path, nargs='?', default=INPUT_ISO, + help='path to the input ISO (optional)') parser.add_argument('-t', '--tag', action='store', type=str, help='add commits from TAG to HEAD to the image (NOTE: also works with commit hashes)') + parser.add_argument('-a', '--add-rpm', action='append', type=str, dest='rpm_paths', + metavar='RPM_PATH', default=[], + help='paths to additional RPMs which will be unpacked to updates image;' + ' can be used multiple times') parser.add_argument('-k', '--ks-file', action='store', type=str, help='path to the kickstart file') parser.add_argument('-b', '--boot-options', action='store', type=str, @@ -222,6 +261,9 @@ def main(): help='boot the updated iso with virt-install') args = parser.parse_args() + # Set INPUT_ISO and UPDATED_ISO based on user preferences + set_input_iso(args) + # Check if we have the input ISO check_input_iso_available() @@ -231,35 +273,36 @@ def main(): # Get sudo, needed for later (mkksiso) warmup_sudo() - # Check if we know git revision for the boot.iso - boot_iso_git_rev = get_boot_iso_revision() + # Check if we know git revision for the ISO + iso_git_rev = get_boot_iso_revision() # Now we need to get the base Git revision for building the updates image. # Every commit after this revision + uncommitted changes will be included - # in the updates image, which will then be itself added to the updated boot.iso + # in the updates image, which will then be itself added to the updated ISO base_git_revision = None if args.tag: print("** using user specified Git revision for the updates image") base_git_revision = args.tag - elif boot_iso_git_rev: - print("** using Git revision from the input boot.iso for the updates image") - base_git_revision = boot_iso_git_rev + elif iso_git_rev: + print("** using Git revision from the input ISO for the updates image") + base_git_revision = iso_git_rev else: print("** error: git revision not specified - please use --tag or make " - "sure the input boot.iso has a matching Git revision file") + "sure the input ISO has a matching Git revision file") sys.exit(1) # Generate updates image - make_updates_image(base_git_revision) + make_updates_image(base_git_revision, args.rpm_paths) # Check updates image has been generated and is in place check_updates_image_available() - # Remove previous updated boot ISO (if it exists) + # Remove previous updated ISO (if it exists) if os.path.exists(UPDATED_ISO): + print(f"Removing previous updated ISO: {UPDATED_ISO}") os.remove(UPDATED_ISO) - # Generate updated boot ISO + # Generate updated ISO generate_updated_iso(args.ks_file, args.boot_options) # Check the updated ISO has been generated From c7a4b6891d1c9d16d768ba7365cefc67c8654b41 Mon Sep 17 00:00:00 2001 From: Jiri Konecny Date: Tue, 13 Aug 2024 13:21:48 +0200 Subject: [PATCH 3/3] infra: Update CONTRIBUTING.rst from `master` --- CONTRIBUTING.rst | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index f85a1d9f799..5016873a6a6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -10,8 +10,8 @@ Setting up development container -------------------------------- The anaconda team uses a containerized development environment using toolbx. -If you can install [toolbx](https://containertoolbx.org/) or -[distrobox](https://distrobox.privatedns.org/) on your system, it is highly +If you can install `toolbx `_ or +`distrobox `_ on your system, it is highly recommended to do that: - It is known to work and gives you reproducible results. @@ -29,7 +29,7 @@ To create and enter a development toolbx for Anaconda just run these commands:: Installing dependencies ----------------------- -If you are using [cockpit/tasks container](https://quay.io/repository/cockpit/tasks) +If you are using `cockpit/tasks container `_ for Web UI development only, you can skip this part. To get all the dependencies and prepare the environment in the container or @@ -150,21 +150,23 @@ Local development workflow ^^^^^^^^^^^^^^^^^^^^^^^^^^ This workflow makes it possible to test changes to the Anaconda source code locally on your machine without any dependencies -on external infrastructure. It uses two scripts, one called ``scripts/rebuild_boot_iso`` to build a fresh bootable installation image (boot.iso) -from Anaconda source code on the given branch and corresponding Fedora/CentOS Stream packages. The second script, called ``scripts/update_boot_iso`` +on external infrastructure. It uses two scripts, one called ``scripts/testing/rebuild_iso`` to build a fresh bootable installation image +from Anaconda source code on the given branch and corresponding Fedora/CentOS Stream packages. The second script, called ``scripts/testing/update_iso`` uses the Anaconda updates image mechanism together with the ``mkksiso`` command provided by the Lorax project to very quickly create an updated version of the boot.iso when Anaconda code is changed. The updated boot.iso can then be booted on a VM or bare metal. -The ``rebuild_boot_iso`` script +The ``rebuild_iso`` script """"""""""""""""""""""""""""""" This is just a simple script that rebuilds the boot.iso from Anaconda source code on the current branch & corresponding Fedora -(on Fedora branches) or CentOS Stream (on RHEL branches) packages. The script makes sure to remove the old images first +(on Fedora branches) or CentoOS Stream (on RHEL branches) packages. The script makes sure to remove the old images first and also records Anaconda Git revision that was used to build the image. This should take about 15 minutes on modern hardware. -The ``update_boot_iso`` script +See --help for further information. + +The ``update_iso`` script """""""""""""""""""""""""""""" This is the main script that enables local development by quickly updating a boot iso with local changes. @@ -172,9 +174,9 @@ This should take a couple seconds on modern hardware. For the most common use case ("I have changed the Anaconda source and want to see what it does.") just do this: -1. run ``scripts/rebuild_boot_iso`` first, this creates ``result/iso/boot.iso`` +1. run ``scripts/testing/rebuild_iso`` first, this creates ``result/iso/boot.iso`` 2. change the Anaconda source code -3. run ``scripts/update_boot_iso`` which creates the ``result/iso/updated_boot.iso`` +3. run ``scripts/testing/update_iso`` which creates the ``result/iso/updated_boot.iso`` 4. start the ``result/iso/updated_boot.iso`` in a VM or on bare metal The script also has a few command line options that might come handy: @@ -183,6 +185,7 @@ The script also has a few command line options that might come handy: * ``-k, --ks-file`` add the specified kickstart file to the updated boot.iso and use it for installation * ``-v, --virt-install`` boot the updated iso in a temporary VM for super fast & simple debugging * ``-t, --tag`` use a specific Git revision when generating the updates image +* You can specify custom ISO image (requirement for Live ISO usage) as optional positional parameter. Running the updated boot.iso """"""""""""""""""""""""""""