diff --git a/build.sh b/build.sh index 5752c4edd1..3fb986ea56 100755 --- a/build.sh +++ b/build.sh @@ -177,6 +177,9 @@ patch_osbuild() { /usr/lib/coreos-assembler/0002-parsing-treat-locations-without-scheme-as-belonging-.patch \ /usr/lib/coreos-assembler/0003-org.osbuild.selinux-support-operating-on-mounts.patch \ /usr/lib/coreos-assembler/0004-org.osbuild.selinux-support-for-specifying-where-fil.patch \ + /usr/lib/coreos-assembler/0001-util-osrelease.py-improve-whitespace-and-quote-strip.patch \ + /usr/lib/coreos-assembler/0002-util-chroot-Add-support-for-custom-directory-bind-mo.patch \ + /usr/lib/coreos-assembler/0003-stages-add-coreos.live-iso-stage.patch \ | patch -d /usr/lib/osbuild -p1 # And then move the files back; supermin appliance creation will need it back diff --git a/src/0001-util-osrelease.py-improve-whitespace-and-quote-strip.patch b/src/0001-util-osrelease.py-improve-whitespace-and-quote-strip.patch new file mode 100644 index 0000000000..75f4ef98b8 --- /dev/null +++ b/src/0001-util-osrelease.py-improve-whitespace-and-quote-strip.patch @@ -0,0 +1,32 @@ +From 106e632a410e7391e1d667f5aa7062776ed19179 Mon Sep 17 00:00:00 2001 +From: Renata Ravanelli +Date: Tue, 12 Nov 2024 15:12:52 -0300 +Subject: [PATCH 1/3] util/osrelease.py: improve whitespace and quote stripping + +- Enhanced the value stripping logic in osrelease parsing +to handle leading and trailing spaces, newlines, tabs, +and both single and double quotes. +- This ensures cleaner and more accurate key-value assignments. + +Signed-off-by: Renata Ravanelli +(cherry picked from commit 066f1ea89fbda6e886a5d88119586c0f09b0a234) +--- + osbuild/util/osrelease.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/osbuild/util/osrelease.py b/osbuild/util/osrelease.py +index b8d56e73..a2b61d26 100644 +--- a/osbuild/util/osrelease.py ++++ b/osbuild/util/osrelease.py +@@ -33,7 +33,7 @@ def parse_files(*paths): + if line[0] == "#": + continue + key, value = line.split("=", 1) +- osrelease[key] = value.strip('"') ++ osrelease[key] = value.strip(" \n\t'\"") + + return osrelease + +-- +2.47.0 + diff --git a/src/0002-util-chroot-Add-support-for-custom-directory-bind-mo.patch b/src/0002-util-chroot-Add-support-for-custom-directory-bind-mo.patch new file mode 100644 index 0000000000..2ebd46b14b --- /dev/null +++ b/src/0002-util-chroot-Add-support-for-custom-directory-bind-mo.patch @@ -0,0 +1,59 @@ +From f5f269eb31100c22390d35fd658bdd236a0a880e Mon Sep 17 00:00:00 2001 +From: Renata Ravanelli +Date: Thu, 31 Oct 2024 14:13:50 -0300 +Subject: [PATCH 2/3] util/chroot: Add support for custom directory bind mounts + + - Add optional bind_mounts parameter to __init__ method; + - Enhanced methods to accept an optional `bind_mounts`. +This allows for more flexible for configurations when setting +up bind mounts. + +Signed-off-by: Renata Ravanelli +(cherry picked from commit 9b5fbadee6b170455d62c57eb315e20d57173110) +--- + osbuild/util/chroot.py | 14 +++++++++++++- + 1 file changed, 13 insertions(+), 1 deletion(-) + +diff --git a/osbuild/util/chroot.py b/osbuild/util/chroot.py +index da14bf44..4090456b 100644 +--- a/osbuild/util/chroot.py ++++ b/osbuild/util/chroot.py +@@ -12,8 +12,9 @@ class Chroot: + This mounts /proc, /dev, and /sys. + """ + +- def __init__(self, root: str): ++ def __init__(self, root: str, bind_mounts=None): + self.root = root ++ self._bind_mounts = bind_mounts or [] + + def __enter__(self): + for d in ["/proc", "/dev", "/sys"]: +@@ -33,6 +34,13 @@ class Chroot: + "sysfs", f"{self.root}/sys"], + check=True) + ++ for d in self._bind_mounts: ++ target_path = os.path.join(self.root, d.lstrip("/")) ++ if not os.path.exists(target_path): ++ print(f"Making missing chroot directory: {d}") ++ os.makedirs(target_path) ++ subprocess.run(["mount", "--rbind", d, target_path], check=True) ++ + return self + + def __exit__(self, exc_type, exc_value, tracebk): +@@ -43,6 +51,10 @@ class Chroot: + if failed_umounts: + print(f"Error unmounting paths from chroot: {failed_umounts}") + ++ for d in self._bind_mounts[::-1]: ++ target_path = os.path.join(self.root, d.lstrip("/")) ++ if subprocess.run(["umount", "--lazy", target_path], check=False).returncode != 0: ++ print(f"Error unmounting paths from chroot: {d}") + def run(self, cmd, **kwargs): + cmd = ["chroot", self.root] + cmd + # pylint: disable=subprocess-run-check +-- +2.47.0 + diff --git a/src/0003-stages-add-coreos.live-iso-stage.patch b/src/0003-stages-add-coreos.live-iso-stage.patch new file mode 100644 index 0000000000..568d284f60 --- /dev/null +++ b/src/0003-stages-add-coreos.live-iso-stage.patch @@ -0,0 +1,867 @@ +From d1b53923ebcfa71069c7d55b297485f49722be65 Mon Sep 17 00:00:00 2001 +From: Jonathan Lebon +Date: Tue, 13 Aug 2024 12:13:53 -0400 +Subject: [PATCH 3/3] stages: add `coreos.live-iso` stage + +This adds a new `org.osbuild.coreos.live-iso` stage to build a CoreOS +live ISO. The code is heavily based on the `cmd-buildextend-live` script +from coreos-assembler, but a lot of things had to be adapted: +- the stage is provided the deployed oscontainer tree, metal, and + metal4k images as inputs +- we use chroot instead of supermin to execute some commands in the + context of the target oscontainer +- we use a loopback device to create `efiboot.img` instead of libguestfs +- a bunch of calls that were wrapped by libguestfs for us (e.g. + mkfs.vfat, mksquashfs), we now have to call ourselves; to retain + maximum compatibility, I ensured that we still effectively use the + same args that libguestfs passed +- we require read-write access to make certain changes, so we use +OverlayFS instead of mounting the filesystem with the rw option. + +And various other minor adjustments. + +Of course, this is not really in the spirit of the osbuild philosophy +of having smaller-scoped stages. The goal should be to eventually break +this down in many stages that we can then assemble into a pipeline. +However, this will take a lot of time and effort. The CoreOS live ISO +has _many_ special things going on and exposes APIs that are heavily +relied on by both OpenShift and end-users. Having this code live here at +least gives a way to do this incrementally. + +It also more quickly unlocks a major benefit: customized CoreOS live +ISOs. The only real top-level input is the oscontainer (the metal +images are derivations from that), and this new stage can be used with +customized CoreOS oscontainers. + +Co-authored-by: Dusty Mabe +Co-authored-by: Renata Ravanelli + +Signed-off-by: Renata Ravanelli +--- + stages/org.osbuild.coreos.live-artifacts.mono | 780 ++++++++++++++++++ + ...build.coreos.live-artifacts.mono.meta.json | 26 + + 2 files changed, 806 insertions(+) + create mode 100755 stages/org.osbuild.coreos.live-artifacts.mono + create mode 100644 stages/org.osbuild.coreos.live-artifacts.mono.meta.json + +diff --git a/stages/org.osbuild.coreos.live-artifacts.mono b/stages/org.osbuild.coreos.live-artifacts.mono +new file mode 100755 +index 00000000..1b59877e +--- /dev/null ++++ b/stages/org.osbuild.coreos.live-artifacts.mono +@@ -0,0 +1,780 @@ ++#!/usr/bin/python3 ++ ++# This stage is based on coreos-assembler's `cmd-buildextend-live`. ++# For historical context and to see its evolution, refer to the original source: ++# https://github.com/coreos/coreos-assembler/blob/43a9c80e1f548269d71d6d586f0d5754c60f6144/src/cmd-buildextend-live ++import glob ++import hashlib ++import json ++import os ++import re ++import shutil ++import struct ++import subprocess ++import sys ++import tarfile ++import tempfile ++import yaml ++ ++import osbuild.api ++from osbuild.util import checksum, osrelease ++from osbuild.util.chroot import Chroot ++ ++ ++# Size of file used to embed an Ignition config within a CPIO. ++IGNITION_IMG_SIZE = 256 * 1024 ++ ++# Size of the file used to embed miniso data. ++MINISO_DATA_FILE_SIZE = 16 * 1024 ++ ++LIVE_EXCLUDE_KARGS = set([ ++ '$ignition_firstboot', # unsubstituted variable in grub config ++ 'console', # no serial console by default on ISO ++ 'ignition.platform.id', # we hardcode "metal" ++ 'ostree', # dracut finds the tree automatically ++]) ++ ++ ++# The kernel requires that uncompressed cpio archives appended to an initrd ++# start on a 4-byte boundary. If there's misalignment, it stops unpacking ++# and says: ++# ++# Initramfs unpacking failed: invalid magic at start of compressed archive ++# ++# Append NUL bytes to destf until its size is a multiple of 4 bytes. ++# ++# https://www.kernel.org/doc/Documentation/early-userspace/buffer-format.txt ++# https://github.com/torvalds/linux/blob/47ec5303/init/initramfs.c#L463 ++def align_initrd_for_uncompressed_append(destf): ++ offset = destf.tell() ++ if offset % 4: ++ destf.write(b'\0' * (4 - offset % 4)) ++ ++ ++# Return OS features table for features.json, which is read by ++# coreos-installer {iso|pxe} customize ++def get_os_features(deployed_tree): ++ features = { ++ # coreos-installer >= 0.12.0 ++ 'installer-config': True, ++ # coreos/fedora-coreos-config@3edd2f28 ++ 'live-initrd-network': True, ++ } ++ ++ with open(os.path.join(deployed_tree, 'usr/share/coreos-installer/example-config.yaml')) as f: ++ example_config_yaml = yaml.safe_load(f) ++ features['installer-config-directives'] = { ++ k: True for k in example_config_yaml ++ } ++ ++ return features ++ ++ ++# https://www.kernel.org/doc/html/latest/admin-guide/initrd.html#compressed-cpio-images ++def mkinitrd_pipe(tmproot, destf, compress=True): ++ if not compress: ++ align_initrd_for_uncompressed_append(destf) ++ files = subprocess.check_output(['find', '.', '-mindepth', '1', '-print0'], ++ cwd=tmproot) ++ file_list = files.split(b'\0') ++ # If there's a root.squashfs, it _must_ be the first file in the cpio ++ # archive, since the dracut 20live module assumes its contents are at ++ # a fixed offset in the archive. ++ squashfs = b'./root.squashfs' ++ if squashfs in file_list: ++ file_list.remove(squashfs) ++ file_list.insert(0, squashfs) ++ cpioproc = subprocess.Popen(['cpio', '-o', '-H', 'newc', '-R', 'root:root', ++ '--quiet', '--reproducible', '--force-local', '--null', ++ '-D', tmproot], stdin=subprocess.PIPE, stdout=subprocess.PIPE) ++ if compress: ++ gzipargs = ['gzip', '-9'] ++ else: ++ gzipargs = ['cat'] ++ gzipproc = subprocess.Popen(gzipargs, stdin=cpioproc.stdout, stdout=destf) ++ cpioproc.stdin.write(b'\0'.join(file_list)) ++ cpioproc.stdin.close() ++ assert cpioproc.wait() == 0, f"cpio exited with {cpioproc.returncode}" ++ assert gzipproc.wait() == 0, f"gzip exited with {gzipproc.returncode}" ++ # Fix up padding so the user can append the rootfs afterward ++ align_initrd_for_uncompressed_append(destf) ++ ++ ++def extend_initrd(initramfs, tmproot, compress=True): ++ with open(initramfs, 'ab') as fdst: ++ mkinitrd_pipe(tmproot, fdst, compress=compress) ++ ++ ++def cp_reflink(src, dest): ++ subprocess.check_call(['cp', '--reflink=auto', src, dest]) ++ ++ ++# Make stream hash for `rdcore stream-hash` ++# https://github.com/coreos/coreos-installer/blob/a8d6f50dea6e/src/bin/rdcore/stream_hash.rs#L26-L41 ++def make_stream_hash(src, dest): ++ bufsize = 2 * 1024 * 1024 ++ with open(src, 'rb') as inf: ++ with open(dest, 'w') as outf: ++ outf.write('stream-hash sha256 {}\n'.format(bufsize)) ++ while True: ++ buf = inf.read(bufsize) ++ if not buf: ++ break ++ outf.write(hashlib.sha256(buf).hexdigest() + '\n') ++ ++ ++def get_os_name(deployed_tree): ++ with open(os.path.join(deployed_tree, 'usr/share/rpm-ostree/treefile.json')) as f: ++ treefile = json.load(f) ++ return treefile['rojig']['name'] ++ ++ ++def ensure_glob(pathname, **kwargs): ++ '''Call glob.glob(), and fail if there are no results.''' ++ ret = glob.glob(pathname, **kwargs) ++ if not ret: ++ raise Exception(f'No matches for {pathname}') ++ return ret ++ ++ ++# This creates efiboot.img, which is a FAT filesystem. ++def make_efi_bootfile(tar_filename, efiboot_lodev): ++ # lo = ctl.loop_for_fd(f.fileno(), autoclear=True) ++ # On RHEL 8, when booting from a disk device (rather than a CD), ++ # https://github.com/systemd/systemd/issues/14408 causes the ++ # hybrid ESP to race with the ISO9660 filesystem for the ++ # /dev/disk/by-label symlink unless the ESP has its own label, ++ # so set EFI-SYSTEM for consistency with the metal image. ++ # This should not be needed on Fedora or RHEL 9, but seems like ++ # a good thing to do anyway. ++ label = 'EFI-SYSTEM' ++ # this matches how virt-make-fs called mkfs ++ subprocess.check_call(['mkfs', '-t', 'vfat', '-I', '--mbr=n', '-n', label, efiboot_lodev]) ++ with tempfile.TemporaryDirectory() as d: ++ try: ++ subprocess.check_call(['mount', '-o', 'utf8', efiboot_lodev, d]) ++ subprocess.check_call(['tar', '-C', d, '-xf', tar_filename]) ++ finally: ++ subprocess.check_call(['umount', d]) ++ ++ ++def main(inputs, devices, tree, options): ++ output_file = os.path.join(tree, options['filename']) ++ img_metal = devices['metal']['path'] ++ img_metal4k = devices['metal4k']['path'] ++ efiboot_img = devices['efiboot_img']['path'] ++ efiboot_img_filename = os.path.join(tree, options['efiboot_img_filename']) ++ squashfs_compression = options['squashfs_compression'] ++ basearch = os.uname().machine ++ ++ base_name = get_os_name(inputs['tree']['path']) ++ os_release = osrelease.parse_files(inputs['tree']['path'] + "/etc/os-release") ++ version = os_release['OSTREE_VERSION'] ++ ++ iso_name = f'{base_name}-{version}-live.{basearch}.iso' ++ name_version = f'{base_name}-{version}' ++ # The short volume ID can only be 32 characters (bytes probably). We may in the future want ++ # to shorten this more intelligently, otherwise we truncate the ++ # version which may impede uniqueness. ++ volid = name_version[0:32] ++ ++ tmpisoroot = os.path.join(tree, 'live') ++ tmpisocoreos = os.path.join(tmpisoroot, 'coreos') ++ tmpisoimages = os.path.join(tmpisoroot, 'images') ++ tmpisoimagespxe = os.path.join(tmpisoimages, 'pxeboot') ++ tmpisoisolinux = os.path.join(tmpisoroot, 'isolinux') ++ # contents of initramfs on both PXE and ISO ++ tmpinitrd_base = os.path.join(tree, 'initrd') ++ # contents of rootfs image ++ tmpinitrd_rootfs = os.path.join(tree, 'initrd-rootfs') ++ ++ tmp_merged_dir = os.path.join(tree, 'merged') ++ tmp_workdir_root = os.path.join(tree, 'workdir-root') ++ tmp_upperdir_root = os.path.join(tree, 'upperdir-root') ++ tmp_workdir_boot = os.path.join(tree, 'workdir-boot') ++ tmp_upperdir_boot = os.path.join(tree, 'upperdir-boot') ++ ++ for d in (tmpisoroot, tmpisocoreos, tmpisoimages, tmpisoimagespxe, ++ tmpisoisolinux, tmpinitrd_base, tmpinitrd_rootfs, tmp_merged_dir, ++ tmp_workdir_root, tmp_upperdir_root, tmp_workdir_boot, tmp_upperdir_boot): ++ os.mkdir(d) ++ ++ # convention for kernel and initramfs names ++ kernel_img = 'vmlinuz' ++ initrd_img = 'initrd.img' ++ # other files ++ rootfs_img = 'rootfs.img' ++ kargs_file = 'kargs.json' ++ igninfo_file = 'igninfo.json' ++ ++ tmpisofile = os.path.join(tree, iso_name) ++ ++ # Find the directory under `/usr/lib/modules/` where the ++ # kernel/initrd live. It will be the only entity in there. ++ modules_dir = os.path.join(inputs['tree']['path'], 'usr/lib/modules') ++ modules_dirents = os.listdir(modules_dir) ++ if len(modules_dirents) != 1: ++ raise Exception(f"expected unique entry in modules dir, found: {modules_dirents}") ++ moduledir = modules_dirents[0] ++ ++ # copy those files out of the ostree into the iso root dir ++ initramfs_img = 'initramfs.img' ++ for file in [kernel_img, initramfs_img]: ++ src = os.path.join(modules_dir, moduledir, file) ++ dst = os.path.join(tmpisoimagespxe, file) ++ if file == initramfs_img: ++ dst = os.path.join(tmpisoimagespxe, initrd_img) ++ shutil.copyfile(src, dst) ++ # initramfs isn't world readable by default so let's open up perms ++ os.chmod(dst, 0o644) ++ ++ # Generate initramfs stamp file indicating that this is a live ++ # initramfs. Store the build ID in it. ++ stamppath = os.path.join(tmpinitrd_base, 'etc/coreos-live-initramfs') ++ os.makedirs(os.path.dirname(stamppath), exist_ok=True) ++ with open(stamppath, 'w') as fh: ++ fh.write(version + '\n') ++ ++ # Generate rootfs stamp file with the build ID, indicating that the ++ # rootfs has been appended and confirming that initramfs and rootfs are ++ # from the same build. ++ stamppath = os.path.join(tmpinitrd_rootfs, 'etc/coreos-live-rootfs') ++ os.makedirs(os.path.dirname(stamppath), exist_ok=True) ++ with open(stamppath, 'w') as fh: ++ fh.write(version + '\n') ++ ++ # Add placeholder for Ignition CPIO file. This allows an external tool, ++ # `coreos-installer iso ignition embed`, to modify an existing ISO image ++ # to embed a user's custom Ignition config. The tool wraps the Ignition ++ # config in a cpio.xz and write it directly into this file in the ISO ++ # image. The cpio.xz will be read into the initramfs filesystem at ++ # runtime and the Ignition Dracut module will ensure that the config is ++ # moved where Ignition will see it. We only handle !s390x here since that's ++ # the simple case (where layered initrds are supported). The s390x case is ++ # handled lower down ++ if basearch != 's390x': ++ with open(os.path.join(tmpisoimages, 'ignition.img'), 'wb') as fdst: ++ fdst.write(bytes(IGNITION_IMG_SIZE)) ++ igninfo_json = {'file': 'images/ignition.img'} ++ ++ # Generate JSON file that lists OS features available to ++ # coreos-installer {iso|pxe} customize. Put it in the initramfs for ++ # pxe customize and the ISO for iso customize. ++ features = json.dumps(get_os_features(inputs['tree']['path']), indent=2, sort_keys=True) + '\n' ++ featurespath = os.path.join(tmpinitrd_base, 'etc/coreos/features.json') ++ os.makedirs(os.path.dirname(featurespath), exist_ok=True) ++ with open(featurespath, 'w') as fh: ++ fh.write(features) ++ with open(os.path.join(tmpisocoreos, 'features.json'), 'w') as fh: ++ fh.write(features) ++ ++ # Add osmet files ++ fast_arg = [] ++ cmd = ['coreos-installer', 'pack', 'osmet'] ++ root = inputs['tree']['path'] ++ if squashfs_compression == 'lz4': ++ fast_arg = ['--fast'] ++ for img in ['metal', 'metal4k']: ++ print(f'Generating osmet file for {img} image') ++ image = devices[img]['path'] ++ img_checksum = checksum.hexdigest_file(image, "sha256") ++ tmp_img_osmet = os.path.join("/tmp", f'{img}.osmet') ++ extra_cmd = [image, '--description', os_release['PRETTY_NAME'], '--checksum', ++ img_checksum, '--output', tmp_img_osmet] ++ with Chroot(root, bind_mounts=["/tmp"]) as chroot: ++ chroot.run(cmd + extra_cmd + fast_arg, check=True) ++ shutil.move(tmp_img_osmet, os.path.join(tmpinitrd_rootfs, f'{img}.osmet')) ++ ++ tmp_squashfs_dir = os.path.join(tree, 'tmp-squashfs-dir') ++ os.mkdir(tmp_squashfs_dir) ++ # Mount manually to avoid conflicts with osmet. ++ # If mounted via the manifest, the stage begins with mounts already in place, ++ # but osmet also performs a mount operation, leading to conflicts due to duplicate ++ # filesystem UUIDs. Perform the manual mount only after the osmet stage ++ subprocess.check_call(['mount', '-o', 'ro', img_metal + 'p4', tmp_squashfs_dir]) ++ subprocess.check_call(['mount', '-o', 'ro', img_metal + 'p3', ++ os.path.join(tmp_squashfs_dir, 'boot')]) ++ if basearch in ['x86_64', 'aarch64']: ++ subprocess.check_call(['mount', '-o', 'ro', img_metal + 'p2', ++ os.path.join(tmp_squashfs_dir, 'boot/efi')]) ++ # Mount the filesystem using OverlayFS in a read-only mode to ensure the image ++ # remains unchanged and consistent ++ # Note OverlayFS does not support traversing across filesystem boundaries ++ # That's the reason we need two mounts here. ++ subprocess.check_call(['mount', '-t', 'overlay', 'overlay', '-o', ++ f'lowerdir={tmp_squashfs_dir},upperdir={tmp_upperdir_root},workdir={tmp_workdir_root}', ++ f'{tmp_merged_dir}']) ++ ++ subprocess.check_call(['mount', '-t', 'overlay', 'overlay', '-o', ++ f'lowerdir={tmp_squashfs_dir}/boot,upperdir={tmp_upperdir_boot},workdir={tmp_workdir_boot}', ++ f'{tmp_merged_dir}/boot']) ++ squashfs_dir_umount_needed = True ++ merged_dir_umount_needed = True ++ ++ # Immplements necessary CoreOS adjustments ++ # including creating hardlinks in the /boot/ filesystem ++ # and modifying the read-only flag in OSTree configurations ++ # Where the contents of rootfs image are stored ++ # Make sure to create it, if it is not created yet. ++ tmpinitrd_rootfs = os.path.join(tree, 'initrd-rootfs') ++ os.makedirs(tmpinitrd_rootfs, exist_ok=True) ++ ++ basearch = os.uname().machine ++ # Remove the sysroot=readonly flag, see https://github.com/coreos/fedora-coreos-tracker/issues/589 ++ subprocess.check_call(['sed', '-i', '/readonly=true/d', f'{tmp_merged_dir}/ostree/repo/config']) ++ # And ensure that the kernel binary and hmac file is in the place that dracut ++ # expects it to be; xref https://issues.redhat.com/browse/OCPBUGS-15843 ++ ++ kernel_binary = glob.glob(f"{tmp_merged_dir}/boot/ostree/*/vmlinuz*")[0] ++ kernel_hmac = glob.glob(f"{tmp_merged_dir}/boot/ostree/*/.*.hmac")[0] ++ kernel_binary_basename = os.path.basename(kernel_binary) ++ kernel_hmac_basename = os.path.basename(kernel_hmac) ++ ++ # Create hard links in the /boot directory ++ os.link(kernel_hmac, f"{tmp_merged_dir}/boot/{kernel_hmac_basename}") ++ os.link(kernel_binary, f"{tmp_merged_dir}/boot/{kernel_binary_basename}") ++ ++ print(f"Kernel binary linked: {tmp_merged_dir}/boot/{kernel_binary_basename}") ++ print(f"Kernel HMAC linked: {tmp_merged_dir}/boot/{kernel_hmac_basename}") ++ # Generate root squashfs ++ print(f'Compressing squashfs with {squashfs_compression}') ++ ++ try: ++ # Name must be exactly "root.squashfs" because the 20live dracut module ++ # makes assumptions about the length of the name in sysroot.mount ++ tmp_squashfs = os.path.join(tmpinitrd_rootfs, 'root.squashfs') ++ # this matches the set of flags we implicitly passed when doing this ++ # through libguestfs' mksquashfs command ++ subprocess.check_call(['mksquashfs', tmp_merged_dir, tmp_squashfs, ++ '-root-becomes', tmp_merged_dir, '-wildcards', '-no-recovery', ++ '-comp', squashfs_compression]) ++ ++ # while it's mounted here, also get the kargs ++ blsentry = ensure_glob(os.path.join(tmp_squashfs_dir, 'boot/loader/entries/*.conf')) ++ if len(blsentry) != 1: ++ raise Exception(f'Found != 1 BLS entries: {blsentry}') ++ blsentry = blsentry[0] ++ blsentry_kargs = [] ++ with open(blsentry, encoding='utf8') as f: ++ for line in f: ++ if line.startswith('options '): ++ blsentry_kargs = line.split(' ', 1)[1].strip().split(' ') ++ break ++ if len(blsentry_kargs) == 0: ++ raise Exception("found no kargs in metal image") ++ finally: ++ if squashfs_dir_umount_needed: ++ subprocess.check_call(['umount', '-R', tmp_squashfs_dir]) ++ if merged_dir_umount_needed: ++ subprocess.check_call(['umount', '-R', tmp_merged_dir]) ++ # Generate rootfs image ++ iso_rootfs = os.path.join(tmpisoimagespxe, rootfs_img) ++ # The rootfs must be uncompressed because the ISO mounts root.squashfs ++ # directly from the middle of the file ++ extend_initrd(iso_rootfs, tmpinitrd_rootfs, compress=False) ++ # Check that the root.squashfs magic number is in the offset hardcoded ++ # in sysroot.mount in 20live/live-generator ++ with open(iso_rootfs, 'rb') as fh: ++ fh.seek(124) ++ if fh.read(4) != b'hsqs': ++ raise Exception("root.squashfs not at expected offset in rootfs image") ++ pxe_rootfs = os.path.join(tree, rootfs_img) ++ # Clone to PXE image ++ cp_reflink(iso_rootfs, pxe_rootfs) ++ # Save stream hash of rootfs for verifying out-of-band fetches ++ os.makedirs(os.path.join(tmpinitrd_base, 'etc'), exist_ok=True) ++ make_stream_hash(pxe_rootfs, os.path.join(tmpinitrd_base, 'etc/coreos-live-want-rootfs')) ++ # Add common content ++ iso_initramfs = os.path.join(tmpisoimagespxe, initrd_img) ++ extend_initrd(iso_initramfs, tmpinitrd_base) ++ # Clone to PXE image ++ pxe_initramfs = os.path.join(tree, initrd_img) ++ cp_reflink(iso_initramfs, pxe_initramfs) ++ ++ # Filter kernel arguments for substituting into ISO bootloader ++ kargs_array = [karg for karg in blsentry_kargs ++ if karg.split('=')[0] not in LIVE_EXCLUDE_KARGS] ++ kargs_array.append(f"coreos.liveiso={volid}") ++ kargs = ' '.join(kargs_array) ++ print(f'Substituting ISO kernel arguments: {kargs}') ++ ++ kargs_json = {'files': []} ++ cmdline = '' ++ karg_embed_area_length = 0 ++ srcdir_prefix = os.path.join(inputs['tree']['path'], 'usr/share/coreos-assembler/live/') ++ # Grab all the contents from the live dir from the configs ++ for srcdir, _, filenames in os.walk(srcdir_prefix): ++ dir_suffix = srcdir.replace(srcdir_prefix, '', 1) ++ dstdir = os.path.join(tmpisoroot, dir_suffix) ++ if not os.path.exists(dstdir): ++ os.mkdir(dstdir) ++ for filename in filenames: ++ # Skip development readmes to avoid confusing users ++ if filename == 'README-devel.md': ++ continue ++ srcfile = os.path.join(srcdir, filename) ++ dstfile = os.path.join(dstdir, filename) ++ # Assumes all files are text ++ with open(srcfile) as fh: ++ buf = fh.read() ++ newbuf = buf.replace('@@KERNEL-ARGS@@', kargs) ++ # if we injected kargs, also check for an embed area ++ if buf != newbuf: ++ karg_area_start = re.search(r'@@KERNEL-ARGS@@', buf) ++ buf = newbuf ++ karg_area_end = re.search(r'(#+)# COREOS_KARG_EMBED_AREA\n', buf) ++ if karg_area_end is not None: ++ file_kargs = buf[karg_area_start.start():karg_area_end.start()] ++ if len(cmdline) == 0: ++ cmdline = file_kargs ++ elif cmdline != file_kargs: ++ raise Exception(f'Default cmdline is different: "{cmdline}" != "{file_kargs}"') ++ ++ length = karg_area_end.start() + len(karg_area_end[1]) - karg_area_start.start() ++ kargs_json['files'].append({ ++ 'path': os.path.join(dir_suffix, filename), ++ 'offset': karg_area_start.start(), ++ 'pad': '#', ++ 'end': '\n', ++ }) ++ if karg_embed_area_length == 0: ++ karg_embed_area_length = length ++ elif length != karg_embed_area_length: ++ raise Exception(f"Karg embed areas of varying length {kargs_json['files']}") ++ with open(dstfile, 'w') as fh: ++ fh.write(buf) ++ shutil.copystat(srcfile, dstfile) ++ print(f'{srcfile} -> {dstfile}') ++ ++ if karg_embed_area_length > 0: ++ assert (karg_embed_area_length > len(cmdline)) ++ kargs_json.update( ++ size=karg_embed_area_length, ++ default=cmdline.strip(), ++ ) ++ ++ # These sections are based on lorax templates ++ # see https://github.com/weldr/lorax/tree/master/share/templates.d/99-generic ++ ++ # Generate the ISO image. Lots of good info here: ++ # https://fedoraproject.org/wiki/User:Pjones/BootableCDsForBIOSAndUEFI ++ genisoargs = ['/usr/bin/genisoimage', '-verbose', ++ '-V', volid, ++ '-volset', f"{name_version}", ++ # For greater portability, consider using both ++ # Joliet and Rock Ridge extensions. Umm, OK :) ++ '-rational-rock', '-J', '-joliet-long'] ++ ++ # For x86_64 legacy boot (BIOS) booting ++ if basearch == "x86_64": ++ # Install binaries from syslinux package ++ isolinuxfiles = [('/usr/share/syslinux/isolinux.bin', 0o755), ++ ('/usr/share/syslinux/ldlinux.c32', 0o755), ++ ('/usr/share/syslinux/libcom32.c32', 0o755), ++ ('/usr/share/syslinux/libutil.c32', 0o755), ++ ('/usr/share/syslinux/vesamenu.c32', 0o755)] ++ for src, mode in isolinuxfiles: ++ dst = os.path.join(tmpisoisolinux, os.path.basename(src)) ++ shutil.copyfile(src, dst) ++ os.chmod(dst, mode) ++ ++ # for legacy bios boot AKA eltorito boot ++ genisoargs += ['-eltorito-boot', 'isolinux/isolinux.bin', ++ '-eltorito-catalog', 'isolinux/boot.cat', ++ '-no-emul-boot', ++ '-boot-load-size', '4', ++ '-boot-info-table'] ++ ++ elif basearch == "ppc64le": ++ os.makedirs(os.path.join(tmpisoroot, 'boot/grub')) ++ # can be EFI/fedora or EFI/redhat ++ grubpath = ensure_glob(os.path.join(tmpisoroot, 'EFI/*/grub.cfg')) ++ if len(grubpath) != 1: ++ raise Exception(f'Found != 1 grub.cfg files: {grubpath}') ++ shutil.move(grubpath[0], os.path.join(tmpisoroot, 'boot/grub/grub.cfg')) ++ for f in kargs_json['files']: ++ if re.match('^EFI/.*/grub.cfg$', f['path']): ++ f['path'] = 'boot/grub/grub.cfg' ++ ++ # safely remove things we don't need in the final ISO tree ++ for d in ['EFI', 'isolinux', 'zipl.prm']: ++ shutil.rmtree(os.path.join(tmpisoroot, d)) ++ ++ # grub2-mkrescue is a wrapper around xorriso ++ genisoargs = ['grub2-mkrescue', '-volid', volid] ++ elif basearch == "s390x": ++ # Reserve 32MB for the kernel, starting memory address of the initramfs ++ # See https://github.com/weldr/lorax/blob/master/share/templates.d/99-generic/s390.tmpl ++ INITRD_ADDRESS = '0x02000000' ++ lorax_templates = '/usr/share/lorax/templates.d/99-generic/config_files/s390' ++ shutil.copy(os.path.join(lorax_templates, 'redhat.exec'), tmpisoimages) ++ with open(os.path.join(lorax_templates, 'generic.ins'), 'r') as fp1: ++ with open(os.path.join(tmpisoroot, 'generic.ins'), 'w') as fp2: ++ [fp2.write(line.replace('@INITRD_LOAD_ADDRESS@', INITRD_ADDRESS)) for line in fp1] ++ for prmfile in ['cdboot.prm', 'genericdvd.prm', 'generic.prm']: ++ with open(os.path.join(tmpisoimages, prmfile), 'w') as fp1: ++ with open(os.path.join(tmpisoroot, 'zipl.prm'), 'r') as fp2: ++ fp1.write(fp2.read().strip()) ++ ++ # s390x's z/VM CMS files are limited to 8 char for filenames and extensions ++ # Also it is nice to keep naming convetion with Fedora/RHEL for existing users and code ++ kernel_dest = os.path.join(tmpisoimagespxe, 'kernel.img') ++ shutil.move(os.path.join(tmpisoimagespxe, kernel_img), kernel_dest) ++ kernel_img = 'kernel.img' ++ ++ if options.get('fixture'): ++ # truncate it to 128k so it includes the offsets to the initrd and kargs ++ # https://github.com/ibm-s390-linux/s390-tools/blob/032304d5034e/netboot/mk-s390image#L21-L24 ++ with open(kernel_dest, 'rb+') as f: ++ f.truncate(128 * 1024) ++ with open(iso_initramfs, 'rb+') as f: ++ f.truncate(1024) ++ ++ # On s390x, we reserve space for the Ignition config in the initrd ++ # image directly since the bootloader doesn't support multiple initrds. ++ # We do this by inflating the initramfs just for the duration of the ++ # `mk-s390image` call. ++ initramfs_size = os.stat(iso_initramfs).st_size ++ # sanity-check it's 4-byte aligned (see align_initrd_for_uncompressed_append) ++ assert initramfs_size % 4 == 0 ++ ++ # combine kernel, initramfs and cmdline using the mk-s390image tool ++ os.truncate(iso_initramfs, initramfs_size + IGNITION_IMG_SIZE) ++ subprocess.check_call(['/usr/bin/mk-s390image', ++ kernel_dest, ++ os.path.join(tmpisoimages, 'cdboot.img'), ++ '-r', iso_initramfs, ++ '-p', os.path.join(tmpisoimages, 'cdboot.prm')]) ++ os.truncate(iso_initramfs, initramfs_size) ++ ++ # Get the kargs and initramfs offsets in the cdboot.img. For more info, see: ++ # https://github.com/ibm-s390-linux/s390-tools/blob/032304d5034e/netboot/mk-s390image#L21-L23 ++ CDBOOT_IMG_OFFS_INITRD_START_BYTES = 66568 ++ CDBOOT_IMG_OFFS_KARGS_START_BYTES = 66688 ++ CDBOOT_IMG_OFFS_KARGS_MAX_SIZE = 896 ++ with open(os.path.join(tmpisoimages, 'cdboot.img'), 'rb') as f: ++ f.seek(CDBOOT_IMG_OFFS_INITRD_START_BYTES) ++ offset = struct.unpack(">Q", f.read(8))[0] ++ ++ # sanity-check we're at the right spot by comparing a few bytes ++ f.seek(offset) ++ with open(iso_initramfs, 'rb') as canonical: ++ if f.read(1024) != canonical.read(1024): ++ raise Exception(f"expected initrd at offset {offset}") ++ ++ igninfo_json = { ++ 'file': 'images/cdboot.img', ++ 'offset': offset + initramfs_size, ++ 'length': IGNITION_IMG_SIZE, ++ } ++ ++ # kargs are part of 'images/cdboot.img' blob ++ kargs_json['files'].append({ ++ 'path': 'images/cdboot.img', ++ 'offset': CDBOOT_IMG_OFFS_KARGS_START_BYTES, ++ 'pad': '\0', ++ 'end': '\0', ++ }) ++ kargs_json.update( ++ size=CDBOOT_IMG_OFFS_KARGS_MAX_SIZE, ++ ) ++ # generate .addrsize file for LPAR ++ with open(os.path.join(tmpisoimages, 'initrd.addrsize'), 'wb') as addrsize: ++ addrsize_data = struct.pack(">iiii", 0, int(INITRD_ADDRESS, 16), 0, ++ os.stat(iso_initramfs).st_size) ++ addrsize.write(addrsize_data) ++ ++ # safely remove things we don't need in the final ISO tree ++ for d in ['EFI', 'isolinux', 'zipl.prm']: ++ shutil.rmtree(os.path.join(tmpisoroot, d)) ++ ++ genisoargs = ['/usr/bin/xorrisofs', '-verbose', ++ '-volid', volid, ++ '-volset', f"{name_version}", ++ '-rational-rock', '-J', '-joliet-long', ++ '-no-emul-boot', '-eltorito-boot', ++ os.path.join(os.path.relpath(tmpisoimages, tmpisoroot), 'cdboot.img')] ++ ++ # For x86_64 and aarch64 UEFI booting ++ if basearch in ("x86_64", "aarch64"): ++ # Create the efiboot.img file. This is a fat32 formatted ++ # filesystem that contains all the files needed for EFI boot ++ # from an ISO. ++ with tempfile.TemporaryDirectory(): ++ ++ # In restrictive environments, setgid, setuid and ownership changes ++ # may be restricted. This sets the file ownership to root and ++ # removes the setgid and setuid bits in the tarball. ++ def strip(tarinfo): ++ tarinfo.uid = 0 ++ tarinfo.gid = 0 ++ if tarinfo.isdir(): ++ tarinfo.mode = 0o755 ++ elif tarinfo.isfile(): ++ tarinfo.mode = 0o0644 ++ return tarinfo ++ ++ tmpimageefidir = os.path.join(tree, "efi") ++ shutil.copytree(os.path.join(inputs['tree']['path'], 'usr/lib/bootupd/updates/EFI'), tmpimageefidir) ++ ++ # Find name of vendor directory ++ vendor_ids = [n for n in os.listdir(tmpimageefidir) if n != "BOOT"] ++ if len(vendor_ids) != 1: ++ raise Exception(f"did not find exactly one EFI vendor ID: {vendor_ids}") ++ vendor_id = vendor_ids[0] ++ ++ # Always replace live/EFI/{vendor} to actual live/EFI/{vendor_id} ++ # https://github.com/openshift/os/issues/954 ++ dfd = os.open(tmpisoroot, os.O_RDONLY) ++ grubfilepath = ensure_glob('EFI/*/grub.cfg', dir_fd=dfd) ++ if len(grubfilepath) != 1: ++ raise Exception(f'Found != 1 grub.cfg files: {grubfilepath}') ++ srcpath = os.path.dirname(grubfilepath[0]) ++ if srcpath != f'EFI/{vendor_id}': ++ print(f"Renaming '{srcpath}' to 'EFI/{vendor_id}'") ++ os.rename(srcpath, f"EFI/{vendor_id}", src_dir_fd=dfd, dst_dir_fd=dfd) ++ # And update kargs.json ++ for file in kargs_json['files']: ++ if file['path'] == grubfilepath[0]: ++ file['path'] = f'EFI/{vendor_id}/grub.cfg' ++ os.close(dfd) ++ ++ # Delete fallback and its CSV file. Its purpose is to create ++ # EFI boot variables, which we don't want when booting from ++ # removable media. ++ # ++ # A future shim release will merge fallback.efi into the main ++ # shim binary and enable the fallback behavior when the CSV ++ # exists. But for now, fail if fallback.efi is missing. ++ for path in ensure_glob(os.path.join(tmpimageefidir, "BOOT", "fb*.efi")): ++ os.unlink(path) ++ for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "BOOT*.CSV")): ++ os.unlink(path) ++ ++ # Drop vendor copies of shim; we already have it in BOOT*.EFI in ++ # BOOT ++ for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "shim*.efi")): ++ os.unlink(path) ++ ++ # Consolidate remaining files into BOOT. shim needs GRUB to be ++ # there, and the rest doesn't hurt. ++ for path in ensure_glob(os.path.join(tmpimageefidir, vendor_id, "*")): ++ shutil.move(path, os.path.join(tmpimageefidir, "BOOT")) ++ os.rmdir(os.path.join(tmpimageefidir, vendor_id)) ++ ++ # Inject a stub grub.cfg pointing to the one in the main ISO image. ++ # ++ # When booting via El Torito, this stub is not used; GRUB reads ++ # the ISO image directly using its own ISO support. This ++ # happens when booting from a CD device, or when the ISO is ++ # copied to a USB stick and booted on EFI firmware which prefers ++ # to boot a hard disk from an El Torito image if it has one. ++ # EDK II in QEMU behaves this way. ++ # ++ # This stub is used with EFI firmware which prefers to boot a ++ # hard disk from an ESP, or which cannot boot a hard disk via El ++ # Torito at all. In that case, GRUB thinks it booted from a ++ # partition of the disk (a fake ESP created by isohybrid, ++ # pointing to efiboot.img) and needs a grub.cfg there. ++ with open(os.path.join(tmpimageefidir, "BOOT", "grub.cfg"), "w") as fh: ++ fh.write(f'''search --label "{volid}" --set root --no-floppy ++set prefix=($root)/EFI/{vendor_id} ++echo "Booting via ESP..." ++configfile $prefix/grub.cfg ++boot ++''') ++ ++ # Install binaries from boot partition ++ # Manually construct the tarball to ensure proper permissions and ownership ++ efitarfile = tempfile.NamedTemporaryFile(suffix=".tar") ++ with tarfile.open(efitarfile.name, "w:", dereference=True) as tar: ++ tar.add(tmpimageefidir, arcname="/EFI", filter=strip) ++ ++ # Create the efiboot.img file in the images/ dir ++ efibootfile = os.path.join(tmpisoimages, 'efiboot.img') ++ make_efi_bootfile(efitarfile.name, efiboot_img) ++ shutil.copyfile(efiboot_img_filename, efibootfile) ++ ++ genisoargs += ['-eltorito-alt-boot', ++ '-efi-boot', 'images/efiboot.img', ++ '-no-emul-boot'] ++ ++ # We've done everything that might affect kargs, so filter out any files ++ # that no longer exist and write out the kargs JSON if it lists any files ++ kargs_json['files'] = [f for f in kargs_json['files'] ++ if os.path.exists(os.path.join(tmpisoroot, f['path']))] ++ kargs_json['files'].sort(key=lambda f: f['path']) ++ if kargs_json['files']: ++ # Store the location of "karg embed areas" for use by ++ # `coreos-installer iso kargs modify` ++ with open(os.path.join(tmpisocoreos, kargs_file), 'w') as fh: ++ json.dump(kargs_json, fh, indent=2, sort_keys=True) ++ fh.write('\n') ++ ++ # Write out the igninfo.json file. This is used by coreos-installer to know ++ # how to embed the Ignition config. ++ with open(os.path.join(tmpisocoreos, igninfo_file), 'w') as fh: ++ json.dump(igninfo_json, fh, indent=2, sort_keys=True) # pylint: disable=E0601 ++ fh.write('\n') ++ ++ # Define inputs and outputs ++ genisoargs_final = genisoargs + ['-o', tmpisofile, tmpisoroot] ++ ++ miniso_data = os.path.join(tmpisocoreos, "miniso.dat") ++ with open(miniso_data, 'wb') as f: ++ f.truncate(MINISO_DATA_FILE_SIZE) ++ ++ if options.get('fixture'): ++ # Replace or delete anything irrelevant to coreos-installer ++ with open(os.path.join(tmpisoimages, 'efiboot.img'), 'w') as fh: ++ fh.write('efiboot.img\n') ++ with open(os.path.join(tmpisoimagespxe, 'rootfs.img'), 'w') as fh: ++ fh.write('rootfs data\n') ++ with open(os.path.join(tmpisoimagespxe, 'initrd.img'), 'w') as fh: ++ fh.write('initrd data\n') ++ with open(os.path.join(tmpisoimagespxe, 'vmlinuz'), 'w') as fh: ++ fh.write('the kernel\n') ++ # this directory doesn't exist on s390x ++ if os.path.isdir(tmpisoisolinux): ++ with open(os.path.join(tmpisoisolinux, 'isolinux.bin'), 'rb+') as fh: ++ flen = fh.seek(0, 2) ++ fh.truncate(0) ++ fh.truncate(flen) ++ fh.seek(64) ++ # isohybrid checks for this magic ++ fh.write(b'\xfb\xc0\x78\x70') ++ for f in ensure_glob(os.path.join(tmpisoisolinux, '*.c32')): ++ os.unlink(f) ++ for f in ensure_glob(os.path.join(tmpisoisolinux, '*.msg')): ++ os.unlink(f) ++ ++ subprocess.check_call(genisoargs_final) ++ ++ # Add MBR, and GPT with ESP, for x86_64 BIOS/UEFI boot when ISO is ++ # copied to a USB stick ++ if basearch == "x86_64": ++ subprocess.check_call(['/usr/bin/isohybrid', '--uefi', tmpisofile]) ++ ++ genisoargs_minimal = genisoargs + ['-o', f'{tmpisofile}.minimal', tmpisoroot] ++ # The only difference with the miniso is that we drop these two files. ++ # Keep everything else the same to maximize file matching between the ++ # two versions so we can get the smallest delta. E.g. we keep the ++ # `coreos.liveiso` karg, even though the miniso doesn't need it. ++ # coreos-installer takes care of removing it. ++ os.unlink(iso_rootfs) ++ os.unlink(miniso_data) ++ subprocess.check_call(genisoargs_minimal) ++ if basearch == "x86_64": ++ subprocess.check_call(['/usr/bin/isohybrid', '--uefi', f'{tmpisofile}.minimal']) ++ # this consumes the minimal image ++ with Chroot(root, bind_mounts=["/tmp"]) as chroot: ++ chroot.run(['coreos-installer', 'pack', 'minimal-iso', tmpisofile, ++ f'{tmpisofile}.minimal', '--consume']) ++ ++ shutil.move(tmpisofile, output_file) ++ ++ ++if __name__ == "__main__": ++ args = osbuild.api.arguments() ++ r = main(args["inputs"], args["devices"], args["tree"], args["options"]) ++ sys.exit(r) +diff --git a/stages/org.osbuild.coreos.live-artifacts.mono.meta.json b/stages/org.osbuild.coreos.live-artifacts.mono.meta.json +new file mode 100644 +index 00000000..9aeff4a6 +--- /dev/null ++++ b/stages/org.osbuild.coreos.live-artifacts.mono.meta.json +@@ -0,0 +1,26 @@ ++{ ++ "summary": "Build a CoreOS live ISO", ++ "description": [ ++ "TODO" ++ ], ++ "schema_2": { ++ "inputs": { ++ "type": "object", ++ "additionalProperties": true ++ }, ++ "options": { ++ "additionalProperties": false, ++ "properties": { ++ "filename": { ++ "type": "string" ++ }, ++ "efiboot_img_filename": { ++ "type": "string" ++ }, ++ "squashfs_compression": { ++ "type": "string" ++ } ++ } ++ } ++ } ++} +-- +2.47.0 + diff --git a/src/cmd-osbuild b/src/cmd-osbuild index aa2b43426a..8c671a6209 100755 --- a/src/cmd-osbuild +++ b/src/cmd-osbuild @@ -15,6 +15,7 @@ declare -A SUPPORTED_PLATFORMS=( ['metal']='raw' ['qemu']='qcow2' ['qemu-secex']='qcow2' + ['live']='iso' ) print_help() { @@ -175,7 +176,7 @@ generate_runvm_osbuild_config() { # Grab a few values from $image_json deploy_via_container=$(getconfig_def "deploy-via-container" "" "${image_json}") extra_kargs="$(python3 -c 'import sys, json; args = json.load(sys.stdin)["extra-kargs"]; print(" ".join(args))' < "${image_json}")" - + squashfs_compression=$(getconfig_def "squashfs-compression" "" "${image_json}") # OStree container ociarchive file path ostree_container="${builddir}/$(meta_key images.ostree.path)" # If no container_imgref was set let's just set it to some professional @@ -219,6 +220,7 @@ extra-kargs-string: "${extra_kargs}" ostree-repo: "${ostree_repo}" metal-image-size: "${metal_image_size_mb}" cloud-image-size: "${cloud_image_size_mb}" +squashfs-compression: "${squashfs_compression}" # Note: this is only used in the secex case; there, the rootfs is # not the last partition on the disk so we need to explicitly size it rootfs-size: "${rootfs_size_mb}" @@ -242,6 +244,7 @@ main() { "cmd-buildextend-qemu") platforms=(qemu);; "cmd-buildextend-qemu-secex") platforms=(qemu-secex);; "cmd-buildextend-secex") platforms=(qemu-secex);; + "cmd-buildextend-live") platforms=(live);; *) fatal "called as unexpected name $0";; esac @@ -396,6 +399,19 @@ main() { # Skip Compression on these platforms as they are already compressed. postprocess_artifact "${platform}" "${imgpath}" "${imgname}" 'True' ;; + live) + # For live we have more artifacts + artifact_types=("iso.${basearch}.${suffix}" "kernel.${basearch}" "rootfs.${basearch}.img" "initramfs.${basearch}.img") + artifact_prefixes=("-iso" "-kernel" "-rootfs" "-initramfs") + for i in "${!artifact_types[@]}"; do + artifact_type="${artifact_types[$i]}" + artifact_prefix="${artifact_prefixes[$i]}" + artifact_name="${name}-${build}-${platform}-${artifact_type}" + imgpath="${outdir}/${platform}/${artifact_name}" + #TODO: For Live, seems we want the full artifact name here, not live-iso + postprocess_artifact "${platform}${artifact_prefix}" "${imgpath}" "${artifact_name}" 'True' + done + ;; qemu-secex) # Massage the generated artifact through an extra VM for secex. This # will also create an Ignition pubkey and store it in the meta.json diff --git a/src/cmdlib.sh b/src/cmdlib.sh index 32a26bc4f6..90c12a2595 100755 --- a/src/cmdlib.sh +++ b/src/cmdlib.sh @@ -731,6 +731,9 @@ runvm() { # include COSA in the image find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles" + echo /usr/lib/osbuild/stages/org.osbuild.dmverity >> "${vmpreparedir}/hostfiles" + echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-artifacts.mono >> "${vmpreparedir}/hostfiles" + echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-artifacts.mono.meta.json >> "${vmpreparedir}/hostfiles" # and include all GPG keys find /etc/pki/rpm-gpg/ -type f >> "${vmpreparedir}/hostfiles" diff --git a/src/deps-x86_64.txt b/src/deps-x86_64.txt index fcfcf71566..62c2063f37 100644 --- a/src/deps-x86_64.txt +++ b/src/deps-x86_64.txt @@ -1,6 +1,3 @@ -# For generating ISO images -syslinux-nonlinux - # For the pipeline and developers to interact with AWS. It's not needed by cosa # itself. This isn't available on s390x at least, so make it x86_64-only. awscli2 diff --git a/src/deps.txt b/src/deps.txt index 1f3d59b37b..e013ff6ec7 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -16,9 +16,6 @@ dumb-init rpm-ostree createrepo_c openssh-clients python3-createrepo_c composefs dnf-utils -# For generating ISO images -genisoimage - # Standard build tools make git rpm-build @@ -100,4 +97,4 @@ bsdtar fedora-repos-ostree # For graphing manifest includes using `manifest_graph` -python-anytree \ No newline at end of file +python-anytree diff --git a/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml index 958969b420..18d0c19b4d 100644 --- a/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.aarch64.mpp.yaml @@ -11,6 +11,7 @@ mpp-vars: extra_kargs: $extra_kargs metal_image_size_mb: $metal_image_size_mb cloud_image_size_mb: $cloud_image_size_mb + squashfs_compression: $squashfs_compression bios_boot_size_mb: 1 ppc_prep_size_mb: 4 reserved_part_size_mb: 1 @@ -780,3 +781,5 @@ pipelines: path: platform.metal.ipp.yaml - mpp-import-pipelines: path: platform.qemu.ipp.yaml + - mpp-import-pipelines: + path: platform.live.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml index 5354a05fc2..15473c75ae 100644 --- a/src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.ppc64le.mpp.yaml @@ -11,6 +11,7 @@ mpp-vars: extra_kargs: $extra_kargs metal_image_size_mb: $metal_image_size_mb cloud_image_size_mb: $cloud_image_size_mb + squashfs_compression: $squashfs_compression bios_boot_size_mb: 1 ppc_prep_size_mb: 4 reserved_part_size_mb: 1 @@ -722,3 +723,5 @@ pipelines: path: platform.metal.ipp.yaml - mpp-import-pipelines: path: platform.qemu.ipp.yaml + - mpp-import-pipelines: + path: platform.live.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml index 680c47f4ae..0be64deb9e 100644 --- a/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.s390x.mpp.yaml @@ -11,6 +11,7 @@ mpp-vars: extra_kargs: $extra_kargs metal_image_size_mb: $metal_image_size_mb cloud_image_size_mb: $cloud_image_size_mb + squashfs_compression: $squashfs_compression bios_boot_size_mb: 1 ppc_prep_size_mb: 4 reserved_part_size_mb: 1 @@ -666,3 +667,5 @@ pipelines: path: platform.qemu.ipp.yaml - mpp-import-pipelines: path: platform.qemu-secex.ipp.yaml + - mpp-import-pipelines: + path: platform.live.ipp.yaml diff --git a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml index 8f4bb57be0..23edc5adea 100644 --- a/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml +++ b/src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml @@ -11,6 +11,7 @@ mpp-vars: extra_kargs: $extra_kargs metal_image_size_mb: $metal_image_size_mb cloud_image_size_mb: $cloud_image_size_mb + squashfs_compression: $squashfs_compression bios_boot_size_mb: 1 ppc_prep_size_mb: 4 reserved_part_size_mb: 1 @@ -784,3 +785,5 @@ pipelines: path: platform.metal.ipp.yaml - mpp-import-pipelines: path: platform.qemu.ipp.yaml + - mpp-import-pipelines: + path: platform.live.ipp.yaml diff --git a/src/osbuild-manifests/platform.live.ipp.yaml b/src/osbuild-manifests/platform.live.ipp.yaml new file mode 100755 index 0000000000..dc2f4959cc --- /dev/null +++ b/src/osbuild-manifests/platform.live.ipp.yaml @@ -0,0 +1,103 @@ +# This file defines the pipeline for building the live ISO. +version: '2' +pipelines: + - name: live-build-tree + build: + mpp-format-string: '{host_as_buildroot}' + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:metal + options: + paths: + - from: + mpp-format-string: "input://tree/{artifact_name_prefix}-metal.{arch}.raw" + to: tree:///metal.raw + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:metal4k + options: + paths: + - from: + mpp-format-string: "input://tree/{artifact_name_prefix}-metal4k.{arch}.raw" + to: tree:///metal4k.raw + # We need to be able to create efiboot.img, a FAT filesystem image. It's + # hard to setup loopback devices from within the stage, so just do it + # here. This should normally be conditional on the architecture but it + # doesn't hurt either since it's a tiny file. + - type: org.osbuild.truncate + options: + filename: efiboot.img + size: + mpp-format-string: '{16 * 1024 * 1024}' + - type: org.osbuild.coreos.live-artifacts.mono + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:deployed-tree + devices: + metal: + type: org.osbuild.loopback + options: + filename: metal.raw + partscan: true + read-only: true + sector-size: + mpp-format-int: "{sector_size}" + metal4k: + type: org.osbuild.loopback + options: + filename: metal4k.raw + partscan: true + read-only: true + sector-size: + mpp-format-int: "{four_k_sector_size}" + efiboot_img: + type: org.osbuild.loopback + options: + filename: efiboot.img + options: + efiboot_img_filename: + mpp-format-string: 'efiboot.img' + filename: + mpp-format-string: 'live.iso' + squashfs_compression: + mpp-format-string: '{squashfs_compression}' + - name: live + stages: + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:live-build-tree + options: + paths: + - from: + mpp-format-string: 'input://tree/live.iso' + to: + mpp-format-string: 'tree:///{artifact_name_prefix}-live-iso.{arch}.iso' + - from: + mpp-format-string: input://tree/live/images/pxeboot/vmlinuz + to: + mpp-format-string: 'tree:///{artifact_name_prefix}-live-kernel.{arch}' + - from: + mpp-format-string: input://tree/live/images//pxeboot/initrd.img + to: + mpp-format-string: 'tree:///{artifact_name_prefix}-live-initramfs.{arch}.img' + - from: + mpp-format-string: input://tree/rootfs.img + to: + mpp-format-string: 'tree:///{artifact_name_prefix}-live-rootfs.{arch}.img' + diff --git a/src/runvm-osbuild b/src/runvm-osbuild index 0d9f167dd3..ca8e0e19e3 100755 --- a/src/runvm-osbuild +++ b/src/runvm-osbuild @@ -3,8 +3,8 @@ set -eux -o pipefail usage() { cat <