From 5a1bd0450a0af95772eef1d63ab4f21b6616d235 Mon Sep 17 00:00:00 2001 From: Dusty Mabe Date: Mon, 9 Oct 2023 22:40:16 -0400 Subject: [PATCH] add proof of concept for building with osbuild This is proof of concept code with many things hardcoded in the coreos.osbuild.mpp.yaml that need to become more dynamically defined. To use this you can set the COSA_USE_OSBUILD env var to have a value. COSA_USE_OSBUILD=1 should work just fine. --- Dockerfile | 1 + build.sh | 8 + ...-mount-etc-containers-for-host-build.patch | 92 +++++++ ...-boot-from-host-in-host-builder-case.patch | 45 ++++ src/cmd-buildextend-metal | 22 +- src/coreos.osbuild.mpp.yaml | 228 ++++++++++++++++++ src/deps.txt | 6 - src/runvm-osbuild | 30 +++ src/vmdeps.txt | 3 + 9 files changed, 422 insertions(+), 13 deletions(-) create mode 100644 src/0001-objectstore-also-mount-etc-containers-for-host-build.patch create mode 100644 src/0002-Mount-boot-from-host-in-host-builder-case.patch create mode 100644 src/coreos.osbuild.mpp.yaml create mode 100755 src/runvm-osbuild diff --git a/Dockerfile b/Dockerfile index 6ff1bff56d..e7266b97eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ COPY ./ /root/containerbuild/ RUN ./build.sh write_archive_info RUN ./build.sh make_and_makeinstall RUN ./build.sh configure_user +RUN ./build.sh patch_osbuild # clean up scripts (it will get cached in layers, but oh well) WORKDIR /srv/ diff --git a/build.sh b/build.sh index 9e0cf7181c..ed0ea105bf 100755 --- a/build.sh +++ b/build.sh @@ -22,6 +22,7 @@ if [ $# -gt 1 ]; then echo " configure_yum_repos" echo " install_rpms" echo " make_and_makeinstall" + echo " patch_osbuild" exit 1 fi @@ -168,6 +169,12 @@ write_archive_info() { prepare_git_artifacts "${srcdir}" /cosa/coreos-assembler-git.json /cosa/coreos-assembler-git.tar.gz } +patch_osbuild() { + # A few patches that either haven't made it into a release or + # that will be obsoleted with other work that will be done soon. + cat /usr/lib/coreos-assembler/*.patch | patch -p1 -d /usr/lib/python3.11/site-packages/ +} + if [ $# -ne 0 ]; then # Run the function specified by the calling script ${1} @@ -182,4 +189,5 @@ else install_ocp_tools trust_redhat_gpg_keys configure_user + patch_osbuild fi diff --git a/src/0001-objectstore-also-mount-etc-containers-for-host-build.patch b/src/0001-objectstore-also-mount-etc-containers-for-host-build.patch new file mode 100644 index 0000000000..6bc1689a2a --- /dev/null +++ b/src/0001-objectstore-also-mount-etc-containers-for-host-build.patch @@ -0,0 +1,92 @@ +From d4b3e3655deb7d55792e52fe6a11c609fb24e3b8 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Tue, 24 Oct 2023 14:08:44 -0400 +Subject: [PATCH] objectstore: also mount /etc/containers for "host" buildroot + +In the case we are not using a buildroot (i.e. we are using +the host as the buildroot) let's also mount in /etc/containers +into the environment. There are sometimes where software running +from /usr can't operate without configuration in /etc and this +will allow it to work. + +An example of software hitting this problem is skopeo. With a +simple config like: + +``` +version: '2' +mpp-vars: + release: 38 +pipelines: + - name: skopeo-tree + # build: name:build + source-epoch: 1659397331 + stages: + - type: org.osbuild.skopeo + inputs: + images: + type: org.osbuild.containers + origin: org.osbuild.source + mpp-resolve-images: + images: + - source: quay.io/fedora/fedora-coreos + tag: stable + name: localhost/fcos + options: + destination: + type: containers-storage + storage-path: /usr/share/containers/storage +``` + +We end up hitting an error like this: + +``` +time="2023-10-24T18:27:14Z" level=fatal msg="Error loading trust policy: open /etc/containers/policy.json: no such file or directory" +Traceback (most recent call last): + File "/run/osbuild/bin/org.osbuild.skopeo", line 90, in + r = main(args["inputs"], args["tree"], args["options"]) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/run/osbuild/bin/org.osbuild.skopeo", line 73, in main + subprocess.run(["skopeo", "copy", image_source, dest], check=True) + File "/usr/lib64/python3.11/subprocess.py", line 571, in run + raise CalledProcessError(retcode, process.args, +subprocess.CalledProcessError: Command '['skopeo', 'copy', 'dir:/tmp/tmp5_qcng99/image', 'containers-storage:[overlay@/run/osbuild/tree/usr/share/containers/storage+/run/containers/storage]localhost/fcos']' returned non-zero exit status 1. +``` + +This PR adds in a mount for /etc/containers from the host so that +/etc/containers/policy.json can be accessed. +--- + osbuild/objectstore.py | 12 ++++++++++-- + 1 file changed, 10 insertions(+), 2 deletions(-) + +diff --git a/osbuild/objectstore.py b/osbuild/objectstore.py +index 4a19ce9..922d5ee 100644 +--- a/osbuild/objectstore.py ++++ b/osbuild/objectstore.py +@@ -283,14 +283,22 @@ class HostTree: + self._root = self.store.tempdir(prefix="host") + + root = self._root.name +- # Create a bare bones root file system +- # with just /usr mounted from the host ++ # Create a bare bones root file system. Starting with just ++ # /usr mounted from the host. + usr = os.path.join(root, "usr") + os.makedirs(usr) ++ # Also add in /etc/containers, which will allow us to access ++ # /etc/containers/policy.json and enable moving containers ++ # (skopeo): https://github.com/osbuild/osbuild/pull/1410 ++ # If https://github.com/containers/image/issues/2157 ever gets ++ # fixed we can probably remove this bind mount. ++ etc_containers = os.path.join(root, "etc", "containers") ++ os.makedirs(etc_containers) + + # ensure / is read-only + mount(root, root) + mount("/usr", usr) ++ mount("/etc/containers", etc_containers) + + @property + def tree(self) -> os.PathLike: +-- +2.41.0 + diff --git a/src/0002-Mount-boot-from-host-in-host-builder-case.patch b/src/0002-Mount-boot-from-host-in-host-builder-case.patch new file mode 100644 index 0000000000..0d441b3c85 --- /dev/null +++ b/src/0002-Mount-boot-from-host-in-host-builder-case.patch @@ -0,0 +1,45 @@ +From 2e34303f2e9ef1d48b965703976ef1029d7309f1 Mon Sep 17 00:00:00 2001 +From: Dusty Mabe +Date: Fri, 1 Sep 2023 12:18:25 -0400 +Subject: [PATCH] Mount boot from host in host builder case + +--- + osbuild/buildroot.py | 2 +- + osbuild/objectstore.py | 3 +++ + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git a/osbuild/buildroot.py b/osbuild/buildroot.py +index 5b47d70..a0f654d 100644 +--- a/osbuild/buildroot.py ++++ b/osbuild/buildroot.py +@@ -196,7 +196,7 @@ class BuildRoot(contextlib.AbstractContextManager): + + # Import directories from the caller-provided root. + imports = ["usr"] +- if self.mount_boot: ++ if True: + imports.insert(0, "boot") + + for p in imports: +diff --git a/osbuild/objectstore.py b/osbuild/objectstore.py +index 922d5ee..6a3f89a 100644 +--- a/osbuild/objectstore.py ++++ b/osbuild/objectstore.py +@@ -294,11 +294,14 @@ class HostTree: + # fixed we can probably remove this bind mount. + etc_containers = os.path.join(root, "etc", "containers") + os.makedirs(etc_containers) ++ boot = os.path.join(root, "boot") ++ os.makedirs(boot) + + # ensure / is read-only + mount(root, root) + mount("/usr", usr) + mount("/etc/containers", etc_containers) ++ mount("/boot", boot) + + @property + def tree(self) -> os.PathLike: +-- +2.41.0 + diff --git a/src/cmd-buildextend-metal b/src/cmd-buildextend-metal index 0648c50128..990210bf57 100755 --- a/src/cmd-buildextend-metal +++ b/src/cmd-buildextend-metal @@ -261,13 +261,21 @@ EOF cat "${image_json}" image-dynamic.json | jq -s add > image-for-disk.json platforms_json="${workdir}/tmp/platforms.json" yaml2json "${configdir}/platforms.yaml" "${platforms_json}" -runvm "${qemu_args[@]}" -- \ - /usr/lib/coreos-assembler/create_disk.sh \ - --config "$(pwd)"/image-for-disk.json \ - --kargs "${kargs}" \ - --platform "${ignition_platform_id}" \ - --platforms-json "${platforms_json}" \ - "${disk_args[@]}" + +if [ "${image_type}" == "qemu" ] && [ "${COSA_USE_OSBUILD:-}" != "" ]; then + runvm -- /usr/lib/coreos-assembler/runvm-osbuild \ + "${ostree_repo}" "${ref}" \ + /usr/lib/coreos-assembler/coreos.osbuild.mpp.yaml \ + "${path}.tmp" +else + runvm "${qemu_args[@]}" -- \ + /usr/lib/coreos-assembler/create_disk.sh \ + --config "$(pwd)"/image-for-disk.json \ + --kargs "${kargs}" \ + --platform "${ignition_platform_id}" \ + --platforms-json "${platforms_json}" \ + "${disk_args[@]}" +fi if [[ $secure_execution -eq "1" && -z "${hostkey}" ]]; then /usr/lib/coreos-assembler/secex-genprotimgvm-scripts/runvm.sh \ diff --git a/src/coreos.osbuild.mpp.yaml b/src/coreos.osbuild.mpp.yaml new file mode 100644 index 0000000000..3b825ce591 --- /dev/null +++ b/src/coreos.osbuild.mpp.yaml @@ -0,0 +1,228 @@ +version: '2' +mpp-vars: + release: 38 +mpp-define-image: + id: image + #10G + size: '10737418240' + table: + uuid: 00000000-0000-4000-a000-000000000001 + label: gpt + partitions: + - id: BIOS-BOOT + size: 2048 + type: 21686148-6449-6E6F-744E-656564454649 + bootable: true + uuid: FAC7F1FB-3E8D-4137-A512-961DE09A5549 + - id: EFI-SYSTEM + size: 260096 + type: C12A7328-F81F-11D2-BA4B-00A0C93EC93B + uuid: 68B2905B-DF3E-4FB3-80FA-49D1E773AA33 + - id: boot + size: 786432 + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + uuid: 61B2905B-DF3E-4FB3-80FA-49D1E773AA32 + - id: root + # XXX: Dynamically set this size in the future + size: 4194304 + type: 0FC63DAF-8483-4772-8E79-3D69D8477DE4 + uuid: CA7D7CCB-63ED-4C53-861C-1742536059CC +pipelines: + - name: image-tree + source-epoch: 1659397331 + stages: + - type: org.osbuild.ostree.init-fs + - type: org.osbuild.ostree.os-init + options: + osname: fedora-coreos + - type: org.osbuild.ostree.config + options: + repo: /ostree/repo + config: + sysroot: + readonly: false + bootloader: none + - type: org.osbuild.mkdir + options: + paths: + - path: /boot/efi + mode: 448 + - type: org.osbuild.ignition + - type: org.osbuild.ostree.deploy + options: + osname: fedora-coreos + remote: fedora + mounts: + - /boot + - /boot/efi + kernel_opts: + - rw + - console=tty0 + - console=ttyS0 + - ignition.platform.id=qemu + - '$ignition_firstboot' + inputs: + commits: + type: org.osbuild.ostree + origin: org.osbuild.source + mpp-resolve-ostree-commits: + commits: + - ref: $ref + remote: + url: $repourl + - type: org.osbuild.ostree.selinux + options: + deployment: + osname: fedora-coreos + ref: ostree/1/1/0 + - type: org.osbuild.grub2 + options: + rootfs: + label: root + bootfs: + label: boot + uefi: + vendor: fedora + install: true + legacy: i386-pc + write_defaults: false + greenboot: false + ignition: true + - name: image + stages: + - type: org.osbuild.truncate + options: + filename: disk.img + size: + mpp-format-string: '{image.size}' + - type: org.osbuild.sfdisk + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + options: + mpp-format-json: '{image.layout}' + - type: org.osbuild.mkfs.fat + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''EFI-SYSTEM''].start}' + size: + mpp-format-int: '{image.layout[''EFI-SYSTEM''].size}' + lock: true + options: + label: EFI-SYSTEM + volid: 7B7795E7 + - type: org.osbuild.mkfs.ext4 + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''boot''].start}' + size: + mpp-format-int: '{image.layout[''boot''].size}' + lock: true + options: + uuid: 96d15588-3596-4b3c-adca-a2ff7279ea63 + label: boot + - type: org.osbuild.mkfs.xfs + devices: + device: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''root''].start}' + size: + mpp-format-int: '{image.layout[''root''].size}' + lock: true + options: + uuid: 910678ff-f77e-4a7d-8d53-86f2ac47a823 + label: root + - type: org.osbuild.copy + inputs: + tree: + type: org.osbuild.tree + origin: org.osbuild.pipeline + references: + - name:image-tree + options: + paths: + - from: input://tree/ + to: mount://root/ + devices: + efi: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''EFI-SYSTEM''].start}' + size: + mpp-format-int: '{image.layout[''EFI-SYSTEM''].size}' + boot: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''boot''].start}' + size: + mpp-format-int: '{image.layout[''boot''].size}' + root: + type: org.osbuild.loopback + options: + filename: disk.img + start: + mpp-format-int: '{image.layout[''root''].start}' + size: + mpp-format-int: '{image.layout[''root''].size}' + mounts: + - name: root + type: org.osbuild.xfs + source: root + target: / + - name: boot + type: org.osbuild.ext4 + source: boot + target: /boot + - name: efi + type: org.osbuild.fat + source: efi + target: /boot/efi + - type: org.osbuild.grub2.inst + options: + platform: i386-pc + filename: disk.img + location: + mpp-format-int: '{image.layout[''BIOS-BOOT''].start}' + core: + type: mkimage + partlabel: gpt + filesystem: ext4 + prefix: + type: partition + partlabel: + mpp-format-string: '{image.layout.label}' + number: + mpp-format-int: '{image.layout[''boot''].index}' + path: /grub2 + - name: qcow2 + stages: + - type: org.osbuild.qemu + inputs: + image: + type: org.osbuild.files + origin: org.osbuild.pipeline + references: + name:image: + file: disk.img + options: + filename: disk.qcow2 + format: + type: qcow2 + compat: '1.1' diff --git a/src/deps.txt b/src/deps.txt index bb5d64e85a..bd1cc80536 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -16,12 +16,6 @@ dumb-init rpm-ostree createrepo_c openssh-clients python3-createrepo_c dnf-utils -# We don't actually use this right now but we intend to share -# code in the future. -# XXX: temporarily disabled until we fix -# https://github.com/osbuild/osbuild-composer/issues/1915 -# osbuild-composer - # For generating ISO images genisoimage diff --git a/src/runvm-osbuild b/src/runvm-osbuild new file mode 100755 index 0000000000..92328a7551 --- /dev/null +++ b/src/runvm-osbuild @@ -0,0 +1,30 @@ +#!/bin/bash +set -eux -o pipefail + +repo=$1 +ref=$2 +mppyaml=$3 +path=$4 + +# Since it doesn't exist create loop-control +mknod /dev/loop-control c 10 237 + +# get away from the virtiofs share because the xattrs that +# are written out by the ostree deploy will cause SELinux denials. +mkdir /root/osbuild && cd /root/osbuild + +# Run through the preprocessor +osbuild-mpp \ + -D ref=\""${ref}"\" \ + -D repourl=\""file://${repo}"\" \ + "${mppyaml}" \ + processed.json + +# Build the image +osbuild --store store/ \ + --output-directory out/ \ + --export qcow2 processed.json + + +# Copy it out to the specified location +cp out/qcow2/disk.qcow2 "${path}" diff --git a/src/vmdeps.txt b/src/vmdeps.txt index a73a00c066..eaa9e95fad 100644 --- a/src/vmdeps.txt +++ b/src/vmdeps.txt @@ -35,3 +35,6 @@ tar # needed for extensions container build podman + +# For running osbuild +osbuild osbuild-ostree osbuild-selinux osbuild-tools python3-pyrsistent