Build live ISO using osbuild

This adds a new osbuild pipeline for building the live ISO using the new
`org.osbuild.coreos.live-iso` mega stage.

Still missing is changing `cmd-buildextend-live` to call osbuild
instead. I think we'll need to ratchet this in using e.g. an env var
like we did for the other osbuild artifacts. E.g. `COSA_OSBUILD_LIVE`
would control whether to use the old logic or the new one. And then turn
it on in rawhide, etc...

The new `cmd-buildextend-live` would also take care of extracting the
live artifacts from the live ISO to insert into `meta.json`.

src/cmdlib.sh

@@ -730,6 +730,9 @@ runvm() {
 
     # include COSA in the image
     find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles"
+    echo /usr/lib/osbuild/stages/org.osbuild.dmverity >> "${vmpreparedir}/hostfiles"
+    echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-iso >> "${vmpreparedir}/hostfiles"
+    echo /usr/lib/osbuild/stages/org.osbuild.coreos.live-iso.meta.json >> "${vmpreparedir}/hostfiles"
 
     # and include all GPG keys
     find /etc/pki/rpm-gpg/ -type f >> "${vmpreparedir}/hostfiles"

src/osbuild-manifests/coreos.osbuild.x86_64.mpp.yaml

@@ -11,13 +11,17 @@ mpp-vars:
   extra_kargs: $extra_kargs
   metal_image_size_mb: $metal_image_size_mb
   cloud_image_size_mb: $cloud_image_size_mb
+  squashfs_compression: $squashfs_compression
   bios_boot_size_mb: 1
   ppc_prep_size_mb: 4
   reserved_part_size_mb: 1
   efi_system_size_mb: 127
   boot_size_mb: 384
   sector_size: 512
   four_k_sector_size: 4096
+  metal_filename: $metal_filename
+  metal4k_filename: $metal4k_filename
+  live_efiboot_img_size_mb: $live_efiboot_img_size_mb
   # Filesystem UUID and label definitions. These UUIDs
   # are looked for on boot and if found replaced with
   # a new random UUID to make each install unique.
@@ -593,3 +597,5 @@ pipelines:
       path: platform.metal.ipp.yaml
   - mpp-import-pipelines:
       path: platform.qemu.ipp.yaml
+  - mpp-import-pipelines:
+      path: platform.live-iso.ipp.yaml

src/osbuild-manifests/platform.live-iso.ipp.yaml

@@ -0,0 +1,96 @@
+# This file defines the pipeline for building the live ISO.
+version: '2'
+pipelines:
+  - name: live-iso
+    stages:
+      # Yuck: here we copy the metal and metal4k images into the tree from the
+      # outside instead of referencing the metal/metal4k osbuild pipelines.
+      # The reason for that is that our tooling and pipeline is currently
+      # very oriented towards building the metal images as separate artifacts
+      # in separate invocations of osbuild, so we need to support the metal
+      # images pre-existing. Also, the invocations in which we currently build
+      # those images are done with -snapshot on the cache qcow so it can be
+      # parallelized, so we can't just checkpoint those stage outputs. And even
+      # if we did, it doesn't feel quite right to rely on whatever is cached
+      # vs the canonical images. I think to improve this, we should move to
+      # building the metal, metal4k and live artifacts in one invocation.
+      #
+      # The mpp-if here is so osbuild-mpp doesn't try to embed the (possibly
+      # non-existent metal images) when building other stuff... We really should
+      # improve how we interact with osbuild.
+      - mpp-if: metal_filename != ''
+        then:
+          type: org.osbuild.copy
+          inputs:
+            metal:
+              type: org.osbuild.files
+              origin: org.osbuild.source
+              mpp-embed:
+                id: metal
+                url:
+                  mpp-format-string: 'file://{metal_filename}'
+          options:
+            paths:
+              - from:
+                  mpp-format-string: input://metal/{embedded['metal']}
+                to: tree:///metal.raw
+      - mpp-if: metal4k_filename != ''
+        then:
+          type: org.osbuild.copy
+          inputs:
+            metal4k:
+              type: org.osbuild.files
+              origin: org.osbuild.source
+              mpp-embed:
+                id: metal4k
+                url:
+                  mpp-format-string: 'file://{metal4k_filename}'
+          options:
+            paths:
+              - from:
+                  mpp-format-string: input://metal4k/{embedded['metal4k']}
+                to: tree:///metal4k.raw
+      # We need to be able to create efiboot.img, a FAT filesystem image. It's
+      # hard to setup loopback devices from within the stage, so just do it
+      # here. This should normally be conditional on the architecture but it
+      # doesn't hurt either since it's a tiny file.
+      - type: org.osbuild.truncate
+        options:
+          filename: efiboot.img
+          size:
+            mpp-format-string: '{live_efiboot_img_size_mb * 1024 * 1024}'
+      - type: org.osbuild.coreos.live-iso
+        inputs:
+          tree:
+            type: org.osbuild.tree
+            origin: org.osbuild.pipeline
+            references:
+              # XXX: we should rename this pipeline now that we don't use it as
+              # a buildroot
+              - name:build
+        devices:
+          metal:
+            type: org.osbuild.loopback
+            options:
+              filename: metal.raw
+              partscan: true
+              read-only: true
+              sector-size: 512
+          metal4k:
+            type: org.osbuild.loopback
+            options:
+              filename: metal4k.raw
+              partscan: true
+              read-only: true
+              sector-size: 4096
+          efiboot_img:
+            type: org.osbuild.loopback
+            options:
+              filename: efiboot.img
+        options:
+          efiboot_img_filename:
+            mpp-format-string: 'efiboot.img'
+          filename:
+            mpp-format-string: '{filename}'
+          squashfs_compression:
+            mpp-format-string: '{squashfs_compression}'

src/runvm-osbuild

@@ -3,8 +3,8 @@ set -eux -o pipefail
 
 usage() {
     cat <<EOC
-${0} creates a supermin virtual machine that then
-runs OSBuild to build a CoreOS style disk image.
+${0} runs in a supermin virtual machine to then
+run OSBuild to build a CoreOS style disk image.
 
 Options:
     --config: JSON-formatted image.yaml
@@ -60,6 +60,10 @@ container_imgref=$(getconfig "container-imgref")
 container_repo=$(getconfig_def "container-repo" "")
 container_tag=$(getconfig_def "container-tag" "")
 extra_kargs=$(getconfig "extra-kargs-string" "")
+squashfs_compression=$(getconfig_def "squashfs-compression" "")
+metal_filename=$(getconfig_def "metal-filename" "")
+metal4k_filename=$(getconfig_def "metal4k-filename" "")
+live_efiboot_img_size_mb=$(getconfig_def "live-efiboot-img-size" "0")
 # If we are not deploying via container let's go ahead and
 # set the OSTree repo and ref information
 ostree_ref=""
@@ -103,6 +107,10 @@ osbuild-mpp                                             \
     -D cloud_image_size_mb="${cloud_image_size_mb}"     \
     -D rootfs_size_mb="${rootfs_size_mb}"               \
     -D qemu_secex=\""${secex}"\"                        \
+    -D squashfs_compression=\""${squashfs_compression}"\" \
+    -D metal_filename=\""${metal_filename}"\"             \
+    -D metal4k_filename=\""${metal4k_filename}"\"         \
+    -D live_efiboot_img_size_mb="${live_efiboot_img_size_mb}" \
     "${mppyaml}" "${processed_json}"
 
 # Build the image