Merge pull request canonical#505 from mihalicyn/ext-qemu-snap

external QEMU snap support

lxd-qemu-snap/README.md

@@ -0,0 +1,69 @@
+# External QEMU snap for LXD snap
+
+## How to build and use
+
+```
+cd lxd-qemu-snap
+
+# clean up previous builds
+rm -f qemu-for-lxd_*.snap
+
+# build
+snapcraft
+
+# install
+sudo snap install qemu-for-lxd_*.snap --devmode
+
+# connect snaps
+sudo snap connect lxd:gpu-2404 mesa-2404:gpu-2404
+sudo snap connect lxd:qemu-external qemu-for-lxd:qemu-external
+```
+
+## How to use with virgl (only specific to this example of snapcraft.yaml)
+
+```
+lxc init images:ubuntu/noble/desktop desktop -c limits.memory=8GiB --vm
+
+# modify instance configuration:
+lxc config edit desktop
+
+# choose an appropriate renderer:
+ls -la /dev/dri/by-path/*-render
+
+# for example, on my system it is /dev/dri/by-path/pci-0000:67:00.0-render
+# lspci | grep -E "(3D|VGA)" shows:
+# 67:00.0 VGA compatible controller: Advanced Micro Devices, Inc. [AMD/ATI] Rembrandt (rev d8)
+
+# add the following lines:
+  raw.apparmor: |-
+    /snap/lxd/*/gpu-2404/** mr,
+    /dev/dri/ r,
+    /dev/dri/card[0-9]* rw,
+    /dev/dri/renderD[0-9]* rw,
+    /run/udev/data/c226:[0-9]* r,  # 226 drm
+    /sys/devices/** r,
+    /sys/bus/** r,
+  raw.qemu: -display egl-headless,rendernode=/dev/dri/by-path/pci-0000:67:00.0-render
+  raw.qemu.conf: |-
+    [device "qemu_gpu"]
+    driver = "virtio-vga-gl"
+
+# try it
+lxc start desktop --console=vga
+
+# you can check output from:
+dmesg | grep -i drm
+# if you see:
+# [drm] features: +virgl ...
+# it's a good sign
+
+glxinfo | grep -i vir
+# it should show something like:
+# Device: virgl ...
+```
+
+## References:
+
+https://github.com/snapcore/snapd/blob/5c8d8431baa425464b279ff26b8c44eecb9aab22/interfaces/builtin/opengl.go#L41
+
+https://gitlab.gnome.org/GNOME/gnome-boxes/-/issues/586

lxd-qemu-snap/snapcraft.yaml

@@ -0,0 +1,235 @@
+name: qemu-for-lxd
+base: core24
+assumes:
+  - snapd2.63
+version: '1'
+grade: devel
+summary: External Qemu for LXD
+description: |-
+ External Qemu for LXD
+
+contact: [email protected]
+issues: https://github.com/canonical/lxd/issues
+source-code: https://github.com/canonical/lxd
+website: https://ubuntu.com/lxd
+confinement: strict
+
+slots:
+  qemu-external:
+    interface: content
+    content: qemu-external-binaries
+    read:
+      - $SNAP/external/qemu
+
+parts:
+  #
+  # This step is extremely important, as it ensures that no mesa libs will
+  # be provided with this snap to prevent any possible version mistmatches
+  # between this snap and mesa-2404 snap.
+  # If you the error like:
+  # DRI driver not from this Mesa build ('24.0.5-1ubuntu1' vs '24.0.9-0ubuntu0.1')
+  # this is a clear sign that you likely ship some mesa library along with your qemu-for-lxd
+  # snap and it has a version conflict with what you have in gpu-2404 interface snap.
+  #
+  gpu-2404:
+    after:
+      - virgl
+      - qemu
+    source: https://github.com/canonical/gpu-snap.git
+    plugin: dump
+    override-prime: |
+      set -ex
+
+      CUSTOM_PREFIX="external/qemu"
+      CRAFT_PRIME_OVERRIDE="${CRAFT_PRIME}/${CUSTOM_PREFIX}"
+
+      # cleanup script expects to see libraries in usr/lib, while we have them in lib
+      mkdir "${CRAFT_PRIME_OVERRIDE}/usr"
+      ln -s "${CRAFT_PRIME_OVERRIDE}/lib" "${CRAFT_PRIME_OVERRIDE}/usr/lib"
+
+      # a bit hacky way to call a cleanup script forcing it to look in a right directory
+      CRAFT_PRIME="${CRAFT_PRIME_OVERRIDE}" ${CRAFT_PART_SRC}/bin/gpu-2404-cleanup mesa-2404 nvidia-2404
+
+      rm -rf "${CRAFT_PRIME_OVERRIDE}/usr"
+
+      set +ex
+
+  virgl:
+    source: https://gitlab.freedesktop.org/virgl/virglrenderer.git
+    source-tag: virglrenderer-1.0.1
+    source-depth: 1
+    plugin: meson
+    meson-parameters:
+      - --prefix=/external/qemu
+    build-packages:
+      - meson
+      - libgbm-dev
+      - libdrm-dev
+
+  qemu:
+    after:
+      - virgl
+    source: https://gitlab.com/qemu-project/qemu.git
+    source-tag: v9.0.1
+    source-depth: 1
+    plugin: autotools
+    autotools-configure-parameters:
+      - --prefix=/external/qemu
+      - --enable-opengl
+      - --enable-virglrenderer
+      - --disable-smartcard
+# options from LXD snap:
+      - --disable-bochs
+      - --disable-cloop
+      - --disable-dmg
+      - --disable-docs
+      - --disable-guest-agent
+      - --disable-parallels
+      - --disable-qed
+      - --disable-slirp
+      - --disable-user
+      - --disable-vdi
+      - --disable-vnc
+      - --disable-xen
+      - --enable-attr
+      - --enable-cap-ng
+      - --enable-kvm
+      - --enable-libusb
+      - --enable-usb-redir
+      - --enable-linux-aio
+#      - --enable-linux-io-uring
+      - --enable-numa
+      - --enable-pie
+      - --enable-rbd
+      - --enable-seccomp
+      - --enable-spice
+      - --enable-system
+      - --enable-tcg
+      - --enable-tools
+      - --enable-vhost-crypto
+      - --enable-vhost-kernel
+      - --enable-vhost-net
+      - --enable-vhost-user
+      - --enable-virtfs
+      - --firmwarepath=/snap/lxd/current/share/qemu/
+      - --localstatedir=/var/
+    build-environment:
+      - ACLOCAL_PATH: /usr/share/aclocal
+      - PKG_CONFIG_PATH: $CRAFT_STAGE/external/qemu/lib/$CRAFT_ARCH_TRIPLET/pkgconfig:$CRAFT_STAGE/usr/lib/$CRAFT_ARCH_TRIPLET/pkgconfig:/usr/lib/$CRAFT_ARCH_TRIPLET/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH}
+      - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/snap/snapcraft/current/bin/scriptlet-bin:$PATH
+      - LD_LIBRARY_PATH: $CRAFT_STAGE/usr/lib/$CRAFT_ARCH_TRIPLET:/usr/lib/$CRAFT_ARCH_TRIPLET${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}
+    override-pull: |
+      [ "$(uname -m)" != "x86_64" ] && [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "ppc64le" ] && [ "$(uname -m)" != "s390x" ] && exit 0
+
+      craftctl default
+    override-build: |
+      [ "$(uname -m)" != "x86_64" ] && [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "ppc64le" ] && [ "$(uname -m)" != "s390x" ] && exit 0
+      set -ex
+      # Mangle the configure a bit
+      QEMUARCH="$(uname -m)"
+      [ "${QEMUARCH}" = "ppc64le" ] && QEMUARCH="ppc64"
+
+      sed -i "s/^unset target_list$/target_list=\"${QEMUARCH}-softmmu\"/" configure
+
+      set +ex
+      craftctl default
+    build-packages:
+      - bison
+      - flex
+      - gettext
+      - libaio-dev
+      - libbluetooth-dev
+      - libbrlapi-dev
+      - libbz2-dev
+      - libcap-dev
+      - libcap-ng-dev
+      - libcacard-dev
+      - libcurl4-gnutls-dev
+      - libepoxy-dev
+      - libfdt-dev
+      - libgtk-3-dev
+      - libglib2.0-dev
+      - libglusterfs-dev
+      - libibverbs-dev
+      - libiscsi-dev
+      - libjemalloc-dev
+      - libjpeg8-dev
+      - liblzo2-dev
+      - libncurses5-dev
+      - libnfs-dev
+      - libnuma-dev
+      - libpango1.0-dev
+      - libpixman-1-dev
+      - libpulse-dev
+      - librbd-dev
+      - librdmacm-dev
+      - libsasl2-dev
+      - libseccomp-dev
+      - libsdl2-dev
+      - libsdl2-image-dev
+      - libsnappy-dev
+      - libspice-protocol-dev
+      - libspice-server-dev
+      - libusb-1.0-0-dev
+      - libusbredirhost-dev
+      - libusbredirparser-dev
+      - libvde-dev
+      - libvdeplug-dev
+      - libvte-2.91-dev
+      - libxml2-dev
+      - libx11-dev
+      - libzstd-dev
+      - ninja-build
+      - zlib1g-dev
+    stage-packages:
+      - libepoxy0
+      - libvte-2.91-0
+      - libsndio7.0
+      - libaio1t64
+      - libasn1-8-heimdal
+      - libboost-iostreams1.74.0
+      - libboost-thread1.74.0
+      - libbrlapi0.8
+      - libcurl3-gnutls
+      - libfdt1
+      - libgfapi0
+      - libgfrpc0
+      - libgfxdr0
+      - libgssapi3-heimdal
+      - libhcrypto5t64-heimdal
+      - libheimbase1-heimdal
+      - libheimntlm0-heimdal
+      - libhx509-5-heimdal
+      - libibverbs1
+      - libiscsi7
+      - libjemalloc2
+      - libkrb5-26-heimdal
+      - libnfs14
+      - libnghttp2-14
+      - libnuma1
+      - librados2
+      - librbd1
+      - librdmacm1
+      - libroken19-heimdal
+      - librtmp1
+      - libsasl2-2
+      - libsdl2-2.0-0
+      - libsdl2-image-2.0-0
+      - libsnappy1v5
+      - libspice-server1
+      - libtirpc3
+      - liburcu8
+      - libusb-1.0-0
+      - libusbredirhost1
+      - libusbredirparser1
+      - libvdeplug2
+      - libwind0-heimdal
+      - seabios
+    organize:
+      lib/: external/qemu/lib/
+      usr/bin/: external/qemu/bin/
+      usr/lib/: external/qemu/lib/
+      usr/local/bin/: external/qemu/bin/
+      usr/local/lib/: external/qemu/lib/
+      usr/local/libexec/: external/qemu/bin/
+      usr/local/share/: external/qemu/share/

snapcraft.yaml

@@ -71,6 +71,20 @@ plugs:
  ovn-chassis:
    interface: content
    target: "$SNAP_DATA/microovn/chassis"
+ gpu-2404:
+   interface: content
+   target: $SNAP/gpu-2404
+#   default-provider: mesa-2404
+ qemu-external:
+   interface: content
+   content: qemu-external-binaries
+   target: $SNAP/external/qemu
+
+layout:
+  /usr/share/libdrm:
+    bind: $SNAP/gpu-2404/libdrm
+  /usr/share/drirc.d:
+    symlink: $SNAP/gpu-2404/drirc.d
 
 apps:
   # Main commands
@@ -82,6 +96,8 @@ apps:
       - system-observe
 
   daemon:
+    command-chain:
+      - bin/gpu-2404-custom-wrapper
     command: commands/daemon.start
     reload-command: commands/daemon.reload
     stop-command: commands/daemon.stop
@@ -1501,6 +1517,15 @@ parts:
     prime:
       - bin/setup-shmounts
 
+  gpu-2404:
+    after:
+      - lxd
+      - qemu
+    source: https://github.com/canonical/gpu-snap.git
+    plugin: dump
+    override-prime: |
+      ${CRAFT_PART_SRC}/bin/gpu-2404-cleanup mesa-2404 nvidia-2404
+
   strip:
     after:
       - btrfs
@@ -1551,6 +1576,7 @@ parts:
         -not -path "${CRAFT_PRIME}/bin/xfs_admin" \
         -not -path "${CRAFT_PRIME}/bin/uefivars.py" \
         -not -path "${CRAFT_PRIME}/bin/lxcfs" \
+        -not -path "${CRAFT_PRIME}/bin/gpu-2404-custom-wrapper" \
         -exec strip -s {} +
 
       # Strip binaries not under bin/ due to being dynamically
@@ -1601,6 +1627,7 @@ parts:
     organize:
       commands/snap-query: bin/
       hooks/: snap/hooks/
+      wrappers/gpu-2404-custom-wrapper: bin/
       wrappers/editor: bin/
       wrappers/remote-viewer: bin/
       wrappers/sshfs: bin/

snapcraft/commands/daemon.start

@@ -180,6 +180,13 @@ echo "==> Preparing /run/bin"
 mkdir -p "/run/bin"
 export PATH="/run/bin:${PATH}"
 
+if [ -e "${SNAP_COMMON}/use-qemu-external-snap" ]; then
+    echo "==> Setting up external QEMU snap integration"
+    export SNAP_QEMU_PREFIX="external/qemu"
+    LD_LPATH_PIPEWIRE="$(readlink -f "${SNAP_CURRENT}"/${SNAP_QEMU_PREFIX}/lib/"${ARCH}"/pipewire-*/)"
+    export LD_LIBRARY_PATH="${SNAP_CURRENT}/${SNAP_QEMU_PREFIX}/lib/${ARCH}:${SNAP_CURRENT}/${SNAP_QEMU_PREFIX}/lib/${ARCH}/pulseaudio:${SNAP_CURRENT}/${SNAP_QEMU_PREFIX}/lib/${ARCH}/ceph:${LD_LPATH_PIPEWIRE:+${LD_LPATH_PIPEWIRE}:}${LD_LIBRARY_PATH}"
+fi
+
 if [ "${ceph_external:-"false"}" = "true" ]; then
     ln -s "${SNAP}/wrappers/run-host" "/run/bin/ceph"
     ln -s "${SNAP}/wrappers/run-host" "/run/bin/radosgw-admin"

snapcraft/hooks/connect-plug-qemu-external

@@ -0,0 +1,16 @@
+#!/bin/sh
+set -eu
+
+# Re-exec outside of apparmor confinement
+if [ -d /sys/kernel/security/apparmor ]; then
+  label="$(cat /proc/self/attr/current 2>/dev/null)"
+  if [ "$label" != "unconfined" ] && [ -n "${label##*(unconfined)}" ]; then
+    exec aa-exec -p unconfined -- "$0" "$@"
+  fi
+fi
+
+echo 1 > "${SNAP_COMMON}/use-qemu-external-snap"
+
+echo reload > "${SNAP_COMMON}/state"
+read -r PID < "${SNAP_COMMON}/lxd.pid"
+kill "$PID"