From c03c8674b89e7da8dc252a7015a19527f14672a2 Mon Sep 17 00:00:00 2001 From: Alexander Mikhalitsyn Date: Thu, 11 Jul 2024 10:05:19 +0200 Subject: [PATCH] lxd-qemu-snap: add example Signed-off-by: Alexander Mikhalitsyn --- lxd-qemu-snap/README.md | 69 ++++++++++ lxd-qemu-snap/snapcraft.yaml | 235 +++++++++++++++++++++++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 lxd-qemu-snap/README.md create mode 100644 lxd-qemu-snap/snapcraft.yaml diff --git a/lxd-qemu-snap/README.md b/lxd-qemu-snap/README.md new file mode 100644 index 000000000..50c9a8617 --- /dev/null +++ b/lxd-qemu-snap/README.md @@ -0,0 +1,69 @@ +# External QEMU snap for LXD snap + +## How to build and use + +``` +cd lxd-qemu-snap + +# clean up previous builds +rm -f qemu-for-lxd_*.snap + +# build +snapcraft + +# install +sudo snap install qemu-for-lxd_*.snap --devmode + +# connect snaps +sudo snap connect lxd:gpu-2404 mesa-2404:gpu-2404 +sudo snap connect lxd:qemu-external qemu-for-lxd:qemu-external +``` + +## How to use with virgl (only specific to this example of snapcraft.yaml) + +``` +lxc init images:ubuntu/noble/desktop desktop -c limits.memory=8GiB --vm + +# modify instance configuration: +lxc config edit desktop + +# choose an appropriate renderer: +ls -la /dev/dri/by-path/*-render + +# for example, on my system it is /dev/dri/by-path/pci-0000:67:00.0-render +# lspci | grep -E "(3D|VGA)" shows: +# 67:00.0 VGA compatible controller: Advanced Micro Devices, Inc. [AMD/ATI] Rembrandt (rev d8) + +# add the following lines: + raw.apparmor: |- + /snap/lxd/*/gpu-2404/** mr, + /dev/dri/ r, + /dev/dri/card[0-9]* rw, + /dev/dri/renderD[0-9]* rw, + /run/udev/data/c226:[0-9]* r, # 226 drm + /sys/devices/** r, + /sys/bus/** r, + raw.qemu: -display egl-headless,rendernode=/dev/dri/by-path/pci-0000:67:00.0-render + raw.qemu.conf: |- + [device "qemu_gpu"] + driver = "virtio-vga-gl" + +# try it +lxc start desktop --console=vga + +# you can check output from: +dmesg | grep -i drm +# if you see: +# [drm] features: +virgl ... +# it's a good sign + +glxinfo | grep -i vir +# it should show something like: +# Device: virgl ... +``` + +## References: + +https://github.com/snapcore/snapd/blob/5c8d8431baa425464b279ff26b8c44eecb9aab22/interfaces/builtin/opengl.go#L41 + +https://gitlab.gnome.org/GNOME/gnome-boxes/-/issues/586 \ No newline at end of file diff --git a/lxd-qemu-snap/snapcraft.yaml b/lxd-qemu-snap/snapcraft.yaml new file mode 100644 index 000000000..6c3236da6 --- /dev/null +++ b/lxd-qemu-snap/snapcraft.yaml @@ -0,0 +1,235 @@ +name: qemu-for-lxd +base: core24 +assumes: + - snapd2.63 +version: '1' +grade: devel +summary: External Qemu for LXD +description: |- + External Qemu for LXD + +contact: lxd@lists.canonical.com +issues: https://github.com/canonical/lxd/issues +source-code: https://github.com/canonical/lxd +website: https://ubuntu.com/lxd +confinement: strict + +slots: + qemu-external: + interface: content + content: qemu-external-binaries + read: + - $SNAP/external/qemu + +parts: + # + # This step is extremely important, as it ensures that no mesa libs will + # be provided with this snap to prevent any possible version mistmatches + # between this snap and mesa-2404 snap. + # If you the error like: + # DRI driver not from this Mesa build ('24.0.5-1ubuntu1' vs '24.0.9-0ubuntu0.1') + # this is a clear sign that you likely ship some mesa library along with your qemu-for-lxd + # snap and it has a version conflict with what you have in gpu-2404 interface snap. + # + gpu-2404: + after: + - virgl + - qemu + source: https://github.com/canonical/gpu-snap.git + plugin: dump + override-prime: | + set -ex + + CUSTOM_PREFIX="external/qemu" + CRAFT_PRIME_OVERRIDE="${CRAFT_PRIME}/${CUSTOM_PREFIX}" + + # cleanup script expects to see libraries in usr/lib, while we have them in lib + mkdir "${CRAFT_PRIME_OVERRIDE}/usr" + ln -s "${CRAFT_PRIME_OVERRIDE}/lib" "${CRAFT_PRIME_OVERRIDE}/usr/lib" + + # a bit hacky way to call a cleanup script forcing it to look in a right directory + CRAFT_PRIME="${CRAFT_PRIME_OVERRIDE}" ${CRAFT_PART_SRC}/bin/gpu-2404-cleanup mesa-2404 nvidia-2404 + + rm -rf "${CRAFT_PRIME_OVERRIDE}/usr" + + set +ex + + virgl: + source: https://gitlab.freedesktop.org/virgl/virglrenderer.git + source-tag: virglrenderer-1.0.1 + source-depth: 1 + plugin: meson + meson-parameters: + - --prefix=/external/qemu + build-packages: + - meson + - libgbm-dev + - libdrm-dev + + qemu: + after: + - virgl + source: https://gitlab.com/qemu-project/qemu.git + source-tag: v9.0.1 + source-depth: 1 + plugin: autotools + autotools-configure-parameters: + - --prefix=/external/qemu + - --enable-opengl + - --enable-virglrenderer + - --disable-smartcard +# options from LXD snap: + - --disable-bochs + - --disable-cloop + - --disable-dmg + - --disable-docs + - --disable-guest-agent + - --disable-parallels + - --disable-qed + - --disable-slirp + - --disable-user + - --disable-vdi + - --disable-vnc + - --disable-xen + - --enable-attr + - --enable-cap-ng + - --enable-kvm + - --enable-libusb + - --enable-usb-redir + - --enable-linux-aio +# - --enable-linux-io-uring + - --enable-numa + - --enable-pie + - --enable-rbd + - --enable-seccomp + - --enable-spice + - --enable-system + - --enable-tcg + - --enable-tools + - --enable-vhost-crypto + - --enable-vhost-kernel + - --enable-vhost-net + - --enable-vhost-user + - --enable-virtfs + - --firmwarepath=/snap/lxd/current/share/qemu/ + - --localstatedir=/var/ + build-environment: + - ACLOCAL_PATH: /usr/share/aclocal + - PKG_CONFIG_PATH: $CRAFT_STAGE/external/qemu/lib/$CRAFT_ARCH_TRIPLET/pkgconfig:$CRAFT_STAGE/usr/lib/$CRAFT_ARCH_TRIPLET/pkgconfig:/usr/lib/$CRAFT_ARCH_TRIPLET/pkgconfig:/usr/lib/pkgconfig:/usr/share/pkgconfig${PKG_CONFIG_PATH:+:$PKG_CONFIG_PATH} + - PATH: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin:/snap/snapcraft/current/bin/scriptlet-bin:$PATH + - LD_LIBRARY_PATH: $CRAFT_STAGE/usr/lib/$CRAFT_ARCH_TRIPLET:/usr/lib/$CRAFT_ARCH_TRIPLET${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} + override-pull: | + [ "$(uname -m)" != "x86_64" ] && [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "ppc64le" ] && [ "$(uname -m)" != "s390x" ] && exit 0 + + craftctl default + override-build: | + [ "$(uname -m)" != "x86_64" ] && [ "$(uname -m)" != "aarch64" ] && [ "$(uname -m)" != "ppc64le" ] && [ "$(uname -m)" != "s390x" ] && exit 0 + set -ex + # Mangle the configure a bit + QEMUARCH="$(uname -m)" + [ "${QEMUARCH}" = "ppc64le" ] && QEMUARCH="ppc64" + + sed -i "s/^unset target_list$/target_list=\"${QEMUARCH}-softmmu\"/" configure + + set +ex + craftctl default + build-packages: + - bison + - flex + - gettext + - libaio-dev + - libbluetooth-dev + - libbrlapi-dev + - libbz2-dev + - libcap-dev + - libcap-ng-dev + - libcacard-dev + - libcurl4-gnutls-dev + - libepoxy-dev + - libfdt-dev + - libgtk-3-dev + - libglib2.0-dev + - libglusterfs-dev + - libibverbs-dev + - libiscsi-dev + - libjemalloc-dev + - libjpeg8-dev + - liblzo2-dev + - libncurses5-dev + - libnfs-dev + - libnuma-dev + - libpango1.0-dev + - libpixman-1-dev + - libpulse-dev + - librbd-dev + - librdmacm-dev + - libsasl2-dev + - libseccomp-dev + - libsdl2-dev + - libsdl2-image-dev + - libsnappy-dev + - libspice-protocol-dev + - libspice-server-dev + - libusb-1.0-0-dev + - libusbredirhost-dev + - libusbredirparser-dev + - libvde-dev + - libvdeplug-dev + - libvte-2.91-dev + - libxml2-dev + - libx11-dev + - libzstd-dev + - ninja-build + - zlib1g-dev + stage-packages: + - libepoxy0 + - libvte-2.91-0 + - libsndio7.0 + - libaio1t64 + - libasn1-8-heimdal + - libboost-iostreams1.74.0 + - libboost-thread1.74.0 + - libbrlapi0.8 + - libcurl3-gnutls + - libfdt1 + - libgfapi0 + - libgfrpc0 + - libgfxdr0 + - libgssapi3-heimdal + - libhcrypto5t64-heimdal + - libheimbase1-heimdal + - libheimntlm0-heimdal + - libhx509-5-heimdal + - libibverbs1 + - libiscsi7 + - libjemalloc2 + - libkrb5-26-heimdal + - libnfs14 + - libnghttp2-14 + - libnuma1 + - librados2 + - librbd1 + - librdmacm1 + - libroken19-heimdal + - librtmp1 + - libsasl2-2 + - libsdl2-2.0-0 + - libsdl2-image-2.0-0 + - libsnappy1v5 + - libspice-server1 + - libtirpc3 + - liburcu8 + - libusb-1.0-0 + - libusbredirhost1 + - libusbredirparser1 + - libvdeplug2 + - libwind0-heimdal + - seabios + organize: + lib/: external/qemu/lib/ + usr/bin/: external/qemu/bin/ + usr/lib/: external/qemu/lib/ + usr/local/bin/: external/qemu/bin/ + usr/local/lib/: external/qemu/lib/ + usr/local/libexec/: external/qemu/bin/ + usr/local/share/: external/qemu/share/