examples/virtio: init

This is an example showing off the virtIO support of libvmm.
It currently only shows off the virtIO console device, with more
devices such as block and graphics to be added later.

.gitmodules

@@ -0,0 +1,8 @@
+[submodule "sddf"]
+	path = examples/virtio/sddf
+	url = https://github.com/au-ts/sddf
+	branch = ivanv/dev
+[submodule "examples/virtio/sddf"]
+	path = examples/virtio/sddf
+	url = https://github.com/au-ts/sddf.git
+	branch = ivanv/dev

ci/examples.sh

@@ -68,6 +68,19 @@ simulate_simple_make() {
     ./ci/buildroot_login.exp ${BUILD_DIR}/loader.img
 }
 
+build_virtio() {
+    BOARD=$1
+    CONFIG=$2
+    echo "CI|INFO: building virtio example via Make with board: $BOARD and config: $CONFIG"
+    BUILD_DIR="${PWD}/build/examples/virtio/make/${BOARD}/${CONFIG}"
+    mkdir -p ${BUILD_DIR}
+    make -C examples/virtio -B \
+        BUILD_DIR=${BUILD_DIR} \
+        CONFIG=${CONFIG} \
+        BOARD=${BOARD} \
+        MICROKIT_SDK=${SDK_PATH}
+}
+
 build_simple_make "qemu_arm_virt" "debug"
 simulate_simple_make "qemu_arm_virt" "debug"
 build_simple_make "qemu_arm_virt" "release"
@@ -89,5 +102,8 @@ simulate_rust "debug"
 build_rust "release"
 simulate_rust "release"
 
+build_virtio "qemu_arm_virt" "debug"
+build_virtio "qemu_arm_virt" "release"
+
 echo ""
 echo "CI|INFO: Passed all VMM tests"

examples/virtio/Makefile

@@ -0,0 +1,197 @@
+#
+# Copyright 2021, Breakaway Consulting Pty. Ltd.
+# Copyright 2022, UNSW (ABN 57 195 873 179)
+#
+# SPDX-License-Identifier: BSD-2-Clause
+#
+
+ifeq ($(strip $(MICROKIT_SDK)),)
+$(error MICROKIT_SDK must be specified)
+endif
+
+ifeq ($(strip $(BOARD)),)
+$(error BOARD must be specified)
+endif
+
+# Default build directory, pass BUILD_DIR=<dir> to override
+BUILD_DIR ?= build
+# Default config is a debug build, pass CONFIG=<debug/release/benchmark> to override
+CONFIG ?= debug
+
+# @ivanv: Check for dependencies and make sure they are installed/in the path
+
+# @ivanv: check that all dependencies exist
+# Specify that we use bash for all shell commands
+SHELL=/bin/bash
+# All dependencies needed to compile the VMM
+QEMU := qemu-system-aarch64
+DTC := dtc
+
+# ifndef TOOLCHAIN
+# 	# Get whether the common toolchain triples exist
+# 	TOOLCHAIN_AARCH64_LINUX_GNU := $(shell command -v aarch64-linux-gnu-gcc 2> /dev/null)
+# 	TOOLCHAIN_AARCH64_UNKNOWN_LINUX_GNU := $(shell command -v aarch64-unknown-linux-gnu-gcc 2> /dev/null)
+# 	# Then check if they are defined and select the appropriate one
+# 	ifdef TOOLCHAIN_AARCH64_LINUX_GNU
+# 		TOOLCHAIN := aarch64-linux-gnu
+# 	else ifdef TOOLCHAIN_AARCH64_UNKNOWN_LINUX_GNU
+# 		TOOLCHAIN := aarch64-unknown-linux-gnu
+# 	else
+# 		$(error "Could not find an AArch64 cross-compiler")
+# 	endif
+# endif
+
+CC := clang
+LD := ld.lld
+MICROKIT_TOOL ?= $(MICROKIT_SDK)/bin/microkit
+
+# @ivanv: need to have a step for putting in the initrd node into the DTB,
+# 		  right now it is unfortunately hard-coded.
+
+# @ivanv: check that the path of SDK_PATH/BOARD exists
+# @ivanv: Have a list of supported boards to check with, if it's not one of those
+# have a helpful message that lists all the support boards.
+
+# @ivanv: incremental builds don't work with IMAGE_DIR changing
+
+BOARD_DIR := $(MICROKIT_SDK)/board/$(BOARD)/$(CONFIG)
+VMM := ../../
+VMM_TOOLS := $(VMM)/tools
+VMM_SRC_DIR := $(VMM)/src
+SYSTEM_DESCRIPTION := board/$(BOARD)/virtio.system
+
+IMAGE_DIR := board/$(BOARD)
+LINUX := $(IMAGE_DIR)/linux
+DTS := $(IMAGE_DIR)/linux.dts
+DTB := $(BUILD_DIR)/linux.dtb
+INITRD := $(IMAGE_DIR)/rootfs.cpio.gz
+
+SDDF := sddf
+SDDF_SERIAL_RINGBUFFER := $(SDDF)/serial/libserialsharedringbuffer
+SDDF_SERIAL_COMPONENTS := $(SDDF)/serial/components
+SDDF_SERIAL_DRIVER := $(SDDF)/drivers/serial/arm
+SDDF_UTIL := $(SDDF)/util
+
+ELFS := vmm.elf serial_mux_tx.elf serial_mux_rx.elf uart_driver.elf
+
+IMAGE_FILE = $(BUILD_DIR)/loader.img
+REPORT_FILE = $(BUILD_DIR)/report.txt
+
+# @ivanv: should only compile printf.o in debug
+VMM_OBJS := vmm.o \
+			printf.o \
+			virq.o \
+			linux.o \
+			guest.o \
+			psci.o \
+			smc.o \
+			fault.o \
+			util.o \
+			vgic.o \
+			vgic_v2.o \
+			package_guest_images.o \
+			tcb.o \
+			vcpu.o \
+			mmio.o \
+			console.o \
+			virtio.o \
+			shared_ringbuffer.o
+
+SERIAL_MUX_TX_OBJS := mux_tx.o shared_ringbuffer.o
+SERIAL_MUX_RX_OBJS := mux_rx.o shared_ringbuffer.o
+SERIAL_DRIVER_OBJS := uart.o shared_ringbuffer.o
+
+# Toolchain flags
+# FIXME: For optimisation we should consider providing the flag -mcpu.
+# FIXME: We should also consider whether -mgeneral-regs-only should be
+# used to avoid the use of the FPU and therefore seL4 does not have to
+# context switch the FPU.
+# Note we only need -Wno-unused-command-line-argument because in Nix
+# passes an extra `--gcc-toolchain` flag which we do not need.
+CFLAGS := -mstrict-align \
+		  -g3 \
+		  -O3 \
+		  -ffreestanding \
+		  -nostdlib \
+		  -Wno-unused-command-line-argument \
+		  -Wall -Wno-unused-function -Werror \
+		  -I$(VMM_SRC_DIR)/arch/aarch64 -I$(VMM_SRC_DIR) -I$(VMM_SRC_DIR)/util -I$(BOARD_DIR)/include \
+		  -I$(SDDF_SERIAL_RINGBUFFER)/include \
+		  -I$(SDDF_SERIAL_DRIVER)/include \
+		  -I$(SDDF) \
+		  -DBOARD_$(BOARD) \
+		  -DCONFIG_$(CONFIG) \
+		  -target aarch64-none-elf
+
+LDFLAGS := -L$(BOARD_DIR)/lib
+LIBS := -lmicrokit -Tmicrokit.ld
+
+all: directories $(IMAGE_FILE)
+
+qemu: all
+	# @ivanv: check that the amount of RAM given to QEMU is at least the number of RAM that QEMU is setup with for seL4.
+	if ! command -v $(QEMU) &> /dev/null; then echo "Could not find dependency: qemu-system-aarch64"; exit 1; fi
+	$(QEMU) -machine virt,virtualization=on,highmem=off,secure=off \
+			-cpu cortex-a53 \
+			-serial mon:stdio \
+			-device loader,file=$(IMAGE_FILE),addr=0x70000000,cpu-num=0 \
+			-m size=2G \
+			-nographic
+
+directories:
+	$(shell mkdir -p $(BUILD_DIR))
+
+$(DTB): $(DTS)
+	if ! command -v $(DTC) &> /dev/null; then echo "Could not find dependency: Device Tree Compiler (dtc)"; exit 1; fi
+	# @ivanv: Shouldn't supress warnings
+	$(DTC) -q -I dts -O dtb $< > $@
+
+$(BUILD_DIR)/package_guest_images.o: $(VMM_TOOLS)/package_guest_images.S $(IMAGE_DIR) $(LINUX) $(INITRD) $(DTB)
+	$(CC) -c -g3 -x assembler-with-cpp \
+					-DGUEST_KERNEL_IMAGE_PATH=\"$(LINUX)\" \
+					-DGUEST_DTB_IMAGE_PATH=\"$(DTB)\" \
+					-DGUEST_INITRD_IMAGE_PATH=\"$(INITRD)\" \
+					-target aarch64-none-elf \
+					$< -o $@
+
+$(BUILD_DIR)/%.o: %.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/util/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/arch/aarch64/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/arch/aarch64/vgic/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/virtio/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(SDDF_SERIAL_COMPONENTS)/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(SDDF_SERIAL_RINGBUFFER)/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/%.o: $(SDDF_SERIAL_DRIVER)/%.c Makefile
+	$(CC) -c $(CFLAGS) $< -o $@
+
+$(BUILD_DIR)/vmm.elf: $(addprefix $(BUILD_DIR)/, $(VMM_OBJS))
+	$(LD) $(LDFLAGS) $^ $(LIBS) -o $@
+
+$(BUILD_DIR)/serial_mux_tx.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_MUX_TX_OBJS))
+	$(LD) $(LDFLAGS) $^ $(LIBS) -o $@
+
+$(BUILD_DIR)/serial_mux_rx.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_MUX_RX_OBJS))
+	$(LD) $(LDFLAGS) $^ $(LIBS) -o $@
+
+$(BUILD_DIR)/uart_driver.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_DRIVER_OBJS))
+	$(LD) $(LDFLAGS) $^ $(LIBS) -o $@
+
+$(IMAGE_FILE) $(REPORT_FILE): $(addprefix $(BUILD_DIR)/, $(ELFS)) $(SYSTEM_DESCRIPTION) $(IMAGE_DIR)
+	$(MICROKIT_TOOL) $(SYSTEM_DESCRIPTION) --search-path $(BUILD_DIR) $(IMAGE_DIR) --board $(BOARD) --config $(CONFIG) -o $(IMAGE_FILE) -r $(REPORT_FILE)

examples/virtio/README.md

@@ -0,0 +1,61 @@
+# Using virtIO with multiple Linux guests
+
+This example shows off the virtIO support that libvmm provides using the
+[seL4 Device Driver Framework](https://github.com/au-ts/sddf) to talk to the
+actual hardware.
+
+The example currently works on the following platforms:
+* QEMU ARM virt
+
+## Building
+
+```sh
+make BOARD=<BOARD> MICROKIT_SDK=/path/to/sdk
+```
+
+Where `<BOARD>` is one of:
+* `qemu_arm_virt`
+
+Other configuration options can be passed to the Makefile such as `CONFIG`
+and `BUILD_DIR`, see the Makefile for details.
+
+If you would like to simulate the QEMU board you can run the following command:
+```sh
+make BOARD=qemu_arm_virt MICROKIT_SDK=/path/to/sdk qemu
+```
+
+This will build the example code as well as run the QEMU command to simulate a
+system running the whole system.
+
+## Running
+
+### virtIO console
+
+This example makes use of the virtIO console device so that neither guest has access
+to any serial device on the platform. The virtIO console support in libvmm talks to
+a serial multiplexor which then talks to a driver for input/output to the physical
+serial device.
+
+When you boot the example, you will see different coloured output for each guest. The
+Linux logs will be interleaving like so:
+```
+Starting klogd: OKStarting klogd: 
+OK
+Running sysctl: Running sysctl: OK
+OKSaving random seed: 
+Saving random seed: [    4.070358] random: crng init done
+[    4.103992] random: crng init done
+OK
+Starting network: OK
+Starting network: OK
+OK
+
+Welcome to Buildroot
+buildroot login: 
+Welcome to Buildroot
+buildroot login:
+```
+
+You will notice that we have two Buildroot login prompts, initially all input is defaulted
+to the guest one (in red). To switch to input into the other guest (green), type in `@2`.
+The `@` symbol is used to switch between clients, in this case the green guest is client 2.

examples/virtio/board/qemu_arm_virt/README.md

@@ -0,0 +1,42 @@
+# QEMU ARM virt images
+
+## Linux kernel
+
+### Details
+* Image name: `linux`
+* Config name: `linux_config`
+* Git remote: https://github.com/torvalds/linux.git
+* Tag: v5.18 (commit hash: `4b0986a3613c92f4ec1bdc7f60ec66fea135991f`)
+* Toolchain: `aarch64-none-elf`
+    * Version: GNU Toolchain for the A-profile Architecture 10.2-2020.11 (arm-10.16)) 10.2.1 20201103
+
+You can also get the Linux config used after booting by running the following
+command in userspace: `zcat /proc/config.gz`.
+
+### Instructions for reproducing
+```
+git clone --depth 1 --branch v5.18 https://github.com/torvalds/linux.git
+cp linux_config linux/.config
+make -C linux ARCH=arm64 CROSS_COMPILE=aarch64-none-elf- all -j$(nproc)
+```
+
+The path to the image is: `linux/arch/arm64/boot/Image`.
+
+## Buildroot RootFS image
+
+### Details
+* Image name: `rootfs.cpio.gz`
+* Config name: `buildroot_config`
+* Version: 2022.08-rc2
+
+### Instructions for reproducing
+
+```
+wget https://buildroot.org/downloads/buildroot-2022.08-rc2.tar.xz
+tar xvf buildroot-2022.08-rc2.tar.xz
+cp buildroot_config buildroot-2022.08-rc2/.config
+make -C buildroot-2022.08-rc2
+```
+
+The root filesystem will be located at: `buildroot-2022.08-rc2/output/images/rootfs.cpio.gz` along
+with the other buildroot artefacts.