From d05b03d4b07feb2afac6e3a8a6b692ceaf60aa50 Mon Sep 17 00:00:00 2001 From: Peter Chubb Date: Tue, 19 Mar 2024 11:35:31 +1100 Subject: [PATCH] Serial queue (#51) * Update mmio.h to match serial queue renaming sharedringbuffer.h has been renamed queue.h Signed-off-by: Peter Chubb * Update for sDDF serial queue renaming Everywhere rename ring->queue, used->active Signed-off-by: Peter Chubb * Updated submodule sDDF to head, to get queue renaming changes Signed-off-by: Peter Chubb * Make the Makefile use the sDDF submodule Signed-off-by: Peter Chubb * More virtio example fixes Rename MUX->Virtualiser, ring->queue Fix Makefile to track dependencies properly, Fix printf.c name clash (the uart driver wants the sddf version, vmm.o wants the libvmm version Signed-off-by: Peter Chubb --------- Signed-off-by: Peter Chubb Signed-off-by: Peter Chubb --- examples/virtio/Makefile | 63 +++++++------- .../virtio/board/qemu_arm_virt/virtio.system | 82 +++++++++---------- examples/virtio/sddf | 2 +- examples/virtio/vmm.c | 36 ++++---- src/virtio/console.c | 28 +++---- src/virtio/console.h | 4 +- src/virtio/mmio.h | 12 +-- src/virtio/virtio.c | 8 +- src/virtio/virtio.h | 4 +- 9 files changed, 118 insertions(+), 121 deletions(-) diff --git a/examples/virtio/Makefile b/examples/virtio/Makefile index 9b059aa8..2710ce2b 100644 --- a/examples/virtio/Makefile +++ b/examples/virtio/Makefile @@ -66,13 +66,12 @@ DTS := $(IMAGE_DIR)/linux.dts DTB := $(BUILD_DIR)/linux.dtb INITRD := $(IMAGE_DIR)/rootfs.cpio.gz -SDDF := sddf -SDDF_SERIAL_RINGBUFFER := $(SDDF)/serial/libserialsharedringbuffer +SDDF ?= sddf SDDF_SERIAL_COMPONENTS := $(SDDF)/serial/components SDDF_SERIAL_DRIVER := $(SDDF)/drivers/serial/arm SDDF_UTIL := $(SDDF)/util -ELFS := vmm.elf serial_mux_tx.elf serial_mux_rx.elf uart_driver.elf +ELFS := vmm.elf serial_virt_tx.elf serial_virt_rx.elf uart_driver.elf IMAGE_FILE = $(BUILD_DIR)/loader.img REPORT_FILE = $(BUILD_DIR)/report.txt @@ -94,14 +93,18 @@ VMM_OBJS := vmm.o \ vcpu.o \ mmio.o \ console.o \ - virtio.o \ - shared_ringbuffer.o + virtio.o -SERIAL_MUX_TX_OBJS := mux_tx.o shared_ringbuffer.o -SERIAL_MUX_RX_OBJS := mux_rx.o shared_ringbuffer.o -SERIAL_DRIVER_OBJS := uart.o shared_ringbuffer.o +SERIAL_VIRT_TX_OBJS := virt_tx.o +SERIAL_VIRT_RX_OBJS := virt_rx.o +SERIAL_DRIVER_OBJS := uart.o sddf_printf.o putchar_debug.o + +ALL_OBJS := $(addprefix ${BUILD_DIR}/,${VMM_OBJS} ${SERIAL_VIRT_TX_OBJS} ${SERIAL_VIRT_RX_OBJS} ${SERIAL_DRIVER_OBJS}) + +DEPENDS := ${ALL_OBJS:.o=.d} # Toolchain flags + # FIXME: For optimisation we should consider providing the flag -mcpu. # FIXME: We should also consider whether -mgeneral-regs-only should be # used to avoid the use of the FPU and therefore seL4 does not have to @@ -115,19 +118,21 @@ CFLAGS := -mstrict-align \ -nostdlib \ -Wno-unused-command-line-argument \ -Wall -Wno-unused-function -Werror \ - -I$(VMM_SRC_DIR)/arch/aarch64 -I$(VMM_SRC_DIR) -I$(VMM_SRC_DIR)/util -I$(BOARD_DIR)/include \ - -I$(SDDF_SERIAL_RINGBUFFER)/include \ + -I$(VMM_SRC_DIR)/arch/aarch64 \ + -I$(VMM_SRC_DIR) -I$(VMM_SRC_DIR)/util \ + -I$(BOARD_DIR)/include \ -I$(SDDF_SERIAL_DRIVER)/include \ -I$(SDDF) \ -I$(SDDF)/include \ -DBOARD_$(BOARD) \ -DCONFIG_$(CONFIG) \ - -target aarch64-none-elf + -target aarch64-none-elf \ + -MD LDFLAGS := -L$(BOARD_DIR)/lib LIBS := -lmicrokit -Tmicrokit.ld -all: directories $(IMAGE_FILE) +all: $(IMAGE_FILE) qemu: all # @ivanv: check that the amount of RAM given to QEMU is at least the number of RAM that QEMU is setup with for seL4. @@ -139,8 +144,9 @@ qemu: all -m size=2G \ -nographic -directories: - $(shell mkdir -p $(BUILD_DIR)) +${BUILD_DIR}/directories: + mkdir -p $(BUILD_DIR) + touch $@ $(DTB): $(DTS) if ! command -v $(DTC) &> /dev/null; then echo "Could not find dependency: Device Tree Compiler (dtc)"; exit 1; fi @@ -155,40 +161,26 @@ $(BUILD_DIR)/package_guest_images.o: $(VMM_TOOLS)/package_guest_images.S $(IMAGE -target aarch64-none-elf \ $< -o $@ -$(BUILD_DIR)/%.o: %.c Makefile - $(CC) -c $(CFLAGS) $< -o $@ - -$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/%.c Makefile - $(CC) -c $(CFLAGS) $< -o $@ - -$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/util/%.c Makefile - $(CC) -c $(CFLAGS) $< -o $@ +VPATH=${VMM_SRC_DIR}:${VMM_SRC_DIR}/util:${VMM_SRC_DIR}/arch/aarch64:${SDDF_SERIAL_DRIVER}:${SDDF_UTIL}:${VMM_SRC_DIR}/arch/aarch64/vgic:${VMM_SRC_DIR}/virtio -$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/arch/aarch64/%.c Makefile +$(BUILD_DIR)/%.o: %.c Makefile $(CC) -c $(CFLAGS) $< -o $@ -$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/arch/aarch64/vgic/%.c Makefile +${BUILD_DIR}/sddf_printf.o: ${SDDF_UTIL}/printf.c $(CC) -c $(CFLAGS) $< -o $@ -$(BUILD_DIR)/%.o: $(VMM_SRC_DIR)/virtio/%.c Makefile - $(CC) -c $(CFLAGS) $< -o $@ $(BUILD_DIR)/%.o: $(SDDF_SERIAL_COMPONENTS)/%.c Makefile $(CC) -c $(CFLAGS) -DSERIAL_NUM_CLIENTS=2 -DSERIAL_TRANSFER_WITH_COLOUR=1 $< -o $@ -$(BUILD_DIR)/%.o: $(SDDF_SERIAL_RINGBUFFER)/%.c Makefile - $(CC) -c $(CFLAGS) $< -o $@ - -$(BUILD_DIR)/%.o: $(SDDF_SERIAL_DRIVER)/%.c Makefile - $(CC) -c $(CFLAGS) $< -o $@ $(BUILD_DIR)/vmm.elf: $(addprefix $(BUILD_DIR)/, $(VMM_OBJS)) $(LD) $(LDFLAGS) $^ $(LIBS) -o $@ -$(BUILD_DIR)/serial_mux_tx.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_MUX_TX_OBJS)) +$(BUILD_DIR)/serial_virt_tx.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_VIRT_TX_OBJS)) $(LD) $(LDFLAGS) $^ $(LIBS) -o $@ -$(BUILD_DIR)/serial_mux_rx.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_MUX_RX_OBJS)) +$(BUILD_DIR)/serial_virt_rx.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_VIRT_RX_OBJS)) $(LD) $(LDFLAGS) $^ $(LIBS) -o $@ $(BUILD_DIR)/uart_driver.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_DRIVER_OBJS)) @@ -196,3 +188,8 @@ $(BUILD_DIR)/uart_driver.elf: $(addprefix $(BUILD_DIR)/, $(SERIAL_DRIVER_OBJS)) $(IMAGE_FILE) $(REPORT_FILE): $(addprefix $(BUILD_DIR)/, $(ELFS)) $(SYSTEM_DESCRIPTION) $(IMAGE_DIR) $(MICROKIT_TOOL) $(SYSTEM_DESCRIPTION) --search-path $(BUILD_DIR) $(IMAGE_DIR) --board $(BOARD) --config $(CONFIG) -o $(IMAGE_FILE) -r $(REPORT_FILE) + +${ALL_OBJS}: ${BUILD_DIR}/directories ${MAKEFILE} + + +-include ${DEPENDS} diff --git a/examples/virtio/board/qemu_arm_virt/virtio.system b/examples/virtio/board/qemu_arm_virt/virtio.system index 3a5419a1..90852206 100644 --- a/examples/virtio/board/qemu_arm_virt/virtio.system +++ b/examples/virtio/board/qemu_arm_virt/virtio.system @@ -14,11 +14,11 @@ - + - + - + @@ -33,11 +33,11 @@ - + - + - + @@ -55,55 +55,55 @@ - + - + - + - + - + - + - + - - - - + + + + - - - + + + - + - - + + - - - - + + + + - - + + - + - + @@ -116,42 +116,42 @@ - + - + - + - + - + - + - + - + - + diff --git a/examples/virtio/sddf b/examples/virtio/sddf index 91685a62..95492dcc 160000 --- a/examples/virtio/sddf +++ b/examples/virtio/sddf @@ -1 +1 @@ -Subproject commit 91685a6215af13bca7fa6080efd53cd7726eaef5 +Subproject commit 95492dcc32abf8f8b4fc34297ba5c26fb181d891 diff --git a/examples/virtio/vmm.c b/examples/virtio/vmm.c index 953bb336..b1f1c90b 100644 --- a/examples/virtio/vmm.c +++ b/examples/virtio/vmm.c @@ -16,7 +16,7 @@ #include "vcpu.h" #include "virtio/virtio.h" #include "virtio/console.h" -#include +#include /* * As this is just an example, for simplicity we just make the size of the @@ -45,18 +45,18 @@ extern char _guest_initrd_image_end[]; uintptr_t guest_ram_vaddr; uintptr_t serial_rx_free; -uintptr_t serial_rx_used; +uintptr_t serial_rx_active; uintptr_t serial_tx_free; -uintptr_t serial_tx_used; +uintptr_t serial_tx_active; uintptr_t serial_rx_data; uintptr_t serial_tx_data; -ring_handle_t serial_rx_ring; -ring_handle_t serial_tx_ring; +serial_queue_handle_t serial_rx_queue; +serial_queue_handle_t serial_tx_queue; -#define SERIAL_MUX_TX_CH 1 -#define SERIAL_MUX_RX_CH 2 +#define SERIAL_VIRT_TX_CH 1 +#define SERIAL_VIRT_RX_CH 2 #define VIRTIO_CONSOLE_IRQ (74) #define VIRTIO_CONSOLE_BASE (0x130000) @@ -91,19 +91,19 @@ void init(void) { LOG_VMM_ERR("Failed to initialise emulated interrupt controller\n"); return; } - /* Initialise our sDDF ring buffers for the serial device */ - ring_init(&serial_rx_ring, (ring_buffer_t *)serial_rx_free, (ring_buffer_t *)serial_rx_used, true, NUM_BUFFERS, NUM_BUFFERS); - for (int i = 0; i < NUM_BUFFERS - 1; i++) { - int ret = enqueue_free(&serial_rx_ring, serial_rx_data + (i * BUFFER_SIZE), BUFFER_SIZE, NULL); + /* Initialise our sDDF queue for the serial device */ + serial_queue_init(&serial_rx_queue, (serial_queue_t *)serial_rx_free, (serial_queue_t *)serial_rx_active, true, NUM_ENTRIES, NUM_ENTRIES); + for (int i = 0; i < NUM_ENTRIES - 1; i++) { + int ret = serial_enqueue_free(&serial_rx_queue, serial_rx_data + (i * BUFFER_SIZE), BUFFER_SIZE); if (ret != 0) { microkit_dbg_puts(microkit_name); microkit_dbg_puts(": server rx buffer population, unable to enqueue buffer\n"); } } - ring_init(&serial_tx_ring, (ring_buffer_t *)serial_tx_free, (ring_buffer_t *)serial_tx_used, true, NUM_BUFFERS, NUM_BUFFERS); - for (int i = 0; i < NUM_BUFFERS - 1; i++) { + serial_queue_init(&serial_tx_queue, (serial_queue_t *)serial_tx_free, (serial_queue_t *)serial_tx_active, true, NUM_ENTRIES, NUM_ENTRIES); + for (int i = 0; i < NUM_ENTRIES - 1; i++) { // Have to start at the memory region left of by the rx ring - int ret = enqueue_free(&serial_tx_ring, serial_tx_data + ((i + NUM_BUFFERS) * BUFFER_SIZE), BUFFER_SIZE, NULL); + int ret = serial_enqueue_free(&serial_tx_queue, serial_tx_data + ((i + NUM_ENTRIES) * BUFFER_SIZE), BUFFER_SIZE); assert(ret == 0); if (ret != 0) { microkit_dbg_puts(microkit_name); @@ -111,11 +111,11 @@ void init(void) { } } /* Neither ring should be plugged and hence all buffers we send should actually end up at the driver. */ - assert(!ring_plugged(serial_tx_ring.free_ring)); - assert(!ring_plugged(serial_tx_ring.used_ring)); + assert(!serial_queue_plugged(serial_tx_queue.free)); + assert(!serial_queue_plugged(serial_tx_queue.active)); /* Initialise virtIO console device */ success = virtio_mmio_device_init(&virtio_console, CONSOLE, VIRTIO_CONSOLE_BASE, VIRTIO_CONSOLE_SIZE, VIRTIO_CONSOLE_IRQ, - &serial_rx_ring, &serial_tx_ring, SERIAL_MUX_TX_CH); + &serial_rx_queue, &serial_tx_queue, SERIAL_VIRT_TX_CH); assert(success); /* Finally start the guest */ guest_start(GUEST_VCPU_ID, kernel_pc, GUEST_DTB_VADDR, GUEST_INIT_RAM_DISK_VADDR); @@ -123,7 +123,7 @@ void init(void) { void notified(microkit_channel ch) { switch (ch) { - case SERIAL_MUX_RX_CH: { + case SERIAL_VIRT_RX_CH: { /* We have received an event from the serial multipelxor, so we * call the virtIO console handling */ virtio_console_handle_rx(&virtio_console); diff --git a/src/virtio/console.c b/src/virtio/console.c index 805a7036..275b3298 100644 --- a/src/virtio/console.c +++ b/src/virtio/console.c @@ -126,10 +126,9 @@ static int virtio_console_handle_tx(struct virtio_device *dev) { /* We first need a free buffer from the TX ring */ uintptr_t sddf_buffer = 0; unsigned int sddf_buffer_len = 0; - void *sddf_cookie = NULL; - LOG_CONSOLE("tx ring free size: 0x%lx, tx ring used size: 0x%lx\n", ring_size(dev->sddf_tx_ring->free_ring), ring_size(dev->sddf_tx_ring->used_ring)); - assert(!ring_empty(dev->sddf_tx_ring->free_ring)); - int ret = dequeue_free(dev->sddf_tx_ring, &sddf_buffer, &sddf_buffer_len, &sddf_cookie); + LOG_CONSOLE("tx ring free size: 0x%lx, tx ring active size: 0x%lx\n", ring_size(dev->sddf_tx_queue->free), serial_queue_size(dev->sddf_tx_queue->active)); + assert(!serial_queue_empty(dev->sddf_tx_queue->free)); + int ret = serial_dequeue_free(dev->sddf_tx_queue, &sddf_buffer, &sddf_buffer_len); assert(!ret); if (ret != 0) { LOG_CONSOLE_ERR("could not dequeue from the TX free ring\n"); @@ -146,15 +145,15 @@ static int virtio_console_handle_tx(struct virtio_device *dev) { * by the multiplexor. */ memcpy((char *) sddf_buffer, (char *) desc.addr, desc.len); - bool is_empty = ring_empty(dev->sddf_tx_ring->used_ring); + bool is_empty = serial_queue_empty(dev->sddf_tx_queue->active); /* Now we can enqueue our buffer into the used TX ring */ - ret = enqueue_used(dev->sddf_tx_ring, sddf_buffer, desc.len, sddf_cookie); + ret = serial_enqueue_active(dev->sddf_tx_queue, sddf_buffer, desc.len); // @ivanv: handle case in release made assert(ret == 0); if (is_empty) { // @ivanv: should we be using the notify_reader/notify_writer API? - microkit_notify(dev->sddf_mux_tx_ch); + microkit_notify(dev->sddf_virt_tx_ch); } /* Lastly, move to the next descriptor in the chain */ @@ -196,11 +195,10 @@ int virtio_console_handle_rx(struct virtio_device *dev) { uintptr_t sddf_buffer = 0; unsigned int sddf_buffer_len = 0; - void *sddf_cookie = NULL; - int ret = dequeue_used(dev->sddf_rx_ring, &sddf_buffer, &sddf_buffer_len, &sddf_cookie); + int ret = serial_dequeue_active(dev->sddf_rx_queue, &sddf_buffer, &sddf_buffer_len); assert(!ret); if (ret != 0) { - LOG_CONSOLE_ERR("could not dequeue from RX used ring\n"); + LOG_CONSOLE_ERR("could not dequeue from RX active queue\n"); // @ivanv: handle properly } @@ -242,7 +240,7 @@ int virtio_console_handle_rx(struct virtio_device *dev) { } // 4. Enqueue sDDF buffer into RX free ring - ret = enqueue_free(dev->sddf_rx_ring, sddf_buffer, BUFFER_SIZE, sddf_cookie); + ret = serial_enqueue_free(dev->sddf_rx_queue, sddf_buffer, BUFFER_SIZE); assert(!ret); // @ivanv: error handle for release mode @@ -261,7 +259,7 @@ virtio_device_funs_t functions = { void virtio_console_init(struct virtio_device *dev, struct virtio_queue_handler *vqs, size_t num_vqs, size_t virq, - ring_handle_t *sddf_rx_ring, ring_handle_t *sddf_tx_ring, size_t sddf_mux_tx_ch) { + serial_queue_handle_t *sddf_rx_queue, serial_queue_handle_t *sddf_tx_queue, size_t sddf_virt_tx_ch) { // @ivanv: check that num_vqs is greater than the minimum vqs to function? dev->data.DeviceID = DEVICE_ID_VIRTIO_CONSOLE; dev->data.VendorID = VIRTIO_MMIO_DEV_VENDOR_ID; @@ -269,7 +267,7 @@ void virtio_console_init(struct virtio_device *dev, dev->vqs = vqs; dev->num_vqs = num_vqs; dev->virq = virq; - dev->sddf_rx_ring = sddf_rx_ring; - dev->sddf_tx_ring = sddf_tx_ring; - dev->sddf_mux_tx_ch = sddf_mux_tx_ch; + dev->sddf_rx_queue = sddf_rx_queue; + dev->sddf_tx_queue = sddf_tx_queue; + dev->sddf_virt_tx_ch = sddf_virt_tx_ch; } diff --git a/src/virtio/console.h b/src/virtio/console.h index 651c47c5..5c217232 100644 --- a/src/virtio/console.h +++ b/src/virtio/console.h @@ -91,5 +91,7 @@ struct virtio_console_control { void virtio_console_init(struct virtio_device *dev, struct virtio_queue_handler *vqs, size_t num_vqs, size_t virq, - ring_handle_t *sddf_rx_ring, ring_handle_t *sddf_tx_ring, size_t sddf_mux_tx_ch); + serial_queue_handle_t *sddf_rx_queue, + serial_queue_handle_t *sddf_tx_queue, + size_t sddf_virt_tx_ch); int virtio_console_handle_rx(struct virtio_device *dev); diff --git a/src/virtio/mmio.h b/src/virtio/mmio.h index c9534190..47569c61 100644 --- a/src/virtio/mmio.h +++ b/src/virtio/mmio.h @@ -8,7 +8,7 @@ #include #include "util/util.h" #include "virtio/virtq.h" -#include +#include // table 4.1 #define VIRTIO_MMIO_DEV_MAGIC 0x74726976 // "virt" @@ -115,14 +115,14 @@ typedef struct virtio_device { size_t num_vqs; /* Virtual IRQ associated with this virtIO device */ size_t virq; - /* Handlers for sDDF ring buffers */ - ring_handle_t *sddf_rx_ring; - ring_handle_t *sddf_tx_ring; - /* Microkit channel to the sDDF TX multiplexor */ + /* Handlers for sDDF queues */ + serial_queue_handle_t *sddf_rx_queue; + serial_queue_handle_t *sddf_tx_queue; + /* Microkit channel to the sDDF TX virtualiser */ // @ivanv: this is microkit specific so maybe should be a callback instead or something. // @ivanv: my worry here is that the device struct is supposed to be for all devices, but // this is specific to device classes such as serial and networking - size_t sddf_mux_tx_ch; + size_t sddf_virt_tx_ch; } virtio_device_t; /** diff --git a/src/virtio/virtio.c b/src/virtio/virtio.c index 99c9d213..14ea6c97 100644 --- a/src/virtio/virtio.c +++ b/src/virtio/virtio.c @@ -21,14 +21,14 @@ bool virtio_mmio_device_init(virtio_device_t *dev, uintptr_t region_base, uintptr_t region_size, size_t virq, - ring_handle_t *sddf_rx_ring, - ring_handle_t *sddf_tx_ring, - size_t sddf_mux_tx_ch) + serial_queue_handle_t *sddf_rx_queue, + serial_queue_handle_t *sddf_tx_queue, + size_t sddf_virt_tx_ch) { bool success = true; switch (type) { case CONSOLE: - virtio_console_init(dev, virtio_console_queues, VIRTIO_CONSOLE_NUM_VIRTQ, virq, sddf_rx_ring, sddf_tx_ring, sddf_mux_tx_ch); + virtio_console_init(dev, virtio_console_queues, VIRTIO_CONSOLE_NUM_VIRTQ, virq, sddf_rx_queue, sddf_tx_queue, sddf_virt_tx_ch); success = fault_register_vm_exception_handler(region_base, region_size, &virtio_mmio_fault_handle, diff --git a/src/virtio/virtio.h b/src/virtio/virtio.h index 54c537a8..444c43a6 100644 --- a/src/virtio/virtio.h +++ b/src/virtio/virtio.h @@ -26,6 +26,6 @@ bool virtio_mmio_device_init(virtio_device_t *dev, uintptr_t region_base, uintptr_t region_size, size_t virq, - ring_handle_t *sddf_rx_ring, - ring_handle_t *sddf_tx_ring, + serial_queue_handle_t *sddf_rx_queue, + serial_queue_handle_t *sddf_tx_queue, size_t sddf_mux_tx_ch);