diff --git a/apps/x86/supermicro/CMakeLists.txt b/apps/x86/supermicro/CMakeLists.txt new file mode 100644 index 00000000..a762f93f --- /dev/null +++ b/apps/x86/supermicro/CMakeLists.txt @@ -0,0 +1,46 @@ + + +# +# Copyright 2022, UNSW (ABN 57 195 873 179) +# +# SPDX-License-Identifier: BSD-2-Clause +# + +cmake_minimum_required(VERSION 3.8.2) + +project(supermicro) + +# Include CAmkES VM helper functions +include(${CAMKES_VM_HELPERS_PATH}) +find_package(camkes-vm-linux REQUIRED) +include(${CAMKES_VM_LINUX_HELPERS_PATH}) + +set(NumberVMs 4) +MATH(EXPR NumberVMsLess1 "${NumberVMs}-1") + +# Declare VM component: Init +foreach(i RANGE ${NumberVMsLess1}) + file(GLOB "init${i}_extra" "src/cross_vm_connections_vm${i}.c") + DeclareCAmkESVM("Init${i}" EXTRA_SOURCES "${init${i}_extra}") +endforeach() + +DeclareCamkesComponent(virtqueueinit) + +# Get 32-bit Linux VM files +GetSupermicroLinuxKernelFile(kernel_file) +GetArchDefaultLinuxRootfsFile("32" rootfs_file) + +# Decompress Linux Kernel image +DecompressLinuxKernel(extract_linux_kernel decompressed_kernel ${kernel_file}) + +# Add Linux VM images into file server +AddToFileServer("bzimage" ${decompressed_kernel} DEPENDS extract_linux_kernel) +AddToFileServer("rootfs.cpio" ${rootfs_file}) + +# Initialise CAmkES Root Server with addition CPP includes +DeclareCAmkESVMRootServer(supermicro.camkes CPP_INCLUDES configurations) + +if(SIMULATION) + include(simulation) + GenerateSimulateScript() +endif() diff --git a/apps/x86/supermicro/README.md b/apps/x86/supermicro/README.md new file mode 100644 index 00000000..32ff1124 --- /dev/null +++ b/apps/x86/supermicro/README.md @@ -0,0 +1,112 @@ + + +# Multi VM Demo Application + +Please refer also to +https://docs.sel4.systems/Tutorials/camkes-vm-linux.html +for a general set up guide. + +## Intro +This is a sample configuration for a Supermicro SYS-5019D-FN8TP +It sets up 4 32-bit Linux virtual machines (VMs). VM0 has passthrough ethernet for LAN0 +(PCI device 66:0:0); the other VMs have no passthrough devices. + +Serial console on port COM2 (the one redirected to IPMI +Serial-over-LAN by the machine's firmware) provides debug output, and +redirection to the VM consoles on their port /dev/ttyS0 at 115200 +baud. + +Each VM has a virtio-net NIC that are connected together via a virtual +switch. These come up unconfigured in the example. + +VMs 0, 1, and 2 have partitions 1, 2, and 3 of the sata disc on the +hardware available as /dev/vda1 + +The root file system is a simple buildroot; login/password root/root + + +## Config + +The passthrough device configuration options in `supermicro.camkes` may need to be changed. These include: + - Physical address of the base of the memory-mapped device and I/O ports + - Size of the memory region + - IRQ-related information (PCI, MSI, etc) + - PCI bus/dev/function information of the device + +## Build and run the application + +The following instructions build the application for x86_64: +``` +mkdir build +cd build +../init-build.sh -DCAMKES_VM_APP=supermicro +ninja +``` + +## Testing + +Some tips for testing the Virtio Net and the passthrough ethernet. + +1. Switching the serial port between different VMs: press `Return`, + then `@`, then the index number of the VM you want to switch to. + +2. Testing Virtio Net +``` +# on vm0 +ifconfig eth0 192.168.1.1 + +# on vm1 +ifconfig eth0 192.168.1.2 +ping 192.168.1.1 +``` + +3. Setting up the passthrough ethernet +``` +# on vm0 +ip link set up dev eth1 +udhcpc -i eth1 +``` + +4. The SATA shared disk + +The default configuration assumes a disc with three partitions. These +are available as `/dev/vda1` in each of vm0, vm1, and vm2. + +The sataserver code currently allows only primary MBR partitions, so +you are restricted to 4 max per disc. + +## ISSUES + +The supermicro machine has multiple PCI roots. We replaced the +recursive scan for PCI devices with a brute force 'scan everything' +approach in PR .../util_libs/pull/132 to get around this without +having to include an ACPI bytecode interpreter in the kernel. + +The supermicro machine has 5 IOAPICS. The gigabit and 10Gb NICS are +on IOAPIC 1; the code for handling other than the 0th IOAPIC seems +buggy. Right now the VM does not get any interrupts. + +This is also true of anything plugged into the only PCI slot. + +The SATA server currently polls for end of operation. On a fast SSD +this isn;t that much of an issue; but should be fixed. + + +These PRs are currently outstanding: + https://github.com/seL4/seL4/pull/324 improvements to allow 64-bit + guests + https://github.com/seL4/seL4_projects_libs/pull/16 Likewise... + https://github.com/seL4/camkes-vm/pull/32 add the SATA config for + the supermicro. + https://github.com/seL4/camkes-vm-examples/pull/27 the supermicro + config itself (where this file is at the moment) + https://github.com/seL4/util_libs/pull/132 use brute force scan for + PCI devices. + +These features do not yet have a PR up: + -- virtIO-socket + -- virtIO-console between VMs diff --git a/apps/x86/supermicro/app_settings.cmake b/apps/x86/supermicro/app_settings.cmake new file mode 100644 index 00000000..7a4bc83e --- /dev/null +++ b/apps/x86/supermicro/app_settings.cmake @@ -0,0 +1,21 @@ +# +# Copyright 2022, UNSW (ABN 57 195 873 179) +# +# SPDX-License-Identifier: BSD-2-Clause +# + +cmake_minimum_required(VERSION 3.8.2) + +# Define kernel config options +set(KernelSel4Arch "x86_64" CACHE STRING "" FORCE) +set(KernelHugePage OFF CACHE BOOL "" FORCE) +set(KernelMaxNumIOAPIC 5 CACHE STRING "" FORCE) +# this is for the Serial-Over-Lan console via IPMI +set(LibPlatSupportX86ConsoleDevice "com2" CACHE STRING "" FORCE) + +ApplyCommonSimulationSettings(${KernelArch}) +set(KernelIOMMU ON CACHE BOOL "" FORCE) + +# Use AHCI by default +set(SataserverUseAHCI ON CACHE STRING "") +set(CAmkESVMDestHardware "supermicro" CACHE STRING "" FORCE) \ No newline at end of file diff --git a/apps/x86/supermicro/src/cross_vm_connections_0.c b/apps/x86/supermicro/src/cross_vm_connections_0.c new file mode 100644 index 00000000..2420bc5e --- /dev/null +++ b/apps/x86/supermicro/src/cross_vm_connections_0.c @@ -0,0 +1,31 @@ +/* + * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230) + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#include +#include +#include +#include + +// these are defined in the dataport's glue code +extern dataport_caps_handle_t dp01_handle; +extern dataport_caps_handle_t dp02_handle; +extern dataport_caps_handle_t dp03_handle; + +static struct camkes_crossvm_connection connections[] = { + {&dp01_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp02_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp03_handle, NULL, {.id = -1, .reg_callback = NULL}} +}; + +int camkes_cross_vm_connections_init(vm_t *vm, vmm_pci_space_t *pci, seL4_CPtr irq_notification, + uintptr_t connection_base_address) +{ + return cross_vm_connections_init(vm, connection_base_address, connections, ARRAY_SIZE(connections), pci, + irq_notification); +} diff --git a/apps/x86/supermicro/src/cross_vm_connections_1.c b/apps/x86/supermicro/src/cross_vm_connections_1.c new file mode 100644 index 00000000..986fd835 --- /dev/null +++ b/apps/x86/supermicro/src/cross_vm_connections_1.c @@ -0,0 +1,31 @@ +/* + * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230) + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#include +#include +#include +#include + +// these are defined in the dataport's glue code +extern dataport_caps_handle_t dp10_handle; +extern dataport_caps_handle_t dp12_handle; +extern dataport_caps_handle_t dp13_handle; + +static struct camkes_crossvm_connection connections[] = { + {&dp10_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp12_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp13_handle, NULL, {.id = -1, .reg_callback = NULL}} +}; + +int camkes_cross_vm_connections_init(vm_t *vm, vmm_pci_space_t *pci, seL4_CPtr irq_notification, + uintptr_t connection_base_address) +{ + return cross_vm_connections_init(vm, connection_base_address, connections, ARRAY_SIZE(connections), pci, + irq_notification); +} diff --git a/apps/x86/supermicro/src/cross_vm_connections_2.c b/apps/x86/supermicro/src/cross_vm_connections_2.c new file mode 100644 index 00000000..b6f44c64 --- /dev/null +++ b/apps/x86/supermicro/src/cross_vm_connections_2.c @@ -0,0 +1,31 @@ +/* + * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230) + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#include +#include +#include +#include + +// these are defined in the dataport's glue code +extern dataport_caps_handle_t dp20_handle; +extern dataport_caps_handle_t dp21_handle; +extern dataport_caps_handle_t dp23_handle; + +static struct camkes_crossvm_connection connections[] = { + {&dp20_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp21_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp23_handle, NULL, {.id = -1, .reg_callback = NULL}} +}; + +int camkes_cross_vm_connections_init(vm_t *vm, vmm_pci_space_t *pci, seL4_CPtr irq_notification, + uintptr_t connection_base_address) +{ + return cross_vm_connections_init(vm, connection_base_address, connections, ARRAY_SIZE(connections), pci, + irq_notification); +} diff --git a/apps/x86/supermicro/src/cross_vm_connections_3.c b/apps/x86/supermicro/src/cross_vm_connections_3.c new file mode 100644 index 00000000..485e8148 --- /dev/null +++ b/apps/x86/supermicro/src/cross_vm_connections_3.c @@ -0,0 +1,31 @@ +/* + * Copyright 2019, Data61, CSIRO (ABN 41 687 119 230) + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +#include +#include + +#include +#include +#include +#include + +// these are defined in the dataport's glue code +extern dataport_caps_handle_t dp30_handle; +extern dataport_caps_handle_t dp31_handle; +extern dataport_caps_handle_t dp32_handle; + +static struct camkes_crossvm_connection connections[] = { + {&dp30_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp31_handle, NULL, {.id = -1, .reg_callback = NULL}}, + {&dp32_handle, NULL, {.id = -1, .reg_callback = NULL}} +}; + +int camkes_cross_vm_connections_init(vm_t *vm, vmm_pci_space_t *pci, seL4_CPtr irq_notification, + uintptr_t connection_base_address) +{ + return cross_vm_connections_init(vm, connection_base_address, connections, ARRAY_SIZE(connections), pci, + irq_notification); +} diff --git a/apps/x86/supermicro/supermicro.camkes b/apps/x86/supermicro/supermicro.camkes new file mode 100644 index 00000000..48deb291 --- /dev/null +++ b/apps/x86/supermicro/supermicro.camkes @@ -0,0 +1,212 @@ +/* + * Copyright 2022, UNSW (ABN 57 195 873 179) + * + * SPDX-License-Identifier: BSD-2-Clause + */ + +import ; +import ; + +#include +#include +#include + +/* +-- 4 VMs +-- virtIO Net to the virtual switch that allows them all to talk to each other +-- virtIO block to three of the VMs each with a separate partition on the disk +-- virtio Console from COM1 on the VMs to COM2 in the serial server + +-- virtIO console from COM2 to COM2 of VM1 and VM2; COM3 tp COM3 from VM3 to VM4; COM4 from VM4 to VM1 and from VM3 to VM2 +-- And virtIO socket as soon as we have it +*/ + +#define SHARED_BUFSIZE 4096 + +#define VM_GUEST_CMDLINE "earlyprintk=ttyS0,115200 console=ttyS0,115200 root=/dev/mem i8042.nokbd=iy \ +i8042.nomux=y i8042.noaux=y io_delay=udelay noisapnp pci=nomsi" + +#define VM0_topology_def(f) f(0,1,2,3) +#define VM1_topology_def(f) f(1,0,2,3) +#define VM2_topology_def(f) f(2,0,1,3) +#define VM3_topology_def(f) f(3,0,1,2) + +#define topology_def(f) \ + VM0_topology_def(f) \ + VM1_topology_def(f) \ + VM2_topology_def(f) \ + VM3_topology_def(f) + +#define VM0_MACADDRESS "02:00:00:00:AA:01" +#define VM1_MACADDRESS "02:00:00:00:AA:02" +#define VM2_MACADDRESS "02:00:00:00:AA:03" +#define VM3_MACADDRESS "02:00:00:00:AA:04" + +component Init0 { + dataport Buf(SHARED_BUFSIZE) dp01; + dataport Buf(SHARED_BUFSIZE) dp02; + dataport Buf(SHARED_BUFSIZE) dp03; + VM_INIT_DEF() + VM_INIT_SATA() + VM_CONNECTION_COMPONENT_DEF(0,topology_def) +} + +component Init1 { + dataport Buf(SHARED_BUFSIZE) dp10; + dataport Buf(SHARED_BUFSIZE) dp12; + dataport Buf(SHARED_BUFSIZE) dp13; + VM_INIT_DEF() + VM_INIT_SATA() + VM_CONNECTION_COMPONENT_DEF(1,topology_def) +} + +component Init2 { + dataport Buf(SHARED_BUFSIZE) dp20; + dataport Buf(SHARED_BUFSIZE) dp21; + dataport Buf(SHARED_BUFSIZE) dp23; + VM_INIT_DEF() + VM_INIT_SATA() + VM_CONNECTION_COMPONENT_DEF(2,topology_def) +} + +component Init3 { + dataport Buf(SHARED_BUFSIZE) dp30; + dataport Buf(SHARED_BUFSIZE) dp31; + dataport Buf(SHARED_BUFSIZE) dp32; + VM_INIT_DEF() + VM_CONNECTION_COMPONENT_DEF(3,topology_def) +} + +assembly { + composition { + VM_COMPOSITION_DEF() + SATA_COMPOSITION_DEF() + + VM_PER_VM_COMP_DEF(0) + VM_SATA_CONNECTIONS(0) + + VM_PER_VM_COMP_DEF(1) + VM_SATA_CONNECTIONS(1) + + VM_PER_VM_COMP_DEF(2) + VM_SATA_CONNECTIONS(2) + + VM_PER_VM_COMP_DEF(3) + + component VirtQueueInit init; + VM_CONNECTION_CONNECT_VMS(init.init, topology_def) + + connection seL4SharedDataWithCaps cross_vm_conn_01(from vm0.dp01, to vm1.dp10); + connection seL4SharedDataWithCaps cross_vm_conn_02(from vm0.dp02, to vm2.dp20); + connection seL4SharedDataWithCaps cross_vm_conn_03(from vm0.dp03, to vm3.dp30); + + connection seL4SharedDataWithCaps cross_vm_conn_12(from vm1.dp12, to vm2.dp21); + connection seL4SharedDataWithCaps cross_vm_conn_13(from vm1.dp13, to vm3.dp31); + + connection seL4SharedDataWithCaps cross_vm_conn_23(from vm2.dp23, to vm3.dp32); + + } + + configuration { + VM_CONNECTION_CONFIG(init.init, topology_def) + VM_CONFIGURATION_DEF() + VM_SATA_CONFIG() + + /* I/O ports for SATA device */ + sataserver.ioports = "0x3020:0x303f,0x3060:0x3063,0x3070:0x3077"; + + /* IOMMU group/bus/dev/fun for SATA device */ + sataserver.iospaces = "0x15:0x0:0x11:5"; + sataserver.pci_bdfs = "0x0:0x11.5"; + sataserver.iospace_id = 0x15; + + sataserver.num_bdfs = 1; + sataserver.drive = 0; + + VM_PER_VM_CONFIG_DEF(0) + vm0.simple_untyped25_pool = 95; + vm0.heap_size = 0x2000000; + vm0.guest_ram_mb = 1900; + vm0.kernel_cmdline = VM_GUEST_CMDLINE; + vm0.kernel_image = "bzimage"; + vm0.kernel_relocs = "bzimage"; + vm0.initrd_image = "rootfs.cpio"; + vm0.iospace_domain = 0x0f; + vm0.pci_devices_iospace = 1; + vm0.sataserver_iface_attributes = "0"; + vm0.sataserver_iface_partitions = [1]; /* Physical Partition assigned to VM0 */ + + + vm0.init_cons = [VM_CONNECTION_INIT_HANDLER(0,topology_def), {"init":"make_virtio_blk"}, + ]; + + VM_PER_VM_CONFIG_DEF(1) + vm1.simple_untyped25_pool = 95; + vm1.heap_size = 0x2000000; + vm1.guest_ram_mb = 1900; + vm1.kernel_cmdline = VM_GUEST_CMDLINE; + vm1.kernel_image = "bzimage"; + vm1.kernel_relocs = "bzimage"; + vm1.initrd_image = "rootfs.cpio"; + vm1.iospace_domain = 0x10; + vm1.pci_devices_iospace = 2; + vm1.sataserver_iface_attributes = "1"; + vm1.sataserver_iface_partitions = [2]; /* Physical Partition assigned to VM1 */ + + vm1.init_cons = [VM_CONNECTION_INIT_HANDLER(1,topology_def), {"init":"make_virtio_blk"}, + ]; + + VM_PER_VM_CONFIG_DEF(2) + vm2.simple_untyped25_pool = 95; + vm2.heap_size = 0x2000000; + vm2.guest_ram_mb = 1900; + vm2.kernel_cmdline = VM_GUEST_CMDLINE; + vm2.kernel_image = "bzimage"; + vm2.kernel_relocs = "bzimage"; + vm2.initrd_image = "rootfs.cpio"; + vm2.iospace_domain = 0x11; + vm2.pci_devices_iospace = 3; + vm2.sataserver_iface_attributes = "2"; + vm2.sataserver_iface_partitions = [3]; /* Physical Partition assigned to VM2 */ + + vm2.init_cons = [VM_CONNECTION_INIT_HANDLER(2,topology_def), {"init":"make_virtio_blk"}, + ]; + + VM_PER_VM_CONFIG_DEF(3) + vm3.simple_untyped25_pool = 95; + vm3.heap_size = 0x2000000; + vm3.guest_ram_mb = 1900; + vm3.kernel_cmdline = VM_GUEST_CMDLINE; + vm3.kernel_image = "bzimage"; + vm3.kernel_relocs = "bzimage"; + vm3.initrd_image = "rootfs.cpio"; + vm3.iospace_domain = 0x12; + vm3.pci_devices_iospace = 4; + + vm3.init_cons = [VM_CONNECTION_INIT_HANDLER(3,topology_def), + ]; + + // Hardware details of the passthrough Ethernet device + vm0.vm_ioports = [ + {"start":0xa060, "end":0xa060 + 32, "pci_device":0, "name":"Ethernet-vm0"}, + ]; + + vm0.pci_devices = [ + { + "name":"Ethernet-vm0", + "bus":0x66, "dev":0, "fun":0, + "irq":"Ethernet-vm0-Irq", + "memory":[ + {"paddr":0xe0d60000, "size":0x20000, "page_bits":12}, + {"paddr":0xe0d8c000, "size":0x4000, "page_bits":12}, + ], + }, + ]; + vm0.vm_irqs = [ + {"name":"Ethernet-vm0-Irq", "ioapic":1, "source":5, "level_trig":1, "active_low":1, "dest":11}, + ]; + + + + } +}