RFC: enable openhcl to run in VTL0, use nested virtualization

Cargo.lock

@@ -6707,6 +6707,7 @@ dependencies = [
  "vga_proxy",
  "video_core",
  "virt",
+ "virt_kvm",
  "virt_mshv_vtl",
  "vm_loader",
  "vm_manifest_builder",
@@ -6790,9 +6791,11 @@ dependencies = [
  "libc",
  "log",
  "nix 0.26.4",
+ "safe_intrinsics",
  "underhill_confidentiality",
  "vergen",
  "walkdir",
+ "x86defs",
 ]
 
 [[package]]
@@ -6801,6 +6804,7 @@ version = "0.0.0"
 dependencies = [
  "anyhow",
  "build_rs_guest_arch",
+ "fs-err",
  "futures",
  "guestmem",
  "hcl",

flowey/flowey_hvlite/src/pipelines/build_igvm.rs

@@ -30,6 +30,9 @@ pub enum OpenhclRecipeCli {
     X64,
     /// X64 OpenHCL, using the dev kernel in VTL2
     X64Devkern,
+    /// X64 OpenHCL running in VTL0, using KVM and nested virtualization to run
+    /// the guest. Uses the dev kernel.
+    X64Nested,
 }
 
 /// Build OpenHCL IGVM files for local development. DO NOT USE IN CI.
@@ -293,6 +296,7 @@ impl IntoPipeline for BuildIgvmCli {
                     OpenhclRecipeCli::X64CvmDevkern => OpenhclIgvmRecipe::X64CvmDevkern,
                     OpenhclRecipeCli::Aarch64 => OpenhclIgvmRecipe::Aarch64,
                     OpenhclRecipeCli::Aarch64Devkern => OpenhclIgvmRecipe::Aarch64Devkern,
+                    OpenhclRecipeCli::X64Nested => OpenhclIgvmRecipe::X64Nested,
                 },
                 release,
 

flowey/flowey_lib_hvlite/src/_jobs/local_build_igvm.rs

@@ -7,6 +7,7 @@ use flowey::node::prelude::*;
 
 use crate::build_openhcl_boot::OpenhclBootOutput;
 use crate::build_openhcl_igvm_from_recipe::IgvmManifestPath;
+use crate::build_openhcl_igvm_from_recipe::InitrdRootfsPath;
 use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipe;
 use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipeDetails;
 use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipeDetailsLocalOnly;
@@ -115,6 +116,7 @@ impl SimpleFlowNode for Node {
                 with_uefi,
                 with_interactive,
                 with_sidecar_details,
+                initrd_rootfs,
             } = &mut recipe_details;
 
             if custom_kernel.is_some() {
@@ -149,12 +151,12 @@ impl SimpleFlowNode for Node {
                 custom_uefi: custom_uefi.map(|p| p.absolute()).transpose()?,
                 custom_kernel: custom_kernel.map(|p| p.absolute()).transpose()?,
                 custom_sidecar: custom_sidecar.map(|p| p.absolute()).transpose()?,
-                custom_extra_rootfs: custom_extra_rootfs
-                    .into_iter()
-                    .map(|p| p.absolute())
-                    .collect::<Result<_, _>>()?,
             });
 
+            for p in custom_extra_rootfs.iter() {
+                initrd_rootfs.push(InitrdRootfsPath::LocalOnlyCustom(p.absolute()?));
+            }
+
             if let Some(p) = override_manifest {
                 *igvm_manifest = IgvmManifestPath::LocalOnlyCustom(p.absolute()?);
             }
@@ -309,6 +311,7 @@ pub fn non_production_build_igvm_tool_out_name(recipe: &OpenhclIgvmRecipe) -> &'
         OpenhclIgvmRecipe::X64CvmDevkern => "x64-cvm-devkern",
         OpenhclIgvmRecipe::Aarch64 => "aarch64",
         OpenhclIgvmRecipe::Aarch64Devkern => "aarch64-devkern",
+        OpenhclIgvmRecipe::X64Nested => "x64-nested",
         OpenhclIgvmRecipe::LocalOnlyCustom(_) => unreachable!(),
     }
 }

flowey/flowey_lib_hvlite/src/artifact_openhcl_igvm_from_recipe.rs

@@ -14,6 +14,7 @@ pub(crate) fn recipe_to_filename(flavor: &OpenhclIgvmRecipe) -> &str {
         OpenhclIgvmRecipe::X64TestLinuxDirectDevkern => "openhcl-direct-dev",
         OpenhclIgvmRecipe::X64Cvm => "openhcl-cvm",
         OpenhclIgvmRecipe::X64CvmDevkern => "openhcl-cvm-dev",
+        OpenhclIgvmRecipe::X64Nested => "openhcl-nested",
         OpenhclIgvmRecipe::Aarch64 => "openhcl-aarch64",
         OpenhclIgvmRecipe::Aarch64Devkern => "openhcl-aarch64-dev",
         OpenhclIgvmRecipe::LocalOnlyCustom(_) => unreachable!(),

flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs

@@ -54,11 +54,20 @@ pub enum IgvmManifestPath {
     LocalOnlyCustom(PathBuf),
 }
 
+#[derive(Serialize, Deserialize, Clone, Debug)]
+pub enum InitrdRootfsPath {
+    /// Name of an in-tree file (located under `openhcl`)
+    InTree(String),
+    /// An absolute path to a custom manifest (for local use only)
+    LocalOnlyCustom(PathBuf),
+}
+
 #[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct OpenhclIgvmRecipeDetails {
     pub local_only: Option<OpenhclIgvmRecipeDetailsLocalOnly>,
 
     pub igvm_manifest: IgvmManifestPath,
+    pub initrd_rootfs: Vec<InitrdRootfsPath>,
     pub openhcl_kernel_package: OpenhclKernelPackage,
     pub openvmm_hcl_features: BTreeSet<OpenvmmHclFeature>,
     pub target: CommonTriple,
@@ -77,7 +86,6 @@ pub struct OpenhclIgvmRecipeDetailsLocalOnly {
     pub custom_uefi: Option<PathBuf>,
     pub custom_kernel: Option<PathBuf>,
     pub custom_sidecar: Option<PathBuf>,
-    pub custom_extra_rootfs: Vec<PathBuf>,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]
@@ -91,6 +99,7 @@ pub enum OpenhclIgvmRecipe {
     X64CvmDevkern,
     Aarch64,
     Aarch64Devkern,
+    X64Nested,
 }
 
 impl OpenhclIgvmRecipe {
@@ -107,6 +116,8 @@ impl OpenhclIgvmRecipe {
             m
         };
 
+        let base_initrd_rootfs = || vec![InitrdRootfsPath::InTree("rootfs.config".into())];
+
         let in_repo_template = |debug_manifest: &'static str, release_manifest: &'static str| {
             IgvmManifestPath::InTree(if matches!(profile, OpenvmmHclBuildProfile::Debug) {
                 debug_manifest.into()
@@ -122,6 +133,7 @@ impl OpenhclIgvmRecipe {
             Self::LocalOnlyCustom(details) => details.clone(),
             Self::X64 => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template("openhcl-x64-dev.json", "openhcl-x64-release.json"),
                 openhcl_kernel_package: OpenhclKernelPackage::Main,
                 openvmm_hcl_features: base_openvmm_hcl_features(),
@@ -133,6 +145,7 @@ impl OpenhclIgvmRecipe {
             },
             Self::X64Devkern => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template("openhcl-x64-dev.json", "openhcl-x64-release.json"),
                 openhcl_kernel_package: OpenhclKernelPackage::Dev,
                 openvmm_hcl_features: base_openvmm_hcl_features(),
@@ -144,6 +157,7 @@ impl OpenhclIgvmRecipe {
             },
             Self::X64CvmDevkern => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template(
                     "openhcl-x64-cvm-dev.json",
                     "openhcl-x64-cvm-release.json",
@@ -158,6 +172,7 @@ impl OpenhclIgvmRecipe {
             },
             Self::X64TestLinuxDirect => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template(
                     "openhcl-x64-direct-dev.json",
                     "openhcl-x64-direct-release.json",
@@ -172,6 +187,7 @@ impl OpenhclIgvmRecipe {
             },
             Self::X64TestLinuxDirectDevkern => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template(
                     "openhcl-x64-direct-dev.json",
                     "openhcl-x64-direct-release.json",
@@ -186,6 +202,7 @@ impl OpenhclIgvmRecipe {
             },
             Self::X64Cvm => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template(
                     "openhcl-x64-cvm-dev.json",
                     "openhcl-x64-cvm-release.json",
@@ -198,8 +215,32 @@ impl OpenhclIgvmRecipe {
                 with_interactive,
                 with_sidecar_details: false,
             },
+            Self::X64Nested => OpenhclIgvmRecipeDetails {
+                local_only: None,
+                initrd_rootfs: {
+                    let mut v = base_initrd_rootfs();
+                    v.push(InitrdRootfsPath::InTree("rootfs.kvm.config".to_owned()));
+                    v
+                },
+                igvm_manifest: in_repo_template(
+                    "openhcl-x64-nested.json",
+                    "openhcl-x64-nested.json",
+                ),
+                openhcl_kernel_package: OpenhclKernelPackage::Dev,
+                openvmm_hcl_features: {
+                    let mut features = base_openvmm_hcl_features();
+                    features.insert(OpenvmmHclFeature::VirtKvm);
+                    features
+                },
+                target: CommonTriple::X86_64_LINUX_MUSL,
+                vtl0_kernel_type: None,
+                with_uefi: true,
+                with_interactive,
+                with_sidecar_details: false,
+            },
             Self::Aarch64 => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template(
                     "openhcl-aarch64-dev.json",
                     "openhcl-aarch64-release.json",
@@ -214,6 +255,7 @@ impl OpenhclIgvmRecipe {
             },
             Self::Aarch64Devkern => OpenhclIgvmRecipeDetails {
                 local_only: None,
+                initrd_rootfs: base_initrd_rootfs(),
                 igvm_manifest: in_repo_template(
                     "openhcl-aarch64-dev.json",
                     "openhcl-aarch64-release.json",
@@ -280,6 +322,7 @@ impl SimpleFlowNode for Node {
 
         let OpenhclIgvmRecipeDetails {
             local_only,
+            initrd_rootfs,
             igvm_manifest,
             openhcl_kernel_package,
             openvmm_hcl_features,
@@ -298,7 +341,6 @@ impl SimpleFlowNode for Node {
             custom_uefi,
             custom_kernel,
             custom_sidecar,
-            custom_extra_rootfs,
         } = local_only.unwrap_or(OpenhclIgvmRecipeDetailsLocalOnly {
             openvmm_hcl_no_strip: false,
             openhcl_initrd_extra_params: None,
@@ -307,7 +349,6 @@ impl SimpleFlowNode for Node {
             custom_uefi: None,
             custom_kernel: None,
             custom_sidecar: None,
-            custom_extra_rootfs: Vec::new(),
         });
 
         let target = custom_target.unwrap_or(target);
@@ -544,14 +585,16 @@ impl SimpleFlowNode for Node {
         openvmm_hcl_bin.write_into(ctx, built_openvmm_hcl, |x| x);
 
         let initrd = {
-            let rootfs_config = [openvmm_repo_path.map(ctx, |p| p.join("openhcl/rootfs.config"))]
+            let rootfs_config = initrd_rootfs
                 .into_iter()
-                .chain(
-                    custom_extra_rootfs
-                        .into_iter()
-                        .map(|p| ReadVar::from_static(p)),
-                )
-                .collect();
+                .map(|p| match p {
+                    InitrdRootfsPath::InTree(path) => {
+                        openvmm_repo_path.map(ctx, |p| p.join("openhcl").join(path))
+                    }
+                    InitrdRootfsPath::LocalOnlyCustom(p) => ReadVar::from_static(p),
+                })
+                .collect::<Vec<_>>();
+
             let openvmm_hcl_bin = openvmm_hcl_bin.map(ctx, |o| o.bin);
 
             ctx.reqv(|v| crate::build_openhcl_initrd::Request {

flowey/flowey_lib_hvlite/src/build_openvmm_hcl.rs

@@ -14,6 +14,7 @@ use std::collections::BTreeSet;
 pub enum OpenvmmHclFeature {
     Gdb,
     Tpm,
+    VirtKvm,
     LocalOnlyCustom(String),
 }
 
@@ -119,6 +120,7 @@ impl FlowNode for Node {
                         match f {
                             OpenvmmHclFeature::Gdb => "gdb",
                             OpenvmmHclFeature::Tpm => "tpm",
+                            OpenvmmHclFeature::VirtKvm => "virt_kvm",
                             OpenvmmHclFeature::LocalOnlyCustom(s) => s,
                         }
                         .into()

openhcl/openvmm_hcl/Cargo.toml

@@ -19,6 +19,9 @@ uidevices = ["openvmm_hcl_resources/uidevices", "openvmm_hcl_resources/vnc_worke
 # Enable NVMe emulation.
 nvme = ["openvmm_hcl_resources/nvme", "underhill_entry/vpci"]
 
+# Enable KVM and nested virtualization support.
+virt_kvm = ["underhill_entry/virt_kvm"]
+
 [target.'cfg(target_os = "linux")'.dependencies]
 underhill_entry.workspace = true
 openvmm_hcl_resources.workspace = true

openhcl/rootfs.config

@@ -25,16 +25,17 @@ dir /lib/modules 0755 0 0
 # Kernel modules are loaded in sort order; put them in directories appropriately
 # to ensure they are loaded in dependency order.
 
-dir /lib/modules/000    0755 0 0
-dir /lib/modules/001    0755 0 0
-dir /lib/modules/999    0755 0 0
+dir /lib/modules/auto        0755 0 0
+dir /lib/modules/auto/000    0755 0 0
+dir /lib/modules/auto/001    0755 0 0
+dir /lib/modules/auto/999    0755 0 0
 
-file /lib/modules/000/pci-hyperv-intf.ko    ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/drivers/pci/controller/pci-hyperv-intf.ko  0644 0 0
-file /lib/modules/001/pci-hyperv.ko         ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/drivers/pci/controller/pci-hyperv.ko       0644 0 0
+file /lib/modules/auto/000/pci-hyperv-intf.ko    ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/drivers/pci/controller/pci-hyperv-intf.ko  0644 0 0
+file /lib/modules/auto/001/pci-hyperv.ko         ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/drivers/pci/controller/pci-hyperv.ko       0644 0 0
 
 # Storvsc is last because it sometimes takes a long time to load and should not
 # block other device startup.
-file /lib/modules/999/hv_storvsc.ko         ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/drivers/scsi/hv_storvsc.ko      0644 0 0
+file /lib/modules/auto/999/hv_storvsc.ko         ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/drivers/scsi/hv_storvsc.ko      0644 0 0
 
 # These nodes are needed for early logging before devfs is mounted.
 nod /dev/null      0666 0 0 c 1  3

openhcl/rootfs.kvm.config

@@ -0,0 +1,3 @@
+file /lib/modules/kvm.ko        ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/arch/x86/kvm/kvm.ko         0644 0 0
+file /lib/modules/kvm-amd.ko    ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/arch/x86/kvm/kvm-amd.ko     0644 0 0
+file /lib/modules/kvm-intel.ko  ${OPENHCL_KERNEL_PATH}/build/native/bin/${OPENHCL_KERNEL_ARCH}/modules/kernel/arch/x86/kvm/kvm-intel.ko   0644 0 0

openhcl/underhill_core/Cargo.toml

@@ -20,6 +20,11 @@ openssl-vendored = ["underhill_attestation/openssl-vendored"]
 # Enable VPCI device support
 vpci = []
 
+# Enable support for running the guest OS via nested virtualization with KVM.
+# (Note that the virt_kvm crate is always a dependency just to avoid build
+# breaks.)
+virt_kvm = []
+
 [target.'cfg(target_os = "linux")'.dependencies]
 vmotherboard = { workspace = true, features = [
     "encryption",
@@ -92,6 +97,7 @@ underhill_threadpool.workspace = true
 bootloader_fdt_parser.workspace = true
 vga_proxy.workspace = true
 video_core.workspace = true
+virt_kvm.workspace = true
 virt_mshv_vtl.workspace = true
 vm_manifest_builder.workspace = true
 vmbus_async.workspace = true

openhcl/underhill_core/src/dispatch/mod.rs

@@ -10,6 +10,7 @@ use self::vtl2_settings_worker::DeviceInterfaces;
 use crate::emuplat::netvsp::RuntimeSavedState;
 use crate::emuplat::EmuplatServicing;
 use crate::nvme_manager::NvmeManager;
+use crate::partition::OpenhclPartition;
 use crate::reference_time::ReferenceTime;
 use crate::servicing;
 use crate::servicing::ServicingState;
@@ -53,9 +54,9 @@ use std::time::Duration;
 use tracing::instrument;
 use tracing::Instrument;
 use uevent::UeventListener;
+use underhill_mem::AccessGuestMemory;
 use underhill_threadpool::AffinitizedThreadpool;
 use virt::IsolationType;
-use virt_mshv_vtl::UhPartition;
 use virt_mshv_vtl::VtlCrash;
 use vm_resource::ResourceResolver;
 use vm_topology::memory::MemoryRangeWithNode;
@@ -105,7 +106,7 @@ pub trait LoadedVmNetworkSettings: Inspect {
         uevent_listener: &UeventListener,
         servicing_netvsp_state: &Option<Vec<crate::emuplat::netvsp::SavedState>>,
         shared_vis_pages_pool: &Option<SharedPool>,
-        partition: Arc<UhPartition>,
+        partition: Arc<dyn OpenhclPartition>,
         state_units: &StateUnits,
         vmbus_server: &Option<VmbusServerHandle>,
     ) -> anyhow::Result<RuntimeSavedState>;
@@ -127,7 +128,7 @@ pub trait LoadedVmNetworkSettings: Inspect {
 pub(crate) struct LoadedVm {
     pub partition_unit: PartitionUnit,
     /// The various guest memory objects.
-    pub memory: underhill_mem::MemoryMappings,
+    pub memory: Box<dyn AccessGuestMemory>,
     pub firmware_type: FirmwareType,
     pub isolation: IsolationType,
     // contain task handles which must be kept live
@@ -143,7 +144,7 @@ pub(crate) struct LoadedVm {
     /// Memory map with IGVM types for each range.
     pub vtl0_memory_map: Vec<(MemoryRangeWithNode, MemoryMapEntryType)>,
 
-    pub partition: Arc<UhPartition>,
+    pub partition: Arc<dyn OpenhclPartition>,
     pub state_units: StateUnits,
     pub last_state_unit_stop: Option<ReferenceTime>,
     pub vmbus_server: Option<VmbusServerHandle>,