From 86d58fc25caa672dccb4dee7a9d2e0b33ab0e366 Mon Sep 17 00:00:00 2001 From: Daniel Prilik Date: Mon, 11 Nov 2024 14:27:15 -0800 Subject: [PATCH 1/3] . --- .../pipelines/custom_vmfirmwareigvm_dll.rs | 91 +++++++++++++++ flowey/flowey_hvlite/src/pipelines/mod.rs | 3 + .../src/pipelines/restore_packages.rs | 28 +---- .../src/pipelines_shared/cfg_common_params.rs | 28 +++++ .../src/_jobs/local_restore_packages.rs | 110 +++++++++--------- .../_jobs/local_vmfirmwareigvm_dll_helper.rs | 70 +++++++++++ flowey/flowey_lib_hvlite/src/_jobs/mod.rs | 1 + 7 files changed, 254 insertions(+), 77 deletions(-) create mode 100644 flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs create mode 100644 flowey/flowey_lib_hvlite/src/_jobs/local_vmfirmwareigvm_dll_helper.rs diff --git a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs new file mode 100644 index 000000000..a4ca5073d --- /dev/null +++ b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! See [`CustomVmfirmwareigvmDllCli`] + +use crate::pipelines_shared::cfg_common_params::CommonArchCli; +use anyhow::Context; +use flowey::node::prelude::ReadVar; +use flowey::pipeline::prelude::*; +use std::path::PathBuf; + +/// Build OpenHCL IGVM files for local development. DO NOT USE IN CI. +#[derive(clap::Args)] +pub struct CustomVmfirmwareigvmDllCli { + /// Path to IGVM payload to encapsulate in the vmfirmwareigvm resource DLL. + pub igvm_payload: PathBuf, + + /// What architecture the DLL should be built for. + /// + /// Defaults to the current host architecture. + #[clap(long)] + pub arch: Option, +} + +impl IntoPipeline for CustomVmfirmwareigvmDllCli { + fn into_pipeline(self, backend_hint: PipelineBackendHint) -> anyhow::Result { + if !matches!(backend_hint, PipelineBackendHint::Local) { + anyhow::bail!("build-igvm is for local use only") + } + + // DEVNOTE: it would be nice to figure out what sort of magic is + // required for the WSL2 case to work. The tricky part is dealing with + // the underlying invocations to `rc.exe` via WSL2. + if !matches!(FlowPlatform::host(backend_hint), FlowPlatform::Windows) { + anyhow::bail!("custom-vmfirmwareigvm-dll only runs on Windows (WSL2 is NOT supported)") + } + + let CustomVmfirmwareigvmDllCli { arch, igvm_payload } = self; + + let arch = match arch { + Some(arch) => arch, + None => FlowArch::host(backend_hint).try_into()?, + }; + let igvm_payload = std::path::absolute(igvm_payload) + .context("could not make path to igvm payload absolute")?; + + let openvmm_repo = flowey_lib_common::git_checkout::RepoSource::ExistingClone( + ReadVar::from_static(crate::repo_root()), + ); + + let mut pipeline = Pipeline::new(); + + let (pub_out_dir, _) = pipeline.new_artifact("custom-vmfirmwareigvm-dll"); + + pipeline + .new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "custom-vmfirmwareigvm-dll", + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) + .dep_on( + |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo, + }, + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: false, + force_nuget_mono: false, // no oss nuget packages + external_nuget_auth: false, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(false), + locked: false, + deny_warnings: false, + }) + .dep_on( + |ctx| flowey_lib_hvlite::_jobs::local_vmfirmwareigvm_dll_helper::Params { + arch: arch.into(), + igvm_payload, + artifact_dir: ctx.publish_artifact(pub_out_dir), + done: ctx.new_done_handle(), + }, + ) + .finish(); + + Ok(pipeline) + } +} diff --git a/flowey/flowey_hvlite/src/pipelines/mod.rs b/flowey/flowey_hvlite/src/pipelines/mod.rs index 2d52f1756..a45f9c496 100644 --- a/flowey/flowey_hvlite/src/pipelines/mod.rs +++ b/flowey/flowey_hvlite/src/pipelines/mod.rs @@ -6,6 +6,7 @@ use restore_packages::RestorePackagesCli; pub mod build_igvm; pub mod checkin_gates; +pub mod custom_vmfirmwareigvm_dll; pub mod restore_packages; #[derive(clap::Subcommand)] @@ -19,6 +20,7 @@ pub enum OpenvmmPipelines { }, BuildIgvm(build_igvm::BuildIgvmCli), + CustomVmfirmwareigvmDll(custom_vmfirmwareigvm_dll::CustomVmfirmwareigvmDllCli), /// Flowey pipelines primarily designed to run in CI. #[clap(subcommand)] @@ -54,6 +56,7 @@ impl IntoPipeline for OpenvmmPipelines { } OpenvmmPipelines::BuildIgvm(cmd) => cmd.into_pipeline(pipeline_hint), + OpenvmmPipelines::CustomVmfirmwareigvmDll(cmd) => cmd.into_pipeline(pipeline_hint), OpenvmmPipelines::Ci(cmd) => match cmd { OpenvmmPipelinesCi::CheckinGates(cmd) => cmd.into_pipeline(pipeline_hint), diff --git a/flowey/flowey_hvlite/src/pipelines/restore_packages.rs b/flowey/flowey_hvlite/src/pipelines/restore_packages.rs index c2266fad0..30eceb05b 100644 --- a/flowey/flowey_hvlite/src/pipelines/restore_packages.rs +++ b/flowey/flowey_hvlite/src/pipelines/restore_packages.rs @@ -1,28 +1,16 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use crate::pipelines_shared::cfg_common_params::CommonArchCli; use flowey::node::prelude::ReadVar; use flowey::pipeline::prelude::*; -use flowey_lib_hvlite::run_cargo_build::common::CommonArch; -#[derive(clap::ValueEnum, Clone, Copy)] -pub enum CommonArchCli { - X86_64, - Aarch64, -} - -impl From for CommonArch { - fn from(value: CommonArchCli) -> Self { - match value { - CommonArchCli::X86_64 => CommonArch::X86_64, - CommonArchCli::Aarch64 => CommonArch::Aarch64, - } - } -} - -#[derive(clap::Args)] /// Download and restore packages needed for building the specified architectures. +#[derive(clap::Args)] pub struct RestorePackagesCli { + /// Specify what architectures to restore packages for. + /// + /// If none are specified, defaults to just the current host architecture. arch: Vec, } @@ -60,11 +48,7 @@ impl IntoPipeline for RestorePackagesCli { let arches = { if self.arch.is_empty() { - vec![match FlowArch::host(backend_hint) { - FlowArch::X86_64 => CommonArchCli::X86_64, - FlowArch::Aarch64 => CommonArchCli::Aarch64, - arch => anyhow::bail!("unsupported arch {arch}"), - }] + vec![FlowArch::host(backend_hint).try_into()?] } else { self.arch } diff --git a/flowey/flowey_hvlite/src/pipelines_shared/cfg_common_params.rs b/flowey/flowey_hvlite/src/pipelines_shared/cfg_common_params.rs index 03893ec05..5bd2b0a55 100644 --- a/flowey/flowey_hvlite/src/pipelines_shared/cfg_common_params.rs +++ b/flowey/flowey_hvlite/src/pipelines_shared/cfg_common_params.rs @@ -5,6 +5,7 @@ use flowey::node::prelude::*; use flowey::pipeline::prelude::*; +use flowey_lib_hvlite::run_cargo_build::common::CommonArch; #[derive(Clone, Default, clap::Args)] #[clap(next_help_heading = "Local Only")] @@ -98,3 +99,30 @@ pub fn get_cfg_common_params( } } } + +#[derive(clap::ValueEnum, Clone, Copy)] +pub enum CommonArchCli { + X86_64, + Aarch64, +} + +impl From for CommonArch { + fn from(value: CommonArchCli) -> Self { + match value { + CommonArchCli::X86_64 => CommonArch::X86_64, + CommonArchCli::Aarch64 => CommonArch::Aarch64, + } + } +} + +impl TryFrom for CommonArchCli { + type Error = anyhow::Error; + + fn try_from(arch: FlowArch) -> anyhow::Result { + Ok(match arch { + FlowArch::X86_64 => CommonArchCli::X86_64, + FlowArch::Aarch64 => CommonArchCli::Aarch64, + arch => anyhow::bail!("unsupported arch {arch}"), + }) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_restore_packages.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_restore_packages.rs index f450bff62..c814a0cac 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/local_restore_packages.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_restore_packages.rs @@ -15,9 +15,9 @@ flowey_request! { } } -new_flow_node!(struct Node); +new_simple_flow_node!(struct Node); -impl FlowNode for Node { +impl SimpleFlowNode for Node { type Request = Request; fn imports(ctx: &mut ImportCtx<'_>) { @@ -28,67 +28,67 @@ impl FlowNode for Node { ctx.import::(); } - fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Request { arch, done } = request; + let mut deps = vec![ctx.reqv(crate::init_openvmm_magicpath_protoc::Request)]; - for req in &requests { - match req.arch { - CommonArch::X86_64 => { - if matches!(ctx.platform(), FlowPlatform::Linux(_)) { - deps.extend_from_slice(&[ctx - .reqv(|v| crate::init_openvmm_magicpath_openhcl_sysroot::Request { - arch: OpenvmmSysrootArch::X64, - path: v, - }) - .into_side_effect()]); - } - deps.extend_from_slice(&[ - ctx.reqv(|done| crate::init_openvmm_magicpath_lxutil::Request { - arch: LxutilArch::X86_64, - done, - }), - ctx.reqv(|done| crate::init_openvmm_magicpath_uefi_mu_msvm::Request { - arch: MuMsvmArch::X86_64, - done, - }), - ctx.reqv( - |done| crate::init_openvmm_magicpath_linux_test_kernel::Request { - arch: OpenvmmLinuxTestKernelArch::X64, - done, - }, - ), - ]); + match arch { + CommonArch::X86_64 => { + if matches!(ctx.platform(), FlowPlatform::Linux(_)) { + deps.extend_from_slice(&[ctx + .reqv(|v| crate::init_openvmm_magicpath_openhcl_sysroot::Request { + arch: OpenvmmSysrootArch::X64, + path: v, + }) + .into_side_effect()]); } - CommonArch::Aarch64 => { - if matches!(ctx.platform(), FlowPlatform::Linux(_)) { - deps.extend_from_slice(&[ctx - .reqv(|v| crate::init_openvmm_magicpath_openhcl_sysroot::Request { - arch: OpenvmmSysrootArch::Aarch64, - path: v, - }) - .into_side_effect()]); - } - deps.extend_from_slice(&[ - ctx.reqv(|done| crate::init_openvmm_magicpath_lxutil::Request { - arch: LxutilArch::Aarch64, + deps.extend_from_slice(&[ + ctx.reqv(|done| crate::init_openvmm_magicpath_lxutil::Request { + arch: LxutilArch::X86_64, + done, + }), + ctx.reqv(|done| crate::init_openvmm_magicpath_uefi_mu_msvm::Request { + arch: MuMsvmArch::X86_64, + done, + }), + ctx.reqv( + |done| crate::init_openvmm_magicpath_linux_test_kernel::Request { + arch: OpenvmmLinuxTestKernelArch::X64, done, - }), - ctx.reqv(|done| crate::init_openvmm_magicpath_uefi_mu_msvm::Request { - arch: MuMsvmArch::Aarch64, - done, - }), - ctx.reqv( - |done| crate::init_openvmm_magicpath_linux_test_kernel::Request { - arch: OpenvmmLinuxTestKernelArch::Aarch64, - done, - }, - ), - ]); + }, + ), + ]); + } + CommonArch::Aarch64 => { + if matches!(ctx.platform(), FlowPlatform::Linux(_)) { + deps.extend_from_slice(&[ctx + .reqv(|v| crate::init_openvmm_magicpath_openhcl_sysroot::Request { + arch: OpenvmmSysrootArch::Aarch64, + path: v, + }) + .into_side_effect()]); } + deps.extend_from_slice(&[ + ctx.reqv(|done| crate::init_openvmm_magicpath_lxutil::Request { + arch: LxutilArch::Aarch64, + done, + }), + ctx.reqv(|done| crate::init_openvmm_magicpath_uefi_mu_msvm::Request { + arch: MuMsvmArch::Aarch64, + done, + }), + ctx.reqv( + |done| crate::init_openvmm_magicpath_linux_test_kernel::Request { + arch: OpenvmmLinuxTestKernelArch::Aarch64, + done, + }, + ), + ]); } } - ctx.emit_side_effect_step(deps, requests.into_iter().map(|x| x.done)); + ctx.emit_side_effect_step(deps, [done]); Ok(()) } diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_vmfirmwareigvm_dll_helper.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_vmfirmwareigvm_dll_helper.rs new file mode 100644 index 000000000..80f8c5b52 --- /dev/null +++ b/flowey/flowey_lib_hvlite/src/_jobs/local_vmfirmwareigvm_dll_helper.rs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::run_cargo_build::common::CommonArch; +use flowey::node::prelude::*; + +flowey_request! { + pub struct Params { + pub igvm_payload: PathBuf, + pub arch: CommonArch, + + pub artifact_dir: ReadVar, + pub done: WriteVar, + } +} + +new_simple_flow_node!(struct Node); + +impl SimpleFlowNode for Node { + type Request = Params; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + } + + fn process_request(request: Self::Request, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let Params { + arch, + igvm_payload, + + artifact_dir, + done, + } = request; + + let built_dll = ctx.reqv(|v| crate::build_vmfirmwareigvm_dll::Request { + arch, + igvm: ReadVar::from_static(crate::run_igvmfilegen::IgvmOutput { + igvm_bin: igvm_payload, + igvm_map: None, + igvm_tdx_json: None, + igvm_snp_json: None, + igvm_vbs_json: None, + }), + // fixed version to signal that this is a custom dll + dll_version: ReadVar::from_static((1, 0, 1337, 0)), + internal_dll_name: "vmfirmwareigvm.dll".into(), + vmfirmwareigvm_dll: v, + }); + + ctx.emit_rust_step("copy resulting vmfirmwareigvm.dll", |ctx| { + done.claim(ctx); + let artifact_dir = artifact_dir.claim(ctx); + let built_dll = built_dll.claim(ctx); + |rt| { + let artifact_dir = rt.read(artifact_dir); + let built_dll = rt.read(built_dll); + + fs_err::copy(built_dll.dll, artifact_dir.join("vmfirmwareigvm.dll"))?; + + for e in fs_err::read_dir(artifact_dir)? { + let e = e?; + log::info!("{}", e.path().display()); + } + Ok(()) + } + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs index ed22234de..c3b7bfd36 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs @@ -31,4 +31,5 @@ pub mod consume_and_test_nextest_unit_tests_archive; pub mod consume_and_test_nextest_vmm_tests_archive; pub mod local_build_igvm; pub mod local_restore_packages; +pub mod local_vmfirmwareigvm_dll_helper; pub mod test_local_flowey_build_igvm; From 5b5b83760128d169424781cf2edb2dd92c1b92a1 Mon Sep 17 00:00:00 2001 From: Daniel Prilik Date: Mon, 11 Nov 2024 14:40:21 -0800 Subject: [PATCH 2/3] little bit of polish --- .../src/pipelines/custom_vmfirmwareigvm_dll.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs index a4ca5073d..219610c61 100644 --- a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs +++ b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs @@ -9,13 +9,23 @@ use flowey::node::prelude::ReadVar; use flowey::pipeline::prelude::*; use std::path::PathBuf; -/// Build OpenHCL IGVM files for local development. DO NOT USE IN CI. +/// Encapsulate an existing pre-built IGVM file into *unsigned* +/// `vmfirmwareigvm.dll` resource DLL. +/// +/// Unlike `build-igvm`, this tool will NOT build OpenHCL from scratch. This +/// tool streamlines the process of building the in-tree `vmfirmwareigvm_dll` +/// crate (which requires setting various env vars, installing certain +/// dependencies, etc...). +/// +/// NOTE: This tool is primarily intended for use by Microsoft employees, as +/// open-source deployments of OpenHCL typically load the IGVM file directly +/// (rather than being encapsulated in a resource-DLL). #[derive(clap::Args)] pub struct CustomVmfirmwareigvmDllCli { /// Path to IGVM payload to encapsulate in the vmfirmwareigvm resource DLL. pub igvm_payload: PathBuf, - /// What architecture the DLL should be built for. + /// Architecture the DLL should be built for. /// /// Defaults to the current host architecture. #[clap(long)] From 5fd5d0d1011e0caab83e90f0fbf589b55e18bea6 Mon Sep 17 00:00:00 2001 From: Daniel Prilik Date: Mon, 11 Nov 2024 14:48:17 -0800 Subject: [PATCH 3/3] whoops, consistency is good --- flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs | 2 +- ...areigvm_dll_helper.rs => local_custom_vmfirmwareigvm_dll.rs} | 0 flowey/flowey_lib_hvlite/src/_jobs/mod.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename flowey/flowey_lib_hvlite/src/_jobs/{local_vmfirmwareigvm_dll_helper.rs => local_custom_vmfirmwareigvm_dll.rs} (100%) diff --git a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs index 219610c61..2781b404c 100644 --- a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs +++ b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs @@ -87,7 +87,7 @@ impl IntoPipeline for CustomVmfirmwareigvmDllCli { deny_warnings: false, }) .dep_on( - |ctx| flowey_lib_hvlite::_jobs::local_vmfirmwareigvm_dll_helper::Params { + |ctx| flowey_lib_hvlite::_jobs::local_custom_vmfirmwareigvm_dll::Params { arch: arch.into(), igvm_payload, artifact_dir: ctx.publish_artifact(pub_out_dir), diff --git a/flowey/flowey_lib_hvlite/src/_jobs/local_vmfirmwareigvm_dll_helper.rs b/flowey/flowey_lib_hvlite/src/_jobs/local_custom_vmfirmwareigvm_dll.rs similarity index 100% rename from flowey/flowey_lib_hvlite/src/_jobs/local_vmfirmwareigvm_dll_helper.rs rename to flowey/flowey_lib_hvlite/src/_jobs/local_custom_vmfirmwareigvm_dll.rs diff --git a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs index c3b7bfd36..d7c38715d 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/mod.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/mod.rs @@ -30,6 +30,6 @@ pub mod consolidate_and_publish_gh_pages; pub mod consume_and_test_nextest_unit_tests_archive; pub mod consume_and_test_nextest_vmm_tests_archive; pub mod local_build_igvm; +pub mod local_custom_vmfirmwareigvm_dll; pub mod local_restore_packages; -pub mod local_vmfirmwareigvm_dll_helper; pub mod test_local_flowey_build_igvm;