From 25664811f1d22fd9c6397f42c90a60bef9b7d339 Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Mon, 23 Dec 2024 19:26:28 +0000 Subject: [PATCH] twoliter: fetch build SDK prior to cargo-make Now that `krane` is run as a static library, we cannot execute it within Makefile.toml. This change moves the SDK fetch to occur prior to executing build make targets. --- Cargo.lock | 11 +++++ Cargo.toml | 1 + twoliter/Cargo.toml | 1 + twoliter/embedded/Makefile.toml | 69 +++----------------------- twoliter/src/cleanup.rs | 86 +++++++++++++++++++++++++++++++++ twoliter/src/cmd/build.rs | 2 + twoliter/src/cmd/fetch.rs | 3 +- twoliter/src/cmd/make.rs | 44 ++++++----------- twoliter/src/cmd/publish_kit.rs | 2 + twoliter/src/docker/commands.rs | 52 +++++++++++++++++++- twoliter/src/preflight.rs | 3 +- twoliter/src/project/mod.rs | 13 +++-- twoliter/src/project/tasks.rs | 53 ++++++++++++++++++++ 13 files changed, 242 insertions(+), 98 deletions(-) create mode 100644 twoliter/src/cleanup.rs create mode 100644 twoliter/src/project/tasks.rs diff --git a/Cargo.lock b/Cargo.lock index 76529523..2c0774f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,6 +1067,16 @@ dependencies = [ "typenum", ] +[[package]] +name = "ctrlc" +version = "3.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" +dependencies = [ + "nix", + "windows-sys 0.52.0", +] + [[package]] name = "daemonize" version = "0.5.0" @@ -3898,6 +3908,7 @@ dependencies = [ "buildsys-config", "bytes", "clap", + "ctrlc", "env_logger", "filetime", "flate2", diff --git a/Cargo.toml b/Cargo.toml index 1e63d0ad..1f96bfb7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -84,6 +84,7 @@ bytes = "1" chrono = { version = "0.4", default-features = false } clap = "4" coldsnap = { version = "0.6", default-features = false } +ctrlc = "3" daemonize = "0.5" duct = "0.13" env_logger = "0.11" diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index b5d06f33..fa245d5b 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -17,6 +17,7 @@ async-trait.workspace = true base64.workspace = true buildsys-config.workspace = true clap = { workspace = true, features = ["derive", "env", "std"] } +ctrlc.workspace = true env_logger.workspace = true filetime.workspace = true flate2.workspace = true diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index e7b50202..168a0836 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -13,7 +13,6 @@ BUILDSYS_BUILD_DIR = "${BUILDSYS_ROOT_DIR}/build" BUILDSYS_PACKAGES_DIR = "${BUILDSYS_BUILD_DIR}/rpms" BUILDSYS_KITS_DIR = "${BUILDSYS_BUILD_DIR}/kits" BUILDSYS_EXTERNAL_KITS_DIR = "${BUILDSYS_BUILD_DIR}/external-kits" -BUILDSYS_EXTERNAL_SDKS_DIR = "${BUILDSYS_BUILD_DIR}/external-sdk-archives" BUILDSYS_STATE_DIR = "${BUILDSYS_BUILD_DIR}/state" BUILDSYS_IMAGES_DIR = "${BUILDSYS_BUILD_DIR}/images" BUILDSYS_LOGS_DIR = "${BUILDSYS_BUILD_DIR}/logs" @@ -291,64 +290,12 @@ mkdir -p ${GO_MOD_CACHE} ''' ] -[tasks.setup-build] -dependencies = ["setup"] -script = [ -''' -for cmd in docker gzip lz4; do - if ! command -v ${cmd} >/dev/null 2>&1 ; then - echo "required program '${cmd}' not found" >&2 - exit 1 - fi -done -''' -] - [tasks.fetch] dependencies = [ - "fetch-sdk", "fetch-sources", "fetch-vendored", ] -[tasks.fetch-sdk] -dependencies = ["setup-build"] -script_runner = "bash" -script = [ -''' - -cleanup() { - [ -n "${SDK_ARCHIVE_PATH}" ] && rm -rf "${SDK_ARCHIVE_PATH}" -} - -trap 'cleanup' EXIT - -SDK_PLATFORM="$(docker version --format '{{.Server.Os}}/{{.Server.Arch}}')" -KRANE="${TWOLITER_TOOLS_DIR}/krane" - -mkdir -p "${BUILDSYS_EXTERNAL_SDKS_DIR}" -SDK_ARCHIVE_PATH="$(mktemp -p ${BUILDSYS_EXTERNAL_SDKS_DIR} bottlerocket-sdk-tmp-archive-XXXXXXXX.tar)" - -if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.sdk-verified" ]; then - echo "Twoliter could not validate '${TLPRIVATE_SDK_IMAGE}', refusing to continue" >&2 - exit 1 -fi - -if ! docker image inspect "${TLPRIVATE_SDK_IMAGE}" >/dev/null 2>&1 ; then - echo "Pulling SDK '${TLPRIVATE_SDK_IMAGE}'" - if ! ${KRANE} pull "${TLPRIVATE_SDK_IMAGE}" "${SDK_ARCHIVE_PATH}" --platform "${SDK_PLATFORM}" ; then - echo "failed to pull '${TLPRIVATE_SDK_IMAGE}'" >&2 - exit 1 - fi - - if ! docker load --input "${SDK_ARCHIVE_PATH}" ; then - echo "failed to load '${TLPRIVATE_SDK_IMAGE}' into docker daemon" >&2 - exit 1 - fi -fi -''' -] - [tasks.fetch-sources] dependencies = ["setup"] script_runner = "bash" @@ -364,7 +311,7 @@ chmod -R o+r ${CARGO_HOME} ] [tasks.fetch-vendored] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script = [ ''' go_fetch() { @@ -384,7 +331,7 @@ done ] [tasks.unit-tests] -dependencies = ["fetch-sdk", "fetch-sources", "fetch-vendored"] +dependencies = ["setup", "fetch-sources", "fetch-vendored"] script = [ ''' export VARIANT="${BUILDSYS_VARIANT}" @@ -639,7 +586,7 @@ fi ] [tasks.build-sbkeys] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -707,7 +654,7 @@ fi ] [tasks.boot-config] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -747,7 +694,7 @@ echo "Boot configuration initrd may be found at ${BOOT_CONFIG}" ] [tasks.validate-boot-config] -dependencies = ["fetch-sdk"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -908,7 +855,7 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${BUILDSYS_OUTPUT_DIR}/latest" ] [tasks.repack-variant] -dependencies = ["fetch-sdk", "build-sbkeys", "publish-setup", "cargo-metadata"] +dependencies = ["setup", "build-sbkeys", "publish-setup", "cargo-metadata"] script = [ ''' export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" @@ -1223,7 +1170,7 @@ pubsys \ ] [tasks.ami] -dependencies = ["setup-build"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' @@ -1569,7 +1516,7 @@ pubsys \ ] [tasks._upload-ova-base] -dependencies = ["setup-build"] +dependencies = ["setup"] script_runner = "bash" script = [ ''' diff --git a/twoliter/src/cleanup.rs b/twoliter/src/cleanup.rs new file mode 100644 index 00000000..28e99d2f --- /dev/null +++ b/twoliter/src/cleanup.rs @@ -0,0 +1,86 @@ +//! Provides a mechanism for cleaning up resources when twoliter is interrupted. + +use anyhow::{Context, Result}; +use std::collections::BTreeMap; +use std::future::Future; +use std::path::PathBuf; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use tempfile::TempPath; +use uuid::Uuid; + +use self::sealed::*; + +lazy_static::lazy_static! { + pub(crate) static ref JANITOR: TempfileJanitor = TempfileJanitor::default(); +} + +impl TempfileJanitor { + /// Run a given async closure using a [`tempfile::TempPath`]. + /// + /// The closure has access to the path where the (closed) tempfile is stored. + /// [`TempfileJanitor`] will ensure that the temporary file is deleted in the case that the + /// current process receives SIGINT. + pub(crate) async fn with_tempfile( + &self, + tmpfile: TempPath, + do_: impl FnOnce(PathBuf) -> Fut, + ) -> Result + where + Fut: Future, + { + let path = tmpfile.to_path_buf(); + let path_id = Uuid::new_v4(); + + self.paths.lock().unwrap().insert(path_id, tmpfile); + + let result = do_(path).await; + + self.paths.lock().unwrap().remove(&path_id); + + Ok(result) + } + + pub(crate) fn try_cleanup(&mut self) { + tracing::info!("Cleaning up temporary resources..."); + if let Ok(mut paths) = self.paths.lock() { + while let Some((_, path)) = paths.pop_first() { + tracing::debug!("Deleting tempfile at '{}'", path.display()); + if let Err(e) = std::fs::remove_file(&path) { + tracing::error!("Failed to clean tempfile '{}': {}", path.display(), e); + } + } + } + tracing::info!("Done cleaning up."); + } + + /// Attempts to install the cleanup process as a SIGINT signal handler + pub(crate) fn setup_signal_handler(&self) -> Result<()> { + let mut handler_ref = Self { + paths: Arc::clone(&self.paths), + }; + + let already_handling = Arc::new(AtomicBool::new(false)); + ctrlc::try_set_handler(move || { + if already_handling + .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + handler_ref.try_cleanup(); + } + // SIGINT is 130 + std::process::exit(130); + }) + .context("Failed to create cleanup signal handler") + } +} + +/// Signal handlers are global -- hide `TempfileJanitor` to encourage use of the static reference. +mod sealed { + use super::*; + + #[derive(Default, Debug)] + pub(crate) struct TempfileJanitor { + pub(super) paths: Arc>>, + } +} diff --git a/twoliter/src/cmd/build.rs b/twoliter/src/cmd/build.rs index b75e836d..4666fcdf 100644 --- a/twoliter/src/cmd/build.rs +++ b/twoliter/src/cmd/build.rs @@ -63,6 +63,7 @@ impl BuildKit { optional_envs.push(("BUILDSYS_LOOKASIDE_CACHE", lookaside_cache)) } + project.fetch_sdk().await?; CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_ARCH", &self.arch) @@ -135,6 +136,7 @@ impl BuildVariant { )) } + project.fetch_sdk().await?; CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_ARCH", &self.arch) diff --git a/twoliter/src/cmd/fetch.rs b/twoliter/src/cmd/fetch.rs index 1e3c9918..80a656f1 100644 --- a/twoliter/src/cmd/fetch.rs +++ b/twoliter/src/cmd/fetch.rs @@ -18,7 +18,8 @@ impl Fetch { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; let project = project.load_lock::().await?; - project.fetch(self.arch.as_str()).await?; + project.fetch_kits(self.arch.as_str()).await?; + project.fetch_sdk().await?; Ok(()) } } diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index 23b80ccf..75f11c7d 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -54,7 +54,19 @@ pub(crate) struct Make { impl Make { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let sdk_source = self.locked_sdk(&project).await?; + + let sdk_source = if self.can_skip_kit_verification(&project) { + let project = project.load_lock::().await?; + project.fetch_sdk().await?; + project.sdk_image() + } else { + let project = project.load_lock::().await?; + project.fetch_sdk().await?; + project.sdk_image() + } + .project_image_uri() + .to_string(); + let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); @@ -75,17 +87,6 @@ impl Make { target_allows_kit_verification_skip && project_has_explicit_sdk_dep } - - /// Returns the locked SDK image for the project. - async fn locked_sdk(&self, project: &project::Project) -> Result { - Ok(if self.can_skip_kit_verification(project) { - project.load_lock::().await?.sdk_image() - } else { - project.load_lock::().await?.sdk_image() - } - .project_image_uri() - .to_string()) - } } #[cfg(test)] @@ -227,6 +228,7 @@ mod test { install_tools(&toolsdir).await.unwrap(); let makefile_path = toolsdir.join("Makefile.toml"); + project.fetch_sdk().await?; CargoMake::new(&sdk_source) .unwrap() .env("CARGO_HOME", project_dir.display().to_string()) @@ -266,24 +268,6 @@ mod test { assert!(!target_can_skip_kit_verification("build-variant").await); } - #[tokio::test] - #[ignore] // integration test - async fn test_fetch_sdk_succeeds_when_only_sdk_verified() { - let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); - assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), false) - .await - .is_ok()); - } - - #[tokio::test] - #[ignore] // integration test - async fn test_fetch_sdk_fails_when_nothing_verified() { - let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); - assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), true) - .await - .is_err()); - } - #[tokio::test] #[ignore] // integration test async fn test_validate_kits_fails_when_only_sdk_verified() { diff --git a/twoliter/src/cmd/publish_kit.rs b/twoliter/src/cmd/publish_kit.rs index a009f555..75bdf7e8 100644 --- a/twoliter/src/cmd/publish_kit.rs +++ b/twoliter/src/cmd/publish_kit.rs @@ -40,6 +40,7 @@ impl PublishKit { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; let project = project.load_lock::().await?; + let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); @@ -48,6 +49,7 @@ impl PublishKit { Some(kit_repo) => kit_repo, None => &self.kit_name, }; + project.fetch_sdk().await?; CargoMake::new(project.sdk_image().project_image_uri().to_string().as_str())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_KIT", &self.kit_name) diff --git a/twoliter/src/docker/commands.rs b/twoliter/src/docker/commands.rs index 866eda1f..a08d393e 100644 --- a/twoliter/src/docker/commands.rs +++ b/twoliter/src/docker/commands.rs @@ -1,11 +1,61 @@ -use crate::common::exec; +use crate::common::{exec, exec_log}; use anyhow::{Context, Result}; use semver::Version; +use std::path::Path; use tokio::process::Command; +use super::ImageUri; + pub(crate) struct Docker; impl Docker { + /// Loads an image tarball into the docker daemon from the given path + pub(crate) async fn load(path: impl AsRef) -> Result<()> { + exec_log( + Command::new("docker") + .args(["load", "-i"]) + .arg(path.as_ref()), + ) + .await + } + + /// Returns whether or not the docker daemon has cached an image with the given URI locally + pub(crate) async fn image_is_cached(image_uri: &ImageUri) -> Result { + let image_hash = exec( + Command::new("docker") + .args(["images", "-q"]) + .arg(image_uri.uri()), + true, + ) + .await + // Convert Result> to Option + .ok() + .flatten() + .map(|s| s.trim().to_string()) + .with_context(|| { + format!( + "Failed to search docker daemon for image '{}'", + image_uri.uri() + ) + })?; + + Ok(!image_hash.is_empty()) + } + + /// Fetches the host platform in the form $OS/$GOARCH, e.g. linux/arm64 + pub(crate) async fn host_platform() -> Result { + exec( + Command::new("docker").args(["version", "--format", "{{.Server.Os}}/{{.Server.Arch}}"]), + true, + ) + .await + // Convert Result> to Option + .ok() + .flatten() + .map(|s| s.trim().to_string()) + .context("Failed to fetch host platform from docker") + } + /// Fetches the version of the docker daemon pub(crate) async fn server_version() -> Result { let version_str = exec( diff --git a/twoliter/src/preflight.rs b/twoliter/src/preflight.rs index 576a9a3e..f49c17d1 100644 --- a/twoliter/src/preflight.rs +++ b/twoliter/src/preflight.rs @@ -28,9 +28,10 @@ lazy_static! { /// Runs all common setup required for twoliter. /// /// * Ensures that any required system tools are installed an accessible. -/// * Sets up interrupt handler to cleanup on SIGINT +/// * Sets up signal handler to cleanup on SIGINT pub(crate) async fn preflight() -> Result<()> { check_environment().await?; + crate::cleanup::JANITOR.setup_signal_handler()?; Ok(()) } diff --git a/twoliter/src/project/mod.rs b/twoliter/src/project/mod.rs index a33772f2..8863fe82 100644 --- a/twoliter/src/project/mod.rs +++ b/twoliter/src/project/mod.rs @@ -1,9 +1,11 @@ mod image; mod lock; +pub(crate) mod tasks; pub(crate) mod vendor; pub(crate) use self::image::{Image, ProjectImage, ValidIdentifier, VendedArtifact, Vendor}; pub(crate) use self::vendor::ArtifactVendor; +use lock::LockedImage; pub(crate) use lock::VerificationTagger; use self::lock::{Lock, LockedSDK, Override}; @@ -171,6 +173,10 @@ impl Project { self.project_dir.join(EXTERNAL_KIT_METADATA) } + pub(crate) fn external_sdk_archive_dir(&self) -> PathBuf { + self.project_dir.join("build/external-sdk-archives") + } + pub(crate) fn schema_version(&self) -> SchemaVersion<1> { self.schema_version } @@ -271,17 +277,16 @@ impl Project { } } -impl Project { +impl Project { pub(crate) fn sdk_image(&self) -> ProjectImage { - let SDKLocked(lock) = &self.lock; - self.as_project_image(&lock.0) + self.as_project_image(self.lock.locked_sdk_image()) .expect("Could not find SDK vendor despite lock resolution succeeding?") } } impl Project { /// Fetches all external kits defined in a Twoliter.lock to the build directory - pub(crate) async fn fetch(&self, arch: &str) -> Result<()> { + pub(crate) async fn fetch_kits(&self, arch: &str) -> Result<()> { let Locked(lock) = &self.lock; lock.fetch(self, arch).await } diff --git a/twoliter/src/project/tasks.rs b/twoliter/src/project/tasks.rs new file mode 100644 index 00000000..eb02392e --- /dev/null +++ b/twoliter/src/project/tasks.rs @@ -0,0 +1,53 @@ +//! This module defines common atomic build tasks that can be performed with a fully loaded project. +use super::{LockedSDKProvider, Project}; +use crate::cleanup::JANITOR; +use crate::docker::Docker; +use anyhow::{Context, Result}; +use krane_static::call_krane_inherited_io; +use tracing::instrument; + +impl Project { + /// Caches the project's SDK into the docker daemon if an image with the same name/tag is not + /// already cached. + #[instrument(level = "trace")] + pub(crate) async fn fetch_sdk(&self) -> Result<()> { + let sdk_uri = self.sdk_image().project_image_uri(); + tracing::info!("Ensuring project SDK '{sdk_uri}' is cached locally."); + + if Docker::image_is_cached(&sdk_uri).await? { + tracing::debug!("SDK '{sdk_uri}' is cached."); + return Ok(()); + } + + let temp_path = tempfile::Builder::new() + .prefix("bottlerocket-sdk-tmp-ardchive-") + .suffix(".tar") + .tempfile_in(self.external_sdk_archive_dir())? + .into_temp_path(); + + let host_platform = Docker::host_platform().await?; + + JANITOR + .with_tempfile(temp_path, |temp_path| async move { + let path_str = temp_path.to_string_lossy().to_string(); + + tracing::info!("Pulling '{sdk_uri}' for platform '{host_platform}'"); + call_krane_inherited_io(&[ + "pull", + &sdk_uri.uri(), + &path_str, + "--platform", + &host_platform, + ]) + .context("Failed to pull SDK image")?; + + tracing::info!("Loading SDK image '{sdk_uri}' into docker daemon"); + Docker::load(&path_str).await?; + + Ok::<_, anyhow::Error>(()) + }) + .await??; + + Ok(()) + } +}