diff --git a/Cargo.lock b/Cargo.lock index 7f93d943..663cab9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2800,6 +2800,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + [[package]] name = "ryu" version = "1.0.18" @@ -3151,6 +3157,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.75", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3836,6 +3864,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "strum", "tar", "tempfile", "testsys", diff --git a/Cargo.toml b/Cargo.toml index f1e8675d..505d681e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ sha2 = "0.10" shell-words = "1" simplelog = "0.12" snafu = "0.8" +strum = "0.26" tabled = "0.10" tar = "0.4" tempfile = "3" diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 20bf9c33..2793ca49 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -28,6 +28,7 @@ semver = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true +strum = { workspace = true, features = ["derive"] } tar.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["fs", "macros", "process", "rt-multi-thread"] } diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index c9dec286..5b993ac7 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -315,6 +315,10 @@ dependencies = ["setup-build"] script_runner = "bash" script = [ ''' +if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.sdk-verified" ]; then + echo "Twoliter could not validate '${TLPRIVATE_SDK_IMAGE}', refusing to continue" >&2 + exit 1 +fi if ! docker image inspect "${TLPRIVATE_SDK_IMAGE}" >/dev/null 2>&1 ; then if ! docker pull "${TLPRIVATE_SDK_IMAGE}" ; then echo "failed to pull '${TLPRIVATE_SDK_IMAGE}'" >&2 @@ -736,6 +740,18 @@ docker run --rm \ ''' ] +[tasks.validate-kits] +dependencies = ["cargo-metadata"] +script_runner = "bash" +script = [ +''' +if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.kits-verified" ]; then + echo "Twoliter could not validate external kits, refusing to continue" >&2 + exit 1 +fi +''' +] + # Reads the project's workspace Cargo dependency graph to a json file. Needed by buildsys when # building packages, kits and variants. [tasks.cargo-metadata] @@ -772,7 +788,7 @@ cargo metadata \ # Builds a package including its build-time and runtime dependency packages. [tasks.build-package] -dependencies = ["check-cargo-version", "fetch", "publish-setup", "cargo-metadata"] +dependencies = ["check-cargo-version", "fetch", "publish-setup", "validate-kits"] script_runner = "bash" script = [ ''' @@ -817,7 +833,7 @@ cargo build \ # Builds a kit including its dependency packages. [tasks.build-kit] -dependencies = ["check-cargo-version", "fetch", "publish-setup", "cargo-metadata"] +dependencies = ["check-cargo-version", "fetch", "publish-setup", "validate-kits"] script_runner = "bash" script = [ ''' @@ -841,7 +857,7 @@ cargo build \ ] [tasks.build-variant] -dependencies = ["fetch", "build-sbkeys", "publish-setup", "cargo-metadata"] +dependencies = ["fetch", "build-sbkeys", "publish-setup", "validate-kits"] script = [ ''' export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" @@ -892,7 +908,7 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${OUTPUT_LOGS_DIR}/latest" ] [tasks.build-all] -dependencies = ["fetch", "build-sbkeys", "publish-setup", "cargo-metadata"] +dependencies = ["fetch", "build-sbkeys", "publish-setup", "validate-kits"] script = [ ''' export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index 2a1c6c96..1ca7ee79 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -1,11 +1,29 @@ use crate::cargo_make::CargoMake; -use crate::lock::Lock; +use crate::lock::{Lock, LockedSDK}; use crate::project::{self}; use crate::tools::install_tools; use anyhow::Result; use clap::Parser; use std::path::PathBuf; +// Most subcommands do not require kits and thus do not need to resolve and verify them against the +// lockfile. +// +// Avoiding that resolution can be useful in CI situations where images are already built and we +// need to perform additional operations using the SDK. +// +// Only twoliter make targets in the following list *require* kit validation; however, kit +// validation will occur in other targets which require the SDK if no SDK is explicitly listed in +// Twoliter.toml. +const MUST_VALIDATE_KITS_TARGETS: &[&str] = &[ + "build-package", + "build-kit", + "build-variant", + "build-all", + "build", + "default", +]; + /// Run a cargo make command in Twoliter's build environment. Known Makefile.toml environment /// variables will be passed-through to the cargo make invocation. #[derive(Debug, Parser)] @@ -37,11 +55,11 @@ pub(crate) struct Make { impl Make { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock = Lock::load(&project).await?; + let sdk_source = self.locked_sdk(&project).await?; let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); - CargoMake::new(&lock.sdk.source)? + CargoMake::new(&sdk_source)? .env("CARGO_HOME", self.cargo_home.display().to_string()) .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_VERSION_IMAGE", project.release_version()) @@ -50,99 +68,231 @@ impl Make { .exec_with_args(&self.makefile_task, self.additional_args.clone()) .await } -} -#[test] -fn test_trailing_args_1() { - let args = Make::try_parse_from([ - "make", - "--cargo-home", - "/tmp/foo", - "--arch", - "x86_64", - "testsys", - "--", - "add", - "secret", - "map", - "--name", - "foo", - "something=bar", - "something-else=baz", - ]) - .unwrap(); - - assert_eq!(args.makefile_task, "testsys"); - assert_eq!(args.additional_args[0], "add"); - assert_eq!(args.additional_args[1], "secret"); - assert_eq!(args.additional_args[2], "map"); - assert_eq!(args.additional_args[3], "--name"); - assert_eq!(args.additional_args[4], "foo"); - assert_eq!(args.additional_args[5], "something=bar"); - assert_eq!(args.additional_args[6], "something-else=baz"); -} + fn can_skip_kit_verification(&self, project: &project::Project) -> bool { + let target_allows_kit_verification_skip = + !MUST_VALIDATE_KITS_TARGETS.contains(&self.makefile_task.as_str()); + let project_has_explicit_sdk_dep = project.sdk_image().is_some(); -#[test] -fn test_trailing_args_2() { - let args = Make::try_parse_from([ - "make", - "--cargo-home", - "/tmp/foo", - "--arch", - "x86_64", - "testsys", - "add", - "secret", - "map", - "--name", - "foo", - "something=bar", - "something-else=baz", - ]) - .unwrap(); - - assert_eq!(args.makefile_task, "testsys"); - assert_eq!(args.additional_args[0], "add"); - assert_eq!(args.additional_args[1], "secret"); - assert_eq!(args.additional_args[2], "map"); - assert_eq!(args.additional_args[3], "--name"); - assert_eq!(args.additional_args[4], "foo"); - assert_eq!(args.additional_args[5], "something=bar"); - assert_eq!(args.additional_args[6], "something-else=baz"); + target_allows_kit_verification_skip && project_has_explicit_sdk_dep + } + + /// Returns the locked SDK image for the project. + async fn locked_sdk(&self, project: &project::Project) -> Result { + let sdk_source = if self.can_skip_kit_verification(project) { + let lock = LockedSDK::load(project).await?; + lock.0.source + } else { + let lock = Lock::load(project).await?; + lock.sdk.source + }; + Ok(sdk_source) + } } -#[test] -fn test_trailing_args_3() { - let args = Make::try_parse_from([ - "make", - "--cargo-home", - "/tmp/foo", - "--arch", - "x86_64", - "testsys", - "--", - "add", - "secret", - "map", - "--", - "--name", - "foo", - "something=bar", - "something-else=baz", - "--", - ]) - .unwrap(); - - assert_eq!(args.makefile_task, "testsys"); - assert_eq!(args.additional_args[0], "add"); - assert_eq!(args.additional_args[1], "secret"); - assert_eq!(args.additional_args[2], "map"); - // The first instance of `--`, between `testsys` and `add`, is not passed through to the - // varargs. After that, instances of `--` are passed through the varargs. - assert_eq!(args.additional_args[3], "--"); - assert_eq!(args.additional_args[4], "--name"); - assert_eq!(args.additional_args[5], "foo"); - assert_eq!(args.additional_args[6], "something=bar"); - assert_eq!(args.additional_args[7], "something-else=baz"); - assert_eq!(args.additional_args[8], "--"); +#[cfg(test)] +mod test { + use std::path::Path; + + use crate::cmd::update::Update; + use crate::lock::VerificationTagger; + + use super::*; + + #[test] + fn test_trailing_args_1() { + let args = Make::try_parse_from([ + "make", + "--cargo-home", + "/tmp/foo", + "--arch", + "x86_64", + "testsys", + "--", + "add", + "secret", + "map", + "--name", + "foo", + "something=bar", + "something-else=baz", + ]) + .unwrap(); + + assert_eq!(args.makefile_task, "testsys"); + assert_eq!(args.additional_args[0], "add"); + assert_eq!(args.additional_args[1], "secret"); + assert_eq!(args.additional_args[2], "map"); + assert_eq!(args.additional_args[3], "--name"); + assert_eq!(args.additional_args[4], "foo"); + assert_eq!(args.additional_args[5], "something=bar"); + assert_eq!(args.additional_args[6], "something-else=baz"); + } + + #[test] + fn test_trailing_args_2() { + let args = Make::try_parse_from([ + "make", + "--cargo-home", + "/tmp/foo", + "--arch", + "x86_64", + "testsys", + "add", + "secret", + "map", + "--name", + "foo", + "something=bar", + "something-else=baz", + ]) + .unwrap(); + + assert_eq!(args.makefile_task, "testsys"); + assert_eq!(args.additional_args[0], "add"); + assert_eq!(args.additional_args[1], "secret"); + assert_eq!(args.additional_args[2], "map"); + assert_eq!(args.additional_args[3], "--name"); + assert_eq!(args.additional_args[4], "foo"); + assert_eq!(args.additional_args[5], "something=bar"); + assert_eq!(args.additional_args[6], "something-else=baz"); + } + + #[test] + fn test_trailing_args_3() { + let args = Make::try_parse_from([ + "make", + "--cargo-home", + "/tmp/foo", + "--arch", + "x86_64", + "testsys", + "--", + "add", + "secret", + "map", + "--", + "--name", + "foo", + "something=bar", + "something-else=baz", + "--", + ]) + .unwrap(); + + assert_eq!(args.makefile_task, "testsys"); + assert_eq!(args.additional_args[0], "add"); + assert_eq!(args.additional_args[1], "secret"); + assert_eq!(args.additional_args[2], "map"); + // The first instance of `--`, between `testsys` and `add`, is not passed through to the + // varargs. After that, instances of `--` are passed through the varargs. + assert_eq!(args.additional_args[3], "--"); + assert_eq!(args.additional_args[4], "--name"); + assert_eq!(args.additional_args[5], "foo"); + assert_eq!(args.additional_args[6], "something=bar"); + assert_eq!(args.additional_args[7], "something-else=baz"); + assert_eq!(args.additional_args[8], "--"); + } + + const PROJECT: &str = "local-kit"; + + async fn twoliter_update(project_path: &Path) { + let command = Update { + project_path: Some(project_path.to_path_buf()), + }; + command.run().await.unwrap(); + } + + async fn run_makefile_target( + target: &str, + project_dir: &Path, + delete_verifier_tags: bool, + ) -> Result<()> { + let project_path = project_dir.join("Twoliter.toml"); + + twoliter_update(&project_path).await; + + let project = project::load_or_find_project(Some(project_path)) + .await + .unwrap(); + let sdk_source = LockedSDK::load(&project).await.unwrap().0.source; + + if delete_verifier_tags { + // Clean up tags so that the build fails + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()) + .await + .unwrap(); + } + + let toolsdir = project.project_dir().join("build/tools"); + install_tools(&toolsdir).await.unwrap(); + let makefile_path = toolsdir.join("Makefile.toml"); + + CargoMake::new(&sdk_source) + .unwrap() + .env("CARGO_HOME", project_dir.display().to_string()) + .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) + .env("BUILDSYS_VERSION_IMAGE", project.release_version()) + .makefile(makefile_path) + .project_dir(project.project_dir()) + .exec_with_args(target, Vec::<&'static str>::new()) + .await + } + + async fn target_can_skip_kit_verification(target_name: &str) -> bool { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + let project_dir = temp_dir.path(); + let project_path = project_dir.join("Twoliter.toml"); + let project = project::load_or_find_project(Some(project_path.clone())) + .await + .unwrap(); + + let make = Make { + project_path: Some(project_path), + cargo_home: project_dir.to_owned(), + arch: "x86_64".to_string(), + makefile_task: target_name.to_string(), + additional_args: Vec::new(), + }; + make.can_skip_kit_verification(&project) + } + + #[tokio::test] + async fn test_repack_variant_can_skip_kit_verification() { + assert!(target_can_skip_kit_verification("repack-variant").await); + } + + #[tokio::test] + async fn test_build_variant_cannot_skip_kit_verification() { + assert!(!target_can_skip_kit_verification("build-variant").await); + } + + #[tokio::test] + #[ignore] // integration test + async fn test_fetch_sdk_succeeds_when_only_sdk_verified() { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), false) + .await + .is_ok()); + } + + #[tokio::test] + #[ignore] // integration test + async fn test_fetch_sdk_fails_when_nothing_verified() { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), true) + .await + .is_err()); + } + + #[tokio::test] + #[ignore] // integration test + async fn test_validate_kits_fails_when_only_sdk_verified() { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + assert!( + run_makefile_target("validate-kits", &temp_dir.path(), false) + .await + .is_err() + ); + } } diff --git a/twoliter/src/lock/image.rs b/twoliter/src/lock/image.rs index 558a3262..4210cf1c 100644 --- a/twoliter/src/lock/image.rs +++ b/twoliter/src/lock/image.rs @@ -3,7 +3,7 @@ use super::views::ManifestListView; use super::Override; use crate::common::fs::create_dir_all; use crate::docker::ImageUri; -use crate::project::{Image, Vendor}; +use crate::project::{Image, Project, ValidIdentifier}; use anyhow::{bail, Context, Result}; use base64::Engine; use futures::{pin_mut, stream, StreamExt, TryStreamExt}; @@ -207,13 +207,23 @@ pub struct ImageResolver { } impl ImageResolver { - pub(crate) fn from_image( - image: &Image, - vendor_name: &str, - vendor: &Vendor, - override_: Option<&Override>, - ) -> Self { - Self { + pub(crate) fn from_image(image: &Image, project: &Project) -> Result { + let vendor_name = image.vendor.0.as_str(); + let vendor = project.vendor().get(&image.vendor).context(format!( + "vendor '{}' is not specified in Twoliter.toml", + image.vendor + ))?; + let override_ = project + .overrides() + .get(&image.vendor.to_string()) + .and_then(|x| x.get(&image.name.to_string())); + if let Some(override_) = override_.as_ref() { + debug!( + ?override_, + "Found override for image '{}' with vendor '{}'", image.name, image.vendor + ); + } + Ok(Self { image_resolver_impl: if let Some(override_) = override_ { Box::new(OverriddenImage { base_uri: ImageUri { @@ -234,16 +244,31 @@ impl ImageResolver { }, }) }, - } + }) } - pub(crate) fn from_locked_image( - locked_image: &LockedImage, - vendor_name: &str, - vendor: &Vendor, - override_: Option<&Override>, - ) -> Self { - Self { + pub(crate) fn from_locked_image(locked_image: &LockedImage, project: &Project) -> Result { + let vendor_name = &locked_image.vendor; + let vendor = project + .vendor() + .get(&ValidIdentifier(vendor_name.clone())) + .context(format!( + "failed to find vendor for kit with name '{}' and vendor '{}'", + locked_image.name, locked_image.vendor + ))?; + let override_ = project + .overrides() + .get(&locked_image.vendor) + .and_then(|x| x.get(&locked_image.name)); + if let Some(override_) = override_.as_ref() { + debug!( + ?override_, + "Found override for image '{}' with vendor '{}'", + locked_image.name, + locked_image.vendor + ); + } + Ok(Self { image_resolver_impl: if let Some(override_) = override_ { Box::new(OverriddenImage { base_uri: ImageUri { @@ -264,7 +289,7 @@ impl ImageResolver { }, }) }, - } + }) } /// Calculate the digest of the locked image diff --git a/twoliter/src/lock/mod.rs b/twoliter/src/lock/mod.rs index 47f70e4f..24532899 100644 --- a/twoliter/src/lock/mod.rs +++ b/twoliter/src/lock/mod.rs @@ -7,9 +7,13 @@ pub mod archive; /// Covers resolution and validation of a single image dependency in a lock file pub mod image; +/// Provides tools for marking artifacts as having been verified against the Twoliter lockfile +pub mod verification; /// Implements view models of common OCI manifest and configuration types pub mod views; +pub(crate) use self::verification::VerificationTagger; + use crate::common::fs::{create_dir_all, read, write}; use crate::project::{Image, Project, ValidIdentifier}; use crate::schema_version::SchemaVersion; @@ -42,6 +46,66 @@ pub(crate) struct Override { pub registry: Option, } +/// A resolved and locked project SDK, typically from the Twoliter.lock file for a project. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LockedSDK(pub LockedImage); + +impl AsRef for LockedSDK { + fn as_ref(&self) -> &LockedImage { + &self.0 + } +} + +impl LockedSDK { + /// Loads the locked SDK for the given project. + /// + /// Re-resolves the project's SDK to ensure that the lockfile matches the state of the world. + #[instrument(level = "trace", skip(project))] + pub(crate) async fn load(project: &Project) -> Result { + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()).await?; + + info!("Resolving project references to check against lock file"); + let resolved_lock = { + // bind `current_lock` in a block so that we can't accidentally pass it to the + // VerificationTagger. + let current_lock = Lock::current_lock_state(project).await?; + let resolved_lock = Self::resolve_sdk(project) + .await? + .context("Project does not have explicit SDK image.")?; + ensure!(¤t_lock.sdk == resolved_lock.as_ref(), "changes have occured to Twoliter.toml or the remote SDK image that require an update to Twoliter.lock"); + resolved_lock + }; + + VerificationTagger::from(&resolved_lock) + .write_tags(project.external_kits_dir()) + .await?; + + Ok(resolved_lock) + } + + /// Creates a project lock referring to only the resolved SDK image from the project. + /// + /// Returns `None` if the project does not have an explicit SDK image. + #[instrument(level = "trace", skip(project))] + async fn resolve_sdk(project: &Project) -> Result> { + debug!("Attempting to resolve workspace SDK"); + let sdk = match project.sdk_image() { + Some(sdk) => sdk, + None => { + debug!("No explicit SDK image provided"); + return Ok(None); + } + }; + + debug!(?sdk, "Resolving workspace SDK"); + let image_tool = ImageTool::from_environment()?; + ImageResolver::from_image(&sdk, project)? + .resolve(&image_tool, true) + .await + .map(|(sdk, _)| Some(Self(sdk))) + } +} + /// Represents the structure of a `Twoliter.lock` lock file. #[derive(Debug, Clone, Eq, Ord, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -79,8 +143,33 @@ impl Lock { Ok(lock_state) } + /// Loads the lockfile for the given project. + /// + /// Re-resolves the project's dependencies to ensure that the lockfile matches the state of the + /// world. #[instrument(level = "trace", skip(project))] pub(crate) async fn load(project: &Project) -> Result { + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()).await?; + + info!("Resolving project references to check against lock file"); + let resolved_lock = { + // bind `current_lock` in a block so that we can't accidentally pass it to the + // VerificationTagger. + let current_lock = Self::current_lock_state(project).await?; + let resolved_lock = Self::resolve(project).await?; + ensure!(current_lock == resolved_lock, "changes have occured to Twoliter.toml or the remote kit images that require an update to Twoliter.lock"); + resolved_lock + }; + + VerificationTagger::from(&resolved_lock) + .write_tags(project.external_kits_dir()) + .await?; + + Ok(resolved_lock) + } + + /// Returns the state of the lockfile for the given `Project` + async fn current_lock_state(project: &Project) -> Result { let lock_file_path = project.project_dir().join(TWOLITER_LOCK); ensure!( lock_file_path.exists(), @@ -92,10 +181,6 @@ impl Lock { .context("failed to read lockfile")?; let lock: Self = toml::from_str(lock_str.as_str()).context("failed to deserialize lockfile")?; - info!("Resolving project references to check against lock file"); - let lock_state = Self::resolve(project).await?; - - ensure!(lock_state == lock, "changes have occured to Twoliter.toml or the remote kit images that require an update to Twoliter.lock"); Ok(lock) } @@ -121,19 +206,7 @@ impl Lock { "Extracting kit dependencies." ); for image in self.kit.iter() { - let vendor = project - .vendor() - .get(&ValidIdentifier(image.vendor.clone())) - .context(format!( - "failed to find vendor for kit with name '{}' and vendor '{}'", - image.name, image.vendor - ))?; - let override_ = project - .overrides() - .get(&image.vendor) - .and_then(|x| x.get(&image.name)); - let resolver = - ImageResolver::from_locked_image(image, image.vendor.as_str(), vendor, override_); + let resolver = ImageResolver::from_locked_image(image, project)?; resolver .extract(&image_tool, &project.external_kits_dir(), arch) .await?; @@ -172,12 +245,11 @@ impl Lock { #[instrument(level = "trace", skip(project))] async fn resolve(project: &Project) -> Result { - let vendor_table = project.vendor(); let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); let image_tool = ImageTool::from_environment()?; - let overrides = project.overrides(); let mut remaining: Vec = project.kits(); + let mut sdk_set: HashSet = HashSet::new(); if let Some(sdk) = project.sdk_image() { // We don't scan over the sdk images as they are not kit images and there is no kit metadata to fetch @@ -193,7 +265,8 @@ impl Lock { let vendor = image.vendor.clone(); ensure!( image.version == *version, - "cannot have multiple versions of the same kit ({name}-{left_version}@{vendor} != {name}-{version}@{vendor}", + "cannot have multiple versions of the same kit ({name}-{left_version}@{vendor} \ + != {name}-{version}@{vendor}", ); debug!( ?image, @@ -201,25 +274,16 @@ impl Lock { ); continue; } - let vendor = vendor_table.get(&image.vendor).context(format!( + ensure!( + project.vendor().get(&image.vendor).is_some(), "vendor '{}' is not specified in Twoliter.toml", image.vendor - ))?; + ); known.insert( (image.name.clone(), image.vendor.clone()), image.version.clone(), ); - let override_ = overrides - .get(&image.vendor.to_string()) - .and_then(|x| x.get(&image.name.to_string())); - if let Some(override_) = override_.as_ref() { - debug!( - ?override_, - "Found override for kit '{}' with vendor '{}'", image.name, image.vendor - ); - } - let image_resolver = - ImageResolver::from_image(image, image.vendor.0.as_str(), vendor, override_); + let image_resolver = ImageResolver::from_image(image, project)?; let (locked_image, metadata) = image_resolver.resolve(&image_tool, false).await?; let metadata = metadata.context(format!( "failed to validate kit image with name {} from vendor {}", @@ -232,7 +296,6 @@ impl Lock { } } } - debug!(?sdk_set, "Resolving workspace SDK"); ensure!( sdk_set.len() <= 1, @@ -247,20 +310,16 @@ impl Lock { .iter() .next() .context("no sdk was found for use, please specify a sdk in Twoliter.toml")?; - let vendor = vendor_table.get(&sdk.vendor).context(format!( - "vendor '{}' is not specified in Twoliter.toml", - sdk.vendor - ))?; - let sdk_override = overrides - .get(&sdk.vendor.to_string()) - .and_then(|x| x.get(&sdk.name.to_string())); - let sdk_resolver = - ImageResolver::from_image(sdk, sdk.vendor.0.as_str(), vendor, sdk_override); - let (sdk, _) = sdk_resolver.resolve(&image_tool, true).await?; + + debug!(?sdk, "Resolving workspace SDK"); + let (sdk, _metadata) = ImageResolver::from_image(sdk, project)? + .resolve(&image_tool, true) + .await?; + Ok(Self { schema_version: project.schema_version(), - sdk, kit: locked, + sdk, }) } } diff --git a/twoliter/src/lock/verification.rs b/twoliter/src/lock/verification.rs new file mode 100644 index 00000000..4ad01668 --- /dev/null +++ b/twoliter/src/lock/verification.rs @@ -0,0 +1,260 @@ +//! This module contains utilities for marking that certain Twoliter artifacts have been resolved +//! and verified against a project's Lockfile. +//! +//! An overview of the contained abstractions: +//! * The [`LockfileVerifier`] trait allows a type to announce that it has resolved and verified +//! a set of artifacts. +//! * Verified artifacts are identified via a [`VerifyTag`]. +//! * Each [`VerifyTag`] has a [`VerificationManifest`] containing a list of the verified artifacts +//! of that tag type. +//! * The [`VerificationTagger`] writes files containing [`VerifyTag`]s that are produced by +//! [`LockfileVerifier`]s. +use super::image::LockedImage; +use super::{Lock, LockedSDK}; +use anyhow::{Context, Result}; +use olpc_cjson::CanonicalFormatter as CanonicalJsonFormatter; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::fmt::Debug; +use std::path::Path; +use strum::{EnumIter, IntoEnumIterator}; +use tracing::{debug, instrument}; + +const SDK_VERIFIED_MARKER_FILE: &str = ".sdk-verified"; +const KITS_VERIFIED_MARKER_FILE: &str = ".kits-verified"; + +/// A tag indicating that Twoliter artifacts have been resolved and verified against the lockfile +#[derive(Debug, PartialEq, Eq, Ord, PartialOrd, EnumIter)] +pub(crate) enum VerifyTag { + Sdk(VerificationManifest), + Kits(VerificationManifest), +} + +impl VerifyTag { + /// Returns the marker file marking an artifact type that has been verified against the lock + pub(crate) fn marker_file_name(&self) -> &'static str { + match self { + VerifyTag::Sdk(_) => SDK_VERIFIED_MARKER_FILE, + VerifyTag::Kits(_) => KITS_VERIFIED_MARKER_FILE, + } + } + + pub(crate) fn manifest(&self) -> &VerificationManifest { + match self { + VerifyTag::Sdk(manifest) => manifest, + VerifyTag::Kits(manifest) => manifest, + } + } +} + +/// A manifest containing the list of elements that were verified by a `LockfileVerifier` +#[derive(Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct VerificationManifest { + verified_images: BTreeSet, +} + +impl VerificationManifest { + fn as_canonical_json(&self) -> Result> { + let mut manifest = Vec::new(); + let mut ser = + serde_json::Serializer::with_formatter(&mut manifest, CanonicalJsonFormatter::new()); + self.serialize(&mut ser) + .context("failed to serialize external kit metadata")?; + Ok(manifest) + } +} + +impl From<&LockedImage> for VerificationManifest { + fn from(image: &LockedImage) -> Self { + [image].as_slice().into() + } +} + +impl From<&[&LockedImage]> for VerificationManifest { + fn from(images: &[&LockedImage]) -> Self { + Self { + verified_images: images.iter().map(ToString::to_string).collect(), + } + } +} + +/// A `LockfileVerifier` can return a set of `VerifyTag` structs, claiming that those artifacts +/// have been resolved and verified against the lockfile. +pub(crate) trait LockfileVerifier { + fn verified(&self) -> BTreeSet; +} + +impl LockfileVerifier for LockedSDK { + fn verified(&self) -> BTreeSet { + [VerifyTag::Sdk((&self.0).into())].into() + } +} + +impl LockfileVerifier for Lock { + fn verified(&self) -> BTreeSet { + [ + VerifyTag::Sdk((&self.sdk).into()), + VerifyTag::Kits(self.kit.iter().collect::>().as_slice().into()), + ] + .into() + } +} + +/// Writes marker files indicating which artifacts have been resolved and verified against the lock +#[derive(Debug)] +pub(crate) struct VerificationTagger { + tags: BTreeSet, +} + +impl From<&V> for VerificationTagger { + fn from(resolver: &V) -> Self { + Self { + tags: resolver.verified(), + } + } +} + +impl VerificationTagger { + /// Creates marker files for artifacts that have been verified against the lockfile + #[instrument(level = "trace", skip(external_kits_dir))] + pub(crate) async fn write_tags>(&self, external_kits_dir: P) -> Result<()> { + let external_kits_dir = external_kits_dir.as_ref(); + Self::cleanup_existing_tags(&external_kits_dir).await?; + + debug!("Writing tag files for verified artifacts"); + tokio::fs::create_dir_all(&external_kits_dir) + .await + .context(format!( + "failed to create external-kits directory at '{}'", + external_kits_dir.display() + ))?; + + for tag in self.tags.iter() { + let flag_file = external_kits_dir.join(tag.marker_file_name()); + debug!( + "Writing tag file for verified artifacts: '{}'", + flag_file.display() + ); + tokio::fs::write(&flag_file, tag.manifest().as_canonical_json()?) + .await + .context(format!( + "failed to write verification tag file: '{}'", + flag_file.display() + ))?; + } + Ok(()) + } + + /// Deletes any existing verifier marker files in the kits directory + #[instrument(level = "trace", skip(external_kits_dir))] + pub(crate) async fn cleanup_existing_tags>(external_kits_dir: P) -> Result<()> { + let external_kits_dir = external_kits_dir.as_ref(); + + debug!("Cleaning up any existing tag files for resolved artifacts",); + for resolve_tag in VerifyTag::iter() { + let flag_file = external_kits_dir.join(resolve_tag.marker_file_name()); + if flag_file.exists() { + debug!( + "Removing existing verification tag file '{}'", + flag_file.display() + ); + tokio::fs::remove_file(&flag_file).await.context(format!( + "failed to remove existing verification tag file: {}", + flag_file.display() + ))?; + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + struct SDKResolver; + + impl LockfileVerifier for SDKResolver { + fn verified(&self) -> BTreeSet { + [VerifyTag::Sdk(VerificationManifest { + verified_images: ["image1".into(), "image2".into()].into(), + })] + .into() + } + } + + struct KitResolver; + + impl LockfileVerifier for KitResolver { + fn verified(&self) -> BTreeSet { + [ + VerifyTag::Sdk(VerificationManifest { + verified_images: ["image1".into(), "image2".into()].into(), + }), + VerifyTag::Kits(VerificationManifest { + verified_images: ["kit1".into(), "kit2".into()].into(), + }), + ] + .into() + } + } + + #[tokio::test] + async fn test_cleanup_existing_tags() { + let kits_dir = tempfile::tempdir().unwrap(); + let flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + tokio::fs::write(&flag_file, "test").await.unwrap(); + + VerificationTagger::cleanup_existing_tags(&kits_dir.path()) + .await + .unwrap(); + assert!(!flag_file.exists()); + } + + #[tokio::test] + async fn test_write_sdk_tags() { + let kits_dir = tempfile::tempdir().unwrap(); + let tagger = VerificationTagger::from(&SDKResolver); + tagger.write_tags(&kits_dir.path()).await.unwrap(); + + let flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + assert!(flag_file.exists()); + let contents = tokio::fs::read_to_string(&flag_file).await.unwrap(); + assert_eq!(contents, r#"["image1","image2"]"#); + } + + #[tokio::test] + async fn test_write_kit_tags() { + let kits_dir = tempfile::tempdir().unwrap(); + let tagger = VerificationTagger::from(&KitResolver); + tagger.write_tags(&kits_dir.path()).await.unwrap(); + + let sdk_flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + assert!(sdk_flag_file.exists()); + let sdk_contents = tokio::fs::read_to_string(&sdk_flag_file).await.unwrap(); + assert_eq!(sdk_contents, r#"["image1","image2"]"#); + + let kit_flag_file = kits_dir.path().join(KITS_VERIFIED_MARKER_FILE); + assert!(kit_flag_file.exists()); + let kit_contents = tokio::fs::read_to_string(&kit_flag_file).await.unwrap(); + assert_eq!(kit_contents, r#"["kit1","kit2"]"#); + } + + #[tokio::test] + async fn test_previous_tags_removed() { + let kits_dir = tempfile::tempdir().unwrap(); + let flag_file = kits_dir.path().join(KITS_VERIFIED_MARKER_FILE); + tokio::fs::write(&flag_file, "test").await.unwrap(); + + let tagger = VerificationTagger::from(&SDKResolver); + tagger.write_tags(&kits_dir.path()).await.unwrap(); + + assert!(!flag_file.exists()); + + let sdk_flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + assert!(sdk_flag_file.exists()); + let sdk_contents = tokio::fs::read_to_string(&sdk_flag_file).await.unwrap(); + assert_eq!(sdk_contents, r#"["image1","image2"]"#); + } +} diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index 5904a1f0..82493717 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -1,6 +1,6 @@ use crate::common::fs::{self, read_to_string}; use crate::docker::ImageUri; -use crate::lock::Override; +use crate::lock::{Override, VerificationTagger}; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; @@ -71,7 +71,15 @@ impl Project { "Unable to deserialize project file '{}'", path.display() ))?; - unvalidated.validate(path).await + let project = unvalidated.validate(path).await?; + + // When projects are resolved, tags are written indicating which artifacts have been checked + // against the lockfile. + // We clean these up as early as possible to avoid situations in which artifacts are + // incorrectly flagged as having been resolved. + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()).await?; + + Ok(project) } /// Recursively search for a file named `Twoliter.toml` starting in `dir`. If it is not found, @@ -152,7 +160,7 @@ impl Project { ))?; Ok(Some(ImageUri::new( Some(vendor.registry.clone()), - kit.name.to_string(), + &kit.name, format!("v{}", kit.version), ))) } else { @@ -253,6 +261,12 @@ impl<'de> Deserialize<'de> for ValidIdentifier { } } +impl AsRef for ValidIdentifier { + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + impl Display for ValidIdentifier { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_str(self.0.as_str())