From 87624fe1ec34bc59fc83737ce23fafbfd1fdc09e Mon Sep 17 00:00:00 2001 From: "Sean P. Kelly" Date: Fri, 6 Sep 2024 07:42:24 +0000 Subject: [PATCH] twoliter: allow partial lockfile validation in some scenarios In some CI scenarios, it's useful to drop previously-built variant images into a Twoliter project's build directory and use Twoliter to publish those variant images. In these cases, Kit dependencies are not necessary, and it can be useful to avoid resolving them and comparing them against Twoliter.lock. This change allows skipping Kit lockfile verification only when executing certain `twoliter make` targets. --- Cargo.lock | 29 +++ Cargo.toml | 1 + twoliter/Cargo.toml | 1 + twoliter/embedded/Makefile.toml | 24 ++- twoliter/src/cmd/make.rs | 338 +++++++++++++++++++++--------- twoliter/src/lock/image.rs | 59 ++++-- twoliter/src/lock/mod.rs | 149 +++++++++---- twoliter/src/lock/verification.rs | 260 +++++++++++++++++++++++ twoliter/src/project.rs | 20 +- 9 files changed, 718 insertions(+), 163 deletions(-) create mode 100644 twoliter/src/lock/verification.rs diff --git a/Cargo.lock b/Cargo.lock index 7f93d9436..663cab9b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2800,6 +2800,12 @@ dependencies = [ "untrusted", ] +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + [[package]] name = "ryu" version = "1.0.18" @@ -3151,6 +3157,28 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.75", +] + [[package]] name = "subtle" version = "2.6.1" @@ -3836,6 +3864,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "strum", "tar", "tempfile", "testsys", diff --git a/Cargo.toml b/Cargo.toml index f1e8675dc..505d681e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ sha2 = "0.10" shell-words = "1" simplelog = "0.12" snafu = "0.8" +strum = "0.26" tabled = "0.10" tar = "0.4" tempfile = "3" diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 20bf9c335..2793ca491 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -28,6 +28,7 @@ semver = { workspace = true, features = ["serde"] } serde = { workspace = true, features = ["derive"] } serde_json.workspace = true sha2.workspace = true +strum = { workspace = true, features = ["derive"] } tar.workspace = true tempfile.workspace = true tokio = { workspace = true, features = ["fs", "macros", "process", "rt-multi-thread"] } diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index c9dec286b..5b993ac7b 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -315,6 +315,10 @@ dependencies = ["setup-build"] script_runner = "bash" script = [ ''' +if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.sdk-verified" ]; then + echo "Twoliter could not validate '${TLPRIVATE_SDK_IMAGE}', refusing to continue" >&2 + exit 1 +fi if ! docker image inspect "${TLPRIVATE_SDK_IMAGE}" >/dev/null 2>&1 ; then if ! docker pull "${TLPRIVATE_SDK_IMAGE}" ; then echo "failed to pull '${TLPRIVATE_SDK_IMAGE}'" >&2 @@ -736,6 +740,18 @@ docker run --rm \ ''' ] +[tasks.validate-kits] +dependencies = ["cargo-metadata"] +script_runner = "bash" +script = [ +''' +if [ ! -s "${BUILDSYS_EXTERNAL_KITS_DIR}/.kits-verified" ]; then + echo "Twoliter could not validate external kits, refusing to continue" >&2 + exit 1 +fi +''' +] + # Reads the project's workspace Cargo dependency graph to a json file. Needed by buildsys when # building packages, kits and variants. [tasks.cargo-metadata] @@ -772,7 +788,7 @@ cargo metadata \ # Builds a package including its build-time and runtime dependency packages. [tasks.build-package] -dependencies = ["check-cargo-version", "fetch", "publish-setup", "cargo-metadata"] +dependencies = ["check-cargo-version", "fetch", "publish-setup", "validate-kits"] script_runner = "bash" script = [ ''' @@ -817,7 +833,7 @@ cargo build \ # Builds a kit including its dependency packages. [tasks.build-kit] -dependencies = ["check-cargo-version", "fetch", "publish-setup", "cargo-metadata"] +dependencies = ["check-cargo-version", "fetch", "publish-setup", "validate-kits"] script_runner = "bash" script = [ ''' @@ -841,7 +857,7 @@ cargo build \ ] [tasks.build-variant] -dependencies = ["fetch", "build-sbkeys", "publish-setup", "cargo-metadata"] +dependencies = ["fetch", "build-sbkeys", "publish-setup", "validate-kits"] script = [ ''' export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" @@ -892,7 +908,7 @@ ln -snf "${BUILDSYS_VERSION_FULL}" "${OUTPUT_LOGS_DIR}/latest" ] [tasks.build-all] -dependencies = ["fetch", "build-sbkeys", "publish-setup", "cargo-metadata"] +dependencies = ["fetch", "build-sbkeys", "publish-setup", "validate-kits"] script = [ ''' export PATH="${TWOLITER_TOOLS_DIR}:${PATH}" diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index 2a1c6c965..1ca7ee798 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -1,11 +1,29 @@ use crate::cargo_make::CargoMake; -use crate::lock::Lock; +use crate::lock::{Lock, LockedSDK}; use crate::project::{self}; use crate::tools::install_tools; use anyhow::Result; use clap::Parser; use std::path::PathBuf; +// Most subcommands do not require kits and thus do not need to resolve and verify them against the +// lockfile. +// +// Avoiding that resolution can be useful in CI situations where images are already built and we +// need to perform additional operations using the SDK. +// +// Only twoliter make targets in the following list *require* kit validation; however, kit +// validation will occur in other targets which require the SDK if no SDK is explicitly listed in +// Twoliter.toml. +const MUST_VALIDATE_KITS_TARGETS: &[&str] = &[ + "build-package", + "build-kit", + "build-variant", + "build-all", + "build", + "default", +]; + /// Run a cargo make command in Twoliter's build environment. Known Makefile.toml environment /// variables will be passed-through to the cargo make invocation. #[derive(Debug, Parser)] @@ -37,11 +55,11 @@ pub(crate) struct Make { impl Make { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock = Lock::load(&project).await?; + let sdk_source = self.locked_sdk(&project).await?; let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); - CargoMake::new(&lock.sdk.source)? + CargoMake::new(&sdk_source)? .env("CARGO_HOME", self.cargo_home.display().to_string()) .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_VERSION_IMAGE", project.release_version()) @@ -50,99 +68,231 @@ impl Make { .exec_with_args(&self.makefile_task, self.additional_args.clone()) .await } -} -#[test] -fn test_trailing_args_1() { - let args = Make::try_parse_from([ - "make", - "--cargo-home", - "/tmp/foo", - "--arch", - "x86_64", - "testsys", - "--", - "add", - "secret", - "map", - "--name", - "foo", - "something=bar", - "something-else=baz", - ]) - .unwrap(); - - assert_eq!(args.makefile_task, "testsys"); - assert_eq!(args.additional_args[0], "add"); - assert_eq!(args.additional_args[1], "secret"); - assert_eq!(args.additional_args[2], "map"); - assert_eq!(args.additional_args[3], "--name"); - assert_eq!(args.additional_args[4], "foo"); - assert_eq!(args.additional_args[5], "something=bar"); - assert_eq!(args.additional_args[6], "something-else=baz"); -} + fn can_skip_kit_verification(&self, project: &project::Project) -> bool { + let target_allows_kit_verification_skip = + !MUST_VALIDATE_KITS_TARGETS.contains(&self.makefile_task.as_str()); + let project_has_explicit_sdk_dep = project.sdk_image().is_some(); -#[test] -fn test_trailing_args_2() { - let args = Make::try_parse_from([ - "make", - "--cargo-home", - "/tmp/foo", - "--arch", - "x86_64", - "testsys", - "add", - "secret", - "map", - "--name", - "foo", - "something=bar", - "something-else=baz", - ]) - .unwrap(); - - assert_eq!(args.makefile_task, "testsys"); - assert_eq!(args.additional_args[0], "add"); - assert_eq!(args.additional_args[1], "secret"); - assert_eq!(args.additional_args[2], "map"); - assert_eq!(args.additional_args[3], "--name"); - assert_eq!(args.additional_args[4], "foo"); - assert_eq!(args.additional_args[5], "something=bar"); - assert_eq!(args.additional_args[6], "something-else=baz"); + target_allows_kit_verification_skip && project_has_explicit_sdk_dep + } + + /// Returns the locked SDK image for the project. + async fn locked_sdk(&self, project: &project::Project) -> Result { + let sdk_source = if self.can_skip_kit_verification(project) { + let lock = LockedSDK::load(project).await?; + lock.0.source + } else { + let lock = Lock::load(project).await?; + lock.sdk.source + }; + Ok(sdk_source) + } } -#[test] -fn test_trailing_args_3() { - let args = Make::try_parse_from([ - "make", - "--cargo-home", - "/tmp/foo", - "--arch", - "x86_64", - "testsys", - "--", - "add", - "secret", - "map", - "--", - "--name", - "foo", - "something=bar", - "something-else=baz", - "--", - ]) - .unwrap(); - - assert_eq!(args.makefile_task, "testsys"); - assert_eq!(args.additional_args[0], "add"); - assert_eq!(args.additional_args[1], "secret"); - assert_eq!(args.additional_args[2], "map"); - // The first instance of `--`, between `testsys` and `add`, is not passed through to the - // varargs. After that, instances of `--` are passed through the varargs. - assert_eq!(args.additional_args[3], "--"); - assert_eq!(args.additional_args[4], "--name"); - assert_eq!(args.additional_args[5], "foo"); - assert_eq!(args.additional_args[6], "something=bar"); - assert_eq!(args.additional_args[7], "something-else=baz"); - assert_eq!(args.additional_args[8], "--"); +#[cfg(test)] +mod test { + use std::path::Path; + + use crate::cmd::update::Update; + use crate::lock::VerificationTagger; + + use super::*; + + #[test] + fn test_trailing_args_1() { + let args = Make::try_parse_from([ + "make", + "--cargo-home", + "/tmp/foo", + "--arch", + "x86_64", + "testsys", + "--", + "add", + "secret", + "map", + "--name", + "foo", + "something=bar", + "something-else=baz", + ]) + .unwrap(); + + assert_eq!(args.makefile_task, "testsys"); + assert_eq!(args.additional_args[0], "add"); + assert_eq!(args.additional_args[1], "secret"); + assert_eq!(args.additional_args[2], "map"); + assert_eq!(args.additional_args[3], "--name"); + assert_eq!(args.additional_args[4], "foo"); + assert_eq!(args.additional_args[5], "something=bar"); + assert_eq!(args.additional_args[6], "something-else=baz"); + } + + #[test] + fn test_trailing_args_2() { + let args = Make::try_parse_from([ + "make", + "--cargo-home", + "/tmp/foo", + "--arch", + "x86_64", + "testsys", + "add", + "secret", + "map", + "--name", + "foo", + "something=bar", + "something-else=baz", + ]) + .unwrap(); + + assert_eq!(args.makefile_task, "testsys"); + assert_eq!(args.additional_args[0], "add"); + assert_eq!(args.additional_args[1], "secret"); + assert_eq!(args.additional_args[2], "map"); + assert_eq!(args.additional_args[3], "--name"); + assert_eq!(args.additional_args[4], "foo"); + assert_eq!(args.additional_args[5], "something=bar"); + assert_eq!(args.additional_args[6], "something-else=baz"); + } + + #[test] + fn test_trailing_args_3() { + let args = Make::try_parse_from([ + "make", + "--cargo-home", + "/tmp/foo", + "--arch", + "x86_64", + "testsys", + "--", + "add", + "secret", + "map", + "--", + "--name", + "foo", + "something=bar", + "something-else=baz", + "--", + ]) + .unwrap(); + + assert_eq!(args.makefile_task, "testsys"); + assert_eq!(args.additional_args[0], "add"); + assert_eq!(args.additional_args[1], "secret"); + assert_eq!(args.additional_args[2], "map"); + // The first instance of `--`, between `testsys` and `add`, is not passed through to the + // varargs. After that, instances of `--` are passed through the varargs. + assert_eq!(args.additional_args[3], "--"); + assert_eq!(args.additional_args[4], "--name"); + assert_eq!(args.additional_args[5], "foo"); + assert_eq!(args.additional_args[6], "something=bar"); + assert_eq!(args.additional_args[7], "something-else=baz"); + assert_eq!(args.additional_args[8], "--"); + } + + const PROJECT: &str = "local-kit"; + + async fn twoliter_update(project_path: &Path) { + let command = Update { + project_path: Some(project_path.to_path_buf()), + }; + command.run().await.unwrap(); + } + + async fn run_makefile_target( + target: &str, + project_dir: &Path, + delete_verifier_tags: bool, + ) -> Result<()> { + let project_path = project_dir.join("Twoliter.toml"); + + twoliter_update(&project_path).await; + + let project = project::load_or_find_project(Some(project_path)) + .await + .unwrap(); + let sdk_source = LockedSDK::load(&project).await.unwrap().0.source; + + if delete_verifier_tags { + // Clean up tags so that the build fails + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()) + .await + .unwrap(); + } + + let toolsdir = project.project_dir().join("build/tools"); + install_tools(&toolsdir).await.unwrap(); + let makefile_path = toolsdir.join("Makefile.toml"); + + CargoMake::new(&sdk_source) + .unwrap() + .env("CARGO_HOME", project_dir.display().to_string()) + .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) + .env("BUILDSYS_VERSION_IMAGE", project.release_version()) + .makefile(makefile_path) + .project_dir(project.project_dir()) + .exec_with_args(target, Vec::<&'static str>::new()) + .await + } + + async fn target_can_skip_kit_verification(target_name: &str) -> bool { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + let project_dir = temp_dir.path(); + let project_path = project_dir.join("Twoliter.toml"); + let project = project::load_or_find_project(Some(project_path.clone())) + .await + .unwrap(); + + let make = Make { + project_path: Some(project_path), + cargo_home: project_dir.to_owned(), + arch: "x86_64".to_string(), + makefile_task: target_name.to_string(), + additional_args: Vec::new(), + }; + make.can_skip_kit_verification(&project) + } + + #[tokio::test] + async fn test_repack_variant_can_skip_kit_verification() { + assert!(target_can_skip_kit_verification("repack-variant").await); + } + + #[tokio::test] + async fn test_build_variant_cannot_skip_kit_verification() { + assert!(!target_can_skip_kit_verification("build-variant").await); + } + + #[tokio::test] + #[ignore] // integration test + async fn test_fetch_sdk_succeeds_when_only_sdk_verified() { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), false) + .await + .is_ok()); + } + + #[tokio::test] + #[ignore] // integration test + async fn test_fetch_sdk_fails_when_nothing_verified() { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + assert!(run_makefile_target("fetch-sdk", &temp_dir.path(), true) + .await + .is_err()); + } + + #[tokio::test] + #[ignore] // integration test + async fn test_validate_kits_fails_when_only_sdk_verified() { + let temp_dir = crate::test::copy_project_to_temp_dir(PROJECT); + assert!( + run_makefile_target("validate-kits", &temp_dir.path(), false) + .await + .is_err() + ); + } } diff --git a/twoliter/src/lock/image.rs b/twoliter/src/lock/image.rs index 558a3262f..4210cf1c6 100644 --- a/twoliter/src/lock/image.rs +++ b/twoliter/src/lock/image.rs @@ -3,7 +3,7 @@ use super::views::ManifestListView; use super::Override; use crate::common::fs::create_dir_all; use crate::docker::ImageUri; -use crate::project::{Image, Vendor}; +use crate::project::{Image, Project, ValidIdentifier}; use anyhow::{bail, Context, Result}; use base64::Engine; use futures::{pin_mut, stream, StreamExt, TryStreamExt}; @@ -207,13 +207,23 @@ pub struct ImageResolver { } impl ImageResolver { - pub(crate) fn from_image( - image: &Image, - vendor_name: &str, - vendor: &Vendor, - override_: Option<&Override>, - ) -> Self { - Self { + pub(crate) fn from_image(image: &Image, project: &Project) -> Result { + let vendor_name = image.vendor.0.as_str(); + let vendor = project.vendor().get(&image.vendor).context(format!( + "vendor '{}' is not specified in Twoliter.toml", + image.vendor + ))?; + let override_ = project + .overrides() + .get(&image.vendor.to_string()) + .and_then(|x| x.get(&image.name.to_string())); + if let Some(override_) = override_.as_ref() { + debug!( + ?override_, + "Found override for image '{}' with vendor '{}'", image.name, image.vendor + ); + } + Ok(Self { image_resolver_impl: if let Some(override_) = override_ { Box::new(OverriddenImage { base_uri: ImageUri { @@ -234,16 +244,31 @@ impl ImageResolver { }, }) }, - } + }) } - pub(crate) fn from_locked_image( - locked_image: &LockedImage, - vendor_name: &str, - vendor: &Vendor, - override_: Option<&Override>, - ) -> Self { - Self { + pub(crate) fn from_locked_image(locked_image: &LockedImage, project: &Project) -> Result { + let vendor_name = &locked_image.vendor; + let vendor = project + .vendor() + .get(&ValidIdentifier(vendor_name.clone())) + .context(format!( + "failed to find vendor for kit with name '{}' and vendor '{}'", + locked_image.name, locked_image.vendor + ))?; + let override_ = project + .overrides() + .get(&locked_image.vendor) + .and_then(|x| x.get(&locked_image.name)); + if let Some(override_) = override_.as_ref() { + debug!( + ?override_, + "Found override for image '{}' with vendor '{}'", + locked_image.name, + locked_image.vendor + ); + } + Ok(Self { image_resolver_impl: if let Some(override_) = override_ { Box::new(OverriddenImage { base_uri: ImageUri { @@ -264,7 +289,7 @@ impl ImageResolver { }, }) }, - } + }) } /// Calculate the digest of the locked image diff --git a/twoliter/src/lock/mod.rs b/twoliter/src/lock/mod.rs index 47f70e4f7..24532899c 100644 --- a/twoliter/src/lock/mod.rs +++ b/twoliter/src/lock/mod.rs @@ -7,9 +7,13 @@ pub mod archive; /// Covers resolution and validation of a single image dependency in a lock file pub mod image; +/// Provides tools for marking artifacts as having been verified against the Twoliter lockfile +pub mod verification; /// Implements view models of common OCI manifest and configuration types pub mod views; +pub(crate) use self::verification::VerificationTagger; + use crate::common::fs::{create_dir_all, read, write}; use crate::project::{Image, Project, ValidIdentifier}; use crate::schema_version::SchemaVersion; @@ -42,6 +46,66 @@ pub(crate) struct Override { pub registry: Option, } +/// A resolved and locked project SDK, typically from the Twoliter.lock file for a project. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct LockedSDK(pub LockedImage); + +impl AsRef for LockedSDK { + fn as_ref(&self) -> &LockedImage { + &self.0 + } +} + +impl LockedSDK { + /// Loads the locked SDK for the given project. + /// + /// Re-resolves the project's SDK to ensure that the lockfile matches the state of the world. + #[instrument(level = "trace", skip(project))] + pub(crate) async fn load(project: &Project) -> Result { + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()).await?; + + info!("Resolving project references to check against lock file"); + let resolved_lock = { + // bind `current_lock` in a block so that we can't accidentally pass it to the + // VerificationTagger. + let current_lock = Lock::current_lock_state(project).await?; + let resolved_lock = Self::resolve_sdk(project) + .await? + .context("Project does not have explicit SDK image.")?; + ensure!(¤t_lock.sdk == resolved_lock.as_ref(), "changes have occured to Twoliter.toml or the remote SDK image that require an update to Twoliter.lock"); + resolved_lock + }; + + VerificationTagger::from(&resolved_lock) + .write_tags(project.external_kits_dir()) + .await?; + + Ok(resolved_lock) + } + + /// Creates a project lock referring to only the resolved SDK image from the project. + /// + /// Returns `None` if the project does not have an explicit SDK image. + #[instrument(level = "trace", skip(project))] + async fn resolve_sdk(project: &Project) -> Result> { + debug!("Attempting to resolve workspace SDK"); + let sdk = match project.sdk_image() { + Some(sdk) => sdk, + None => { + debug!("No explicit SDK image provided"); + return Ok(None); + } + }; + + debug!(?sdk, "Resolving workspace SDK"); + let image_tool = ImageTool::from_environment()?; + ImageResolver::from_image(&sdk, project)? + .resolve(&image_tool, true) + .await + .map(|(sdk, _)| Some(Self(sdk))) + } +} + /// Represents the structure of a `Twoliter.lock` lock file. #[derive(Debug, Clone, Eq, Ord, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -79,8 +143,33 @@ impl Lock { Ok(lock_state) } + /// Loads the lockfile for the given project. + /// + /// Re-resolves the project's dependencies to ensure that the lockfile matches the state of the + /// world. #[instrument(level = "trace", skip(project))] pub(crate) async fn load(project: &Project) -> Result { + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()).await?; + + info!("Resolving project references to check against lock file"); + let resolved_lock = { + // bind `current_lock` in a block so that we can't accidentally pass it to the + // VerificationTagger. + let current_lock = Self::current_lock_state(project).await?; + let resolved_lock = Self::resolve(project).await?; + ensure!(current_lock == resolved_lock, "changes have occured to Twoliter.toml or the remote kit images that require an update to Twoliter.lock"); + resolved_lock + }; + + VerificationTagger::from(&resolved_lock) + .write_tags(project.external_kits_dir()) + .await?; + + Ok(resolved_lock) + } + + /// Returns the state of the lockfile for the given `Project` + async fn current_lock_state(project: &Project) -> Result { let lock_file_path = project.project_dir().join(TWOLITER_LOCK); ensure!( lock_file_path.exists(), @@ -92,10 +181,6 @@ impl Lock { .context("failed to read lockfile")?; let lock: Self = toml::from_str(lock_str.as_str()).context("failed to deserialize lockfile")?; - info!("Resolving project references to check against lock file"); - let lock_state = Self::resolve(project).await?; - - ensure!(lock_state == lock, "changes have occured to Twoliter.toml or the remote kit images that require an update to Twoliter.lock"); Ok(lock) } @@ -121,19 +206,7 @@ impl Lock { "Extracting kit dependencies." ); for image in self.kit.iter() { - let vendor = project - .vendor() - .get(&ValidIdentifier(image.vendor.clone())) - .context(format!( - "failed to find vendor for kit with name '{}' and vendor '{}'", - image.name, image.vendor - ))?; - let override_ = project - .overrides() - .get(&image.vendor) - .and_then(|x| x.get(&image.name)); - let resolver = - ImageResolver::from_locked_image(image, image.vendor.as_str(), vendor, override_); + let resolver = ImageResolver::from_locked_image(image, project)?; resolver .extract(&image_tool, &project.external_kits_dir(), arch) .await?; @@ -172,12 +245,11 @@ impl Lock { #[instrument(level = "trace", skip(project))] async fn resolve(project: &Project) -> Result { - let vendor_table = project.vendor(); let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); let image_tool = ImageTool::from_environment()?; - let overrides = project.overrides(); let mut remaining: Vec = project.kits(); + let mut sdk_set: HashSet = HashSet::new(); if let Some(sdk) = project.sdk_image() { // We don't scan over the sdk images as they are not kit images and there is no kit metadata to fetch @@ -193,7 +265,8 @@ impl Lock { let vendor = image.vendor.clone(); ensure!( image.version == *version, - "cannot have multiple versions of the same kit ({name}-{left_version}@{vendor} != {name}-{version}@{vendor}", + "cannot have multiple versions of the same kit ({name}-{left_version}@{vendor} \ + != {name}-{version}@{vendor}", ); debug!( ?image, @@ -201,25 +274,16 @@ impl Lock { ); continue; } - let vendor = vendor_table.get(&image.vendor).context(format!( + ensure!( + project.vendor().get(&image.vendor).is_some(), "vendor '{}' is not specified in Twoliter.toml", image.vendor - ))?; + ); known.insert( (image.name.clone(), image.vendor.clone()), image.version.clone(), ); - let override_ = overrides - .get(&image.vendor.to_string()) - .and_then(|x| x.get(&image.name.to_string())); - if let Some(override_) = override_.as_ref() { - debug!( - ?override_, - "Found override for kit '{}' with vendor '{}'", image.name, image.vendor - ); - } - let image_resolver = - ImageResolver::from_image(image, image.vendor.0.as_str(), vendor, override_); + let image_resolver = ImageResolver::from_image(image, project)?; let (locked_image, metadata) = image_resolver.resolve(&image_tool, false).await?; let metadata = metadata.context(format!( "failed to validate kit image with name {} from vendor {}", @@ -232,7 +296,6 @@ impl Lock { } } } - debug!(?sdk_set, "Resolving workspace SDK"); ensure!( sdk_set.len() <= 1, @@ -247,20 +310,16 @@ impl Lock { .iter() .next() .context("no sdk was found for use, please specify a sdk in Twoliter.toml")?; - let vendor = vendor_table.get(&sdk.vendor).context(format!( - "vendor '{}' is not specified in Twoliter.toml", - sdk.vendor - ))?; - let sdk_override = overrides - .get(&sdk.vendor.to_string()) - .and_then(|x| x.get(&sdk.name.to_string())); - let sdk_resolver = - ImageResolver::from_image(sdk, sdk.vendor.0.as_str(), vendor, sdk_override); - let (sdk, _) = sdk_resolver.resolve(&image_tool, true).await?; + + debug!(?sdk, "Resolving workspace SDK"); + let (sdk, _metadata) = ImageResolver::from_image(sdk, project)? + .resolve(&image_tool, true) + .await?; + Ok(Self { schema_version: project.schema_version(), - sdk, kit: locked, + sdk, }) } } diff --git a/twoliter/src/lock/verification.rs b/twoliter/src/lock/verification.rs new file mode 100644 index 000000000..4ad01668c --- /dev/null +++ b/twoliter/src/lock/verification.rs @@ -0,0 +1,260 @@ +//! This module contains utilities for marking that certain Twoliter artifacts have been resolved +//! and verified against a project's Lockfile. +//! +//! An overview of the contained abstractions: +//! * The [`LockfileVerifier`] trait allows a type to announce that it has resolved and verified +//! a set of artifacts. +//! * Verified artifacts are identified via a [`VerifyTag`]. +//! * Each [`VerifyTag`] has a [`VerificationManifest`] containing a list of the verified artifacts +//! of that tag type. +//! * The [`VerificationTagger`] writes files containing [`VerifyTag`]s that are produced by +//! [`LockfileVerifier`]s. +use super::image::LockedImage; +use super::{Lock, LockedSDK}; +use anyhow::{Context, Result}; +use olpc_cjson::CanonicalFormatter as CanonicalJsonFormatter; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::fmt::Debug; +use std::path::Path; +use strum::{EnumIter, IntoEnumIterator}; +use tracing::{debug, instrument}; + +const SDK_VERIFIED_MARKER_FILE: &str = ".sdk-verified"; +const KITS_VERIFIED_MARKER_FILE: &str = ".kits-verified"; + +/// A tag indicating that Twoliter artifacts have been resolved and verified against the lockfile +#[derive(Debug, PartialEq, Eq, Ord, PartialOrd, EnumIter)] +pub(crate) enum VerifyTag { + Sdk(VerificationManifest), + Kits(VerificationManifest), +} + +impl VerifyTag { + /// Returns the marker file marking an artifact type that has been verified against the lock + pub(crate) fn marker_file_name(&self) -> &'static str { + match self { + VerifyTag::Sdk(_) => SDK_VERIFIED_MARKER_FILE, + VerifyTag::Kits(_) => KITS_VERIFIED_MARKER_FILE, + } + } + + pub(crate) fn manifest(&self) -> &VerificationManifest { + match self { + VerifyTag::Sdk(manifest) => manifest, + VerifyTag::Kits(manifest) => manifest, + } + } +} + +/// A manifest containing the list of elements that were verified by a `LockfileVerifier` +#[derive(Debug, Clone, Default, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] +#[serde(transparent)] +pub(crate) struct VerificationManifest { + verified_images: BTreeSet, +} + +impl VerificationManifest { + fn as_canonical_json(&self) -> Result> { + let mut manifest = Vec::new(); + let mut ser = + serde_json::Serializer::with_formatter(&mut manifest, CanonicalJsonFormatter::new()); + self.serialize(&mut ser) + .context("failed to serialize external kit metadata")?; + Ok(manifest) + } +} + +impl From<&LockedImage> for VerificationManifest { + fn from(image: &LockedImage) -> Self { + [image].as_slice().into() + } +} + +impl From<&[&LockedImage]> for VerificationManifest { + fn from(images: &[&LockedImage]) -> Self { + Self { + verified_images: images.iter().map(ToString::to_string).collect(), + } + } +} + +/// A `LockfileVerifier` can return a set of `VerifyTag` structs, claiming that those artifacts +/// have been resolved and verified against the lockfile. +pub(crate) trait LockfileVerifier { + fn verified(&self) -> BTreeSet; +} + +impl LockfileVerifier for LockedSDK { + fn verified(&self) -> BTreeSet { + [VerifyTag::Sdk((&self.0).into())].into() + } +} + +impl LockfileVerifier for Lock { + fn verified(&self) -> BTreeSet { + [ + VerifyTag::Sdk((&self.sdk).into()), + VerifyTag::Kits(self.kit.iter().collect::>().as_slice().into()), + ] + .into() + } +} + +/// Writes marker files indicating which artifacts have been resolved and verified against the lock +#[derive(Debug)] +pub(crate) struct VerificationTagger { + tags: BTreeSet, +} + +impl From<&V> for VerificationTagger { + fn from(resolver: &V) -> Self { + Self { + tags: resolver.verified(), + } + } +} + +impl VerificationTagger { + /// Creates marker files for artifacts that have been verified against the lockfile + #[instrument(level = "trace", skip(external_kits_dir))] + pub(crate) async fn write_tags>(&self, external_kits_dir: P) -> Result<()> { + let external_kits_dir = external_kits_dir.as_ref(); + Self::cleanup_existing_tags(&external_kits_dir).await?; + + debug!("Writing tag files for verified artifacts"); + tokio::fs::create_dir_all(&external_kits_dir) + .await + .context(format!( + "failed to create external-kits directory at '{}'", + external_kits_dir.display() + ))?; + + for tag in self.tags.iter() { + let flag_file = external_kits_dir.join(tag.marker_file_name()); + debug!( + "Writing tag file for verified artifacts: '{}'", + flag_file.display() + ); + tokio::fs::write(&flag_file, tag.manifest().as_canonical_json()?) + .await + .context(format!( + "failed to write verification tag file: '{}'", + flag_file.display() + ))?; + } + Ok(()) + } + + /// Deletes any existing verifier marker files in the kits directory + #[instrument(level = "trace", skip(external_kits_dir))] + pub(crate) async fn cleanup_existing_tags>(external_kits_dir: P) -> Result<()> { + let external_kits_dir = external_kits_dir.as_ref(); + + debug!("Cleaning up any existing tag files for resolved artifacts",); + for resolve_tag in VerifyTag::iter() { + let flag_file = external_kits_dir.join(resolve_tag.marker_file_name()); + if flag_file.exists() { + debug!( + "Removing existing verification tag file '{}'", + flag_file.display() + ); + tokio::fs::remove_file(&flag_file).await.context(format!( + "failed to remove existing verification tag file: {}", + flag_file.display() + ))?; + } + } + + Ok(()) + } +} + +#[cfg(test)] +mod test { + use super::*; + struct SDKResolver; + + impl LockfileVerifier for SDKResolver { + fn verified(&self) -> BTreeSet { + [VerifyTag::Sdk(VerificationManifest { + verified_images: ["image1".into(), "image2".into()].into(), + })] + .into() + } + } + + struct KitResolver; + + impl LockfileVerifier for KitResolver { + fn verified(&self) -> BTreeSet { + [ + VerifyTag::Sdk(VerificationManifest { + verified_images: ["image1".into(), "image2".into()].into(), + }), + VerifyTag::Kits(VerificationManifest { + verified_images: ["kit1".into(), "kit2".into()].into(), + }), + ] + .into() + } + } + + #[tokio::test] + async fn test_cleanup_existing_tags() { + let kits_dir = tempfile::tempdir().unwrap(); + let flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + tokio::fs::write(&flag_file, "test").await.unwrap(); + + VerificationTagger::cleanup_existing_tags(&kits_dir.path()) + .await + .unwrap(); + assert!(!flag_file.exists()); + } + + #[tokio::test] + async fn test_write_sdk_tags() { + let kits_dir = tempfile::tempdir().unwrap(); + let tagger = VerificationTagger::from(&SDKResolver); + tagger.write_tags(&kits_dir.path()).await.unwrap(); + + let flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + assert!(flag_file.exists()); + let contents = tokio::fs::read_to_string(&flag_file).await.unwrap(); + assert_eq!(contents, r#"["image1","image2"]"#); + } + + #[tokio::test] + async fn test_write_kit_tags() { + let kits_dir = tempfile::tempdir().unwrap(); + let tagger = VerificationTagger::from(&KitResolver); + tagger.write_tags(&kits_dir.path()).await.unwrap(); + + let sdk_flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + assert!(sdk_flag_file.exists()); + let sdk_contents = tokio::fs::read_to_string(&sdk_flag_file).await.unwrap(); + assert_eq!(sdk_contents, r#"["image1","image2"]"#); + + let kit_flag_file = kits_dir.path().join(KITS_VERIFIED_MARKER_FILE); + assert!(kit_flag_file.exists()); + let kit_contents = tokio::fs::read_to_string(&kit_flag_file).await.unwrap(); + assert_eq!(kit_contents, r#"["kit1","kit2"]"#); + } + + #[tokio::test] + async fn test_previous_tags_removed() { + let kits_dir = tempfile::tempdir().unwrap(); + let flag_file = kits_dir.path().join(KITS_VERIFIED_MARKER_FILE); + tokio::fs::write(&flag_file, "test").await.unwrap(); + + let tagger = VerificationTagger::from(&SDKResolver); + tagger.write_tags(&kits_dir.path()).await.unwrap(); + + assert!(!flag_file.exists()); + + let sdk_flag_file = kits_dir.path().join(SDK_VERIFIED_MARKER_FILE); + assert!(sdk_flag_file.exists()); + let sdk_contents = tokio::fs::read_to_string(&sdk_flag_file).await.unwrap(); + assert_eq!(sdk_contents, r#"["image1","image2"]"#); + } +} diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index 5904a1f0c..824937179 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -1,6 +1,6 @@ use crate::common::fs::{self, read_to_string}; use crate::docker::ImageUri; -use crate::lock::Override; +use crate::lock::{Override, VerificationTagger}; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; @@ -71,7 +71,15 @@ impl Project { "Unable to deserialize project file '{}'", path.display() ))?; - unvalidated.validate(path).await + let project = unvalidated.validate(path).await?; + + // When projects are resolved, tags are written indicating which artifacts have been checked + // against the lockfile. + // We clean these up as early as possible to avoid situations in which artifacts are + // incorrectly flagged as having been resolved. + VerificationTagger::cleanup_existing_tags(project.external_kits_dir()).await?; + + Ok(project) } /// Recursively search for a file named `Twoliter.toml` starting in `dir`. If it is not found, @@ -152,7 +160,7 @@ impl Project { ))?; Ok(Some(ImageUri::new( Some(vendor.registry.clone()), - kit.name.to_string(), + &kit.name, format!("v{}", kit.version), ))) } else { @@ -253,6 +261,12 @@ impl<'de> Deserialize<'de> for ValidIdentifier { } } +impl AsRef for ValidIdentifier { + fn as_ref(&self) -> &str { + self.0.as_str() + } +} + impl Display for ValidIdentifier { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.write_str(self.0.as_str())