diff --git a/tests/projects/local-kit/Twoliter.override b/tests/projects/local-kit/Twoliter.override new file mode 100644 index 000000000..297482244 --- /dev/null +++ b/tests/projects/local-kit/Twoliter.override @@ -0,0 +1,2 @@ +[bottlerocket.bottlerocket-sdk] +registry = "public.ecr.aws/bottlerocket" diff --git a/tests/projects/local-kit/Twoliter.toml b/tests/projects/local-kit/Twoliter.toml index b0d76c7bb..c1ed993be 100644 --- a/tests/projects/local-kit/Twoliter.toml +++ b/tests/projects/local-kit/Twoliter.toml @@ -7,4 +7,6 @@ vendor = "bottlerocket" version = "0.41.0" [vendor.bottlerocket] -registry = "public.ecr.aws/bottlerocket" +# This test will fail if Twoliter overrides aren't working +# See `Twoliter.override` +registry = "this-definitely-wont-resolve/bottlerocket" diff --git a/tools/oci-cli-wrapper/src/lib.rs b/tools/oci-cli-wrapper/src/lib.rs index b673ec72d..81a64691b 100644 --- a/tools/oci-cli-wrapper/src/lib.rs +++ b/tools/oci-cli-wrapper/src/lib.rs @@ -142,7 +142,7 @@ impl ImageTool { } #[async_trait] -pub trait ImageToolImpl: std::fmt::Debug { +pub trait ImageToolImpl: std::fmt::Debug + Send + Sync + 'static { /// Pull an image archive to disk async fn pull_oci_image(&self, path: &Path, uri: &str) -> Result<()>; /// Fetch the image config diff --git a/twoliter/src/cmd/build.rs b/twoliter/src/cmd/build.rs index 3efb4dae9..b75e836d9 100644 --- a/twoliter/src/cmd/build.rs +++ b/twoliter/src/cmd/build.rs @@ -1,7 +1,7 @@ use super::build_clean::BuildClean; use crate::cargo_make::CargoMake; use crate::common::fs; -use crate::project; +use crate::project::{self, Locked}; use crate::tools::install_tools; use anyhow::{Context, Result}; use clap::Parser; @@ -52,7 +52,7 @@ pub(crate) struct BuildKit { impl BuildKit { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock = project.load_lock().await?; + let project = project.load_lock::().await?; let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); @@ -63,7 +63,7 @@ impl BuildKit { optional_envs.push(("BUILDSYS_LOOKASIDE_CACHE", lookaside_cache)) } - CargoMake::new(&lock.sdk.source)? + CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_ARCH", &self.arch) .env("BUILDSYS_KIT", &self.kit) @@ -112,7 +112,7 @@ pub(crate) struct BuildVariant { impl BuildVariant { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock = project.load_lock().await?; + let project = project.load_lock::().await?; let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); @@ -135,7 +135,7 @@ impl BuildVariant { )) } - CargoMake::new(&lock.sdk.source)? + CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_ARCH", &self.arch) .env("BUILDSYS_VARIANT", &self.variant) diff --git a/twoliter/src/cmd/build_clean.rs b/twoliter/src/cmd/build_clean.rs index 9aa3140c2..4707b62bf 100644 --- a/twoliter/src/cmd/build_clean.rs +++ b/twoliter/src/cmd/build_clean.rs @@ -1,5 +1,5 @@ use crate::cargo_make::CargoMake; -use crate::project; +use crate::project::{self, Locked}; use crate::tools; use anyhow::Result; use clap::Parser; @@ -15,12 +15,12 @@ pub(crate) struct BuildClean { impl BuildClean { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock = project.load_lock().await?; + let project = project.load_lock::().await?; let toolsdir = project.project_dir().join("build/tools"); tools::install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); - CargoMake::new(&lock.sdk.source)? + CargoMake::new(&project.sdk_image().project_image_uri().to_string())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .makefile(makefile_path) .project_dir(project.project_dir()) diff --git a/twoliter/src/cmd/fetch.rs b/twoliter/src/cmd/fetch.rs index 089d2f49b..1e3c99183 100644 --- a/twoliter/src/cmd/fetch.rs +++ b/twoliter/src/cmd/fetch.rs @@ -1,4 +1,4 @@ -use crate::project; +use crate::project::{self, Locked}; use anyhow::Result; use clap::Parser; use std::path::PathBuf; @@ -17,8 +17,8 @@ pub(crate) struct Fetch { impl Fetch { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock_file = project.load_lock().await?; - lock_file.fetch(&project, self.arch.as_str()).await?; + let project = project.load_lock::().await?; + project.fetch(self.arch.as_str()).await?; Ok(()) } } diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index 1a4bfb028..23b80ccf6 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -1,5 +1,5 @@ use crate::cargo_make::CargoMake; -use crate::project::{self}; +use crate::project::{self, Locked, SDKLocked, Unlocked}; use crate::tools::install_tools; use anyhow::Result; use clap::Parser; @@ -68,24 +68,23 @@ impl Make { .await } - fn can_skip_kit_verification(&self, project: &project::Project) -> bool { + fn can_skip_kit_verification(&self, project: &project::Project) -> bool { let target_allows_kit_verification_skip = !MUST_VALIDATE_KITS_TARGETS.contains(&self.makefile_task.as_str()); - let project_has_explicit_sdk_dep = project.sdk_image().is_some(); + let project_has_explicit_sdk_dep = project.direct_sdk_image_dep().is_some(); target_allows_kit_verification_skip && project_has_explicit_sdk_dep } /// Returns the locked SDK image for the project. - async fn locked_sdk(&self, project: &project::Project) -> Result { - let sdk_source = if self.can_skip_kit_verification(project) { - let lock = project.load_locked_sdk().await?; - lock.0.source + async fn locked_sdk(&self, project: &project::Project) -> Result { + Ok(if self.can_skip_kit_verification(project) { + project.load_lock::().await?.sdk_image() } else { - let lock = project.load_lock().await?; - lock.sdk.source - }; - Ok(sdk_source) + project.load_lock::().await?.sdk_image() + } + .project_image_uri() + .to_string()) } } @@ -214,7 +213,8 @@ mod test { let project = project::load_or_find_project(Some(project_path)) .await .unwrap(); - let sdk_source = project.load_locked_sdk().await.unwrap().0.source; + let project = project.load_lock::().await.unwrap(); + let sdk_source = project.sdk_image().project_image_uri().to_string(); if delete_verifier_tags { // Clean up tags so that the build fails diff --git a/twoliter/src/cmd/publish_kit.rs b/twoliter/src/cmd/publish_kit.rs index f223fb9f7..085f60d09 100644 --- a/twoliter/src/cmd/publish_kit.rs +++ b/twoliter/src/cmd/publish_kit.rs @@ -1,5 +1,5 @@ use crate::cargo_make::CargoMake; -use crate::project; +use crate::project::{self, Locked}; use crate::tools::install_tools; use anyhow::Result; use clap::Parser; @@ -36,12 +36,12 @@ pub(crate) struct PublishKit { impl PublishKit { pub(super) async fn run(&self) -> Result<()> { let project = project::load_or_find_project(self.project_path.clone()).await?; - let lock = project.load_lock().await?; + let project = project.load_lock::().await?; let toolsdir = project.project_dir().join("build/tools"); install_tools(&toolsdir).await?; let makefile_path = toolsdir.join("Makefile.toml"); - CargoMake::new(&lock.sdk.source)? + CargoMake::new(project.sdk_image().project_image_uri().to_string().as_str())? .env("TWOLITER_TOOLS_DIR", toolsdir.display().to_string()) .env("BUILDSYS_KIT", &self.kit_name) .env("BUILDSYS_VERSION_IMAGE", project.release_version()) diff --git a/twoliter/src/project/lock/image.rs b/twoliter/src/project/lock/image.rs index e243e25be..81dafa5b2 100644 --- a/twoliter/src/project/lock/image.rs +++ b/twoliter/src/project/lock/image.rs @@ -1,12 +1,7 @@ use super::archive::OCIArchive; use super::views::ManifestListView; -use super::Override; use crate::common::fs::create_dir_all; -use crate::docker::ImageUri; -use crate::project::{ - ArtifactVendor, Image, OverriddenVendor, Project, ValidIdentifier, VendedArtifact, - VerbatimVendor, -}; +use crate::project::{Image, ProjectImage, ValidIdentifier, VendedArtifact}; use anyhow::{bail, Context, Result}; use base64::Engine; use futures::{pin_mut, stream, StreamExt, TryStreamExt}; @@ -141,177 +136,16 @@ impl Debug for EncodedKitMetadata { } } -pub trait ImageResolverImpl: Debug { - fn name(&self) -> String; - fn version(&self) -> Result; - fn vendor(&self) -> String; - fn source(&self) -> String; - fn uri(&self) -> ImageUri; -} - -#[derive(Debug)] -pub struct VerbatimImage { - uri: ImageUri, - vendor: String, -} - -impl VerbatimImage { - fn from_vendor(artifact: &V, vendor: VerbatimVendor) -> Self { - let vendor_name = &vendor.vendor_name; - let vendor = vendor.vendor; - - VerbatimImage { - vendor: vendor_name.as_ref().to_string(), - uri: ImageUri { - registry: Some(vendor.registry.clone()), - repo: artifact.artifact_name().to_string(), - tag: format!("v{}", artifact.version()), - }, - } - } -} - -impl ImageResolverImpl for VerbatimImage { - fn name(&self) -> String { - self.uri.repo.clone() - } - - fn version(&self) -> Result { - Version::parse(self.uri.tag.trim_start_matches('v')).context("invalid version tag") - } - - fn source(&self) -> String { - self.uri.to_string() - } - - fn vendor(&self) -> String { - self.vendor.clone() - } - - fn uri(&self) -> ImageUri { - self.uri.clone() - } -} - -#[derive(Debug)] -pub struct OverriddenImage { - base_uri: ImageUri, - vendor: String, - override_: Override, -} - -impl OverriddenImage { - fn from_vendor(artifact: &V, vendor: OverriddenVendor) -> Self { - let original_vendor = vendor.original_vendor; - - OverriddenImage { - base_uri: ImageUri { - registry: Some(original_vendor.registry.clone()), - repo: artifact.artifact_name().to_string(), - tag: format!("v{}", artifact.version()), - }, - vendor: vendor.original_vendor_name.as_ref().to_string(), - override_: vendor.override_.clone(), - } - } -} - -impl ImageResolverImpl for OverriddenImage { - fn name(&self) -> String { - self.base_uri.repo.clone() - } - - fn version(&self) -> Result { - Version::parse(self.base_uri.tag.trim_start_matches('v')).context("invalid version tag") - } - - fn source(&self) -> String { - self.base_uri.to_string() - } - - fn vendor(&self) -> String { - self.vendor.clone() - } - - fn uri(&self) -> ImageUri { - ImageUri { - registry: self - .override_ - .registry - .clone() - .or(self.base_uri.registry.clone()), - repo: self - .override_ - .name - .clone() - .unwrap_or(self.base_uri.repo.clone()), - tag: self.base_uri.tag.clone(), - } - } -} - #[derive(Debug)] pub struct ImageResolver { - image_resolver_impl: Box, + image: ProjectImage, skip_metadata_retrieval: bool, } impl ImageResolver { - pub(crate) fn from_image(image: &Image, project: &Project) -> Result { - let vendor = project.vendor_for(image).with_context(|| { - format!( - "failed to find vendor for image with name '{}' and vendor '{}'", - image.name, image.vendor, - ) - })?; - - let image_resolver_impl = match vendor { - ArtifactVendor::Verbatim(vendor) => { - Box::new(VerbatimImage::from_vendor(image, vendor)) as Box - } - - ArtifactVendor::Overridden(vendor) => { - debug!( - vendor_override = ?vendor, - "Found override for image '{}' with vendor '{}'", image.name, image.vendor - ); - Box::new(OverriddenImage::from_vendor(image, vendor)) as Box - } - }; - - Ok(Self { - image_resolver_impl, - skip_metadata_retrieval: false, - }) - } - - pub(crate) fn from_locked_image(locked_image: &LockedImage, project: &Project) -> Result { - let vendor = project.vendor_for(locked_image).with_context(|| { - format!( - "failed to find vendor for image with name '{}' and vendor '{}'", - locked_image.name, locked_image.vendor, - ) - })?; - - let image_resolver_impl = match vendor { - ArtifactVendor::Verbatim(vendor) => { - Box::new(VerbatimImage::from_vendor(locked_image, vendor)) - as Box - } - ArtifactVendor::Overridden(vendor) => { - debug!( - vendor_override = ?vendor, - "Found override for image '{}' with vendor '{}'", - locked_image.name, - locked_image.vendor - ); - Box::new(OverriddenImage::from_vendor(locked_image, vendor)) - as Box - } - }; - + pub(crate) fn from_image(image: &ProjectImage) -> Result { Ok(Self { - image_resolver_impl, + image: image.clone(), skip_metadata_retrieval: false, }) } @@ -326,7 +160,7 @@ impl ImageResolver { /// Calculate the digest of the locked image async fn calculate_digest(&self, image_tool: &ImageTool) -> Result { - let image_uri = self.image_resolver_impl.uri(); + let image_uri = self.image.project_image_uri(); let image_uri_str = image_uri.to_string(); let manifest_bytes = image_tool.get_manifest(image_uri_str.as_str()).await?; let digest = sha2::Sha256::digest(manifest_bytes.as_slice()); @@ -340,7 +174,7 @@ impl ImageResolver { } async fn get_manifest(&self, image_tool: &ImageTool) -> Result { - let uri = self.image_resolver_impl.uri().to_string(); + let uri = self.image.project_image_uri().to_string(); let manifest_bytes = image_tool.get_manifest(uri.as_str()).await?; serde_json::from_slice(manifest_bytes.as_slice()) .context("failed to deserialize manifest list") @@ -351,7 +185,7 @@ impl ImageResolver { image_tool: &ImageTool, ) -> Result<(LockedImage, Option)> { // First get the manifest list - let uri = self.image_resolver_impl.uri(); + let uri = self.image.project_image_uri(); let manifest_list = self.get_manifest(image_tool).await?; let registry = uri .registry @@ -359,11 +193,11 @@ impl ImageResolver { .context("no registry found for image")?; let locked_image = LockedImage { - name: self.image_resolver_impl.name().parse()?, - version: self.image_resolver_impl.version()?, - vendor: self.image_resolver_impl.vendor().parse()?, + name: self.image.name().to_owned(), + version: self.image.version().to_owned(), + vendor: self.image.vendor_name().to_owned(), // The source is the image uri without the tag, which is the digest - source: self.image_resolver_impl.source(), + source: self.image.original_source_uri().to_string(), digest: self.calculate_digest(image_tool).await?, }; @@ -407,7 +241,7 @@ impl ImageResolver { #[instrument( level = "trace", - fields(uri = %self.image_resolver_impl.uri(), path = %path.as_ref().display()) + fields(uri = %self.image.project_image_uri(), path = %path.as_ref().display()) )] pub(crate) async fn extract

(&self, image_tool: &ImageTool, path: P, arch: &str) -> Result<()> where @@ -415,20 +249,20 @@ impl ImageResolver { { info!( "Extracting kit '{}' to '{}'", - self.image_resolver_impl.name(), + self.image.name(), path.as_ref().display() ); let target_path = path.as_ref().join(format!( "{}/{}/{arch}", - self.image_resolver_impl.vendor(), - self.image_resolver_impl.name() + self.image.vendor_name(), + self.image.name() )); let cache_path = path.as_ref().join("cache"); create_dir_all(&target_path).await?; create_dir_all(&cache_path).await?; // First get the manifest for the specific requested architecture - let uri = self.image_resolver_impl.uri(); + let uri = self.image.project_image_uri(); let manifest_list = self.get_manifest(image_tool).await?; let docker_arch = DockerArchitecture::try_from(arch)?; let manifest = manifest_list diff --git a/twoliter/src/project/lock/mod.rs b/twoliter/src/project/lock/mod.rs index c6a430ef6..7c85f583e 100644 --- a/twoliter/src/project/lock/mod.rs +++ b/twoliter/src/project/lock/mod.rs @@ -15,7 +15,7 @@ mod views; pub(crate) use self::verification::VerificationTagger; use crate::common::fs::{create_dir_all, read, write}; -use crate::project::{Image, Project, ValidIdentifier}; +use crate::project::{Project, ValidIdentifier}; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use image::{ImageResolver, LockedImage}; @@ -30,6 +30,8 @@ use std::mem::take; use tokio::fs::read_to_string; use tracing::{debug, info, instrument}; +use super::{Locked, ProjectLock, Unlocked}; + const TWOLITER_LOCK: &str = "Twoliter.lock"; #[derive(Serialize, Debug)] @@ -39,7 +41,7 @@ struct ExternalKitMetadata { kits: Vec, } -#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Hash)] #[serde(rename_all = "kebab-case")] pub(crate) struct Override { pub name: Option, @@ -61,7 +63,7 @@ impl LockedSDK { /// /// Re-resolves the project's SDK to ensure that the lockfile matches the state of the world. #[instrument(level = "trace", skip(project))] - pub(super) async fn load(project: &Project) -> Result { + pub(super) async fn load(project: &Project) -> Result { info!("Resolving project references to check against lock file"); let current_lock = Lock::current_lock_state(project).await?; @@ -77,10 +79,10 @@ impl LockedSDK { /// /// Returns `None` if the project does not have an explicit SDK image. #[instrument(level = "trace", skip(project))] - async fn resolve_sdk(project: &Project) -> Result> { + async fn resolve_sdk(project: &Project) -> Result> { debug!("Attempting to resolve workspace SDK"); - let sdk = match project.sdk_image() { - Some(sdk) => sdk, + let sdk = match project.direct_sdk_image_dep() { + Some(sdk) => sdk?, None => { debug!("No explicit SDK image provided"); return Ok(None); @@ -89,7 +91,7 @@ impl LockedSDK { debug!(?sdk, "Resolving workspace SDK"); let image_tool = ImageTool::from_environment()?; - ImageResolver::from_image(&sdk, project)? + ImageResolver::from_image(&sdk)? .skip_metadata_retrieval() // SDKs don't have metadata .resolve(&image_tool) .await @@ -120,7 +122,7 @@ impl PartialEq for Lock { #[allow(dead_code)] impl Lock { #[instrument(level = "trace", skip(project))] - pub(super) async fn create(project: &Project) -> Result { + pub(super) async fn create(project: &Project) -> Result { let lock_file_path = project.project_dir().join(TWOLITER_LOCK); info!("Resolving project references to create lock file"); @@ -139,7 +141,7 @@ impl Lock { /// Re-resolves the project's dependencies to ensure that the lockfile matches the state of the /// world. #[instrument(level = "trace", skip(project))] - pub(super) async fn load(project: &Project) -> Result { + pub(super) async fn load(project: &Project) -> Result { info!("Resolving project references to check against lock file"); let current_lock = Self::current_lock_state(project).await?; @@ -150,7 +152,7 @@ impl Lock { } /// Returns the state of the lockfile for the given `Project` - async fn current_lock_state(project: &Project) -> Result { + async fn current_lock_state(project: &Project) -> Result { let lock_file_path = project.project_dir().join(TWOLITER_LOCK); ensure!( lock_file_path.exists(), @@ -174,7 +176,7 @@ impl Lock { /// Fetches all external kits defined in a Twoliter.lock to the build directory #[instrument(level = "trace", skip_all)] - pub(crate) async fn fetch(&self, project: &Project, arch: &str) -> Result<()> { + pub(crate) async fn fetch(&self, project: &Project, arch: &str) -> Result<()> { let image_tool = ImageTool::from_environment()?; let target_dir = project.external_kits_dir(); create_dir_all(&target_dir).await.context(format!( @@ -187,7 +189,8 @@ impl Lock { "Extracting kit dependencies." ); for image in self.kit.iter() { - let resolver = ImageResolver::from_locked_image(image, project)?; + let image = project.as_project_image(image)?; + let resolver = ImageResolver::from_image(&image)?; resolver .extract(&image_tool, &project.external_kits_dir(), arch) .await?; @@ -196,7 +199,7 @@ impl Lock { self.synchronize_metadata(project).await } - pub(crate) async fn synchronize_metadata(&self, project: &Project) -> Result<()> { + pub(crate) async fn synchronize_metadata(&self, project: &Project) -> Result<()> { let mut kit_list = Vec::new(); let mut ser = serde_json::Serializer::with_formatter(&mut kit_list, CanonicalJsonFormatter::new()); @@ -225,55 +228,53 @@ impl Lock { } #[instrument(level = "trace", skip(project))] - async fn resolve(project: &Project) -> Result { + async fn resolve(project: &Project) -> Result { let mut known: HashMap<(ValidIdentifier, ValidIdentifier), Version> = HashMap::new(); let mut locked: Vec = Vec::new(); let image_tool = ImageTool::from_environment()?; - let mut remaining: Vec = project.kits(); + let mut remaining = project.direct_kit_deps()?; - let mut sdk_set: HashSet = HashSet::new(); - if let Some(sdk) = project.sdk_image() { + let mut sdk_set = HashSet::new(); + if let Some(sdk) = project.direct_sdk_image_dep() { // We don't scan over the sdk images as they are not kit images and there is no kit metadata to fetch - sdk_set.insert(sdk.clone()); + sdk_set.insert(sdk?.clone()); } while !remaining.is_empty() { let working_set: Vec<_> = take(&mut remaining); for image in working_set.iter() { - debug!(%image, "Resolving kit '{}'", image.name); - if let Some(version) = known.get(&(image.name.clone(), image.vendor.clone())) { - let name = image.name.clone(); - let left_version = image.version.clone(); - let vendor = image.vendor.clone(); + debug!(%image, "Resolving kit '{}'", image.name()); + if let Some(version) = + known.get(&(image.name().clone(), image.vendor_name().clone())) + { + let name = image.name().clone(); + let left_version = image.version().clone(); + let vendor = image.vendor_name().clone(); ensure!( - image.version == *version, + image.version() == version, "cannot have multiple versions of the same kit ({name}-{left_version}@{vendor} \ != {name}-{version}@{vendor}", ); debug!( ?image, - "Skipping kit '{}' as it has already been resolved", image.name + "Skipping kit '{}' as it has already been resolved", + image.name() ); continue; } - ensure!( - project.vendor_for(image).is_some(), - "vendor '{}' is not specified in Twoliter.toml", - image.vendor - ); known.insert( - (image.name.clone(), image.vendor.clone()), - image.version.clone(), + (image.name().clone(), image.vendor_name().clone()), + image.version().clone(), ); - let image_resolver = ImageResolver::from_image(image, project)?; + let image_resolver = ImageResolver::from_image(image)?; let (locked_image, metadata) = image_resolver.resolve(&image_tool).await?; let metadata = metadata.context(format!( "failed to validate kit image with name {} from vendor {}", locked_image.name, locked_image.vendor ))?; locked.push(locked_image); - sdk_set.insert(metadata.sdk); + sdk_set.insert(project.as_project_image(&metadata.sdk)?); for dep in metadata.kits { - remaining.push(dep); + remaining.push(project.as_project_image(&dep)?); } } } @@ -293,7 +294,7 @@ impl Lock { .context("no sdk was found for use, please specify a sdk in Twoliter.toml")?; debug!(?sdk, "Resolving workspace SDK"); - let (sdk, _metadata) = ImageResolver::from_image(sdk, project)? + let (sdk, _metadata) = ImageResolver::from_image(sdk)? .skip_metadata_retrieval() // SDKs don't have metadata .resolve(&image_tool) .await?; diff --git a/twoliter/src/project/lock/verification.rs b/twoliter/src/project/lock/verification.rs index 4ad01668c..02744a96a 100644 --- a/twoliter/src/project/lock/verification.rs +++ b/twoliter/src/project/lock/verification.rs @@ -101,12 +101,23 @@ impl LockfileVerifier for Lock { } } +/// A `LockfileVerifier` can return a set of `VerifyTag` structs, claiming that those artifacts +/// have been resolved and verified against the lockfile. + /// Writes marker files indicating which artifacts have been resolved and verified against the lock #[derive(Debug)] pub(crate) struct VerificationTagger { tags: BTreeSet, } +impl VerificationTagger { + pub fn no_verifications() -> Self { + Self { + tags: BTreeSet::new(), + } + } +} + impl From<&V> for VerificationTagger { fn from(resolver: &V) -> Self { Self { diff --git a/twoliter/src/project/mod.rs b/twoliter/src/project/mod.rs index 7303c5b07..4df653e3e 100644 --- a/twoliter/src/project/mod.rs +++ b/twoliter/src/project/mod.rs @@ -1,5 +1,7 @@ mod lock; +pub(crate) mod vendor; +pub(crate) use self::vendor::ArtifactVendor; pub(crate) use lock::VerificationTagger; use self::lock::{Lock, LockedSDK, Override}; @@ -8,6 +10,7 @@ use crate::docker::ImageUri; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; +use async_trait::async_trait; use async_walkdir::WalkDir; use buildsys_config::{EXTERNAL_KIT_DIRECTORY, EXTERNAL_KIT_METADATA}; use futures::stream::StreamExt; @@ -16,7 +19,7 @@ use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::collections::BTreeMap; use std::ffi::OsStr; -use std::fmt::{Display, Formatter}; +use std::fmt::{Debug, Display, Formatter}; use std::hash::Hash; use std::path::{Path, PathBuf}; use std::str::FromStr; @@ -29,7 +32,7 @@ const TWOLITER_OVERRIDES: &str = "Twoliter.override"; /// we use it, otherwise we search for the file. Returns the `Project` and the path at which it was /// found (this is the same as `user_path` if provided). #[instrument(level = "trace")] -pub(crate) async fn load_or_find_project(user_path: Option) -> Result { +pub(crate) async fn load_or_find_project(user_path: Option) -> Result> { let project = match user_path { None => Project::find_and_load(".").await?, Some(p) => Project::load(&p).await?, @@ -43,7 +46,7 @@ pub(crate) async fn load_or_find_project(user_path: Option) -> Result

{ filepath: PathBuf, project_dir: PathBuf, @@ -63,9 +66,12 @@ pub(crate) struct Project { kit: Vec, overrides: BTreeMap>, + + /// The resolved and locked dependencies of the project. + lock: L, } -impl Project { +impl Project { /// Load a `Twoliter.toml` file from the given file path (it can have any filename). pub(crate) async fn load>(path: P) -> Result { let path = fs::canonicalize(path).await?; @@ -116,32 +122,39 @@ impl Project { Self::find_and_load(parent).await } - pub(crate) async fn create_lock(&self) -> Result { - Lock::create(self).await + pub(crate) async fn create_lock(self) -> Result> { + let lock = Lock::create(&self).await?; + Ok(self.with_new_lock(lock)) } - pub(crate) async fn load_lock(&self) -> Result { + pub(crate) async fn load_lock(&self) -> Result> { VerificationTagger::cleanup_existing_tags(self.external_kits_dir()).await?; - let resolved_lock = Lock::load(self).await?; + let resolved_lock = NL::load_lock(self, private::SealToken).await?; - VerificationTagger::from(&resolved_lock) + resolved_lock + .verification_tagger(private::SealToken) .write_tags(self.external_kits_dir()) .await?; - Ok(resolved_lock) + Ok(self.with_new_lock(resolved_lock)) } +} - pub(crate) async fn load_locked_sdk(&self) -> Result { - VerificationTagger::cleanup_existing_tags(self.external_kits_dir()).await?; - - let resolved_lock = LockedSDK::load(self).await?; - - VerificationTagger::from(&resolved_lock) - .write_tags(self.external_kits_dir()) - .await?; - - Ok(resolved_lock) +impl Project { + /// Private function to create a new `Project` after resolving a different lock level. + fn with_new_lock>(&self, new_lock: T) -> Project { + Project { + filepath: self.filepath.clone(), + project_dir: self.project_dir.clone(), + schema_version: self.schema_version, + release_version: self.release_version.clone(), + sdk: self.sdk.clone(), + vendor: self.vendor.clone(), + kit: self.kit.clone(), + overrides: self.overrides.clone(), + lock: new_lock.into(), + } } pub(crate) fn filepath(&self) -> PathBuf { @@ -168,10 +181,18 @@ impl Project { self.release_version.as_str() } - pub(crate) fn vendor_for<'proj, 'arti, V: VendedArtifact>( - &'proj self, - artifact: &'arti V, - ) -> Option> { + pub(crate) fn direct_kit_deps(&self) -> Result> { + self.kit + .iter() + .map(|kit| self.as_project_image(kit)) + .collect() + } + + pub(crate) fn direct_sdk_image_dep(&self) -> Option> { + self.sdk.as_ref().map(|sdk| self.as_project_image(sdk)) + } + + pub(crate) fn vendor_for(&self, artifact: &V) -> Option { let artifact_name = artifact.artifact_name(); let vendor_name = artifact.vendor_name(); let vendor = self.vendor.get(vendor_name)?; @@ -180,41 +201,26 @@ impl Project { .get(vendor_name.as_ref()) .and_then(|vendor_overrides| vendor_overrides.get(artifact_name.as_ref())) .map(|override_| { - ArtifactVendor::Overridden(OverriddenVendor { - original_vendor_name: vendor_name, - original_vendor: vendor, - override_, - }) + ArtifactVendor::overridden(vendor_name.clone(), vendor.clone(), override_.clone()) }) - .or(Some(ArtifactVendor::Verbatim(VerbatimVendor { - vendor_name, - vendor, - }))) - } - - pub(crate) fn kits(&self) -> Vec { - self.kit.clone() - } - - pub(crate) fn sdk_image(&self) -> Option { - self.sdk.clone() + .or(Some(ArtifactVendor::verbatim( + vendor_name.clone(), + vendor.clone(), + ))) } - #[allow(unused)] - pub(crate) fn kit(&self, name: &str) -> Result> { - if let Some(kit) = self.kit.iter().find(|y| y.name.to_string() == name) { - let vendor = self.vendor.get(&kit.vendor).context(format!( - "vendor '{}' was not specified in Twoliter.toml", - kit.vendor - ))?; - Ok(Some(ImageUri::new( - Some(vendor.registry.clone()), - &kit.name, - format!("v{}", kit.version), - ))) - } else { - Ok(None) - } + pub(crate) fn as_project_image<'proj, 'arti: 'proj>( + &'proj self, + image: &'arti impl VendedArtifact, + ) -> Result { + let vendor = self + .vendor_for(image) + .with_context(|| format!("Could not find defined vendor for image '{:?}'", &image))?; + + Ok(ProjectImage { + image: Image::from_vended_artifact(image), + vendor, + }) } /// Returns a list of the names of Go modules by searching the `sources` directory for `go.mod` @@ -264,34 +270,92 @@ impl Project { } } -/// This represents a container registry vendor that is used in resolving the kits and also -/// now the bottlerocket sdk -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd)] -#[serde(rename_all = "kebab-case")] -pub(crate) struct Vendor { - pub registry: String, +impl Project { + pub(crate) fn sdk_image(&self) -> ProjectImage { + let SDKLocked(lock) = &self.lock; + self.as_project_image(&lock.0) + .expect("Could not find SDK vendor despite lock resolution succeeding?") + } } -/// `ArtifactVendor` represents a vendor associated with an image artifact used in a project. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] -pub(crate) enum ArtifactVendor<'proj, 'arti> { - /// The project only knows of the given vendor as it is written in Twoliter.toml - Verbatim(VerbatimVendor<'proj, 'arti>), - /// The project has an override expressed in Twoliter.override - Overridden(OverriddenVendor<'proj, 'arti>), +impl Project { + /// Fetches all external kits defined in a Twoliter.lock to the build directory + pub(crate) async fn fetch(&self, arch: &str) -> Result<()> { + let Locked(lock) = &self.lock; + lock.fetch(self, arch).await + } + + #[expect(dead_code)] + pub(crate) fn kits(&self) -> Vec { + let Locked(lock) = &self.lock; + lock.kit + .iter() + .map(|kit| self.as_project_image(kit)) + .collect::>() + .expect("Could not find kit vendor despite lock resolution succeeding?") + } + + pub(crate) fn sdk_image(&self) -> ProjectImage { + let Locked(lock) = &self.lock; + self.as_project_image(&lock.sdk) + .expect("Could not find SDK vendor despite lock resolution succeeding?") + } } -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] -pub(crate) struct VerbatimVendor<'proj, 'arti> { - vendor_name: &'arti ValidIdentifier, - vendor: &'proj Vendor, +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub(crate) struct ProjectImage { + image: Image, + vendor: ArtifactVendor, } -#[derive(Debug, Clone, Copy, Eq, PartialEq, Ord, PartialOrd)] -pub(crate) struct OverriddenVendor<'proj, 'arti> { - original_vendor_name: &'arti ValidIdentifier, - original_vendor: &'proj Vendor, - override_: &'proj Override, +impl Display for ProjectImage { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.vendor { + ArtifactVendor::Overridden(_) => write!( + f, + "{} (overridden-to: {})", + self.image, + self.project_image_uri() + ), + ArtifactVendor::Verbatim(_) => write!(f, "{}", self.image), + } + } +} + +impl ProjectImage { + pub(crate) fn name(&self) -> &ValidIdentifier { + &self.image.name + } + + pub(crate) fn version(&self) -> &Version { + self.image.version() + } + + pub(crate) fn vendor_name(&self) -> &ValidIdentifier { + self.vendor.vendor_name() + } + + /// Returns the URI for the original vendor. + pub(crate) fn original_source_uri(&self) -> ImageUri { + match &self.vendor { + ArtifactVendor::Overridden(overridden) => { + let original = ArtifactVendor::Verbatim(overridden.original_vendor()); + original.image_uri_for(&self.image) + } + ArtifactVendor::Verbatim(_) => self.vendor.image_uri_for(&self.image), + } + } + + /// Returns the image URI that the project will use for this image + /// + /// This could be different than the source_uri if overridden. + pub(crate) fn project_image_uri(&self) -> ImageUri { + ImageUri { + registry: Some(self.vendor.registry().to_string()), + repo: self.vendor.repo_for(&self.image).to_string(), + tag: format!("v{}", self.image.version()), + } + } } /// An artifact/vendor name combination used to identify an artifact resolved by Twoliter. @@ -368,6 +432,14 @@ fn is_valid_id_char(c: char) -> bool { } } +/// This represents a container registry vendor that is used in resolving the kits and also +/// now the bottlerocket sdk +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[serde(rename_all = "kebab-case")] +pub(crate) struct Vendor { + pub registry: String, +} + /// This represents a dependency on a container, primarily used for kits #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Ord, PartialOrd, Hash)] #[serde(rename_all = "kebab-case")] @@ -377,6 +449,16 @@ pub(crate) struct Image { pub vendor: ValidIdentifier, } +impl Image { + fn from_vended_artifact(artifact: &impl VendedArtifact) -> Self { + Self { + name: artifact.artifact_name().clone(), + vendor: artifact.vendor_name().clone(), + version: artifact.version().clone(), + } + } +} + impl Display for Image { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { write!(f, "{}-{}@{}", self.name, self.version, self.vendor) @@ -413,7 +495,7 @@ struct UnvalidatedProject { impl UnvalidatedProject { /// Constructs a [`Project`] from an [`UnvalidatedProject`] after validating fields. - async fn validate(self, path: impl AsRef) -> Result { + async fn validate(self, path: impl AsRef) -> Result> { let filepath: PathBuf = path.as_ref().into(); let project_dir = filepath .parent() @@ -436,6 +518,7 @@ impl UnvalidatedProject { vendor: self.vendor.unwrap_or_default(), kit: self.kit.unwrap_or_default(), overrides, + lock: Unlocked, }) } @@ -525,6 +608,80 @@ impl UnvalidatedProject { } } +/// Marker trait that dictates what artifacts have been validated in the lock. +#[async_trait] +pub(crate) trait ProjectLock: Sized + Debug + Send + Sync + 'static { + /// Loads the project lock for the given project. + async fn load_lock(project: &Project, _: private::SealToken) -> Result; + + /// Returns a `VerificationTagger` for this lock type. + fn verification_tagger(&self, _: private::SealToken) -> VerificationTagger; +} + +/// Indicates a project which has not resolved and validated the lockfile. +#[derive(Debug)] +pub struct Unlocked; + +#[async_trait] +impl ProjectLock for Unlocked { + async fn load_lock(_project: &Project, _: private::SealToken) -> Result { + Ok(Unlocked) + } + + fn verification_tagger(&self, _: private::SealToken) -> VerificationTagger { + VerificationTagger::no_verifications() + } +} + +/// Indicates a project which has resolved and verified only the SDK. +#[derive(Debug)] +pub struct SDKLocked(LockedSDK); + +#[async_trait] +impl ProjectLock for SDKLocked { + async fn load_lock(project: &Project, _: private::SealToken) -> Result { + LockedSDK::load(project).await.map(Self) + } + + fn verification_tagger(&self, _: private::SealToken) -> VerificationTagger { + (&self.0).into() + } +} + +impl From for SDKLocked { + fn from(lock: LockedSDK) -> Self { + SDKLocked(lock) + } +} + +/// Indicates a project which has resolved and verified all dependencies. +#[derive(Debug)] +pub struct Locked(Lock); + +#[async_trait] +impl ProjectLock for Locked { + async fn load_lock(project: &Project, _: private::SealToken) -> Result { + Lock::load(project).await.map(Self) + } + + fn verification_tagger(&self, _: private::SealToken) -> VerificationTagger { + (&self.0).into() + } +} + +impl From for Locked { + fn from(lock: Lock) -> Self { + Locked(lock) + } +} + +/// Seal the `ProjectLock` trait -- only this module is allowed to define new lock types. +mod private { + /// A marker type that, when used in a method signature, makes it impossible for other modules + /// to implement the `ProjectLock` trait. + pub struct SealToken; +} + #[cfg(test)] mod test { use super::*; @@ -633,23 +790,30 @@ mod test { let path = data_dir().join("override/Twoliter-override-1.toml"); let project = Project::load(path).await.unwrap(); - let sdk = project.sdk.as_ref().unwrap(); - - let vendor = project.vendor_for(sdk).unwrap(); + let sdk = project.direct_sdk_image_dep().unwrap().unwrap(); assert_eq!( - vendor, - ArtifactVendor::Overridden(OverriddenVendor { - original_vendor_name: &sdk.vendor, - original_vendor: &Vendor { + &sdk.vendor, + &ArtifactVendor::overridden( + sdk.vendor_name().clone(), + Vendor { registry: "a.com/b".parse().unwrap(), }, - override_: &Override { + Override { name: Some("my-overridden-sdk".parse().unwrap()), registry: Some("c.com/d".parse().unwrap()), }, - }) + ) ); + + assert_eq!( + sdk.project_image_uri(), + ImageUri { + registry: Some("c.com/d".into()), + repo: "my-overridden-sdk".into(), + tag: "v1.2.3".into(), + } + ) } #[tokio::test] diff --git a/twoliter/src/project/vendor.rs b/twoliter/src/project/vendor.rs new file mode 100644 index 000000000..421d96fb0 --- /dev/null +++ b/twoliter/src/project/vendor.rs @@ -0,0 +1,114 @@ +//! Utilities for interacting with Twoliter vendors. +//! +//! Most users of this module will need [`ArtifactVendor`], which represents a vendor which may have +//! been overridden in a `Twoliter.override` file. +use super::{Override, ValidIdentifier, VendedArtifact, Vendor}; +use crate::docker::ImageUri; +use std::fmt::Debug; + +/// `ArtifactVendor` represents a vendor associated with an image artifact used in a project. +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) enum ArtifactVendor { + /// The project only knows of the given vendor as it is written in Twoliter.toml + Verbatim(VerbatimVendor), + /// The project has an override expressed in Twoliter.override + Overridden(OverriddenVendor), +} + +impl ArtifactVendor { + pub(crate) fn registry(&self) -> &str { + match self { + ArtifactVendor::Verbatim(vendor) => vendor.registry(), + ArtifactVendor::Overridden(vendor) => vendor.registry(), + } + } + + pub(crate) fn repo_for<'a, V: VendedArtifact>(&'a self, image: &'a V) -> &'a str { + match self { + ArtifactVendor::Verbatim(vendor) => vendor.repo_for(image), + ArtifactVendor::Overridden(vendor) => vendor.repo_for(image), + } + } + + pub(crate) fn image_uri_for(&self, image: &V) -> ImageUri { + ImageUri { + registry: Some(self.registry().to_string()), + repo: self.repo_for(image).to_string(), + tag: format!("v{}", image.version()), + } + } + + pub(crate) fn vendor_name(&self) -> &ValidIdentifier { + match self { + ArtifactVendor::Verbatim(vendor) => &vendor.vendor_name, + ArtifactVendor::Overridden(vendor) => &vendor.original_vendor_name, + } + } + + pub(crate) fn overridden( + original_vendor_name: ValidIdentifier, + original_vendor: Vendor, + override_: Override, + ) -> Self { + Self::Overridden(OverriddenVendor { + original_vendor_name, + original_vendor, + override_, + }) + } + + pub(crate) fn verbatim(vendor_name: ValidIdentifier, vendor: Vendor) -> Self { + Self::Verbatim(VerbatimVendor { + vendor_name, + vendor, + }) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct VerbatimVendor { + vendor_name: ValidIdentifier, + vendor: Vendor, +} + +impl VerbatimVendor { + /// The name of the vendor as it appears in the Twoliter.toml file + pub(crate) fn registry(&self) -> &str { + &self.vendor.registry + } + + pub(crate) fn repo_for<'a, V: VendedArtifact>(&'a self, image: &'a V) -> &'a str { + image.artifact_name().as_ref() + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct OverriddenVendor { + original_vendor_name: ValidIdentifier, + original_vendor: Vendor, + override_: Override, +} + +impl OverriddenVendor { + /// The name of the vendor as it appears in the Twoliter.toml file + pub(crate) fn registry(&self) -> &str { + self.override_ + .registry + .as_ref() + .unwrap_or(&self.original_vendor.registry) + } + + pub(crate) fn repo_for<'a, V: VendedArtifact>(&'a self, image: &'a V) -> &str { + self.override_ + .name + .as_deref() + .unwrap_or(image.artifact_name().as_ref()) + } + + pub(crate) fn original_vendor(&self) -> VerbatimVendor { + VerbatimVendor { + vendor_name: self.original_vendor_name.clone(), + vendor: self.original_vendor.clone(), + } + } +}