From 48a94143359ad00465c7eec46466e1392ccf1c29 Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Thu, 30 Nov 2023 16:31:10 -0800 Subject: [PATCH 1/6] add release-version to twoliter.toml --- tests/projects/project1/Twoliter.toml | 1 + twoliter/src/project.rs | 8 ++++++++ twoliter/src/test/data/Twoliter-1.toml | 3 +-- twoliter/src/test/data/Twoliter-invalid-version.toml | 3 +-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/projects/project1/Twoliter.toml b/tests/projects/project1/Twoliter.toml index 93b341d68..0d8c9ea73 100644 --- a/tests/projects/project1/Twoliter.toml +++ b/tests/projects/project1/Twoliter.toml @@ -1,4 +1,5 @@ schema-version = 1 +release-version = "1.0.0" [sdk] registry = "twoliter.alpha" diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index 4528bf43b..e88a1d2fb 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -37,6 +37,9 @@ pub(crate) struct Project { /// The version of this schema struct. schema_version: SchemaVersion<1>, + /// The version that will be given to released artifacts such as kits and variants. + release_version: String, + /// The Bottlerocket SDK container image. sdk: Option, @@ -101,6 +104,10 @@ impl Project { self.project_dir.clone() } + pub(crate) fn _release_version(&self) -> &str { + self.release_version.as_str() + } + pub(crate) fn sdk_name(&self) -> Option<&ImageName> { self.sdk.as_ref() } @@ -287,6 +294,7 @@ mod test { filepath: Default::default(), project_dir: Default::default(), schema_version: Default::default(), + release_version: String::from("1.0.0"), sdk: Some(ImageName { registry: Some("example.com".try_into().unwrap()), name: "foo-abc".try_into().unwrap(), diff --git a/twoliter/src/test/data/Twoliter-1.toml b/twoliter/src/test/data/Twoliter-1.toml index 7ed1fbd8a..236d68f57 100644 --- a/twoliter/src/test/data/Twoliter-1.toml +++ b/twoliter/src/test/data/Twoliter-1.toml @@ -1,6 +1,5 @@ schema-version = 1 -project-name = "sample-project" -project-version = "v1.0.0" +release-version = "1.0.0" [sdk] registry = "a.com/b" diff --git a/twoliter/src/test/data/Twoliter-invalid-version.toml b/twoliter/src/test/data/Twoliter-invalid-version.toml index 6107c06dd..51d63e93c 100644 --- a/twoliter/src/test/data/Twoliter-invalid-version.toml +++ b/twoliter/src/test/data/Twoliter-invalid-version.toml @@ -1,6 +1,5 @@ schema-version = 4294967295 -project-name = "sample-project" -project-version = "v1.0.0" +release-version = "1.0.0" [sdk] registry = "example.com/my-repos" From 93b426d36138930dd2e6d7c3c4a95e3d1cb62d4c Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Thu, 30 Nov 2023 16:52:20 -0800 Subject: [PATCH 2/6] move schema version type to its own file --- twoliter/src/main.rs | 1 + twoliter/src/project.rs | 66 ++-------------------------------- twoliter/src/schema_version.rs | 64 +++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 64 deletions(-) create mode 100644 twoliter/src/schema_version.rs diff --git a/twoliter/src/main.rs b/twoliter/src/main.rs index 9a3ea4be6..5c70326ef 100644 --- a/twoliter/src/main.rs +++ b/twoliter/src/main.rs @@ -7,6 +7,7 @@ mod cmd; mod common; mod docker; mod project; +mod schema_version; mod tools; /// Test code that should only be compiled when running tests. diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index e88a1d2fb..eb00168d9 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -1,12 +1,11 @@ use crate::docker::ImageArchUri; +use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; use log::{debug, trace}; use non_empty_string::NonEmptyString; -use serde::de::Error; -use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha512}; -use std::fmt; use std::path::{Path, PathBuf}; use tokio::fs; @@ -164,67 +163,6 @@ impl ImageName { } } -/// We need to constrain the `Project` struct to a valid version. Unfortunately `serde` does not -/// have an after-deserialization validation hook, so we have this struct to limit the version to a -/// single acceptable value. -#[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -pub(crate) struct SchemaVersion; - -impl SchemaVersion { - pub(crate) fn get(&self) -> u32 { - N - } - - pub(crate) fn get_static() -> u32 { - N - } -} - -impl From> for u32 { - fn from(value: SchemaVersion) -> Self { - value.get() - } -} - -impl fmt::Debug for SchemaVersion { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { - fmt::Debug::fmt(&self.get(), f) - } -} - -impl fmt::Display for SchemaVersion { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { - fmt::Display::fmt(&self.get(), f) - } -} - -impl Serialize for SchemaVersion { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - serializer.serialize_u32(self.get()) - } -} - -impl<'de, const N: u32> Deserialize<'de> for SchemaVersion { - fn deserialize(deserializer: D) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let value: u32 = Deserialize::deserialize(deserializer)?; - if value != Self::get_static() { - Err(Error::custom(format!( - "Incorrect project schema_version: got '{}', expected '{}'", - value, - Self::get_static() - ))) - } else { - Ok(Self) - } - } -} - #[cfg(test)] mod test { use super::*; diff --git a/twoliter/src/schema_version.rs b/twoliter/src/schema_version.rs new file mode 100644 index 000000000..24c6484cb --- /dev/null +++ b/twoliter/src/schema_version.rs @@ -0,0 +1,64 @@ +use serde::de::Error; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; + +/// We need to constrain the `Project` struct to a valid version. Unfortunately `serde` does not +/// have an after-deserialization validation hook, so we have this struct to limit the version to a +/// single acceptable value. +#[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +pub(crate) struct SchemaVersion; + +impl SchemaVersion { + pub(crate) fn get(&self) -> u32 { + N + } + + pub(crate) fn get_static() -> u32 { + N + } +} + +impl From> for u32 { + fn from(value: SchemaVersion) -> Self { + value.get() + } +} + +impl fmt::Debug for SchemaVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { + fmt::Debug::fmt(&self.get(), f) + } +} + +impl fmt::Display for SchemaVersion { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> std::fmt::Result { + fmt::Display::fmt(&self.get(), f) + } +} + +impl Serialize for SchemaVersion { + fn serialize(&self, serializer: S) -> std::result::Result + where + S: Serializer, + { + serializer.serialize_u32(self.get()) + } +} + +impl<'de, const N: u32> Deserialize<'de> for SchemaVersion { + fn deserialize(deserializer: D) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let value: u32 = Deserialize::deserialize(deserializer)?; + if value != Self::get_static() { + Err(Error::custom(format!( + "Incorrect project schema_version: got '{}', expected '{}'", + value, + Self::get_static() + ))) + } else { + Ok(Self) + } + } +} From 67ef875a7e003578d99bb31fd79d88e1c335e202 Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Thu, 30 Nov 2023 18:07:56 -0800 Subject: [PATCH 3/6] deprecate release.toml When loading a project, issue a deprecation warning if Release.toml is found. If it is found, ensure that its version field is aligned with Twoliter.toml --- twoliter/src/project.rs | 100 ++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 13 deletions(-) diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index eb00168d9..a4b932a4d 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -2,12 +2,13 @@ use crate::docker::ImageArchUri; use crate::schema_version::SchemaVersion; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; -use log::{debug, trace}; +use log::{debug, info, trace, warn}; use non_empty_string::NonEmptyString; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha512}; use std::path::{Path, PathBuf}; use tokio::fs; +use toml::Table; /// Common functionality in commands, if the user gave a path to the `Twoliter.toml` file, /// we use it, otherwise we search for the file. Returns the `Project` and the path at which it was @@ -25,7 +26,7 @@ pub(crate) async fn load_or_find_project(user_path: Option) -> Result

, + release_version: String, + sdk: Option, + toolchain: Option, +} + +impl UnvalidatedProject { + /// Constructs a [`Project`] from an [`UnvalidatedProject`] after validating fields. + async fn validate(self, path: impl AsRef) -> Result { + let filepath: PathBuf = path.as_ref().into(); + let project_dir = filepath + .parent() + .context(format!( + "Unable to find the parent directory of '{}'", + filepath.display(), + ))? + .to_path_buf(); + + self.check_release_toml(&project_dir).await?; + + Ok(Project { + filepath, + project_dir, + schema_version: self.schema_version, + release_version: self.release_version, + sdk: self.sdk, + toolchain: self.toolchain, + }) + } + + /// Issues a warning if `Release.toml` is found and, if so, ensures that it contains the same + /// version (i.e. `release-version`) as the `Twoliter.toml` project file. + async fn check_release_toml(&self, project_dir: &Path) -> Result<()> { + let path = project_dir.join("Release.toml"); + if !path.exists() || !path.is_file() { + // There is no Release.toml file. This is a good thing! + trace!("This project does not have a Release.toml file (this is not a problem)"); + return Ok(()); + } + warn!( + "A Release.toml file was found. Release.toml is deprecated. Please remove it from \ + your project." + ); + let content = fs::read_to_string(&path).await.context(format!( + "Error while checking Release.toml file at '{}'", + path.display() + ))?; + let toml: Table = match toml::from_str(&content) { + Ok(toml) => toml, + Err(e) => { + warn!( + "Unable to parse Release.toml to ensure that its version matches the \ + release-version in Twoliter.toml: {e}", + ); + return Ok(()); + } + }; + let version = match toml.get("version") { + Some(version) => version, + None => { + info!("Release.toml does not contain a version key. Ignoring it."); + return Ok(()); + } + } + .to_string(); + ensure!( + version == self.release_version, + "The version found in Release.toml, '{version}', does not match the release-version \ + found in Twoliter.toml '{}'", + self.release_version + ); + Ok(()) + } +} + #[cfg(test)] mod test { use super::*; From ba4185aace26fd55a499fafe08fe0e4c4599f0c3 Mon Sep 17 00:00:00 2001 From: ecpullen Date: Tue, 14 Nov 2023 19:50:21 +0000 Subject: [PATCH 4/6] twoliter: remove release.toml requirement It is no longer required to have a Release.toml file in the project. Co-authored-by: Ethan Pullen Co-authored-by: Matthew James Briggs --- Cargo.lock | 1 + tests/projects/project1/Release.toml | 1 - tests/projects/project1/variants/Cargo.lock | 8 -------- twoliter/Cargo.toml | 1 + twoliter/embedded/Makefile.toml | 4 +++- twoliter/src/cmd/build.rs | 1 + twoliter/src/common.rs | 19 +++++++++++++------ twoliter/src/docker/container.rs | 3 ++- twoliter/src/project.rs | 2 +- twoliter/src/test/mod.rs | 1 + 10 files changed, 23 insertions(+), 18 deletions(-) delete mode 100644 tests/projects/project1/Release.toml diff --git a/Cargo.lock b/Cargo.lock index 439493159..588eb58d4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3487,6 +3487,7 @@ dependencies = [ "pubsys", "pubsys-setup", "serde", + "serde_json", "sha2", "tar", "tempfile", diff --git a/tests/projects/project1/Release.toml b/tests/projects/project1/Release.toml deleted file mode 100644 index 5820a7290..000000000 --- a/tests/projects/project1/Release.toml +++ /dev/null @@ -1 +0,0 @@ -version = "0.0.1" diff --git a/tests/projects/project1/variants/Cargo.lock b/tests/projects/project1/variants/Cargo.lock index 9291e5672..54165822b 100644 --- a/tests/projects/project1/variants/Cargo.lock +++ b/tests/projects/project1/variants/Cargo.lock @@ -2,14 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "aws-test-1" -version = "0.1.0" -dependencies = [ - "hello-agent", - "hello-go", -] - [[package]] name = "hello-agent" version = "0.1.0" diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index ff1e85cd3..db7a8c81a 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -19,6 +19,7 @@ hex = "0.4" log = "0.4" non-empty-string = { version = "0.2", features = [ "serde" ] } serde = { version = "1", features = ["derive"] } +serde_json = "1" sha2 = "0.10" tar = "0.4" tempfile = "3" diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index 87b2c554a..d71d9d49f 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -22,7 +22,7 @@ BUILDSYS_VERSION_BUILD = { script = ["git describe --always --dirty --exclude '* # later in this section. You have to edit the path here in Makefile.toml to # use a different Release.toml. BUILDSYS_RELEASE_CONFIG_PATH = "${BUILDSYS_ROOT_DIR}/Release.toml" -BUILDSYS_VERSION_IMAGE = { script = ["awk -F '[ =\"]+' '$1 == \"version\" {print $2}' ${BUILDSYS_RELEASE_CONFIG_PATH}"] } + # This can be overridden with -e to build a different variant from the variants/ directory BUILDSYS_VARIANT = { script = ['echo "${BUILDSYS_VARIANT:-aws-k8s-1.24}"'] } # Product name used for file and directory naming @@ -137,6 +137,8 @@ TESTSYS_LOG_LEVEL = "info" # Certain variables are defined here to allow us to override a component value # on the command line. +BUILDSYS_VERSION_IMAGE = { script = ["awk -F '[ =\"]+' '$1 == \"version\" {print $2}' ${BUILDSYS_RELEASE_CONFIG_PATH}"], condition = { env_not_set = ["BUILDSYS_VERSION_IMAGE"]}} + # Depends on ${BUILDSYS_JOBS}. CARGO_MAKE_CARGO_LIMIT_JOBS = "--jobs ${BUILDSYS_JOBS}" CARGO_MAKE_CARGO_ARGS = "--offline --locked" diff --git a/twoliter/src/cmd/build.rs b/twoliter/src/cmd/build.rs index 36e78b430..2dc3f6b32 100644 --- a/twoliter/src/cmd/build.rs +++ b/twoliter/src/cmd/build.rs @@ -107,6 +107,7 @@ impl BuildVariant { .env("BUILDSYS_ARCH", &self.arch) .env("BUILDSYS_VARIANT", &self.variant) .env("BUILDSYS_SBKEYS_DIR", sbkeys_dir.display().to_string()) + .env("BUILDSYS_VERSION_IMAGE", project.release_version()) .makefile(makefile_path) .project_dir(project.project_dir()) .exec("build") diff --git a/twoliter/src/common.rs b/twoliter/src/common.rs index 9378614ee..1c0484765 100644 --- a/twoliter/src/common.rs +++ b/twoliter/src/common.rs @@ -9,15 +9,16 @@ pub(crate) async fn exec_log(cmd: &mut Command) -> Result<()> { log::max_level(), LevelFilter::Off | LevelFilter::Error | LevelFilter::Warn ); - exec(cmd, quiet).await + exec(cmd, quiet).await?; + Ok(()) } /// Run a `tokio::process::Command` and return a `Result` letting us know whether or not it worked. /// `quiet` determines whether or not the command output will be piped to `stdout/stderr`. When -/// `quiet=true`, no output will be shown. -pub(crate) async fn exec(cmd: &mut Command, quiet: bool) -> Result<()> { +/// `quiet=true`, no output will be shown and will be returned instead. +pub(crate) async fn exec(cmd: &mut Command, quiet: bool) -> Result> { debug!("Running: {:?}", cmd); - if quiet { + Ok(if quiet { // For quiet levels of logging we capture stdout and stderr let output = cmd .output() @@ -30,6 +31,11 @@ pub(crate) async fn exec(cmd: &mut Command, quiet: bool) -> Result<()> { String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr) ); + + Some( + String::from_utf8(output.stdout) + .context("Unable to convert command output to `String`")?, + ) } else { // For less quiet log levels we stream to stdout and stderr. let status = cmd @@ -42,6 +48,7 @@ pub(crate) async fn exec(cmd: &mut Command, quiet: bool) -> Result<()> { "Command was unsuccessful, exit code {}", status.code().unwrap_or(1), ); - } - Ok(()) + + None + }) } diff --git a/twoliter/src/docker/container.rs b/twoliter/src/docker/container.rs index 7ccec0961..b16e51344 100644 --- a/twoliter/src/docker/container.rs +++ b/twoliter/src/docker/container.rs @@ -51,7 +51,8 @@ impl DockerContainer { let mut args = vec!["cp".to_string()]; args.push(format!("{}:{}", self.name, src.as_ref().display())); args.push(dest.as_ref().display().to_string()); - exec(Command::new("docker").args(args), true).await + exec(Command::new("docker").args(args), true).await?; + Ok(()) } } diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index a4b932a4d..b81e7a7b3 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -95,7 +95,7 @@ impl Project { self.project_dir.clone() } - pub(crate) fn _release_version(&self) -> &str { + pub(crate) fn release_version(&self) -> &str { self.release_version.as_str() } diff --git a/twoliter/src/test/mod.rs b/twoliter/src/test/mod.rs index bead937ef..b5b32884c 100644 --- a/twoliter/src/test/mod.rs +++ b/twoliter/src/test/mod.rs @@ -5,6 +5,7 @@ be compiled for `cfg(test)`, which is accomplished at its declaration in `main.r !*/ mod cargo_make; + use std::path::PathBuf; /// Return the canonical path to the directory where we store test data. From 8b481864770598f54a7ade310fc9cc4f01a205e2 Mon Sep 17 00:00:00 2001 From: ecpullen Date: Wed, 6 Dec 2023 18:07:27 +0000 Subject: [PATCH 5/6] twoliter: User Twoliter.toml for version in `make` --- twoliter/src/cmd/make.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index f8201d261..f297bf63c 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -39,6 +39,7 @@ impl Make { CargoMake::new(&project, &self.arch)? .env("CARGO_HOME", self.cargo_home.display().to_string()) .env("TWOLITER_TOOLS_DIR", tempdir.path().display().to_string()) + .env("BUILDSYS_VERSION_IMAGE", project.release_version()) .makefile(makefile_path) .project_dir(project.project_dir()) .exec_with_args(&self.makefile_task, self.additional_args.clone()) From 13493e82f83f311b139058318c307e0b30acf1d0 Mon Sep 17 00:00:00 2001 From: ecpullen Date: Wed, 6 Dec 2023 18:09:45 +0000 Subject: [PATCH 6/6] twoliter: Make `BUILDSYS_VERSION_IMAGE` required --- twoliter/embedded/Makefile.toml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index d71d9d49f..044960d49 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -22,7 +22,6 @@ BUILDSYS_VERSION_BUILD = { script = ["git describe --always --dirty --exclude '* # later in this section. You have to edit the path here in Makefile.toml to # use a different Release.toml. BUILDSYS_RELEASE_CONFIG_PATH = "${BUILDSYS_ROOT_DIR}/Release.toml" - # This can be overridden with -e to build a different variant from the variants/ directory BUILDSYS_VARIANT = { script = ['echo "${BUILDSYS_VARIANT:-aws-k8s-1.24}"'] } # Product name used for file and directory naming @@ -137,8 +136,6 @@ TESTSYS_LOG_LEVEL = "info" # Certain variables are defined here to allow us to override a component value # on the command line. -BUILDSYS_VERSION_IMAGE = { script = ["awk -F '[ =\"]+' '$1 == \"version\" {print $2}' ${BUILDSYS_RELEASE_CONFIG_PATH}"], condition = { env_not_set = ["BUILDSYS_VERSION_IMAGE"]}} - # Depends on ${BUILDSYS_JOBS}. CARGO_MAKE_CARGO_LIMIT_JOBS = "--jobs ${BUILDSYS_JOBS}" CARGO_MAKE_CARGO_ARGS = "--offline --locked" @@ -265,6 +262,13 @@ if [[ -z "${TLPRIVATE_SDK_IMAGE}" || -z "{TLPRIVATE_TOOLCHAIN}" ]];then exit 1 fi +# Ensure BUILDSYS_VERSION_IMAGE is set +if [[ -z "${BUILDSYS_VERSION_IMAGE}" ]];then + echo "BUILDSYS_VERSION_IMAGE must be defined and must be non-zero in length." + echo "Are you using Twoliter? It is a bug if Twoliter has invoked cargo make without this." + exit 1 +fi + mkdir -p ${BUILDSYS_BUILD_DIR} mkdir -p ${BUILDSYS_OUTPUT_DIR} mkdir -p ${BUILDSYS_PACKAGES_DIR}