diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index acaf2cdb5..a3c9b92c5 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -23,4 +23,5 @@ jobs: steps: - uses: actions/checkout@v3 - run: cargo install cargo-deny + - run: cargo install cargo-make - run: make build diff --git a/twoliter/src/cargo_make.rs b/twoliter/src/cargo_make.rs new file mode 100644 index 000000000..1a344fb66 --- /dev/null +++ b/twoliter/src/cargo_make.rs @@ -0,0 +1,260 @@ +use crate::common::exec_log; +use crate::docker::ImageArchUri; +use crate::project::Project; +use anyhow::{bail, Result}; +use log::trace; +use std::path::PathBuf; +use tokio::process::Command; + +/// A struct used to invoke `cargo make` tasks with `twoliter`'s `Makefile.toml`. +/// ```rust +/// # use crate::project::Project; +/// # use crate::test::data_dir; +/// # use self::CargoMake; +/// # let project_path = data_dir().join("Twoliter-1.toml"); +/// # let makefile_path = data_dir().join("Makefile.toml"); +/// # let project_dir = data_dir(); +/// +/// // First create a twoliter project. +/// let project = Project::load(project_path).await.unwrap(); +/// // Add the architecture that cargo make will be invoked for. +/// // This is required so that the correct sdk and toolchain are selected. +/// let arch = "x86_64"; +/// +/// // Create the `cargo make` command. +/// let cargo_make_command = CargoMake::new(&project, arch) +/// .unwrap() +/// // Specify path to the `Makefile.toml` (Default: `Makefile.toml`) +/// .makefile(makefile_path) +/// // Specify the project directory (Default: `.`) +/// .project_dir(project_dir) +/// // Add environment variable to the command +/// .env("FOO", "bar") +/// // Add cargo make arguments such as `-q` (quiet) +/// ._arg("-q"); +/// +/// // Run the `cargo make` task +/// cargo_make_command.clone() +/// ._exec("verify-twoliter-env") +/// .await +/// .unwrap(); +/// +/// // Run the `cargo make` task with args +/// cargo_make_command +/// .exec_with_args("verify-env-set-with-arg", ["FOO"]) +/// .await +/// .unwrap(); +/// ``` +#[derive(Debug, Clone, Default)] +pub struct CargoMake { + makefile_path: Option, + project_dir: Option, + args: Vec, +} + +impl CargoMake { + /// Create a new `cargo make` command. The sdk and toolchain environment variables will be set + /// based on the definitions in `Twoliter.toml` and `arch`. + pub(crate) fn new(project: &Project, arch: S) -> Result + where + S: Into, + { + let (sdk, toolchain) = require_sdk(project, &arch.into())?; + Ok(Self::default() + .env("TLPRIVATE_SDK_IMAGE", sdk) + .env("TLPRIVATE_TOOLCHAIN", toolchain)) + } + + /// Specify the path to the `Makefile.toml` for the `cargo make` command + pub(crate) fn makefile

(mut self, makefile_path: P) -> Self + where + P: Into, + { + self.makefile_path = Some(makefile_path.into()); + self + } + + /// Specify the project directory for the `cargo make` command + pub(crate) fn project_dir

(mut self, project_dir: P) -> Self + where + P: Into, + { + self.project_dir = Some(project_dir.into()); + self + } + + /// Specify environment variables that should be applied for this comand + pub(crate) fn env(mut self, key: S1, value: S2) -> Self + where + S1: Into, + S2: Into, + { + self.args + .push(format!("-e={}={}", key.into(), value.into())); + self + } + + /// Specify environment variables that should be applied for this comand + pub(crate) fn _envs(mut self, env_vars: V) -> Self + where + S1: Into, + S2: Into, + V: Into>, + { + for (key, value) in env_vars.into() { + self.args + .push(format!("-e={}={}", key.into(), value.into())); + } + self + } + + /// Specify `cargo make` arguments that should be applied for this comand + pub(crate) fn _arg(mut self, arg: S) -> Self + where + S: Into, + { + self.args.push(arg.into()); + self + } + + /// Specify `cargo make` arguments that should be applied for this comand + pub(crate) fn _args(mut self, args: V) -> Self + where + S: Into, + V: Into>, + { + self.args.extend(args.into().into_iter().map(Into::into)); + self + } + + /// Execute the `cargo make` task + pub(crate) async fn _exec(&self, task: S) -> Result<()> + where + S: Into, + { + self.exec_with_args(task, Vec::::new()).await + } + + /// Execute the `cargo make` task with arguments provided + pub(crate) async fn exec_with_args(&self, task: S1, args: I) -> Result<()> + where + S1: Into, + S2: Into, + I: IntoIterator, + { + exec_log( + Command::new("cargo") + .arg("make") + .arg("--disable-check-for-updates") + .args( + self.makefile_path.iter().flat_map(|path| { + vec!["--makefile".to_string(), path.display().to_string()] + }), + ) + .args( + self.project_dir + .iter() + .flat_map(|path| vec!["--cwd".to_string(), path.display().to_string()]), + ) + .args(build_system_env_vars()?) + .args(&self.args) + .arg(task.into()) + .args(args.into_iter().map(Into::into)), + ) + .await + } +} + +fn require_sdk(project: &Project, arch: &str) -> Result<(ImageArchUri, ImageArchUri)> { + match (project.sdk(arch), project.toolchain(arch)) { + (Some(s), Some(t)) => Ok((s, t)), + _ => bail!( + "When using twoliter make, it is required that the SDK and toolchain be specified in \ + Twoliter.toml" + ), + } +} + +fn build_system_env_vars() -> Result> { + let mut args = Vec::new(); + for (key, val) in std::env::vars() { + if is_build_system_env(key.as_str()) { + trace!("Passing env var {} to cargo make", key); + args.push("-e".to_string()); + args.push(format!("{}={}", key, val)); + } + + // To avoid confusion, environment variables whose values have been moved to + // Twoliter.toml are expressly disallowed here. + check_for_disallowed_var(&key)?; + } + Ok(args) +} + +/// A list of environment variables that don't conform to naming conventions but need to be passed +/// through to the `cargo make` invocation. +const ENV_VARS: [&str; 12] = [ + "ALLOW_MISSING_KEY", + "AMI_DATA_FILE_SUFFIX", + "CARGO_MAKE_CARGO_ARGS", + "CARGO_MAKE_CARGO_LIMIT_JOBS", + "CARGO_MAKE_DEFAULT_TESTSYS_KUBECONFIG_PATH", + "CARGO_MAKE_TESTSYS_ARGS", + "CARGO_MAKE_TESTSYS_KUBECONFIG_ARG", + "MARK_OVA_AS_TEMPLATE", + "RELEASE_START_TIME", + "SSM_DATA_FILE_SUFFIX", + "VMWARE_IMPORT_SPEC_PATH", + "VMWARE_VM_NAME_DEFAULT", +]; + +const DISALLOWED_SDK_VARS: [&str; 4] = [ + "BUILDSYS_SDK_NAME", + "BUILDSYS_SDK_VERSION", + "BUILDSYS_REGISTRY", + "BUILDSYS_TOOLCHAIN", +]; + +/// Returns `true` if `key` is an environment variable that needs to be passed to `cargo make`. +fn is_build_system_env(key: impl AsRef) -> bool { + let key = key.as_ref(); + key.starts_with("BUILDSYS_") + || key.starts_with("PUBLISH_") + || key.starts_with("REPO_") + || key.starts_with("TESTSYS_") + || key.starts_with("BOOT_CONFIG") + || key.starts_with("AWS_") + || ENV_VARS.contains(&key) +} + +fn check_for_disallowed_var(key: &str) -> Result<()> { + if DISALLOWED_SDK_VARS.contains(&key) { + bail!( + "The environment variable '{}' can no longer be used. Specify the SDK in Twoliter.toml", + key + ) + } + Ok(()) +} + +#[test] +fn test_is_build_system_env() { + assert!(is_build_system_env( + "CARGO_MAKE_DEFAULT_TESTSYS_KUBECONFIG_PATH" + )); + assert!(is_build_system_env("BUILDSYS_PRETTY_NAME")); + assert!(is_build_system_env("PUBLISH_FOO_BAR")); + assert!(is_build_system_env("TESTSYS_!")); + assert!(is_build_system_env("BOOT_CONFIG!")); + assert!(is_build_system_env("BOOT_CONFIG_INPUT")); + assert!(is_build_system_env("AWS_REGION")); + assert!(!is_build_system_env("PATH")); + assert!(!is_build_system_env("HOME")); + assert!(!is_build_system_env("COLORTERM")); +} + +#[test] +fn test_check_for_disallowed_var() { + assert!(check_for_disallowed_var("BUILDSYS_REGISTRY").is_err()); + assert!(check_for_disallowed_var("BUILDSYS_PRETTY_NAME").is_ok()); +} diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index e3b58e785..f8201d261 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -1,13 +1,9 @@ -use crate::common::exec_log; -use crate::docker::ImageArchUri; +use crate::cargo_make::CargoMake; use crate::project; -use crate::project::Project; use crate::tools::{install_tools, tools_tempdir}; -use anyhow::{bail, ensure, Result}; +use anyhow::Result; use clap::Parser; -use log::trace; use std::path::PathBuf; -use tokio::process::Command; /// Run a cargo make command in Twoliter's build environment. Known Makefile.toml environment /// variables will be passed-through to the cargo make invocation. @@ -29,6 +25,9 @@ pub(crate) struct Make { /// Uninspected arguments to be passed to cargo make after the target name. For example, --foo /// in the following command : cargo make test --foo. additional_args: Vec, + + #[clap(env = "BUILDSYS_ARCH")] + arch: String, } impl Make { @@ -37,134 +36,12 @@ impl Make { let tempdir = tools_tempdir()?; install_tools(&tempdir).await?; let makefile_path = tempdir.path().join("Makefile.toml"); - - let mut args = vec![ - "make".to_string(), - "--disable-check-for-updates".to_string(), - "--makefile".to_string(), - makefile_path.display().to_string(), - "--cwd".to_string(), - project.project_dir().display().to_string(), - ]; - - let mut arch = String::new(); - - for (key, val) in std::env::vars() { - if is_build_system_env(key.as_str()) { - trace!("Passing env var {} to cargo make", key); - args.push("-e".to_string()); - args.push(format!("{}={}", key, val)); - } - - // To avoid confusion, environment variables whose values have been moved to - // Twoliter.toml are expressly disallowed here. - check_for_disallowed_var(&key)?; - - if key == "BUILDSYS_ARCH" { - arch = val.clone(); - } - } - - ensure!( - !arch.is_empty(), - "It is required to pass a non-zero string as the value of environment variable \ - 'BUILDSYS_ARCH' when running twoliter make" - ); - - let (sdk, toolchain) = require_sdk(&project, &arch)?; - - args.push(format!("-e=TLPRIVATE_SDK_IMAGE={}", sdk)); - args.push(format!("-e=TLPRIVATE_TOOLCHAIN={}", toolchain)); - args.push(format!("-e=CARGO_HOME={}", self.cargo_home.display())); - args.push(format!( - "-e=TWOLITER_TOOLS_DIR={}", - tempdir.path().display() - )); - - args.push(self.makefile_task.clone()); - - for cargo_make_arg in &self.additional_args { - args.push(cargo_make_arg.clone()); - } - - exec_log(Command::new("cargo").args(args)).await - } -} - -/// A list of environment variables that don't conform to naming conventions but need to be passed -/// through to the `cargo make` invocation. -const ENV_VARS: [&str; 12] = [ - "ALLOW_MISSING_KEY", - "AMI_DATA_FILE_SUFFIX", - "CARGO_MAKE_CARGO_ARGS", - "CARGO_MAKE_CARGO_LIMIT_JOBS", - "CARGO_MAKE_DEFAULT_TESTSYS_KUBECONFIG_PATH", - "CARGO_MAKE_TESTSYS_ARGS", - "CARGO_MAKE_TESTSYS_KUBECONFIG_ARG", - "MARK_OVA_AS_TEMPLATE", - "RELEASE_START_TIME", - "SSM_DATA_FILE_SUFFIX", - "VMWARE_IMPORT_SPEC_PATH", - "VMWARE_VM_NAME_DEFAULT", -]; - -const DISALLOWED_SDK_VARS: [&str; 4] = [ - "BUILDSYS_SDK_NAME", - "BUILDSYS_SDK_VERSION", - "BUILDSYS_REGISTRY", - "BUILDSYS_TOOLCHAIN", -]; - -/// Returns `true` if `key` is an environment variable that needs to be passed to `cargo make`. -fn is_build_system_env(key: impl AsRef) -> bool { - let key = key.as_ref(); - key.starts_with("BUILDSYS_") - || key.starts_with("PUBLISH_") - || key.starts_with("REPO_") - || key.starts_with("TESTSYS_") - || key.starts_with("BOOT_CONFIG") - || key.starts_with("AWS_") - || ENV_VARS.contains(&key) -} - -fn check_for_disallowed_var(key: &str) -> Result<()> { - if DISALLOWED_SDK_VARS.contains(&key) { - bail!( - "The environment variable '{}' can no longer be used. Specify the SDK in Twoliter.toml", - key - ) + CargoMake::new(&project, &self.arch)? + .env("CARGO_HOME", self.cargo_home.display().to_string()) + .env("TWOLITER_TOOLS_DIR", tempdir.path().display().to_string()) + .makefile(makefile_path) + .project_dir(project.project_dir()) + .exec_with_args(&self.makefile_task, self.additional_args.clone()) + .await } - Ok(()) -} - -fn require_sdk(project: &Project, arch: &str) -> Result<(ImageArchUri, ImageArchUri)> { - match (project.sdk(arch), project.toolchain(arch)) { - (Some(s), Some(t)) => Ok((s, t)), - _ => bail!( - "When using twoliter make, it is required that the SDK and toolchain be specified in \ - Twoliter.toml" - ), - } -} - -#[test] -fn test_is_build_system_env() { - assert!(is_build_system_env( - "CARGO_MAKE_DEFAULT_TESTSYS_KUBECONFIG_PATH" - )); - assert!(is_build_system_env("BUILDSYS_PRETTY_NAME")); - assert!(is_build_system_env("PUBLISH_FOO_BAR")); - assert!(is_build_system_env("TESTSYS_!")); - assert!(is_build_system_env("BOOT_CONFIG!")); - assert!(is_build_system_env("BOOT_CONFIG_INPUT")); - assert!(is_build_system_env("AWS_REGION")); - assert!(!is_build_system_env("PATH")); - assert!(!is_build_system_env("HOME")); - assert!(!is_build_system_env("COLORTERM")); -} - -#[test] -fn test_check_for_disallowed_var() { - assert!(check_for_disallowed_var("BUILDSYS_REGISTRY").is_err()); - assert!(check_for_disallowed_var("BUILDSYS_PRETTY_NAME").is_ok()); } diff --git a/twoliter/src/docker/image.rs b/twoliter/src/docker/image.rs index 795e3d768..aff570ff0 100644 --- a/twoliter/src/docker/image.rs +++ b/twoliter/src/docker/image.rs @@ -104,6 +104,12 @@ impl Display for ImageArchUri { } } +impl From for String { + fn from(value: ImageArchUri) -> Self { + value.to_string() + } +} + #[test] fn image_arch_uri_no_registry() { let uri = ImageArchUri::new(None, "my-sdk", "i386", "v0.33.1"); diff --git a/twoliter/src/main.rs b/twoliter/src/main.rs index cb4d8c28c..9a3ea4be6 100644 --- a/twoliter/src/main.rs +++ b/twoliter/src/main.rs @@ -2,6 +2,7 @@ use crate::cmd::{init_logger, Args}; use anyhow::Result; use clap::Parser; +mod cargo_make; mod cmd; mod common; mod docker; diff --git a/twoliter/src/test/cargo_make.rs b/twoliter/src/test/cargo_make.rs new file mode 100644 index 000000000..73e025af7 --- /dev/null +++ b/twoliter/src/test/cargo_make.rs @@ -0,0 +1,51 @@ +use crate::{cargo_make::CargoMake, project::Project, test::data_dir}; + +#[tokio::test] +async fn test_cargo_make() { + let path = data_dir().join("Twoliter-1.toml"); + let project = Project::load(path).await.unwrap(); + let cargo_make = CargoMake::new(&project, "arch") + .unwrap() + .makefile(data_dir().join("Makefile.toml")); + cargo_make._exec("verify-twoliter-env").await.unwrap(); + cargo_make + .clone() + .env("FOO", "bar") + .exec_with_args("verify-env-set-with-arg", ["FOO"]) + .await + .unwrap(); + cargo_make + .clone() + .env("FOO", "bar") + .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"]) + .await + .unwrap(); + cargo_make + .clone() + .project_dir(data_dir()) + .exec_with_args( + "verify-current-dir-with-arg", + [data_dir().display().to_string()], + ) + .await + .unwrap(); + cargo_make + .clone() + ._arg("--env") + ._arg("FOO=bar") + .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"]) + .await + .unwrap(); + cargo_make + .clone() + ._args(["--env", "FOO=bar"]) + .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"]) + .await + .unwrap(); + cargo_make + .clone() + ._envs([("FOO", "bar"), ("BAR", "baz")]) + .exec_with_args("verify-env-value-with-arg", ["BAR", "baz"]) + .await + .unwrap(); +} diff --git a/twoliter/src/test/data/Makefile.toml b/twoliter/src/test/data/Makefile.toml new file mode 100644 index 000000000..c8975d09b --- /dev/null +++ b/twoliter/src/test/data/Makefile.toml @@ -0,0 +1,41 @@ +[env] +PRESET_ENV_VARIABLE = "set" + +[tasks.verify-twoliter-env] +script_runner = "bash" +script = [''' + if ! [[ -v TLPRIVATE_SDK_IMAGE ]]; then + echo "TLPRIVATE_SDK_IMAGE is not set" + exit 1 + fi + if ! [[ -v TLPRIVATE_TOOLCHAIN ]]; then + echo "TLPRIVATE_TOOLCHAIN is not set" + exit 1 + fi +'''] + +[tasks.verify-env-set-with-arg] +script_runner = "bash" +script = [''' + if ! [[ -v ${1} ]]; then + exit 1 + fi +'''] + +[tasks.verify-env-value-with-arg] +script_runner = "bash" +script = [''' + if ! [ ${!1} = ${2} ]; then + echo "${!1} != ${2}" + exit 1 + fi +'''] + +[tasks.verify-current-dir-with-arg] +script_runner = "bash" +script = [''' + if ! [ $(pwd) = ${1} ]; then + echo "$(pwd) != ${1}" + exit 1 + fi +'''] diff --git a/twoliter/src/test/mod.rs b/twoliter/src/test/mod.rs index 87e5f50f5..bead937ef 100644 --- a/twoliter/src/test/mod.rs +++ b/twoliter/src/test/mod.rs @@ -4,7 +4,7 @@ This directory and module are for tests, test data, and re-usable test code. Thi be compiled for `cfg(test)`, which is accomplished at its declaration in `main.rs`. !*/ - +mod cargo_make; use std::path::PathBuf; /// Return the canonical path to the directory where we store test data.