From d12d4afa190046a879ae53284dc677a641249507 Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Thu, 28 Sep 2023 17:26:33 -0700 Subject: [PATCH 1/2] add sdk to project schema Add the ability to specify the desired Bottlerocket SDK in Twoliter.toml. --- Cargo.lock | 22 +++ twoliter/Cargo.toml | 1 + twoliter/src/cmd/build.rs | 6 +- twoliter/src/docker/image.rs | 20 ++- twoliter/src/docker/mod.rs | 5 - twoliter/src/project.rs | 126 +++++++++++++++--- twoliter/src/test/data/Twoliter-1.toml | 9 +- .../test/data/Twoliter-invalid-version.toml | 7 +- 8 files changed, 161 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 416f48f79..93c9cac38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1020,6 +1020,17 @@ dependencies = [ "parking_lot_core", ] +[[package]] +name = "delegate" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "082a24a9967533dc5d743c602157637116fc1b52806d694a5a45e6f32567fcdd" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "deranged" version = "0.3.8" @@ -1895,6 +1906,16 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" +[[package]] +name = "non-empty-string" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb438f1eb418cb32cdf63f67ce050d821354faaea45a0869f707a9eeaccaea7a" +dependencies = [ + "delegate", + "serde", +] + [[package]] name = "nonzero_ext" version = "0.3.0" @@ -3526,6 +3547,7 @@ dependencies = [ "flate2", "hex", "log", + "non-empty-string", "pubsys", "pubsys-setup", "serde", diff --git a/twoliter/Cargo.toml b/twoliter/Cargo.toml index 742570dad..cdb1dede9 100644 --- a/twoliter/Cargo.toml +++ b/twoliter/Cargo.toml @@ -17,6 +17,7 @@ env_logger = "0.10" flate2 = "1" hex = "0.4" log = "0.4" +non-empty-string = { version = "0.2", features = [ "serde" ] } serde = { version = "1", features = ["derive"] } sha2 = "0.10" tar = "0.4" diff --git a/twoliter/src/cmd/build.rs b/twoliter/src/cmd/build.rs index a80896d15..3625d890c 100644 --- a/twoliter/src/cmd/build.rs +++ b/twoliter/src/cmd/build.rs @@ -1,5 +1,5 @@ -use crate::docker; -use crate::project::{Project, Sdk}; +use crate::project::Project; +use crate::{docker, project}; use anyhow::Result; use clap::Parser; use log::debug; @@ -44,7 +44,7 @@ impl BuildVariant { Some(p) => Project::load(p).await?, }; // TODO - get smart about sdk: https://github.com/bottlerocket-os/twoliter/issues/11 - let sdk = Sdk::default(); + let sdk = project::default_sdk(); let _ = docker::create_twoliter_image_if_not_exists(&sdk.uri(&self.arch)).await?; Ok(()) } diff --git a/twoliter/src/docker/image.rs b/twoliter/src/docker/image.rs index 1beda8c8e..98b48fd09 100644 --- a/twoliter/src/docker/image.rs +++ b/twoliter/src/docker/image.rs @@ -1,3 +1,5 @@ +use std::fmt::{Display, Formatter}; + /// Represents a docker image URI such as `public.ecr.aws/myregistry/myrepo:v0.1.0`. The registry is /// optional as it is when using `docker`. That is, it will be looked for locally first, then at /// `dockerhub.io` when the registry is absent. @@ -71,15 +73,15 @@ impl ImageArchUri { /// Create a new `ImageArchUri`. pub(crate) fn new(registry: Option, name: S1, arch: S2, tag: S3) -> Self where - S1: Into, - S2: Into, - S3: Into, + S1: AsRef, + S2: AsRef, + S3: AsRef, { Self { registry, - name: name.into(), - arch: arch.into(), - tag: tag.into(), + name: name.as_ref().into(), + arch: arch.as_ref().into(), + tag: tag.as_ref().into(), } } @@ -93,6 +95,12 @@ impl ImageArchUri { } } +impl Display for ImageArchUri { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + Display::fmt(&self.uri(), f) + } +} + #[test] fn image_arch_uri_no_registry() { let uri = ImageArchUri::new(None, "my-sdk", "i386", "v0.33.1"); diff --git a/twoliter/src/docker/mod.rs b/twoliter/src/docker/mod.rs index 39e99dbe8..079dec8ad 100644 --- a/twoliter/src/docker/mod.rs +++ b/twoliter/src/docker/mod.rs @@ -5,8 +5,3 @@ mod twoliter; pub(crate) use self::commands::DockerBuild; pub(crate) use self::image::{ImageArchUri, ImageUri}; pub(crate) use self::twoliter::create_twoliter_image_if_not_exists; - -pub(super) const DEFAULT_REGISTRY: &str = "public.ecr.aws/bottlerocket"; -pub(super) const DEFAULT_SDK_NAME: &str = "bottlerocket-sdk"; -// TODO - get this from lock file: https://github.com/bottlerocket-os/twoliter/issues/11 -pub(super) const DEFAULT_SDK_VERSION: &str = "v0.33.0"; diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index 8dcca73af..bee333675 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -1,13 +1,18 @@ -use crate::docker::{ImageArchUri, DEFAULT_REGISTRY, DEFAULT_SDK_NAME, DEFAULT_SDK_VERSION}; +use crate::docker::ImageArchUri; use anyhow::{ensure, Context, Result}; use async_recursion::async_recursion; use log::{debug, trace}; +use non_empty_string::NonEmptyString; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use std::fmt; use std::path::{Path, PathBuf}; use tokio::fs; +const DEFAULT_REGISTRY: &str = "public.ecr.aws/bottlerocket"; +const DEFAULT_SDK_NAME: &str = "bottlerocket-sdk"; +const DEFAULT_SDK_VERSION: &str = "v0.34.1"; + /// Common functionality in commands, if the user gave a path to the `Twoliter.toml` file, /// we use it, otherwise we search for the file. Returns the `Project` and the path at which it was /// found (this is the same as `user_path` if provided). @@ -24,14 +29,22 @@ pub(crate) async fn load_or_find_project(user_path: Option) -> Result

, + + /// The version of this schema struct. + schema_version: SchemaVersion<1>, + + /// The Bottlerocket SDK container image. + sdk: Option, + + /// The Bottlerocket Toolchain container image. + toolchain: Option, } impl Project { @@ -90,29 +103,61 @@ impl Project { pub(crate) fn project_dir(&self) -> PathBuf { self.project_dir.clone() } + + pub(crate) fn sdk_name(&self) -> Option<&ImageName> { + self.sdk.as_ref() + } + + pub(crate) fn toolchain_name(&self) -> Option<&ImageName> { + self.toolchain.as_ref() + } + + pub(crate) fn sdk(&self, arch: &str) -> Option { + self.sdk_name().map(|s| s.uri(arch)) + } + + pub(crate) fn toolchain(&self, arch: &str) -> Option { + self.toolchain_name().map(|s| s.uri(arch)) + } } -#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize, Deserialize)] +/// A base name for an image that can be suffixed using a naming convention. For example, +/// `registry=public.ecr.aws/bottlerocket`, `name=bottlerocket`, `version=v0.50.0` can be suffixed +/// via naming convention to produce: +/// - `registry=public.ecr.aws/bottlerocket/bottlerocket-sdk-x86_64:v0.50.0` +/// - `registry=public.ecr.aws/bottlerocket/bottlerocket-toolchain-aarch64:v0.50.0` +#[derive(Debug, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] -pub(crate) struct Sdk { - pub(crate) registry: Option, - pub(crate) name: String, - pub(crate) version: String, +pub(crate) struct ImageName { + /// The registry, e.g. `public.ecr.aws/bottlerocket`. Optional because locally cached images may + /// not specify a registry. + pub(crate) registry: Option, + /// The base name of the image that can be suffixed. For example `bottlerocket` can become + /// `bottlerocket-sdk` or `bottlerocket-toolchain`. + pub(crate) name: NonEmptyString, + /// The version tag, for example `v0.50.0` + pub(crate) version: NonEmptyString, } -impl Default for Sdk { - fn default() -> Self { - Self { - registry: Some(DEFAULT_REGISTRY.to_string()), - name: DEFAULT_SDK_NAME.to_string(), - version: DEFAULT_SDK_VERSION.to_string(), - } +pub(crate) fn default_sdk() -> ImageName { + ImageName { + registry: Some(DEFAULT_REGISTRY.try_into().unwrap()), + name: DEFAULT_SDK_NAME.try_into().unwrap(), + version: DEFAULT_SDK_VERSION.try_into().unwrap(), } } -impl Sdk { - pub(crate) fn uri>(&self, arch: S) -> ImageArchUri { - ImageArchUri::new(self.registry.clone(), &self.name, arch, &self.version) +impl ImageName { + pub(crate) fn uri(&self, arch: S) -> ImageArchUri + where + S: AsRef, + { + ImageArchUri::new( + self.registry.as_ref().map(|s| s.to_string()), + self.name.clone(), + arch.as_ref(), + &self.version, + ) } } @@ -192,6 +237,23 @@ mod test { // Add checks here as desired to validate deserialization. assert_eq!(SchemaVersion::<1>::default(), deserialized.schema_version); + let sdk_name = deserialized.sdk_name().unwrap(); + let toolchain_name = deserialized.toolchain_name().unwrap(); + assert_eq!("a.com/b", sdk_name.registry.as_ref().unwrap().as_str()); + assert_eq!( + "my-bottlerocket-sdk", + deserialized.sdk_name().unwrap().name.as_str() + ); + assert_eq!("v1.2.3", deserialized.sdk_name().unwrap().version.as_str()); + assert_eq!("c.co/d", toolchain_name.registry.as_ref().unwrap().as_str()); + assert_eq!( + "toolchainz", + deserialized.toolchain_name().unwrap().name.as_str() + ); + assert_eq!( + "v3.4.5", + deserialized.toolchain_name().unwrap().version.as_str() + ); } /// Ensure that a `Twoliter.toml` cannot be serialized if the `schema_version` is incorrect. @@ -222,4 +284,32 @@ mod test { // Ensure that the file we loaded was the one we expected to load. assert_eq!(project.filepath(), twoliter_toml_path); } + + #[test] + fn test_sdk_toolchain_uri() { + let project = Project { + filepath: Default::default(), + project_dir: Default::default(), + schema_version: Default::default(), + sdk: Some(ImageName { + registry: Some("example.com".try_into().unwrap()), + name: "foo-abc".try_into().unwrap(), + version: "version1".try_into().unwrap(), + }), + toolchain: Some(ImageName { + registry: Some("example.com".try_into().unwrap()), + name: "foo-def".try_into().unwrap(), + version: "version2".try_into().unwrap(), + }), + }; + + assert_eq!( + "example.com/foo-abc-x86_64:version1", + project.sdk("x86_64").unwrap().to_string() + ); + assert_eq!( + "example.com/foo-def-aarch64:version2", + project.toolchain("aarch64").unwrap().to_string() + ); + } } diff --git a/twoliter/src/test/data/Twoliter-1.toml b/twoliter/src/test/data/Twoliter-1.toml index 8912428c3..7ed1fbd8a 100644 --- a/twoliter/src/test/data/Twoliter-1.toml +++ b/twoliter/src/test/data/Twoliter-1.toml @@ -3,6 +3,11 @@ project-name = "sample-project" project-version = "v1.0.0" [sdk] -registry = "example.com/my-repos" +registry = "a.com/b" name = "my-bottlerocket-sdk" -version = "1.2.3" +version = "v1.2.3" + +[toolchain] +registry = "c.co/d" +name = "toolchainz" +version = "v3.4.5" diff --git a/twoliter/src/test/data/Twoliter-invalid-version.toml b/twoliter/src/test/data/Twoliter-invalid-version.toml index 455b8878b..6107c06dd 100644 --- a/twoliter/src/test/data/Twoliter-invalid-version.toml +++ b/twoliter/src/test/data/Twoliter-invalid-version.toml @@ -5,4 +5,9 @@ project-version = "v1.0.0" [sdk] registry = "example.com/my-repos" name = "my-bottlerocket-sdk" -version = "1.2.3" +version = "v1.2.3" + +[toolchain] +registry = "example.com/my-repos" +name = "my-bottlerocket-toolchain" +version = "v1.2.3" From af96ef7e0c441ddbf80326b61748d3aa88f277ed Mon Sep 17 00:00:00 2001 From: Matthew James Briggs Date: Thu, 28 Sep 2023 17:27:13 -0700 Subject: [PATCH 2/2] use project file sdk in twoliter make Require the SDK to be specified in Twoliter.toml when using twoliter make. --- tools/buildsys/src/builder.rs | 5 ++- tools/buildsys/src/constants.rs | 2 + tools/buildsys/src/gomod.rs | 5 +-- tools/buildsys/src/main.rs | 1 + twoliter/embedded/Makefile.toml | 66 ++++++++++++++++++--------------- twoliter/src/cmd/make.rs | 57 +++++++++++++++++++++++++++- 6 files changed, 100 insertions(+), 36 deletions(-) create mode 100644 tools/buildsys/src/constants.rs diff --git a/tools/buildsys/src/builder.rs b/tools/buildsys/src/builder.rs index 38f8bb727..74a2ba4f4 100644 --- a/tools/buildsys/src/builder.rs +++ b/tools/buildsys/src/builder.rs @@ -7,6 +7,7 @@ the repository's top-level Dockerfile. pub(crate) mod error; use error::Result; +use crate::constants::{SDK_VAR, TOOLCHAIN_VAR}; use duct::cmd; use lazy_static::lazy_static; use nonzero_ext::nonzero; @@ -283,8 +284,8 @@ fn build( let tag = format!("{}-{}", tag, token); // Our SDK and toolchain are picked by the external `cargo make` invocation. - let sdk = getenv("BUILDSYS_SDK_IMAGE")?; - let toolchain = getenv("BUILDSYS_TOOLCHAIN")?; + let sdk = getenv(SDK_VAR)?; + let toolchain = getenv(TOOLCHAIN_VAR)?; // Avoid using a cached layer from a previous build. let nocache = rand::thread_rng().gen::(); diff --git a/tools/buildsys/src/constants.rs b/tools/buildsys/src/constants.rs new file mode 100644 index 000000000..a3f0bb0d8 --- /dev/null +++ b/tools/buildsys/src/constants.rs @@ -0,0 +1,2 @@ +pub(crate) const SDK_VAR: &str = "TLPRIVATE_SDK_IMAGE"; +pub(crate) const TOOLCHAIN_VAR: &str = "TLPRIVATE_TOOLCHAIN"; diff --git a/tools/buildsys/src/gomod.rs b/tools/buildsys/src/gomod.rs index 59d2ebcb3..c0a835a7e 100644 --- a/tools/buildsys/src/gomod.rs +++ b/tools/buildsys/src/gomod.rs @@ -18,6 +18,7 @@ when the docker-go script is invoked. pub(crate) mod error; use error::Result; +use crate::constants::SDK_VAR; use buildsys::manifest; use duct::cmd; use snafu::{ensure, OptionExt, ResultExt}; @@ -111,9 +112,7 @@ impl GoMod { ); // Our SDK and toolchain are picked by the external `cargo make` invocation. - let sdk = env::var("BUILDSYS_SDK_IMAGE").context(error::EnvironmentSnafu { - var: "BUILDSYS_SDK_IMAGE", - })?; + let sdk = env::var(SDK_VAR).context(error::EnvironmentSnafu { var: SDK_VAR })?; let args = DockerGoArgs { module_path: package_dir, diff --git a/tools/buildsys/src/main.rs b/tools/buildsys/src/main.rs index 5ea01121f..8f457305b 100644 --- a/tools/buildsys/src/main.rs +++ b/tools/buildsys/src/main.rs @@ -10,6 +10,7 @@ The implementation is closely tied to the top-level Dockerfile. */ mod builder; mod cache; +mod constants; mod gomod; mod project; mod spec; diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index 4eaf43b08..eaf60f21d 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -31,12 +31,6 @@ BUILDSYS_NAME = "bottlerocket" # If you're building a Bottlerocket remix, you'd want to set this to something like # "Bottlerocket Remix by ${CORP}" or "${CORP}'s Bottlerocket Remix" BUILDSYS_PRETTY_NAME = "Bottlerocket OS" -# SDK name used for building -BUILDSYS_SDK_NAME="bottlerocket" -# SDK version used for building -BUILDSYS_SDK_VERSION="v0.33.0" -# Site for fetching the SDK -BUILDSYS_REGISTRY="public.ecr.aws/bottlerocket" # These can be overridden with -e to change configuration for pubsys (`cargo # make repo`). In addition, you can set RELEASE_START_TIME to determine when @@ -142,11 +136,6 @@ TESTSYS_LOG_LEVEL = "info" # Certain variables are defined here to allow us to override a component value # on the command line. -# Depends on ${BUILDSYS_ARCH}, ${BUILDSYS_REGISTRY}, ${BUILDSYS_SDK_NAME}, and -# ${BUILDSYS_SDK_VERSION}. -BUILDSYS_SDK_IMAGE = { script = [ "echo ${BUILDSYS_REGISTRY}/${BUILDSYS_SDK_NAME}-sdk-${BUILDSYS_ARCH}:${BUILDSYS_SDK_VERSION}" ] } -BUILDSYS_TOOLCHAIN = { script = [ "echo ${BUILDSYS_REGISTRY}/${BUILDSYS_SDK_NAME}-toolchain-${BUILDSYS_ARCH}:${BUILDSYS_SDK_VERSION}" ] } - # Depends on ${BUILDSYS_JOBS}. CARGO_MAKE_CARGO_LIMIT_JOBS = "--jobs ${BUILDSYS_JOBS}" CARGO_MAKE_CARGO_ARGS = "--offline --locked" @@ -238,7 +227,17 @@ fi ''' ] } +# These are variables that are not meant to be set by users of `twoliter make`. These are intended +# to be set only by Twoliter itself when it invokes `cargo make`. +[env.private] +# The URIs for the SDK image and the toolchain image must be provided. +TLPRIVATE_SDK_IMAGE = "" +TLPRIVATE_TOOLCHAIN = "" + +#################################################################################################### + [tasks.setup] +script_runner = "bash" script = [ ''' # Ensure we use a supported architecture @@ -256,6 +255,13 @@ if [ -z "${TWOLITER_TOOLS_DIR}" ];then exit 1 fi +# Ensure TLPRIVATE_SDK_IMAGE and TLPRIVATE_TOOLCHAIN are set +if [[ -z "${TLPRIVATE_SDK_IMAGE}" || -z "{TLPRIVATE_TOOLCHAIN}" ]];then + echo "TLPRIVATE_SDK_IMAGE and TLPRIVATE_TOOLCHAIN must be defined and must be non-zero in length." + echo "Are you using Twoliter? It is a bug if Twoliter has invoked cargo make without these." + exit 1 +fi + mkdir -p ${BUILDSYS_BUILD_DIR} mkdir -p ${BUILDSYS_OUTPUT_DIR} mkdir -p ${BUILDSYS_PACKAGES_DIR} @@ -290,9 +296,9 @@ dependencies = ["setup-build"] script_runner = "bash" script = [ ''' -if ! docker image inspect "${BUILDSYS_SDK_IMAGE}" >/dev/null 2>&1 ; then - if ! docker pull "${BUILDSYS_SDK_IMAGE}" ; then - echo "failed to pull '${BUILDSYS_SDK_IMAGE}'" >&2 +if ! docker image inspect "${TLPRIVATE_SDK_IMAGE}" >/dev/null 2>&1 ; then + if ! docker pull "${TLPRIVATE_SDK_IMAGE}" ; then + echo "failed to pull '${TLPRIVATE_SDK_IMAGE}'" >&2 exit 1 fi fi @@ -304,7 +310,7 @@ dependencies = ["setup-build"] script_runner = "bash" script = [ ''' -if docker image inspect "${BUILDSYS_TOOLCHAIN}-${BUILDSYS_ARCH}" >/dev/null 2>&1 ; then +if docker image inspect "${TLPRIVATE_TOOLCHAIN}-${BUILDSYS_ARCH}" >/dev/null 2>&1 ; then exit 0 fi @@ -315,14 +321,14 @@ esac # We want the image with the target's native toolchain, rather than one that matches the # host architecture. -if ! docker pull --platform "${docker_arch}" "${BUILDSYS_TOOLCHAIN}" ; then - echo "could not pull '${BUILDSYS_TOOLCHAIN}' for ${docker_arch}" >&2 +if ! docker pull --platform "${docker_arch}" "${TLPRIVATE_TOOLCHAIN}" ; then + echo "could not pull '${TLPRIVATE_TOOLCHAIN}' for ${docker_arch}" >&2 exit 1 fi # Apply a tag to distinguish the image from other architectures. -if ! docker tag "${BUILDSYS_TOOLCHAIN}" "${BUILDSYS_TOOLCHAIN}-${BUILDSYS_ARCH}" ; then - echo "could not tag '${BUILDSYS_TOOLCHAIN}-${BUILDSYS_ARCH}'" >&2 +if ! docker tag "${TLPRIVATE_TOOLCHAIN}" "${TLPRIVATE_TOOLCHAIN}-${BUILDSYS_ARCH}" ; then + echo "could not tag '${TLPRIVATE_TOOLCHAIN}-${BUILDSYS_ARCH}'" >&2 exit 1 fi ''' @@ -350,7 +356,7 @@ go_fetch() { module="${1:?}" ${TWOLITER_TOOLS_DIR}/docker-go \ --module-path "${BUILDSYS_SOURCES_DIR}/${module}" \ - --sdk-image ${BUILDSYS_SDK_IMAGE} \ + --sdk-image ${TLPRIVATE_SDK_IMAGE} \ --go-mod-cache ${GO_MOD_CACHE} \ --command "go list -mod=readonly ./... >/dev/null && go mod vendor" } @@ -379,7 +385,7 @@ test_go_module() { module="${1:?}" ${TWOLITER_TOOLS_DIR}/docker-go \ --module-path "${BUILDSYS_SOURCES_DIR}/${module}" \ - --sdk-image ${BUILDSYS_SDK_IMAGE} \ + --sdk-image ${TLPRIVATE_SDK_IMAGE} \ --go-mod-cache ${GO_MOD_CACHE} \ --command "cd cmd/$module; go test -v" } @@ -410,7 +416,7 @@ go_fmt() { module="${1:?}" ${TWOLITER_TOOLS_DIR}/docker-go \ --module-path "${BUILDSYS_SOURCES_DIR}/${module}" \ - --sdk-image ${BUILDSYS_SDK_IMAGE} \ + --sdk-image ${TLPRIVATE_SDK_IMAGE} \ --go-mod-cache ${GO_MOD_CACHE} \ --command "gofmt -l cmd/$module" } @@ -429,7 +435,7 @@ if ! docker run --rm \ -e CARGO_HOME="/tmp/.cargo" \ -v "${CARGO_HOME}":/tmp/.cargo \ -v "${BUILDSYS_ROOT_DIR}/sources":/tmp/sources \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ cargo fmt \ --manifest-path /tmp/sources/Cargo.toml \ --message-format short \ @@ -466,7 +472,7 @@ if ! docker run --rm \ -v "${CARGO_HOME}":/tmp/.cargo \ -v "${BUILDSYS_ROOT_DIR}/sources":/tmp/sources \ -e VARIANT \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ cargo clippy \ --manifest-path /tmp/sources/Cargo.toml \ --locked -- -D warnings --no-deps; then @@ -491,7 +497,7 @@ if ! docker run --rm \ --user "$(id -u):$(id -g)" \ --security-opt="label=disable" \ -v "${BUILDSYS_TOOLS_DIR}":/tmp/tools \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ bash -c \ 'flagged_scripts=0 && \ cd /tmp/tools && \ @@ -657,7 +663,7 @@ echo "Generating local keys." >&2 mkdir -p "${BUILDSYS_SBKEYS_PROFILE_DIR}" ${BUILDSYS_SBKEYS_DIR}/generate-local-sbkeys \ - --sdk-image "${BUILDSYS_SDK_IMAGE}" \ + --sdk-image "${TLPRIVATE_SDK_IMAGE}" \ --output-dir "${BUILDSYS_SBKEYS_PROFILE_DIR}" ''' ] @@ -714,7 +720,7 @@ docker run --rm \ --security-opt="label=disable" \ -v "${BOOT_CONFIG_INPUT}":/tmp/bootconfig-input \ -v "${boot_config}":/tmp/bootconfig.data \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ bootconfig -a /tmp/bootconfig-input /tmp/bootconfig.data if [ -e "${boot_config_tmp}" ] ; then @@ -734,7 +740,7 @@ docker run --rm \ --user "$(id -u):$(id -g)" \ --security-opt="label=disable" \ -v "${BOOT_CONFIG}":/tmp/bootconfig.data \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ bootconfig -l /tmp/bootconfig.data ''' ] @@ -816,7 +822,7 @@ docker run --rm \ -e CARGO_HOME="/tmp/.cargo" \ -v "${CARGO_HOME}":/tmp/.cargo \ -v "${BUILDSYS_ROOT_DIR}/sources":/tmp/sources \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ bash -c "${run_cargo_deny}" [ "${?}" -eq 0 ] || [ "${BUILDSYS_ALLOW_FAILED_LICENSE_CHECK}" = "true" ] ''' @@ -854,7 +860,7 @@ docker run --rm \ -v "${CARGO_HOME}":/tmp/.cargo \ -v "${BUILDSYS_ROOT_DIR}/licenses:/tmp/licenses" \ -v "${BUILDSYS_ROOT_DIR}/Licenses.toml:/tmp/Licenses.toml" \ - "${BUILDSYS_SDK_IMAGE}" \ + "${TLPRIVATE_SDK_IMAGE}" \ bash -c "${run_fetch_licenses}" ''' ] diff --git a/twoliter/src/cmd/make.rs b/twoliter/src/cmd/make.rs index b77d9b274..04731ea0d 100644 --- a/twoliter/src/cmd/make.rs +++ b/twoliter/src/cmd/make.rs @@ -1,7 +1,9 @@ use crate::common::exec; +use crate::docker::ImageArchUri; use crate::project; +use crate::project::Project; use crate::tools::{install_tools, tools_tempdir}; -use anyhow::Result; +use anyhow::{bail, ensure, Result}; use clap::Parser; use log::trace; use std::path::PathBuf; @@ -45,14 +47,34 @@ impl Make { project.project_dir().display().to_string(), ]; + let mut arch = String::new(); + for (key, val) in std::env::vars() { if is_build_system_env(key.as_str()) { trace!("Passing env var {} to cargo make", key); args.push("-e".to_string()); args.push(format!("{}={}", key, val)); } + + // To avoid confusion, environment variables whose values have been moved to + // Twoliter.toml are expressly disallowed here. + check_for_disallowed_var(&key)?; + + if key == "BUILDSYS_ARCH" { + arch = val.clone(); + } } + ensure!( + !arch.is_empty(), + "It is required to pass a non-zero string as the value of environment variable \ + 'BUILDSYS_ARCH' when running twoliter make" + ); + + let (sdk, toolchain) = require_sdk(&project, &arch)?; + + args.push(format!("-e=TLPRIVATE_SDK_IMAGE={}", sdk)); + args.push(format!("-e=TLPRIVATE_TOOLCHAIN={}", toolchain)); args.push(format!("-e=CARGO_HOME={}", self.cargo_home.display())); args.push(format!( "-e=TWOLITER_TOOLS_DIR={}", @@ -86,6 +108,13 @@ const ENV_VARS: [&str; 12] = [ "VMWARE_VM_NAME_DEFAULT", ]; +const DISALLOWED_SDK_VARS: [&str; 4] = [ + "BUILDSYS_SDK_NAME", + "BUILDSYS_SDK_VERSION", + "BUILDSYS_REGISTRY", + "BUILDSYS_TOOLCHAIN", +]; + /// Returns `true` if `key` is an environment variable that needs to be passed to `cargo make`. fn is_build_system_env(key: impl AsRef) -> bool { let key = key.as_ref(); @@ -98,6 +127,26 @@ fn is_build_system_env(key: impl AsRef) -> bool { || ENV_VARS.contains(&key) } +fn check_for_disallowed_var(key: &str) -> Result<()> { + if DISALLOWED_SDK_VARS.contains(&key) { + bail!( + "The environment variable '{}' can no longer be used. Specify the SDK in Twoliter.toml", + key + ) + } + Ok(()) +} + +fn require_sdk(project: &Project, arch: &str) -> Result<(ImageArchUri, ImageArchUri)> { + match (project.sdk(arch), project.toolchain(arch)) { + (Some(s), Some(t)) => Ok((s, t)), + _ => bail!( + "When using twoliter make, it is required that the SDK and toolchain be specified in \ + Twoliter.toml" + ), + } +} + #[test] fn test_is_build_system_env() { assert!(is_build_system_env( @@ -113,3 +162,9 @@ fn test_is_build_system_env() { assert!(!is_build_system_env("HOME")); assert!(!is_build_system_env("COLORTERM")); } + +#[test] +fn test_check_for_disallowed_var() { + assert!(check_for_disallowed_var("BUILDSYS_REGISTRY").is_err()); + assert!(check_for_disallowed_var("BUILDSYS_PRETTY_NAME").is_ok()); +}