diff --git a/twoliter/embedded/Dockerfile b/twoliter/embedded/Dockerfile index a0c814c02..5abe8119a 100644 --- a/twoliter/embedded/Dockerfile +++ b/twoliter/embedded/Dockerfile @@ -144,7 +144,6 @@ USER builder RUN --mount=source=.cargo,target=/home/builder/.cargo \ --mount=type=cache,target=/home/builder/.cache,from=cache,source=/cache \ --mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/models/src/variant,from=variantcache,source=/variantcache \ - --mount=type=cache,target=/home/builder/rpmbuild/BUILD/sources/logdog/conf/current,from=variantcache,source=/variantcache \ --mount=source=sources,target=/home/builder/rpmbuild/BUILD/sources \ rpmbuild -ba --clean \ --undefine _auto_set_build_flags \ diff --git a/twoliter/embedded/Makefile.toml b/twoliter/embedded/Makefile.toml index 929e0dc60..87b2c554a 100644 --- a/twoliter/embedded/Makefile.toml +++ b/twoliter/embedded/Makefile.toml @@ -97,7 +97,8 @@ BUILDSYS_JOBS = "8" CARGO_HOME = "${BUILDSYS_ROOT_DIR}/.cargo" # This needs to end with pkg/mod so that we can mount the parent of pkg/mod as GOPATH. GO_MOD_CACHE = "${BUILDSYS_ROOT_DIR}/.gomodcache/pkg/mod" -GO_MODULES = "ecs-gpu-init host-ctr" +# Dynamically load a list of go modules from ${BUILDSYS_SOURCE_DIR} +GO_MODULES = { script = ['find ${BUILDSYS_SOURCES_DIR} -name go.mod -type f -printf "%h\n" | xargs -n1 basename'] } DOCKER_BUILDKIT = "1" # This is the filename suffix for operations that write out AMI information to diff --git a/twoliter/embedded/rpm2img b/twoliter/embedded/rpm2img index 93e4cb4d9..36355d5a0 100755 --- a/twoliter/embedded/rpm2img +++ b/twoliter/embedded/rpm2img @@ -238,7 +238,6 @@ INVENTORY_DATA="$(jq --slurp 'sort_by(.Name)' <<< "${INVENTORY_DATA}" | jq '{"Co printf "%s\n" "${INVENTORY_DATA}" > "${ROOT_MOUNT}/usr/share/bottlerocket/application-inventory.json" # install licenses -install -p -m 0644 /host/{COPYRIGHT,LICENSE-APACHE,LICENSE-MIT} "${ROOT_MOUNT}"/usr/share/licenses/ mksquashfs \ "${ROOT_MOUNT}"/usr/share/licenses \ "${ROOT_MOUNT}"/usr/share/bottlerocket/licenses.squashfs \ diff --git a/twoliter/src/cargo_make.rs b/twoliter/src/cargo_make.rs index 1a344fb66..9c6f88ba5 100644 --- a/twoliter/src/cargo_make.rs +++ b/twoliter/src/cargo_make.rs @@ -94,41 +94,8 @@ impl CargoMake { self } - /// Specify environment variables that should be applied for this comand - pub(crate) fn _envs(mut self, env_vars: V) -> Self - where - S1: Into, - S2: Into, - V: Into>, - { - for (key, value) in env_vars.into() { - self.args - .push(format!("-e={}={}", key.into(), value.into())); - } - self - } - - /// Specify `cargo make` arguments that should be applied for this comand - pub(crate) fn _arg(mut self, arg: S) -> Self - where - S: Into, - { - self.args.push(arg.into()); - self - } - - /// Specify `cargo make` arguments that should be applied for this comand - pub(crate) fn _args(mut self, args: V) -> Self - where - S: Into, - V: Into>, - { - self.args.extend(args.into().into_iter().map(Into::into)); - self - } - /// Execute the `cargo make` task - pub(crate) async fn _exec(&self, task: S) -> Result<()> + pub(crate) async fn exec(&self, task: S) -> Result<()> where S: Into, { diff --git a/twoliter/src/cmd/build.rs b/twoliter/src/cmd/build.rs index c3f17820c..36e78b430 100644 --- a/twoliter/src/cmd/build.rs +++ b/twoliter/src/cmd/build.rs @@ -1,6 +1,14 @@ -use anyhow::Result; +use crate::cargo_make::CargoMake; +use crate::docker::DockerContainer; +use crate::project; +use crate::tools::{install_tools, tools_tempdir}; +use anyhow::{Context, Result}; use clap::Parser; -use std::path::PathBuf; +use log::debug; +use std::fs; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; +use tokio::fs::{remove_dir_all, remove_file}; #[derive(Debug, Parser)] pub(crate) enum BuildCommand { @@ -25,10 +33,95 @@ pub(crate) struct BuildVariant { /// The architecture to build for. #[clap(long = "arch", default_value = "x86_64")] arch: String, + + /// The variant to build. + variant: String, } impl BuildVariant { pub(super) async fn run(&self) -> Result<()> { - Ok(()) + let project = project::load_or_find_project(self.project_path.clone()).await?; + let token = project.token(); + let tempdir = tools_tempdir()?; + install_tools(&tempdir).await?; + let makefile_path = tempdir.path().join("Makefile.toml"); + // A temporary directory in the `build` directory + let build_temp_dir = TempDir::new_in(project.project_dir()) + .context("Unable to create a tempdir for Twoliter's build")?; + let packages_dir = build_temp_dir.path().join("sdk_rpms"); + fs::create_dir_all(&packages_dir)?; + + let sdk_container = DockerContainer::new( + format!("sdk-{}", token), + project + .sdk(&self.arch) + .context(format!( + "No SDK defined in {} for {}", + project.filepath().display(), + &self.arch + ))? + .uri(), + ) + .await?; + sdk_container + .cp_out(Path::new("twoliter/alpha/build/rpms"), &packages_dir) + .await?; + + let rpms_dir = project.project_dir().join("build").join("rpms"); + fs::create_dir_all(&rpms_dir)?; + debug!("Moving rpms to build dir"); + for maybe_file in fs::read_dir(packages_dir.join("rpms"))? { + let file = maybe_file?; + debug!("Moving '{}'", file.path().display()); + fs::rename(file.path(), rpms_dir.join(file.file_name()))?; + } + + let mut created_files = Vec::new(); + + let sbkeys_dir = project.project_dir().join("sbkeys"); + if !sbkeys_dir.is_dir() { + // Create a sbkeys directory in the main project + debug!("sbkeys dir not found. Creating a temporary directory"); + fs::create_dir_all(&sbkeys_dir)?; + sdk_container + .cp_out( + Path::new("twoliter/alpha/sbkeys/generate-local-sbkeys"), + &sbkeys_dir, + ) + .await?; + }; + + // TODO: Remove once models is no longer conditionally compiled. + // Create the models directory for the sdk to mount + let models_dir = project.project_dir().join("sources/models"); + if !models_dir.is_dir() { + debug!("models source dir not found. Creating a temporary directory"); + fs::create_dir_all(&models_dir.join("src/variant")) + .context("Unable to create models source directory")?; + created_files.push(models_dir) + } + + // Hold the result of the cargo make call so we can clean up the project directory first. + let res = CargoMake::new(&project, &self.arch)? + .env("TWOLITER_TOOLS_DIR", tempdir.path().display().to_string()) + .env("BUILDSYS_ARCH", &self.arch) + .env("BUILDSYS_VARIANT", &self.variant) + .env("BUILDSYS_SBKEYS_DIR", sbkeys_dir.display().to_string()) + .makefile(makefile_path) + .project_dir(project.project_dir()) + .exec("build") + .await; + + // Clean up all of the files we created + for file_name in created_files { + let added = Path::new(&file_name); + if added.is_file() { + remove_file(added).await?; + } else if added.is_dir() { + remove_dir_all(added).await?; + } + } + + res } } diff --git a/twoliter/src/docker/container.rs b/twoliter/src/docker/container.rs new file mode 100644 index 000000000..7ccec0961 --- /dev/null +++ b/twoliter/src/docker/container.rs @@ -0,0 +1,74 @@ +use crate::common::exec; +use anyhow::Result; +use log::{debug, log, Level}; +use std::path::Path; +use tokio::process::Command; + +pub(crate) struct DockerContainer { + name: String, +} + +impl DockerContainer { + /// Create a docker image with the given name from the image by using `docker create`. + pub(crate) async fn new(container_name: S1, image: S2) -> Result + where + S1: Into, + S2: Into, + { + let name = container_name.into(); + let image = image.into(); + + // Make sure previous versions of this container are stopped deleted. + cleanup_container(&name, Level::Trace).await; + + debug!("Creating docker container '{name}' from image '{image}'"); + + // Create the new container. + let args = vec![ + "create".to_string(), + "--rm".to_string(), + "--name".to_string(), + name.to_string(), + image.to_string(), + ]; + + exec(Command::new("docker").args(args), true).await?; + Ok(Self { name }) + } + + /// Copy the data from this container to a local destination. + pub(crate) async fn cp_out(&self, src: P1, dest: P2) -> Result<()> + where + P1: AsRef, + P2: AsRef, + { + debug!( + "Copying '{}' from '{}' to '{}'", + src.as_ref().display(), + self.name, + dest.as_ref().display() + ); + let mut args = vec!["cp".to_string()]; + args.push(format!("{}:{}", self.name, src.as_ref().display())); + args.push(dest.as_ref().display().to_string()); + exec(Command::new("docker").args(args), true).await + } +} + +impl Drop for DockerContainer { + fn drop(&mut self) { + let name = self.name.clone(); + tokio::task::spawn(async move { cleanup_container(&name, Level::Error).await }); + } +} + +async fn cleanup_container(name: &str, log_level: Level) { + let args = vec!["stop".to_string(), name.to_string()]; + if let Err(e) = exec(Command::new("docker").args(args), true).await { + log!(log_level, "Unable to stop container '{}': {e}", name) + } + let args = vec!["rm".to_string(), name.to_string()]; + if let Err(e) = exec(Command::new("docker").args(args), true).await { + log!(log_level, "Unable to remove container '{}': {e}", name) + } +} diff --git a/twoliter/src/docker/mod.rs b/twoliter/src/docker/mod.rs index 59b30d249..5f2f873e2 100644 --- a/twoliter/src/docker/mod.rs +++ b/twoliter/src/docker/mod.rs @@ -1,5 +1,7 @@ mod commands; +mod container; mod image; -#[allow(unused)] +pub(crate) use self::container::DockerContainer; +#[allow(unused_imports)] pub(crate) use self::image::{ImageArchUri, ImageUri}; diff --git a/twoliter/src/project.rs b/twoliter/src/project.rs index c457a649b..a04c501db 100644 --- a/twoliter/src/project.rs +++ b/twoliter/src/project.rs @@ -5,6 +5,7 @@ use log::{debug, trace}; use non_empty_string::NonEmptyString; use serde::de::Error; use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use sha2::{Digest, Sha512}; use std::fmt; use std::path::{Path, PathBuf}; use tokio::fs; @@ -115,6 +116,13 @@ impl Project { pub(crate) fn toolchain(&self, arch: &str) -> Option { self.toolchain_name().map(|s| s.uri(arch)) } + + pub(crate) fn token(&self) -> String { + let mut d = Sha512::new(); + d.update(self.filepath().display().to_string()); + let digest = hex::encode(d.finalize()); + (digest[..12]).to_string() + } } /// A base name for an image that can be suffixed using a naming convention. For example, diff --git a/twoliter/src/test/cargo_make.rs b/twoliter/src/test/cargo_make.rs index 73e025af7..4faf3a8c7 100644 --- a/twoliter/src/test/cargo_make.rs +++ b/twoliter/src/test/cargo_make.rs @@ -7,7 +7,7 @@ async fn test_cargo_make() { let cargo_make = CargoMake::new(&project, "arch") .unwrap() .makefile(data_dir().join("Makefile.toml")); - cargo_make._exec("verify-twoliter-env").await.unwrap(); + cargo_make.exec("verify-twoliter-env").await.unwrap(); cargo_make .clone() .env("FOO", "bar") @@ -29,23 +29,4 @@ async fn test_cargo_make() { ) .await .unwrap(); - cargo_make - .clone() - ._arg("--env") - ._arg("FOO=bar") - .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"]) - .await - .unwrap(); - cargo_make - .clone() - ._args(["--env", "FOO=bar"]) - .exec_with_args("verify-env-value-with-arg", ["FOO", "bar"]) - .await - .unwrap(); - cargo_make - .clone() - ._envs([("FOO", "bar"), ("BAR", "baz")]) - .exec_with_args("verify-env-value-with-arg", ["BAR", "baz"]) - .await - .unwrap(); }