Skip to content

Commit

Permalink
Merge pull request #357 from bcressey/build-all-variants
Browse files Browse the repository at this point in the history 
add support for a "build-all" task
  • Loading branch information
@bcressey
bcressey authored Sep 6, 2024
2 parents 5f265b4 + 350b5fb commit f835018
Show file tree
Hide file tree
Showing 8 changed files with 100 additions and 165 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ targets = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"]
[workspace.dependencies]
bottlerocket-types = { version = "0.0.14", git = "https://github.com/bottlerocket-os/bottlerocket-test-system", tag = "v0.0.14" }
bottlerocket-variant = { version = "0.1", path = "tools/bottlerocket-variant" }
buildsys = { version = "0.1", path = "tools/buildsys", lib = true, artifact = [ "bin:buildsys", "bin:bottlerocket-variant" ] }
buildsys = { version = "0.1", path = "tools/buildsys", lib = true, artifact = [ "bin:buildsys" ] }
buildsys-config = { version = "0.1", path = "tools/buildsys-config" }
oci-cli-wrapper = { version = "0.1", path = "tools/oci-cli-wrapper" }
parse-datetime = { version = "0.1", path = "tools/parse-datetime" }
Expand Down
37 changes: 12 additions & 25 deletions tools/buildsys/src/args.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,19 +15,18 @@ use url::Url;
/// variable changes. The build type is represented with bit flags so that we can easily list
/// multiple build types for a single variable. See `[BuildType]` and `[rerun_for_envs]` below to
/// see how this list is used.
const REBUILD_VARS: [(&str, u8); 16] = [
const REBUILD_VARS: [(&str, u8); 15] = [
("BUILDSYS_ARCH", PACKAGE | KIT | VARIANT),
("BUILDSYS_CACERTS_BUNDLE_OVERRIDE", VARIANT),
("BUILDSYS_KITS_DIR", KIT),
("BUILDSYS_EXTERNAL_KITS_DIR", PACKAGE | KIT | VARIANT),
("BUILDSYS_NAME", VARIANT),
("BUILDSYS_OUTPUT_DIR", VARIANT),
("BUILDSYS_IMAGES_DIR", VARIANT),
("BUILDSYS_OUTPUT_GENERATION_ID", PACKAGE | KIT | VARIANT),
("BUILDSYS_PACKAGES_DIR", PACKAGE),
("BUILDSYS_PRETTY_NAME", VARIANT),
("BUILDSYS_ROOT_DIR", PACKAGE | KIT | VARIANT),
("BUILDSYS_STATE_DIR", PACKAGE | KIT | VARIANT),
("BUILDSYS_TIMESTAMP", VARIANT),
("BUILDSYS_VARIANT", VARIANT),
("BUILDSYS_VERSION_BUILD", KIT | VARIANT),
("BUILDSYS_VERSION_IMAGE", KIT | VARIANT),
Expand Down Expand Up @@ -69,18 +68,12 @@ pub(crate) struct Common {
#[arg(long, env = "BUILDSYS_CARGO_METADATA_PATH")]
pub(crate) cargo_metadata_path: PathBuf,

#[arg(long, env = "BUILDSYS_OUTPUT_DIR")]
pub(crate) image_arch_variant_dir: PathBuf,

#[arg(long, env = "BUILDSYS_ROOT_DIR")]
pub(crate) root_dir: PathBuf,

#[arg(long, env = "BUILDSYS_STATE_DIR")]
pub(crate) state_dir: PathBuf,

#[arg(long, env = "BUILDSYS_TIMESTAMP")]
pub(crate) timestamp: String,

#[arg(long, env = "BUILDSYS_VERSION_FULL")]
pub(crate) version_full: String,

Expand Down Expand Up @@ -169,27 +162,18 @@ pub(crate) struct BuildVariantArgs {
#[arg(long, env = "BUILDSYS_PRETTY_NAME")]
pub(crate) pretty_name: String,

#[arg(long, env = "BUILDSYS_VARIANT")]
pub(crate) variant: String,

#[arg(long, env = "BUILDSYS_VARIANT_PLATFORM")]
pub(crate) variant_platform: String,

#[arg(long, env = "BUILDSYS_VARIANT_RUNTIME")]
pub(crate) variant_runtime: String,

#[arg(long, env = "BUILDSYS_VARIANT_FAMILY")]
pub(crate) variant_family: String,

#[arg(long, env = "BUILDSYS_VARIANT_FLAVOR")]
pub(crate) variant_flavor: String,
#[arg(long, env = "CARGO_MANIFEST_DIR")]
pub(crate) variant: PathBuf,

#[arg(long, env = "BUILDSYS_VERSION_BUILD")]
pub(crate) version_build: String,

#[arg(long, env = "BUILDSYS_VERSION_IMAGE")]
pub(crate) version_image: String,

#[arg(long, env = "BUILDSYS_IMAGES_DIR")]
pub(crate) image_dir: PathBuf,

#[command(flatten)]
pub(crate) common: Common,
}
Expand All @@ -200,15 +184,18 @@ pub(crate) struct RepackVariantArgs {
#[arg(long, env = "BUILDSYS_NAME")]
pub(crate) name: String,

#[arg(long, env = "BUILDSYS_VARIANT")]
pub(crate) variant: String,
#[arg(long, env = "CARGO_MANIFEST_DIR")]
pub(crate) variant: PathBuf,

#[arg(long, env = "BUILDSYS_VERSION_BUILD")]
pub(crate) version_build: String,

#[arg(long, env = "BUILDSYS_VERSION_IMAGE")]
pub(crate) version_image: String,

#[arg(long, env = "BUILDSYS_IMAGES_DIR")]
pub(crate) image_dir: PathBuf,

#[command(flatten)]
pub(crate) common: Common,
}
Expand Down
70 changes: 0 additions & 70 deletions tools/buildsys/src/bin/bottlerocket-variant/main.rs
View file

This file was deleted.

61 changes: 41 additions & 20 deletions tools/buildsys/src/builder.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ the repository's top-level Dockerfile.
pub(crate) mod error;

use crate::args::{BuildKitArgs, BuildPackageArgs, BuildVariantArgs, RepackVariantArgs};
use bottlerocket_variant::Variant;
use buildsys::manifest::{
ExternalKitMetadataView, ImageFeature, ImageFormat, ImageLayout, Manifest, PartitionPlan,
SupportedArch,
Expand Down Expand Up @@ -419,22 +420,28 @@ impl DockerBuild {
let (os_image_publish_size_gib, data_image_publish_size_gib) =
image_layout.publish_image_sizes_gib();

let variant = filename(args.variant);

let v = Variant::new(&variant).context(error::VariantParseSnafu)?;
let variant_platform = v.platform().into();
let variant_runtime = v.runtime().into();
let variant_family = v.family().into();
let variant_flavor = v.variant_flavor().unwrap_or("").into();

Ok(Self {
dockerfile: args.common.tools_dir.join("build.Dockerfile"),
context: args.common.root_dir.clone(),
target: "variant".to_string(),
tag: append_token(
format!(
"buildsys-var-{variant}-{arch}",
variant = args.variant,
arch = args.common.arch
),
format!("buildsys-var-{variant}-{arch}", arch = args.common.arch),
&args.common.root_dir,
),
root_dir: args.common.root_dir.clone(),
artifacts_dirs: vec![args.common.image_arch_variant_dir],
artifacts_dirs: vec![args
.image_dir
.join(format!("{}-{}", args.common.arch, variant))],
state_dir: args.common.state_dir,
artifact_name: args.variant.clone(),
artifact_name: variant.clone(),
common_build_args: CommonBuildArgs::new(
&args.common.root_dir,
args.common.sdk_image,
Expand Down Expand Up @@ -477,11 +484,11 @@ impl DockerBuild {
}
.to_string(),
pretty_name: args.pretty_name,
variant: args.variant,
variant_family: args.variant_family,
variant_flavor: args.variant_flavor,
variant_platform: args.variant_platform,
variant_runtime: args.variant_runtime,
variant,
variant_family,
variant_flavor,
variant_platform,
variant_runtime,
version_build: args.version_build,
version_image: args.version_image,
}),
Expand All @@ -502,22 +509,22 @@ impl DockerBuild {
let (os_image_publish_size_gib, data_image_publish_size_gib) =
image_layout.publish_image_sizes_gib();

let variant = filename(args.variant);

Ok(Self {
dockerfile: args.common.tools_dir.join("build.Dockerfile"),
context: args.common.root_dir.clone(),
target: "repack".to_string(),
tag: append_token(
format!(
"buildsys-repack-{variant}-{arch}",
variant = args.variant,
arch = args.common.arch
),
format!("buildsys-repack-{variant}-{arch}", arch = args.common.arch),
&args.common.root_dir,
),
root_dir: args.common.root_dir.clone(),
artifacts_dirs: vec![args.common.image_arch_variant_dir],
artifacts_dirs: vec![args
.image_dir
.join(format!("{}-{}", args.common.arch, variant))],
state_dir: args.common.state_dir,
artifact_name: args.variant.clone(),
artifact_name: variant.clone(),
common_build_args: CommonBuildArgs::new(
&args.common.root_dir,
args.common.sdk_image,
Expand All @@ -542,7 +549,7 @@ impl DockerBuild {
PartitionPlan::Unified => "unified",
}
.to_string(),
variant: args.variant,
variant,
version_build: args.version_build,
version_image: args.version_image,
}),
Expand Down Expand Up @@ -1036,3 +1043,17 @@ where
self.as_ref().split(' ').map(String::from).collect()
}
}

// =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^= =^..^=

/// Helper to extract the file name from a path.
fn filename(p: impl AsRef<Path>) -> String {
let path = p.as_ref();
path.file_name()
.with_context(|| error::BadFilenameSnafu {
path: PathBuf::from(path),
})
.unwrap()
.to_string_lossy()
.to_string()
}
8 changes: 8 additions & 0 deletions tools/buildsys/src/builder/error.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ pub(crate) enum Error {
#[snafu(display("Failed to read CA certificates bundle '{}'", ca_bundle_path.display()))]
BadCaBundle { ca_bundle_path: PathBuf },

#[snafu(display("Failed to get file name for '{}'", path.display()))]
BadFilename { path: PathBuf },

#[snafu(display("Failed to read repo root '{}'", root_json_path.display()))]
BadRootJson { root_json_path: PathBuf },

Expand Down Expand Up @@ -83,6 +86,11 @@ pub(crate) enum Error {
prefix: PathBuf,
source: std::path::StripPrefixError,
},

#[snafu(display("Failed to parse variant: {source}"))]
VariantParse {
source: bottlerocket_variant::error::Error,
},
}

pub(super) type Result<T> = std::result::Result<T, Error>;
36 changes: 11 additions & 25 deletions tools/buildsys/src/main.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ use std::path::{Path, PathBuf};
use std::process;

mod error {
use buildsys::manifest::SupportedArch;
use snafu::Snafu;
use std::path::PathBuf;

Expand Down Expand Up @@ -72,16 +71,6 @@ mod error {
source: crate::builder::error::Error,
},

#[snafu(display(
"Unsupported architecture {}, this variant supports {}",
arch,
supported_arches.join(", ")
))]
UnsupportedArch {
arch: SupportedArch,
supported_arches: Vec<String>,
},

#[snafu(display(
"The manifest for package {} has a package.metadata.build-package.package-features \
section. This functionality has been removed from the build system. Packages are no \
Expand Down Expand Up @@ -256,7 +245,7 @@ fn build_variant(args: BuildVariantArgs) -> Result<()> {
)
.context(error::ManifestParseSnafu)?;

supported_arch(manifest.info(), args.common.arch)?;
check_arch_support(manifest.info(), args.common.arch);

if args.common.cicd_hack {
return Ok(());
Expand All @@ -277,7 +266,7 @@ fn repack_variant(args: RepackVariantArgs) -> Result<()> {
)
.context(error::ManifestParseSnafu)?;

supported_arch(manifest.info(), args.common.arch)?;
check_arch_support(manifest.info(), args.common.arch);

if args.common.cicd_hack {
return Ok(());
Expand All @@ -290,20 +279,17 @@ fn repack_variant(args: RepackVariantArgs) -> Result<()> {
}

/// Ensure that the current arch is supported by the current variant
fn supported_arch(manifest: &ManifestInfo, arch: SupportedArch) -> Result<()> {
fn check_arch_support(manifest: &ManifestInfo, arch: SupportedArch) {
if let Some(supported_arches) = manifest.supported_arches() {
ensure!(
supported_arches.contains(&arch),
error::UnsupportedArchSnafu {
arch,
supported_arches: supported_arches
.iter()
.map(|a| a.to_string())
.collect::<Vec<String>>()
}
)
if !supported_arches.contains(&arch) {
let supported_arches = supported_arches
.iter()
.map(|a| a.to_string())
.collect::<Vec<String>>();
println!("cargo:warning={arch} is not one of the supported architectures ({supported_arches:?})");
std::process::exit(0);
}
}
Ok(())
}

/// Prior to the release of Kits as a build feature, packages could, and did, declare themselves
Expand Down
Loading

0 comments on commit f835018

Please sign in to comment.