diff --git a/lib/Cargo.toml b/lib/Cargo.toml index e3db965a..b1673f83 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -50,6 +50,7 @@ tokio-util = { features = ["io-util"], version = "0.7" } tokio-stream = { features = ["sync"], version = "0.1.8" } tracing = "0.1" zstd = { version = "0.13.1", features = ["pkg-config"] } +indexmap = { version = "2.2.2", features = ["serde"] } indoc = { version = "2", optional = true } xshell = { version = "0.2", optional = true } diff --git a/lib/src/chunking.rs b/lib/src/chunking.rs index 4b047e74..abe5f60c 100644 --- a/lib/src/chunking.rs +++ b/lib/src/chunking.rs @@ -3,7 +3,7 @@ // SPDX-License-Identifier: Apache-2.0 OR MIT use std::borrow::{Borrow, Cow}; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet}; use std::fmt::Write; use std::hash::{Hash, Hasher}; use std::num::NonZeroU32; @@ -19,6 +19,7 @@ use camino::Utf8PathBuf; use containers_image_proxy::oci_spec; use gvariant::aligned_bytes::TryAsAligned; use gvariant::{Marker, Structure}; +use indexmap::IndexMap; use ostree::{gio, glib}; use serde::{Deserialize, Serialize}; @@ -53,9 +54,9 @@ pub(crate) struct Chunk { pub struct ObjectSourceMetaSized { /// The original metadata #[serde(flatten)] - meta: ObjectSourceMeta, + pub meta: ObjectSourceMeta, /// Total size of associated objects - size: u64, + pub size: u64, } impl Hash for ObjectSourceMetaSized { @@ -89,7 +90,7 @@ impl ObjectMetaSized { let map = meta.map; let mut set = meta.set; // Maps content id -> total size of associated objects - let mut sizes = HashMap::<&str, u64>::new(); + let mut sizes = BTreeMap::<&str, u64>::new(); // Populate two mappings above, iterating over the object -> contentid mapping for (checksum, contentid) in map.iter() { let finfo = repo.query_file(checksum, cancellable)?.0; @@ -308,7 +309,7 @@ impl Chunking { } // Reverses `contentmeta.map` i.e. contentid -> Vec - let mut rmap = HashMap::>::new(); + let mut rmap = IndexMap::>::new(); for (checksum, contentid) in meta.map.iter() { rmap.entry(Rc::clone(contentid)).or_default().push(checksum); } @@ -577,12 +578,12 @@ fn basic_packing_with_prior_build<'a>( let mut curr_build = curr_build?; // View the packages as unordered sets for lookups and differencing - let prev_pkgs_set: HashSet = curr_build + let prev_pkgs_set: BTreeSet = curr_build .iter() .flat_map(|v| v.iter().cloned()) .filter(|name| !name.is_empty()) .collect(); - let curr_pkgs_set: HashSet = components + let curr_pkgs_set: BTreeSet = components .iter() .map(|pkg| pkg.meta.name.to_string()) .collect(); @@ -597,13 +598,13 @@ fn basic_packing_with_prior_build<'a>( } // Handle removed packages - let removed: HashSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect(); + let removed: BTreeSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect(); for bin in curr_build.iter_mut() { bin.retain(|pkg| !removed.contains(pkg)); } // Handle updated packages - let mut name_to_component: HashMap = HashMap::new(); + let mut name_to_component: BTreeMap = BTreeMap::new(); for component in components.iter() { name_to_component .entry(component.meta.name.to_string()) @@ -821,6 +822,8 @@ mod test { } fn create_manifest(prev_expected_structure: Vec>) -> oci_spec::image::ImageManifest { + use std::collections::HashMap; + let mut p = prev_expected_structure .iter() .map(|b| { diff --git a/lib/src/cli.rs b/lib/src/cli.rs index c9f91cb4..6bbfcdf2 100644 --- a/lib/src/cli.rs +++ b/lib/src/cli.rs @@ -12,6 +12,7 @@ use cap_std_ext::cap_std; use cap_std_ext::prelude::CapStdExtDirExt; use clap::{Parser, Subcommand}; use fn_error_context::context; +use indexmap::IndexMap; use io_lifetimes::AsFd; use ostree::{gio, glib}; use std::borrow::Cow; @@ -19,16 +20,20 @@ use std::collections::BTreeMap; use std::ffi::OsString; use std::fs::File; use std::io::{BufReader, BufWriter, Write}; +use std::num::NonZeroU32; use std::path::PathBuf; use std::process::Command; use tokio::sync::mpsc::Receiver; +use crate::chunking::{ObjectMetaSized, ObjectSourceMetaSized}; use crate::commit::container_commit; use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport}; use crate::container::{self as ostree_container, ManifestDiff}; use crate::container::{Config, ImageReference, OstreeImageReference}; +use crate::objectsource::ObjectSourceMeta; use crate::sysroot::SysrootLock; use ostree_container::store::{ImageImporter, PrepareResult}; +use serde::{Deserialize, Serialize}; /// Parse an [`OstreeImageReference`] from a CLI arguemnt. pub fn parse_imgref(s: &str) -> Result { @@ -165,6 +170,10 @@ pub(crate) enum ContainerOpts { /// Compress at the fastest level (e.g. gzip level 1) #[clap(long)] compression_fast: bool, + + /// Path to a JSON-formatted content meta object. + #[clap(long)] + contentmeta: Option, }, /// Perform build-time checking and canonicalization. @@ -699,6 +708,33 @@ async fn container_import( Ok(()) } +/// Grouping of metadata about an object. +#[derive(Debug, Default, Serialize, Deserialize)] +pub struct RawMeta { + /// The metadata format version. Should be set to 1. + pub version: u32, + /// The image creation timestamp. Format is YYYY-MM-DDTHH:MM:SSZ. + /// Should be synced with the label io.container.image.created. + pub created: Option, + /// Top level labels, to be prefixed to the ones with --label + /// Applied to both the outer config annotations and the inner config labels. + pub labels: Option>, + /// The output layers ordered. Provided as an ordered mapping of a unique + /// machine readable strings to a human readable name (e.g., the layer contents). + /// The human-readable name is placed in a layer annotation. + pub layers: IndexMap, + /// The layer contents. The key is an ostree hash and the value is the + /// machine readable string of the layer the hash belongs to. + /// WARNING: needs to contain all ostree hashes in the input commit. + pub mapping: IndexMap, + /// Whether the mapping is ordered. If true, the output tar stream of the + /// layers will reflect the order of the hashes in the mapping. + /// Otherwise, a deterministic ordering will be used regardless of mapping + /// order. Potentially useful for optimizing zstd:chunked compression. + /// WARNING: not currently supported. + pub ordered: Option, +} + /// Export a container image with an encapsulated ostree commit. #[allow(clippy::too_many_arguments)] async fn container_export( @@ -712,22 +748,85 @@ async fn container_export( container_config: Option, cmd: Option>, compression_fast: bool, + contentmeta: Option, ) -> Result<()> { - let config = Config { - labels: Some(labels), - cmd, - }; let container_config = if let Some(container_config) = container_config { serde_json::from_reader(File::open(container_config).map(BufReader::new)?)? } else { None }; + + let mut contentmeta_data = None; + let mut created = None; + let mut labels = labels.clone(); + if let Some(contentmeta) = contentmeta { + let buf = File::open(contentmeta).map(BufReader::new); + let raw: RawMeta = serde_json::from_reader(buf?)?; + + // Check future variables are set correctly + let supported_version = 1; + if raw.version != supported_version { + return Err(anyhow::anyhow!( + "Unsupported metadata version: {}. Currently supported: {}", + raw.version, + supported_version + )); + } + if let Some(ordered) = raw.ordered { + if ordered { + return Err(anyhow::anyhow!("Ordered mapping not currently supported.")); + } + } + + created = raw.created; + contentmeta_data = Some(ObjectMetaSized { + map: raw + .mapping + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + sizes: raw + .layers + .into_iter() + .map(|(k, v)| ObjectSourceMetaSized { + meta: ObjectSourceMeta { + identifier: k.clone().into(), + name: v.into(), + srcid: k.clone().into(), + change_frequency: if k == "unpackaged" { std::u32::MAX } else { 1 }, + change_time_offset: 1, + }, + size: 1, + }) + .collect(), + }); + + // Merge --label args to the labels from the metadata + labels.extend(raw.labels.into_iter().flatten()); + } + + // Use enough layers so that each package ends in its own layer + // while respecting the layer ordering. + let max_layers = if let Some(contentmeta_data) = &contentmeta_data { + NonZeroU32::new((contentmeta_data.sizes.len() + 1).try_into().unwrap()) + } else { + None + }; + + let config = Config { + labels: Some(labels), + cmd, + }; + let opts = crate::container::ExportOpts { copy_meta_keys, copy_meta_opt_keys, container_config, authfile, skip_compression: compression_fast, // TODO rename this in the struct at the next semver break + contentmeta: contentmeta_data.as_ref(), + max_layers, + created, ..Default::default() }; let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?; @@ -958,6 +1057,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> { config, cmd, compression_fast, + contentmeta, } => { let labels: Result> = labels .into_iter() @@ -980,6 +1080,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> { config, cmd, compression_fast, + contentmeta, ) .await } diff --git a/lib/src/container/encapsulate.rs b/lib/src/container/encapsulate.rs index 3f055f4f..e7964fcf 100644 --- a/lib/src/container/encapsulate.rs +++ b/lib/src/container/encapsulate.rs @@ -186,16 +186,19 @@ fn build_oci( let mut ctrcfg = opts.container_config.clone().unwrap_or_default(); let mut imgcfg = oci_image::ImageConfiguration::default(); - imgcfg.set_created(Some( - commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string(), - )); - let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default); + + let created_at = opts + .created + .clone() + .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string()); + imgcfg.set_created(Some(created_at)); + let mut labels = HashMap::new(); commit_meta_to_labels( &commit_meta, opts.copy_meta_keys.iter().map(|k| k.as_str()), opts.copy_meta_opt_keys.iter().map(|k| k.as_str()), - labels, + &mut labels, )?; let mut manifest = ocidir::new_empty_manifest().build().unwrap(); @@ -244,7 +247,7 @@ fn build_oci( writer, &mut manifest, &mut imgcfg, - labels, + &mut labels, chunking, &opts, &description, @@ -261,9 +264,14 @@ fn build_oci( ctrcfg.set_cmd(Some(cmd.clone())); } + ctrcfg + .labels_mut() + .get_or_insert_with(Default::default) + .extend(labels.clone()); imgcfg.set_config(Some(ctrcfg)); let ctrcfg = writer.write_config(imgcfg)?; manifest.set_config(ctrcfg); + manifest.set_annotations(Some(labels)); let platform = oci_image::Platform::default(); if let Some(tag) = tag { writer.insert_manifest(manifest, Some(tag), platform)?; @@ -375,6 +383,8 @@ pub struct ExportOpts<'m, 'o> { /// Metadata mapping between objects and their owning component/package; /// used to optimize packing. pub contentmeta: Option<&'o ObjectMetaSized>, + /// Sets the created tag in the image manifest. + pub created: Option, } impl<'m, 'o> ExportOpts<'m, 'o> { diff --git a/lib/src/fixture.rs b/lib/src/fixture.rs index 769e8bf8..7a2ba921 100644 --- a/lib/src/fixture.rs +++ b/lib/src/fixture.rs @@ -302,7 +302,7 @@ fn build_mapping_recurse( dir: &gio::File, ret: &mut ObjectMeta, ) -> Result<()> { - use std::collections::btree_map::Entry; + use indexmap::map::Entry; let cancellable = gio::Cancellable::NONE; let e = dir.enumerate_children( "standard::name,standard::type", diff --git a/lib/src/objectsource.rs b/lib/src/objectsource.rs index 64a3eb13..f32c56ea 100644 --- a/lib/src/objectsource.rs +++ b/lib/src/objectsource.rs @@ -2,8 +2,9 @@ //! //! This is used to help split up containers into distinct layers. +use indexmap::IndexMap; use std::borrow::Borrow; -use std::collections::{BTreeMap, HashSet}; +use std::collections::HashSet; use std::hash::Hash; use std::rc::Rc; @@ -78,7 +79,7 @@ impl Borrow for ObjectSourceMeta { pub type ObjectMetaSet = HashSet; /// Maps from an ostree content object digest to the `ContentSet` key. -pub type ObjectMetaMap = BTreeMap; +pub type ObjectMetaMap = IndexMap; /// Grouping of metadata about an object. #[derive(Debug, Default)]