Merge pull request #652 from hhd-dev/main

feat: Add external input support for container encapsulation

lib/Cargo.toml

@@ -50,6 +50,7 @@ tokio-util = { features = ["io-util"], version = "0.7" }
 tokio-stream = { features = ["sync"], version = "0.1.8" }
 tracing = "0.1"
 zstd = { version = "0.13.1", features = ["pkg-config"] }
+indexmap = { version = "2.2.2", features = ["serde"] }
 
 indoc = { version = "2", optional = true }
 xshell = { version = "0.2", optional = true }

lib/src/chunking.rs

@@ -3,7 +3,7 @@
 // SPDX-License-Identifier: Apache-2.0 OR MIT
 
 use std::borrow::{Borrow, Cow};
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeMap, BTreeSet};
 use std::fmt::Write;
 use std::hash::{Hash, Hasher};
 use std::num::NonZeroU32;
@@ -19,6 +19,7 @@ use camino::Utf8PathBuf;
 use containers_image_proxy::oci_spec;
 use gvariant::aligned_bytes::TryAsAligned;
 use gvariant::{Marker, Structure};
+use indexmap::IndexMap;
 use ostree::{gio, glib};
 use serde::{Deserialize, Serialize};
 
@@ -53,9 +54,9 @@ pub(crate) struct Chunk {
 pub struct ObjectSourceMetaSized {
     /// The original metadata
     #[serde(flatten)]
-    meta: ObjectSourceMeta,
+    pub meta: ObjectSourceMeta,
     /// Total size of associated objects
-    size: u64,
+    pub size: u64,
 }
 
 impl Hash for ObjectSourceMetaSized {
@@ -89,7 +90,7 @@ impl ObjectMetaSized {
         let map = meta.map;
         let mut set = meta.set;
         // Maps content id -> total size of associated objects
-        let mut sizes = HashMap::<&str, u64>::new();
+        let mut sizes = BTreeMap::<&str, u64>::new();
         // Populate two mappings above, iterating over the object -> contentid mapping
         for (checksum, contentid) in map.iter() {
             let finfo = repo.query_file(checksum, cancellable)?.0;
@@ -308,7 +309,7 @@ impl Chunking {
         }
 
         // Reverses `contentmeta.map` i.e. contentid -> Vec<checksum>
-        let mut rmap = HashMap::<ContentID, Vec<&String>>::new();
+        let mut rmap = IndexMap::<ContentID, Vec<&String>>::new();
         for (checksum, contentid) in meta.map.iter() {
             rmap.entry(Rc::clone(contentid)).or_default().push(checksum);
         }
@@ -577,12 +578,12 @@ fn basic_packing_with_prior_build<'a>(
     let mut curr_build = curr_build?;
 
     // View the packages as unordered sets for lookups and differencing
-    let prev_pkgs_set: HashSet<String> = curr_build
+    let prev_pkgs_set: BTreeSet<String> = curr_build
         .iter()
         .flat_map(|v| v.iter().cloned())
         .filter(|name| !name.is_empty())
         .collect();
-    let curr_pkgs_set: HashSet<String> = components
+    let curr_pkgs_set: BTreeSet<String> = components
         .iter()
         .map(|pkg| pkg.meta.name.to_string())
         .collect();
@@ -597,13 +598,13 @@ fn basic_packing_with_prior_build<'a>(
     }
 
     // Handle removed packages
-    let removed: HashSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect();
+    let removed: BTreeSet<&String> = prev_pkgs_set.difference(&curr_pkgs_set).collect();
     for bin in curr_build.iter_mut() {
         bin.retain(|pkg| !removed.contains(pkg));
     }
 
     // Handle updated packages
-    let mut name_to_component: HashMap<String, &ObjectSourceMetaSized> = HashMap::new();
+    let mut name_to_component: BTreeMap<String, &ObjectSourceMetaSized> = BTreeMap::new();
     for component in components.iter() {
         name_to_component
             .entry(component.meta.name.to_string())
@@ -821,6 +822,8 @@ mod test {
     }
 
     fn create_manifest(prev_expected_structure: Vec<Vec<&str>>) -> oci_spec::image::ImageManifest {
+        use std::collections::HashMap;
+
         let mut p = prev_expected_structure
             .iter()
             .map(|b| {

lib/src/cli.rs

@@ -12,23 +12,28 @@ use cap_std_ext::cap_std;
 use cap_std_ext::prelude::CapStdExtDirExt;
 use clap::{Parser, Subcommand};
 use fn_error_context::context;
+use indexmap::IndexMap;
 use io_lifetimes::AsFd;
 use ostree::{gio, glib};
 use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::ffi::OsString;
 use std::fs::File;
 use std::io::{BufReader, BufWriter, Write};
+use std::num::NonZeroU32;
 use std::path::PathBuf;
 use std::process::Command;
 use tokio::sync::mpsc::Receiver;
 
+use crate::chunking::{ObjectMetaSized, ObjectSourceMetaSized};
 use crate::commit::container_commit;
 use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport};
 use crate::container::{self as ostree_container, ManifestDiff};
 use crate::container::{Config, ImageReference, OstreeImageReference};
+use crate::objectsource::ObjectSourceMeta;
 use crate::sysroot::SysrootLock;
 use ostree_container::store::{ImageImporter, PrepareResult};
+use serde::{Deserialize, Serialize};
 
 /// Parse an [`OstreeImageReference`] from a CLI arguemnt.
 pub fn parse_imgref(s: &str) -> Result<OstreeImageReference> {
@@ -165,6 +170,10 @@ pub(crate) enum ContainerOpts {
         /// Compress at the fastest level (e.g. gzip level 1)
         #[clap(long)]
         compression_fast: bool,
+
+        /// Path to a JSON-formatted content meta object.
+        #[clap(long)]
+        contentmeta: Option<Utf8PathBuf>,
     },
 
     /// Perform build-time checking and canonicalization.
@@ -699,6 +708,33 @@ async fn container_import(
     Ok(())
 }
 
+/// Grouping of metadata about an object.
+#[derive(Debug, Default, Serialize, Deserialize)]
+pub struct RawMeta {
+    /// The metadata format version. Should be set to 1.
+    pub version: u32,
+    /// The image creation timestamp. Format is YYYY-MM-DDTHH:MM:SSZ.
+    /// Should be synced with the label io.container.image.created.
+    pub created: Option<String>,
+    /// Top level labels, to be prefixed to the ones with --label
+    /// Applied to both the outer config annotations and the inner config labels.
+    pub labels: Option<BTreeMap<String, String>>,
+    /// The output layers ordered. Provided as an ordered mapping of a unique
+    /// machine readable strings to a human readable name (e.g., the layer contents).
+    /// The human-readable name is placed in a layer annotation.
+    pub layers: IndexMap<String, String>,
+    /// The layer contents. The key is an ostree hash and the value is the
+    /// machine readable string of the layer the hash belongs to.
+    /// WARNING: needs to contain all ostree hashes in the input commit.
+    pub mapping: IndexMap<String, String>,
+    /// Whether the mapping is ordered. If true, the output tar stream of the
+    /// layers will reflect the order of the hashes in the mapping.
+    /// Otherwise, a deterministic ordering will be used regardless of mapping
+    /// order. Potentially useful for optimizing zstd:chunked compression.
+    /// WARNING: not currently supported.
+    pub ordered: Option<bool>,
+}
+
 /// Export a container image with an encapsulated ostree commit.
 #[allow(clippy::too_many_arguments)]
 async fn container_export(
@@ -712,22 +748,85 @@ async fn container_export(
     container_config: Option<Utf8PathBuf>,
     cmd: Option<Vec<String>>,
     compression_fast: bool,
+    contentmeta: Option<Utf8PathBuf>,
 ) -> Result<()> {
-    let config = Config {
-        labels: Some(labels),
-        cmd,
-    };
     let container_config = if let Some(container_config) = container_config {
         serde_json::from_reader(File::open(container_config).map(BufReader::new)?)?
     } else {
         None
     };
+
+    let mut contentmeta_data = None;
+    let mut created = None;
+    let mut labels = labels.clone();
+    if let Some(contentmeta) = contentmeta {
+        let buf = File::open(contentmeta).map(BufReader::new);
+        let raw: RawMeta = serde_json::from_reader(buf?)?;
+
+        // Check future variables are set correctly
+        let supported_version = 1;
+        if raw.version != supported_version {
+            return Err(anyhow::anyhow!(
+                "Unsupported metadata version: {}. Currently supported: {}",
+                raw.version,
+                supported_version
+            ));
+        }
+        if let Some(ordered) = raw.ordered {
+            if ordered {
+                return Err(anyhow::anyhow!("Ordered mapping not currently supported."));
+            }
+        }
+
+        created = raw.created;
+        contentmeta_data = Some(ObjectMetaSized {
+            map: raw
+                .mapping
+                .into_iter()
+                .map(|(k, v)| (k.into(), v.into()))
+                .collect(),
+            sizes: raw
+                .layers
+                .into_iter()
+                .map(|(k, v)| ObjectSourceMetaSized {
+                    meta: ObjectSourceMeta {
+                        identifier: k.clone().into(),
+                        name: v.into(),
+                        srcid: k.clone().into(),
+                        change_frequency: if k == "unpackaged" { std::u32::MAX } else { 1 },
+                        change_time_offset: 1,
+                    },
+                    size: 1,
+                })
+                .collect(),
+        });
+
+        // Merge --label args to the labels from the metadata
+        labels.extend(raw.labels.into_iter().flatten());
+    }
+
+    // Use enough layers so that each package ends in its own layer
+    // while respecting the layer ordering.
+    let max_layers = if let Some(contentmeta_data) = &contentmeta_data {
+        NonZeroU32::new((contentmeta_data.sizes.len() + 1).try_into().unwrap())
+    } else {
+        None
+    };
+
+    let config = Config {
+        labels: Some(labels),
+        cmd,
+    };
+
     let opts = crate::container::ExportOpts {
         copy_meta_keys,
         copy_meta_opt_keys,
         container_config,
         authfile,
         skip_compression: compression_fast, // TODO rename this in the struct at the next semver break
+        contentmeta: contentmeta_data.as_ref(),
+        max_layers,
+        created,
         ..Default::default()
     };
     let pushed = crate::container::encapsulate(repo, rev, &config, Some(opts), imgref).await?;
@@ -958,6 +1057,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
                 config,
                 cmd,
                 compression_fast,
+                contentmeta,
             } => {
                 let labels: Result<BTreeMap<_, _>> = labels
                     .into_iter()
@@ -980,6 +1080,7 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
                     config,
                     cmd,
                     compression_fast,
+                    contentmeta,
                 )
                 .await
             }

lib/src/container/encapsulate.rs

@@ -186,16 +186,19 @@ fn build_oci(
 
     let mut ctrcfg = opts.container_config.clone().unwrap_or_default();
     let mut imgcfg = oci_image::ImageConfiguration::default();
-    imgcfg.set_created(Some(
-        commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string(),
-    ));
-    let labels = ctrcfg.labels_mut().get_or_insert_with(Default::default);
+
+    let created_at = opts
+        .created
+        .clone()
+        .unwrap_or_else(|| commit_timestamp.format("%Y-%m-%dT%H:%M:%SZ").to_string());
+    imgcfg.set_created(Some(created_at));
+    let mut labels = HashMap::new();
 
     commit_meta_to_labels(
         &commit_meta,
         opts.copy_meta_keys.iter().map(|k| k.as_str()),
         opts.copy_meta_opt_keys.iter().map(|k| k.as_str()),
-        labels,
+        &mut labels,
     )?;
 
     let mut manifest = ocidir::new_empty_manifest().build().unwrap();
@@ -244,7 +247,7 @@ fn build_oci(
         writer,
         &mut manifest,
         &mut imgcfg,
-        labels,
+        &mut labels,
         chunking,
         &opts,
         &description,
@@ -261,9 +264,14 @@ fn build_oci(
         ctrcfg.set_cmd(Some(cmd.clone()));
     }
 
+    ctrcfg
+        .labels_mut()
+        .get_or_insert_with(Default::default)
+        .extend(labels.clone());
     imgcfg.set_config(Some(ctrcfg));
     let ctrcfg = writer.write_config(imgcfg)?;
     manifest.set_config(ctrcfg);
+    manifest.set_annotations(Some(labels));
     let platform = oci_image::Platform::default();
     if let Some(tag) = tag {
         writer.insert_manifest(manifest, Some(tag), platform)?;
@@ -375,6 +383,8 @@ pub struct ExportOpts<'m, 'o> {
     /// Metadata mapping between objects and their owning component/package;
     /// used to optimize packing.
     pub contentmeta: Option<&'o ObjectMetaSized>,
+    /// Sets the created tag in the image manifest.
+    pub created: Option<String>,
 }
 
 impl<'m, 'o> ExportOpts<'m, 'o> {

lib/src/fixture.rs

@@ -302,7 +302,7 @@ fn build_mapping_recurse(
     dir: &gio::File,
     ret: &mut ObjectMeta,
 ) -> Result<()> {
-    use std::collections::btree_map::Entry;
+    use indexmap::map::Entry;
     let cancellable = gio::Cancellable::NONE;
     let e = dir.enumerate_children(
         "standard::name,standard::type",

lib/src/objectsource.rs

@@ -2,8 +2,9 @@
 //!
 //! This is used to help split up containers into distinct layers.
 
+use indexmap::IndexMap;
 use std::borrow::Borrow;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::HashSet;
 use std::hash::Hash;
 use std::rc::Rc;
 
@@ -78,7 +79,7 @@ impl Borrow<str> for ObjectSourceMeta {
 pub type ObjectMetaSet = HashSet<ObjectSourceMeta>;
 
 /// Maps from an ostree content object digest to the `ContentSet` key.
-pub type ObjectMetaMap = BTreeMap<String, ContentID>;
+pub type ObjectMetaMap = IndexMap<String, ContentID>;
 
 /// Grouping of metadata about an object.
 #[derive(Debug, Default)]