container: Add support for re-exporting a fetched container

The status quo today is basically that with a "pure ostree"
container image, one can pull it, and one *can* re-export
it with `ostree container encapsulate`...but doing so
loses *all chunking* i.e. you end up with a single
giant layer again.

Further, we don't support exporting derived images at all.

Fix both of these with a new CLI and API, for example:

```
$ ostree container image export --repo=/path/to/repo registry:quay.io/exampleos/someos:latest containers-storage:localhost/exampleos
```

Now...before one gets too excited, this is still suboptimal
in a bunch of ways:

- Copying to `containers-storage` is super inefficient, we indirect
  through a local `oci` directory because of the lack of "push"
  support in containers-image-proxy, *and* we end up with a full
  physical copy of the files even when we *could* reflink;
  cc containers/storage#1849
- Because we don't currently save tar-split data, the use
  case of pushing to a registry is virtually guaranteed to produce
  changed diffids, and we'll hence end up duplicating layers
  on the registry

Now what is more interesting is that this code is going to help
us a bit for the use case of "recommitting" a derived container
image.

Signed-off-by: Colin Walters <[email protected]>

lib/Cargo.toml

@@ -53,6 +53,7 @@ zstd = { version = "0.13.1", features = ["pkg-config"] }
 
 indoc = { version = "2", optional = true }
 xshell = { version = "0.2", optional = true }
+similar-asserts = { version = "1.5.0", optional = true }
 
 [dev-dependencies]
 quickcheck = "1"
@@ -66,4 +67,4 @@ features = ["dox"]
 [features]
 docgen = ["clap_mangen"]
 dox = ["ostree/dox"]
-internal-testing-api = ["xshell", "indoc"]
+internal-testing-api = ["xshell", "indoc", "similar-asserts"]

lib/src/chunking.rs

@@ -31,7 +31,7 @@ pub(crate) const MAX_CHUNKS: u32 = 64;
 const MIN_CHUNKED_LAYERS: u32 = 4;
 
 /// A convenient alias for a reference-counted, immutable string.
-type RcStr = Rc<str>;
+pub(crate) type RcStr = Rc<str>;
 /// Maps from a checksum to its size and file names (multiple in the case of
 /// hard links).
 pub(crate) type ChunkMapping = BTreeMap<RcStr, (u64, Vec<Utf8PathBuf>)>;
@@ -215,7 +215,7 @@ impl Chunk {
         }
     }
 
-    fn move_obj(&mut self, dest: &mut Self, checksum: &str) -> bool {
+    pub(crate) fn move_obj(&mut self, dest: &mut Self, checksum: &str) -> bool {
         // In most cases, we expect the object to exist in the source.  However, it's
         // conveneient here to simply ignore objects which were already moved into
         // a chunk.

lib/src/cli.rs

@@ -24,7 +24,7 @@ use std::process::Command;
 use tokio::sync::mpsc::Receiver;
 
 use crate::commit::container_commit;
-use crate::container::store::{ImportProgress, LayerProgress, PreparedImport};
+use crate::container::store::{ExportToOCIOpts, ImportProgress, LayerProgress, PreparedImport};
 use crate::container::{self as ostree_container, ManifestDiff};
 use crate::container::{Config, ImageReference, OstreeImageReference};
 use crate::sysroot::SysrootLock;
@@ -277,6 +277,30 @@ pub(crate) enum ContainerImageOpts {
         imgref: OstreeImageReference,
     },
 
+    /// Re-export a fetched ostree container.
+    #[clap(alias = "export")]
+    Export {
+        /// Path to the repository
+        #[clap(long, value_parser)]
+        repo: Utf8PathBuf,
+
+        /// Source image reference, e.g. registry:quay.io/exampleos/exampleos:latest
+        #[clap(value_parser = parse_base_imgref)]
+        src_imgref: ImageReference,
+
+        /// Destination image reference, e.g. registry:quay.io/exampleos/exampleos:latest
+        #[clap(value_parser = parse_base_imgref)]
+        dest_imgref: ImageReference,
+
+        #[clap(long)]
+        /// Path to Docker-formatted authentication file.
+        authfile: Option<PathBuf>,
+
+        /// Compress at the fastest level (e.g. gzip level 1)
+        #[clap(long)]
+        compression_fast: bool,
+    },
+
     /// Replace the detached metadata (e.g. to add a signature)
     ReplaceDetachedMetadata {
         /// Path to the source repository
@@ -969,6 +993,29 @@ async fn run_from_opt(opt: Opt) -> Result<()> {
                     let repo = parse_repo(&repo)?;
                     container_store(&repo, &imgref, proxyopts, quiet, check).await
                 }
+                ContainerImageOpts::Export {
+                    repo,
+                    src_imgref,
+                    dest_imgref,
+                    authfile,
+                    compression_fast,
+                } => {
+                    let repo = &parse_repo(&repo)?;
+                    let opts = ExportToOCIOpts {
+                        authfile,
+                        skip_compression: compression_fast,
+                        ..Default::default()
+                    };
+                    let digest = ostree_container::store::export(
+                        repo,
+                        &src_imgref,
+                        &dest_imgref,
+                        Some(opts),
+                    )
+                    .await?;
+                    println!("Exported: {digest}");
+                    Ok(())
+                }
                 ContainerImageOpts::History { repo, imgref } => {
                     let repo = parse_repo(&repo)?;
                     container_history(&repo, &imgref).await

lib/src/container/encapsulate.rs

@@ -102,7 +102,7 @@ fn export_chunks(
 /// Write an ostree commit to an OCI blob
 #[context("Writing ostree root to blob")]
 #[allow(clippy::too_many_arguments)]
-fn export_chunked(
+pub(crate) fn export_chunked(
     repo: &ostree::Repo,
     commit: &str,
     ociw: &mut OciDir,

lib/src/container/store.rs

@@ -6,14 +6,18 @@
 //! base.  See [`encapsulate`][`super::encapsulate()`] for more information on encaspulation of images.
 
 use super::*;
+use crate::chunking::{self, Chunk};
 use crate::logging::system_repo_journal_print;
 use crate::refescape;
 use crate::sysroot::SysrootLock;
 use crate::utils::ResultExt;
 use anyhow::{anyhow, Context};
 use camino::{Utf8Path, Utf8PathBuf};
+use cap_std_ext::cap_std;
 use cap_std_ext::cap_std::fs::{Dir, MetadataExt};
+use cap_std_ext::cmdext::CapStdExtCommandExt;
 use containers_image_proxy::{ImageProxy, OpenedImage};
+use flate2::Compression;
 use fn_error_context::context;
 use futures_util::TryFutureExt;
 use oci_spec::image::{self as oci_image, Descriptor, History, ImageConfiguration, ImageManifest};
@@ -1209,6 +1213,187 @@ pub async fn copy(
     Ok(())
 }
 
+/// Options controlling commit export into OCI
+#[derive(Clone, Debug, Default)]
+#[non_exhaustive]
+pub struct ExportToOCIOpts {
+    /// If true, do not perform gzip compression of the tar layers.
+    pub skip_compression: bool,
+    /// Path to Docker-formatted authentication file.
+    pub authfile: Option<std::path::PathBuf>,
+}
+
+/// The way we store "chunk" layers in ostree is by writing a commit
+/// whose filenames are their own object identifier. This function parses
+/// what is written by the `ImporterMode::ObjectSet` logic, turning
+/// it back into a "chunked" structure that is used by the export code.
+fn chunking_from_layer_committed(
+    repo: &ostree::Repo,
+    l: &Descriptor,
+    chunking: &mut chunking::Chunking,
+) -> Result<()> {
+    let mut chunk = Chunk::default();
+    let layer_ref = &ref_for_layer(l)?;
+    let root = repo.read_commit(&layer_ref, gio::Cancellable::NONE)?.0;
+    let e = root.enumerate_children(
+        "standard::name,standard::size",
+        gio::FileQueryInfoFlags::NOFOLLOW_SYMLINKS,
+        gio::Cancellable::NONE,
+    )?;
+    for child in e.clone() {
+        let child = &child?;
+        // The name here should be a valid checksum
+        let name = child.name();
+        // SAFETY: ostree doesn't give us non-UTF8 filenames
+        let name = Utf8Path::from_path(&name).unwrap();
+        ostree::validate_checksum_string(name.as_str())?;
+        chunking.remainder.move_obj(&mut chunk, name.as_str());
+    }
+    chunking.chunks.push(chunk);
+    Ok(())
+}
+
+/// Export an imported container image to a target OCI directory.
+#[context("Copying image")]
+pub(crate) fn export_to_oci(
+    repo: &ostree::Repo,
+    imgref: &ImageReference,
+    dest_oci: &Dir,
+    tag: Option<&str>,
+    opts: ExportToOCIOpts,
+) -> Result<Descriptor> {
+    let srcinfo = query_image(repo, imgref)?.ok_or_else(|| anyhow!("No such image"))?;
+    let (commit_layer, component_layers, remaining_layers) =
+        parse_manifest_layout(&srcinfo.manifest, &srcinfo.configuration)?;
+    let commit_chunk_ref = ref_for_layer(commit_layer)?;
+    let commit_chunk_rev = repo.require_rev(&commit_chunk_ref)?;
+    let mut chunking = chunking::Chunking::new(repo, &commit_chunk_rev)?;
+    for layer in component_layers {
+        chunking_from_layer_committed(repo, layer, &mut chunking)?;
+    }
+    // Unfortunately today we can't guarantee we reserialize the same tar stream
+    // or compression, so we'll need to generate a new copy of the manifest and config
+    // with the layers reset.
+    let mut new_manifest = srcinfo.manifest.clone();
+    new_manifest.layers_mut().clear();
+    let mut new_config = srcinfo.configuration.clone();
+    new_config.history_mut().clear();
+
+    let mut dest_oci = ocidir::OciDir::ensure(&dest_oci)?;
+
+    let opts = ExportOpts {
+        skip_compression: opts.skip_compression,
+        authfile: opts.authfile,
+        ..Default::default()
+    };
+
+    let mut labels = HashMap::new();
+
+    // Given the object chunking information we recomputed from what
+    // we found on disk, re-serialize to layers (tarballs).
+    export_chunked(
+        repo,
+        &srcinfo.base_commit,
+        &mut dest_oci,
+        &mut new_manifest,
+        &mut new_config,
+        &mut labels,
+        chunking,
+        &opts,
+        "",
+    )?;
+
+    // Now, handle the non-ostree layers; this is a simple conversion of
+    //
+    let compression = opts.skip_compression.then_some(Compression::none());
+    for (i, layer) in remaining_layers.iter().enumerate() {
+        let layer_ref = &ref_for_layer(layer)?;
+        let mut target_blob = dest_oci.create_raw_layer(compression)?;
+        // Sadly the libarchive stuff isn't exposed via Rust due to type unsafety,
+        // so we'll just fork off the CLI.
+        let repo_dfd = repo.dfd_borrow();
+        let repo_dir = cap_std_ext::cap_std::fs::Dir::reopen_dir(&repo_dfd)?;
+        let mut subproc = std::process::Command::new("ostree")
+            .args(["--repo=.", "export", layer_ref.as_str()])
+            .stdout(std::process::Stdio::piped())
+            .cwd_dir(repo_dir)
+            .spawn()?;
+        // SAFETY: we piped just above
+        let mut stdout = subproc.stdout.take().unwrap();
+        std::io::copy(&mut stdout, &mut target_blob).context("Creating blob")?;
+        let layer = target_blob.complete()?;
+        let previous_annotations = srcinfo
+            .manifest
+            .layers()
+            .get(i)
+            .and_then(|l| l.annotations().as_ref())
+            .cloned();
+        let previous_description = srcinfo
+            .configuration
+            .history()
+            .get(i)
+            .and_then(|h| h.comment().as_deref())
+            .unwrap_or_default();
+        dest_oci.push_layer(
+            &mut new_manifest,
+            &mut new_config,
+            layer,
+            previous_description,
+            previous_annotations,
+        )
+    }
+
+    let new_config = dest_oci.write_config(new_config)?;
+    new_manifest.set_config(new_config);
+
+    dest_oci.insert_manifest(new_manifest, tag, oci_image::Platform::default())
+}
+
+/// Given a container image reference which is stored in `repo`, export it to the
+/// target image location.
+#[context("Export")]
+pub async fn export(
+    repo: &ostree::Repo,
+    src_imgref: &ImageReference,
+    dest_imgref: &ImageReference,
+    opts: Option<ExportToOCIOpts>,
+) -> Result<String> {
+    let target_oci = dest_imgref.transport == Transport::OciDir;
+    let tempdir = if !target_oci {
+        let vartmp = cap_std::fs::Dir::open_ambient_dir("/var/tmp", cap_std::ambient_authority())?;
+        let td = cap_std_ext::cap_tempfile::TempDir::new_in(&vartmp)?;
+        // Always skip compression when making a temporary copy
+        let opts = ExportToOCIOpts {
+            skip_compression: true,
+            ..Default::default()
+        };
+        export_to_oci(repo, src_imgref, &td, None, opts)?;
+        td
+    } else {
+        let opts = opts.unwrap_or_default();
+        let (path, tag) = parse_oci_path_and_tag(dest_imgref.name.as_str());
+        tracing::debug!("using OCI path={path} tag={tag:?}");
+        let path = Dir::open_ambient_dir(path, cap_std::ambient_authority())
+            .with_context(|| format!("Opening {path}"))?;
+        let descriptor = export_to_oci(repo, src_imgref, &path, tag, opts)?;
+        return Ok(descriptor.digest().clone());
+    };
+    // Pass the temporary oci directory as the current working directory for the skopeo process
+    let target_fd = 3i32;
+    let tempoci = ImageReference {
+        transport: Transport::OciDir,
+        name: format!("/proc/self/fd/{target_fd}"),
+    };
+    let authfile = opts.as_ref().and_then(|o| o.authfile.as_deref());
+    skopeo::copy(
+        &tempoci,
+        dest_imgref,
+        authfile,
+        Some((std::sync::Arc::new(tempdir.try_clone()?.into()), target_fd)),
+    )
+    .await
+}
+
 /// Iterate over deployment commits, returning the manifests from
 /// commits which point to a container image.
 #[context("Listing deployment manifests")]