app: Add rpm-ostree compose extensions

This adds support for a new `rpm-ostree compose extensions` command`
which takes a treefile, a new extensions YAML file, and an OSTree repo
and ref. It performs a depsolve and downloads the extensions to a
provided output directory.

This is intended to replace cosa's `download-extensions`:
https://github.com/coreos/coreos-assembler/blob/master/src/download-extensions

The input YAML schema matches the one accepted by that script.

Some differences from the script:
- We have a guaranteed depsolve match and thus can avoid silly issues
  we've hit in RHCOS (like downloading the wrong `libprotobuf` for
  `usbguard` -- rhbz#1889694).
- We seamlessly re-use the same repos defined in the treefile, whereas
  the cosa script uses `reposdir=$dir` which doesn't have the same
  semantics (repo enablement is in that case purely based on the
  `enabled` flag in those repos, which may be different than what the
  rpm-ostree compose ran with).
- We perform more sanity-checks against the requested extensions, such
  as whether the extension is already in the base.
- We support no-change detection via a state SHA512 file for better
  integration in cosa and pipelines.
- We support a `match-base-evr` key, which forces the extension to have
  the same EVR as the one from a base package: this is helpful in the
  case of extensions which complement a base package, esp. those which
  may not have strong enough reldeps to enforce matching EVRs by
  depsolve alone (`kernel-headers` is an example of this).
- We don't try to organize the RPMs into separate directories by
  extension because IMO it's not at the right level. Instead, we should
  work towards higher-level metadata to represent extensions (see
  openshift/os#409 which is related to this).

Closes: coreos#2055

Cargo.toml

@@ -24,7 +24,7 @@ tempfile = "3.1.0"
 clap = "2.33.3"
 structopt = "0.3.21"
 openat = "0.1.19"
-openat-ext = "^0.1.9"
+openat-ext = "^0.1.11"
 curl = "0.4.34"
 rayon = "1.5.0"
 c_utf8 = "0.1.0"

docs/CONTRIBUTING.md

@@ -1,5 +1,5 @@
 ---
-nav_order: 9
+nav_order: 10
 ---
 
 # Contributing

docs/HACKING.md

@@ -1,5 +1,5 @@
 ---
-nav_order: 6
+nav_order: 7
 ---
 
 # Hacking on rpm-ostree

docs/RELEASE.md

@@ -1,5 +1,5 @@
 ---
-nav_order: 7
+nav_order: 8
 ---
 
 # Releasing rpm-ostree

docs/extensions.md

@@ -0,0 +1,56 @@
+---
+nav_order: 6
+---
+
+# Extensions
+
+Extensions are additional packages which client machines can
+install using package layering. While rpm-ostree itself is
+indifferent on the subject, most rpm-ostree-based distros
+encourage a containerized workflow for better separation of
+host and application layers. But sometimes, containerization
+is not ideal for some software, and yet it may not be
+desirable to bake them into the OSTree commit by default.
+
+Package layering normally fetches such extensions from
+remote repos. However in some architectures there may be a
+better way to transfer them, or one may simply want tighter
+control over them and stronger binding between OSTree commit
+and extension versions (e.g. for reproducibility, guaranteed
+depsolve, QE, releng, etc..).
+
+`rpm-ostree compose extensions` takes an `extensions.yaml`
+file describing OS extensions (packages) and a base OSTree
+commit. After performing a depsolve, it downloads the
+extension packages and places them in an output directory.
+
+## extensions.yaml
+
+The format of the `extensions.yaml` file is as follow:
+
+```yaml
+# The top-level object is a dict. The only supported key
+# right now is `extensions`, which is a dict of extension
+# names to extension objects.
+extensions:
+  # This can be whatever name you'd like. The name itself
+  # isn't used by rpm-ostree.
+  sooper-dooper-tracers:
+    # List of packages for this extension
+    packages:
+        - strace
+        - ltrace
+    # Optional list of architectures on which this extension
+    # is valid. These are RPM basearches. If omitted,
+    # defaults to all architectures.
+    architectures:
+        - x86_64
+        - aarch64
+  kernel-dev:
+    packages:
+      - kernel-devel
+      - kernel-headers
+    # Optional name of a base package used to constrain the
+    # EVR of all the packages in this extension.
+    match-base-evr: kernel
+```

docs/repo_structure.md

@@ -1,5 +1,5 @@
 ---
-nav_order: 8
+nav_order: 9
 ---
 
 # Repository structure

rust/src/extensions.rs

@@ -0,0 +1,194 @@
+//! Core logic for extensions.yaml file.
+
+/*
+ * Copyright (C) 2020 Red Hat, Inc.
+ *
+ * SPDX-License-Identifier: Apache-2.0 OR MIT
+ */
+
+use anyhow::{bail, Context, Result};
+use openat_ext::OpenatDirExt;
+use serde_derive::Deserialize;
+use std::collections::HashMap;
+
+use crate::cxxrsutil::*;
+use crate::ffi::StringMapping;
+use crate::utils;
+
+const RPMOSTREE_EXTENSIONS_STATE_FILE: &str = ".rpm-ostree-state-chksum";
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct Extensions {
+    extensions: HashMap<String, Extension>,
+}
+
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "kebab-case")]
+pub struct Extension {
+    packages: Vec<String>,
+    architectures: Option<Vec<String>>,
+    match_base_evr: Option<String>,
+}
+
+fn extensions_load_stream(
+    stream: &mut impl std::io::Read,
+    basearch: &str,
+    base_pkgs: &Vec<StringMapping>,
+) -> Result<Box<Extensions>> {
+    let mut parsed: Extensions = serde_yaml::from_reader(stream)?;
+
+    parsed.extensions.retain(|_, ext| {
+        ext.architectures
+            .as_ref()
+            .map(|v| v.iter().any(|a| a == basearch))
+            .unwrap_or(true)
+    });
+
+    let base_pkgs: HashMap<&str, &str> = base_pkgs
+        .iter()
+        .map(|i| (i.k.as_str(), i.v.as_str()))
+        .collect();
+
+    for (_, ext) in parsed.extensions.iter_mut() {
+        for pkg in &ext.packages {
+            if base_pkgs.contains_key(pkg.as_str()) {
+                bail!("package {} already present in base", pkg);
+            }
+        }
+        if let Some(ref matched_base_pkg) = ext.match_base_evr {
+            let evr = base_pkgs
+                .get(matched_base_pkg.as_str())
+                .with_context(|| format!("couldn't find base package {}", matched_base_pkg))?;
+            let pkgs = ext
+                .packages
+                .iter()
+                .map(|pkg| format!("{}-{}", pkg, evr))
+                .collect();
+            ext.packages = pkgs;
+        }
+    }
+
+    Ok(Box::new(parsed))
+}
+
+pub(crate) fn extensions_load(
+    path: &str,
+    basearch: &str,
+    base_pkgs: &Vec<StringMapping>,
+) -> Result<Box<Extensions>> {
+    let f = utils::open_file(path)?;
+    let mut f = std::io::BufReader::new(f);
+    extensions_load_stream(&mut f, basearch, base_pkgs).with_context(|| format!("parsing {}", path))
+}
+
+impl Extensions {
+    pub(crate) fn get_packages(&self) -> Vec<String> {
+        self.extensions
+            .iter()
+            .flat_map(|(_, ext)| ext.packages.iter().cloned())
+            .collect()
+    }
+
+    pub(crate) fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> CxxResult<bool> {
+        let output_dir = openat::Dir::open(output_dir)?;
+        if let Some(prev_chksum) =
+            output_dir.read_to_string_optional(RPMOSTREE_EXTENSIONS_STATE_FILE)?
+        {
+            Ok(prev_chksum != chksum)
+        } else {
+            Ok(true)
+        }
+    }
+
+    pub(crate) fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> CxxResult<()> {
+        let output_dir = openat::Dir::open(output_dir)?;
+        Ok(output_dir
+            .write_file_contents(RPMOSTREE_EXTENSIONS_STATE_FILE, 0o644, chksum)
+            .with_context(|| format!("updating state file {}", RPMOSTREE_EXTENSIONS_STATE_FILE))?)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn base_rpmdb() -> Vec<StringMapping> {
+        vec![
+            StringMapping {
+                k: "systemd".into(),
+                v: "246.9-3".into(),
+            },
+            StringMapping {
+                k: "foobar".into(),
+                v: "1.2-3".into(),
+            },
+        ]
+    }
+
+    #[test]
+    fn basic() {
+        let buf = r###"
+extensions:
+    bazboo:
+        packages:
+            - bazboo
+"###;
+        let mut input = std::io::BufReader::new(buf.as_bytes());
+        let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
+        assert!(extensions.get_packages() == vec!["bazboo"]);
+    }
+
+    #[test]
+    fn ext_in_base() {
+        let buf = r###"
+extensions:
+    foobar:
+        packages:
+            - foobar
+"###;
+        let mut input = std::io::BufReader::new(buf.as_bytes());
+        match extensions_load_stream(&mut input, "x86_64", &base_rpmdb()) {
+            Ok(_) => panic!("expected failure from extension in base"),
+            Err(ref e) => assert!(e.to_string() == "package foobar already present in base"),
+        }
+    }
+
+    #[test]
+    fn basearch_filter() {
+        let buf = r###"
+extensions:
+    bazboo:
+        packages:
+            - bazboo
+        architectures:
+            - x86_64
+    dodo:
+        packages:
+            - dodo
+            - dada
+        architectures:
+            - s390x
+"###;
+        let mut input = std::io::BufReader::new(buf.as_bytes());
+        let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
+        assert!(extensions.get_packages() == vec!["bazboo"]);
+        let mut input = std::io::BufReader::new(buf.as_bytes());
+        let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap();
+        assert!(extensions.get_packages() == vec!["dodo", "dada"]);
+    }
+
+    #[test]
+    fn matching_evr() {
+        let buf = r###"
+extensions:
+    foobar-ext:
+        packages:
+            - foobar-ext
+        match-base-evr: foobar
+"###;
+        let mut input = std::io::BufReader::new(buf.as_bytes());
+        let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap();
+        assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]);
+    }
+}

rust/src/lib.rs

@@ -180,6 +180,19 @@ mod ffi {
     extern "Rust" {
         fn countme_entrypoint(argv: Vec<String>) -> Result<()>;
     }
+
+    // extensions.rs
+    extern "Rust" {
+        type Extensions;
+        fn extensions_load(
+            path: &str,
+            basearch: &str,
+            base_pkgs: &Vec<StringMapping>,
+        ) -> Result<Box<Extensions>>;
+        fn get_packages(&self) -> Vec<String>;
+        fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result<bool>;
+        fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>;
+    }
 }
 
 mod client;
@@ -193,6 +206,8 @@ pub(crate) use composepost::*;
 mod core;
 use crate::core::*;
 mod dirdiff;
+mod extensions;
+pub(crate) use extensions::*;
 #[cfg(feature = "fedora-integration")]
 mod fedora_integration;
 mod history;

rust/src/treefile.rs

@@ -1477,6 +1477,16 @@ mod ffi {
         ref_from_raw_ptr(tf).serialized.as_ptr()
     }
 
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_repos(tf: *mut Treefile) -> *mut *mut libc::c_char {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref repos) = tf.parsed.repos {
+            repos.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
     #[no_mangle]
     pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char {
         let tf = ref_from_raw_ptr(tf);

src/app/rpmostree-builtin-compose.cxx

@@ -41,6 +41,9 @@ static RpmOstreeCommand compose_subcommands[] = {
   { "commit", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT),
     "Commit a target path to an OSTree repository",
     rpmostree_compose_builtin_commit },
+  { "extensions", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD,
+    "Download RPM packages guaranteed to depsolve with a base OSTree",
+    rpmostree_compose_builtin_extensions },
 #ifdef BUILDOPT_ROJIG
   { "rojig", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_HIDDEN),
     "EXPERIMENTAL: Build a rojig RPM from a treefile, output to a local rpm-md repo",