app: Add rpm-ostree compose extensions

This adds support for a new `package-extensions` key in the treefile as
well as a new `rpm-ostree compose extensions` command which takes a
treefile and a base OSTree rev, performs a depsolve, and downloads the
extensions to a provided output directory.

This is intended to replace cosa's `download-extensions`:
https://github.com/coreos/coreos-assembler/blob/master/src/download-extensions

The major advantage here is that we have a guaranteed depsolve match and
thus can avoid silly issues we've hit in RHCOS (like downloading the
wrong `libprotobuf` for `usbguard` -- rhbz#1889694).

Closes: coreos#2055

rust/src/treefile.rs

@@ -115,6 +115,10 @@ fn treefile_parse_stream<R: io::Read>(
         }
     }
 
+    if let Some(pkgs) = treefile.package_extensions.take() {
+        treefile.package_extensions = Some(whitespace_split_packages(&pkgs)?);
+    }
+
     treefile.packages = Some(pkgs);
     treefile = treefile.migrate_legacy_fields()?;
     Ok(treefile)
@@ -320,6 +324,7 @@ fn treefile_merge(dest: &mut TreeComposeConfig, src: &mut TreeComposeConfig) {
         repos,
         lockfile_repos,
         packages,
+        package_extensions,
         bootstrap_packages,
         exclude_packages,
         ostree_layers,
@@ -771,6 +776,9 @@ struct TreeComposeConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(rename = "exclude-packages")]
     exclude_packages: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    #[serde(rename = "package-extensions")]
+    package_extensions: Option<Vec<String>>,
 
     // Content installation opts
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -1477,6 +1485,26 @@ mod ffi {
         ref_from_raw_ptr(tf).serialized.as_ptr()
     }
 
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_repos(tf: *mut Treefile) -> *mut *mut libc::c_char {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref repos) = tf.parsed.repos {
+            repos.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
+    #[no_mangle]
+    pub extern "C" fn ror_treefile_get_package_extensions(tf: *mut Treefile) -> *mut *mut libc::c_char {
+        let tf = ref_from_raw_ptr(tf);
+        if let Some(ref pkgs) = tf.parsed.package_extensions {
+            pkgs.to_glib_full()
+        } else {
+            ptr::null_mut()
+        }
+    }
+
     #[no_mangle]
     pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char {
         let tf = ref_from_raw_ptr(tf);

src/app/rpmostree-builtin-compose.cxx

@@ -41,6 +41,9 @@ static RpmOstreeCommand compose_subcommands[] = {
   { "commit", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT),
     "Commit a target path to an OSTree repository",
     rpmostree_compose_builtin_commit },
+  { "extensions", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD,
+    "Download RPM packages guaranteed to depsolve with a base OSTree",
+    rpmostree_compose_builtin_extensions },
 #ifdef BUILDOPT_ROJIG
   { "rojig", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_HIDDEN),
     "EXPERIMENTAL: Build a rojig RPM from a treefile, output to a local rpm-md repo",

src/app/rpmostree-compose-builtin-tree.cxx

@@ -83,6 +83,9 @@ static char **opt_lockfiles;
 static gboolean opt_lockfile_strict;
 static char *opt_parent;
 
+static char *opt_extensions_output_dir;
+static char *opt_extensions_base_rev;
+
 /* shared by both install & commit */
 static GOptionEntry common_option_entries[] = {
   { "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" },
@@ -124,6 +127,13 @@ static GOptionEntry commit_option_entries[] = {
   { NULL }
 };
 
+static GOptionEntry extensions_option_entries[] = {
+  { "output-dir", 0, 0, G_OPTION_ARG_STRING, &opt_extensions_output_dir, "Path to extensions output directory", "PATH" },
+  { "base-rev", 0, 0, G_OPTION_ARG_STRING, &opt_extensions_base_rev, "Base OSTree revision", "REV" },
+  { "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" },
+  { NULL }
+};
+
 typedef struct {
   RpmOstreeContext *corectx;
   GFile *treefile_path;
@@ -1430,3 +1440,132 @@ rpmostree_compose_builtin_tree (int             argc,
 
   return TRUE;
 }
+
+gboolean
+rpmostree_compose_builtin_extensions (int             argc,
+                                      char          **argv,
+                                      RpmOstreeCommandInvocation *invocation,
+                                      GCancellable   *cancellable,
+                                      GError        **error)
+{
+  g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE");
+  g_option_context_add_main_entries (context, common_option_entries, NULL);
+  g_option_context_add_main_entries (context, extensions_option_entries, NULL);
+
+  if (!rpmostree_option_context_parse (context,
+                                       NULL,
+                                       &argc, &argv,
+                                       invocation,
+                                       cancellable,
+                                       NULL, NULL, NULL, NULL, NULL,
+                                       error))
+    return FALSE;
+
+  if (argc < 2)
+    {
+      rpmostree_usage_error (context, "TREEFILE must be specified", error);
+      return FALSE;
+    }
+  if (!opt_repo)
+    {
+      rpmostree_usage_error (context, "--repo must be specified", error);
+      return FALSE;
+    }
+  if (!opt_extensions_output_dir)
+    {
+      rpmostree_usage_error (context, "--output-dir must be specified", error);
+      return FALSE;
+    }
+
+  g_autofree char *arch = rpm_ostree_get_basearch ();
+  g_autoptr(RORTreefile) treefile = ror_treefile_new (argv[1], arch, -1, error);
+  if (!treefile)
+    return glnx_prefix_error (error, "Failed to load treefile");
+
+  g_autoptr(OstreeRepo) repo = ostree_repo_open_at (AT_FDCWD, opt_repo, cancellable, error);
+  if (!repo)
+    return FALSE;
+
+  /* this is a similar construction to what's in rpm_ostree_compose_context_new() */
+  g_auto(GLnxTmpDir) cachedir_tmp = { 0, };
+  glnx_autofd int cachedir_dfd = -1;
+  if (opt_cachedir)
+    {
+      if (!glnx_opendirat (AT_FDCWD, opt_cachedir, TRUE, &cachedir_dfd, error))
+        return glnx_prefix_error (error, "Opening cachedir");
+    }
+  else
+    {
+      if (!glnx_mkdtempat (ostree_repo_get_dfd (repo),
+                           "tmp/rpm-ostree-compose.XXXXXX", 0700,
+                           &cachedir_tmp, error))
+        return FALSE;
+
+      cachedir_dfd = fcntl (cachedir_tmp.fd, F_DUPFD_CLOEXEC, 3);
+      if (cachedir_dfd < 0)
+        return glnx_throw_errno_prefix (error, "fcntl");
+    }
+
+  g_autofree char *base_rev = NULL;
+  if (!ostree_repo_resolve_rev (repo, opt_extensions_base_rev, FALSE, &base_rev, error))
+    return FALSE;
+
+  g_autoptr(GVariant) commit = NULL;
+  if (!ostree_repo_load_commit (repo, base_rev, &commit, NULL, error))
+    return FALSE;
+
+  g_autoptr(GPtrArray) packages =
+      rpm_ostree_db_query_all (repo, opt_extensions_base_rev, cancellable, error);
+  if (!packages)
+      return FALSE;
+
+  g_autoptr(RpmOstreeContext) ctx =
+      rpmostree_context_new_tree (cachedir_dfd, repo, cancellable, error);
+  if (!ctx)
+      return FALSE;
+
+  { int tf_dfd = ror_treefile_get_dfd (treefile);
+    g_autofree char *abs_tf_path = glnx_fdrel_abspath (tf_dfd, ".");
+    dnf_context_set_repo_dir (rpmostree_context_get_dnf (ctx), abs_tf_path);
+  }
+
+#define TMP_EXTENSIONS_ROOTFS "rpmostree-extensions.tmp"
+
+  if (!glnx_shutil_rm_rf_at (cachedir_dfd, TMP_EXTENSIONS_ROOTFS, cancellable, error))
+    return FALSE;
+
+  g_print ("Checking out %.7s... ", base_rev);
+  OstreeRepoCheckoutAtOptions opts = { .mode = OSTREE_REPO_CHECKOUT_MODE_USER };
+  if (!ostree_repo_checkout_at (repo, &opts, cachedir_dfd, TMP_EXTENSIONS_ROOTFS,
+                                base_rev, cancellable, error))
+    return FALSE;
+  g_print ("done!\n");
+
+  g_autoptr(RpmOstreeTreespec) spec = NULL;
+  { char **pkgs = ror_treefile_get_package_extensions (treefile);
+    if (!pkgs || !*pkgs)
+      return glnx_throw (error, "No package-extensions defined in treefile!");
+    char **repos = ror_treefile_get_repos (treefile);
+    g_autoptr(GKeyFile) treespec = g_key_file_new ();
+    g_key_file_set_string_list (treespec, "tree", "packages",
+                                (const char* const*)pkgs, g_strv_length (pkgs));
+    g_key_file_set_string_list (treespec, "tree", "repos",
+                                (const char* const*)repos,
+                                g_strv_length (repos));
+    spec = rpmostree_treespec_new_from_keyfile (treespec, NULL);
+  }
+
+  g_autofree char *checkout_path = glnx_fdrel_abspath (cachedir_dfd, TMP_EXTENSIONS_ROOTFS);
+  if (!rpmostree_context_setup (ctx, checkout_path, checkout_path, spec, cancellable, error))
+      return FALSE;
+
+  if (!rpmostree_context_prepare (ctx, cancellable, error))
+    return FALSE;
+
+  if (!rpmostree_context_download (ctx, opt_extensions_output_dir, cancellable, error))
+    return FALSE;
+
+#undef TMP_EXTENSIONS_ROOTFS
+
+  return TRUE;
+}

src/app/rpmostree-compose-builtins.h

@@ -31,6 +31,7 @@ gboolean rpmostree_compose_builtin_rojig (int argc, char **argv, RpmOstreeComman
 gboolean rpmostree_compose_builtin_install (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
 gboolean rpmostree_compose_builtin_postprocess (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
 gboolean rpmostree_compose_builtin_commit (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
+gboolean rpmostree_compose_builtin_extensions (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error);
 
 G_END_DECLS