diff --git a/Cargo.lock b/Cargo.lock index 8543e6927c..8ec022a56f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -772,9 +772,9 @@ dependencies = [ [[package]] name = "openat-ext" -version = "0.1.10" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7ac688336340b9ce22dd83e3b26d9d9063ceef5990679f75176b7e17f4e6a51" +checksum = "5157ebc7a2da568f161a0d51d355b8520451fffb66c416617236f7c8dda733be" dependencies = [ "drop_bomb", "libc", diff --git a/Cargo.toml b/Cargo.toml index d99ade8671..5630c66e99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ tempfile = "3.1.0" clap = "2.33.3" structopt = "0.3.21" openat = "0.1.19" -openat-ext = "^0.1.9" +openat-ext = "^0.1.11" curl = "0.4.34" rayon = "1.5.0" c_utf8 = "0.1.0" diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 0068b0b5d8..f306d0a235 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -1,5 +1,5 @@ --- -nav_order: 9 +nav_order: 10 --- # Contributing diff --git a/docs/HACKING.md b/docs/HACKING.md index a5a32fe74c..efe3c84f13 100644 --- a/docs/HACKING.md +++ b/docs/HACKING.md @@ -1,5 +1,5 @@ --- -nav_order: 6 +nav_order: 7 --- # Hacking on rpm-ostree diff --git a/docs/RELEASE.md b/docs/RELEASE.md index 803f9e7506..a6d48b5ff6 100644 --- a/docs/RELEASE.md +++ b/docs/RELEASE.md @@ -1,5 +1,5 @@ --- -nav_order: 7 +nav_order: 8 --- # Releasing rpm-ostree diff --git a/docs/extensions.md b/docs/extensions.md new file mode 100644 index 0000000000..b8960f03cd --- /dev/null +++ b/docs/extensions.md @@ -0,0 +1,56 @@ +--- +nav_order: 6 +--- + +# Extensions + +Extensions are additional packages which client machines can +install using package layering. While rpm-ostree itself is +indifferent on the subject, most rpm-ostree-based distros +encourage a containerized workflow for better separation of +host and application layers. But sometimes, containerization +is not ideal for some software, and yet it may not be +desirable to bake them into the OSTree commit by default. + +Package layering normally fetches such extensions from +remote repos. However in some architectures there may be a +better way to transfer them, or one may simply want tighter +control over them and stronger binding between OSTree commit +and extension versions (e.g. for reproducibility, guaranteed +depsolve, QE, releng, etc..). + +`rpm-ostree compose extensions` takes an `extensions.yaml` +file describing OS extensions (packages) and a base OSTree +commit. After performing a depsolve, it downloads the +extension packages and places them in an output directory. + +## extensions.yaml + +The format of the `extensions.yaml` file is as follow: + +```yaml +# The top-level object is a dict. The only supported key +# right now is `extensions`, which is a dict of extension +# names to extension objects. +extensions: + # This can be whatever name you'd like. The name itself + # isn't used by rpm-ostree. + sooper-dooper-tracers: + # List of packages for this extension + packages: + - strace + - ltrace + # Optional list of architectures on which this extension + # is valid. These are RPM basearches. If omitted, + # defaults to all architectures. + architectures: + - x86_64 + - aarch64 + kernel-dev: + packages: + - kernel-devel + - kernel-headers + # Optional name of a base package used to constrain the + # EVR of all the packages in this extension. + match-base-evr: kernel +``` diff --git a/docs/repo_structure.md b/docs/repo_structure.md index b06e593db1..a02656d456 100644 --- a/docs/repo_structure.md +++ b/docs/repo_structure.md @@ -1,5 +1,5 @@ --- -nav_order: 8 +nav_order: 9 --- # Repository structure diff --git a/rust/src/extensions.rs b/rust/src/extensions.rs new file mode 100644 index 0000000000..349c2f1283 --- /dev/null +++ b/rust/src/extensions.rs @@ -0,0 +1,194 @@ +//! Core logic for extensions.yaml file. + +/* + * Copyright (C) 2020 Red Hat, Inc. + * + * SPDX-License-Identifier: Apache-2.0 OR MIT + */ + +use anyhow::{bail, Context, Result}; +use openat_ext::OpenatDirExt; +use serde_derive::Deserialize; +use std::collections::HashMap; + +use crate::cxxrsutil::*; +use crate::ffi::StringMapping; +use crate::utils; + +const RPMOSTREE_EXTENSIONS_STATE_FILE: &str = ".rpm-ostree-state-chksum"; + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +pub struct Extensions { + extensions: HashMap, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case")] +pub struct Extension { + packages: Vec, + architectures: Option>, + match_base_evr: Option, +} + +fn extensions_load_stream( + stream: &mut impl std::io::Read, + basearch: &str, + base_pkgs: &Vec, +) -> Result> { + let mut parsed: Extensions = serde_yaml::from_reader(stream)?; + + parsed.extensions.retain(|_, ext| { + ext.architectures + .as_ref() + .map(|v| v.iter().any(|a| a == basearch)) + .unwrap_or(true) + }); + + let base_pkgs: HashMap<&str, &str> = base_pkgs + .iter() + .map(|i| (i.k.as_str(), i.v.as_str())) + .collect(); + + for (_, ext) in parsed.extensions.iter_mut() { + for pkg in &ext.packages { + if base_pkgs.contains_key(pkg.as_str()) { + bail!("package {} already present in base", pkg); + } + } + if let Some(ref matched_base_pkg) = ext.match_base_evr { + let evr = base_pkgs + .get(matched_base_pkg.as_str()) + .with_context(|| format!("couldn't find base package {}", matched_base_pkg))?; + let pkgs = ext + .packages + .iter() + .map(|pkg| format!("{}-{}", pkg, evr)) + .collect(); + ext.packages = pkgs; + } + } + + Ok(Box::new(parsed)) +} + +pub(crate) fn extensions_load( + path: &str, + basearch: &str, + base_pkgs: &Vec, +) -> Result> { + let f = utils::open_file(path)?; + let mut f = std::io::BufReader::new(f); + extensions_load_stream(&mut f, basearch, base_pkgs).with_context(|| format!("parsing {}", path)) +} + +impl Extensions { + pub(crate) fn get_packages(&self) -> Vec { + self.extensions + .iter() + .flat_map(|(_, ext)| ext.packages.iter().cloned()) + .collect() + } + + pub(crate) fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> CxxResult { + let output_dir = openat::Dir::open(output_dir)?; + if let Some(prev_chksum) = + output_dir.read_to_string_optional(RPMOSTREE_EXTENSIONS_STATE_FILE)? + { + Ok(prev_chksum != chksum) + } else { + Ok(true) + } + } + + pub(crate) fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> CxxResult<()> { + let output_dir = openat::Dir::open(output_dir)?; + Ok(output_dir + .write_file_contents(RPMOSTREE_EXTENSIONS_STATE_FILE, 0o644, chksum) + .with_context(|| format!("updating state file {}", RPMOSTREE_EXTENSIONS_STATE_FILE))?) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn base_rpmdb() -> Vec { + vec![ + StringMapping { + k: "systemd".into(), + v: "246.9-3".into(), + }, + StringMapping { + k: "foobar".into(), + v: "1.2-3".into(), + }, + ] + } + + #[test] + fn basic() { + let buf = r###" +extensions: + bazboo: + packages: + - bazboo +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_packages() == vec!["bazboo"]); + } + + #[test] + fn ext_in_base() { + let buf = r###" +extensions: + foobar: + packages: + - foobar +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + match extensions_load_stream(&mut input, "x86_64", &base_rpmdb()) { + Ok(_) => panic!("expected failure from extension in base"), + Err(ref e) => assert!(e.to_string() == "package foobar already present in base"), + } + } + + #[test] + fn basearch_filter() { + let buf = r###" +extensions: + bazboo: + packages: + - bazboo + architectures: + - x86_64 + dodo: + packages: + - dodo + - dada + architectures: + - s390x +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_packages() == vec!["bazboo"]); + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "s390x", &base_rpmdb()).unwrap(); + assert!(extensions.get_packages() == vec!["dodo", "dada"]); + } + + #[test] + fn matching_evr() { + let buf = r###" +extensions: + foobar-ext: + packages: + - foobar-ext + match-base-evr: foobar +"###; + let mut input = std::io::BufReader::new(buf.as_bytes()); + let extensions = extensions_load_stream(&mut input, "x86_64", &base_rpmdb()).unwrap(); + assert!(extensions.get_packages() == vec!["foobar-ext-1.2-3"]); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index fe567fff75..545edde67e 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -180,6 +180,19 @@ mod ffi { extern "Rust" { fn countme_entrypoint(argv: Vec) -> Result<()>; } + + // extensions.rs + extern "Rust" { + type Extensions; + fn extensions_load( + path: &str, + basearch: &str, + base_pkgs: &Vec, + ) -> Result>; + fn get_packages(&self) -> Vec; + fn state_checksum_changed(&self, chksum: &str, output_dir: &str) -> Result; + fn update_state_checksum(&self, chksum: &str, output_dir: &str) -> Result<()>; + } } mod client; @@ -193,6 +206,8 @@ pub(crate) use composepost::*; mod core; use crate::core::*; mod dirdiff; +mod extensions; +pub(crate) use extensions::*; #[cfg(feature = "fedora-integration")] mod fedora_integration; mod history; diff --git a/rust/src/treefile.rs b/rust/src/treefile.rs index 80cb33fa17..10ea6418e1 100644 --- a/rust/src/treefile.rs +++ b/rust/src/treefile.rs @@ -1477,6 +1477,16 @@ mod ffi { ref_from_raw_ptr(tf).serialized.as_ptr() } + #[no_mangle] + pub extern "C" fn ror_treefile_get_repos(tf: *mut Treefile) -> *mut *mut libc::c_char { + let tf = ref_from_raw_ptr(tf); + if let Some(ref repos) = tf.parsed.repos { + repos.to_glib_full() + } else { + ptr::null_mut() + } + } + #[no_mangle] pub extern "C" fn ror_treefile_get_ostree_layers(tf: *mut Treefile) -> *mut *mut libc::c_char { let tf = ref_from_raw_ptr(tf); diff --git a/src/app/rpmostree-builtin-compose.cxx b/src/app/rpmostree-builtin-compose.cxx index 0d0bc91730..96e134071a 100644 --- a/src/app/rpmostree-builtin-compose.cxx +++ b/src/app/rpmostree-builtin-compose.cxx @@ -41,6 +41,9 @@ static RpmOstreeCommand compose_subcommands[] = { { "commit", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_REQUIRES_ROOT), "Commit a target path to an OSTree repository", rpmostree_compose_builtin_commit }, + { "extensions", RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD, + "Download RPM packages guaranteed to depsolve with a base OSTree", + rpmostree_compose_builtin_extensions }, #ifdef BUILDOPT_ROJIG { "rojig", (RpmOstreeBuiltinFlags)(RPM_OSTREE_BUILTIN_FLAG_LOCAL_CMD | RPM_OSTREE_BUILTIN_FLAG_HIDDEN), "EXPERIMENTAL: Build a rojig RPM from a treefile, output to a local rpm-md repo", diff --git a/src/app/rpmostree-compose-builtin-tree.cxx b/src/app/rpmostree-compose-builtin-tree.cxx index 57ccb5b0cd..f72ad80c8e 100644 --- a/src/app/rpmostree-compose-builtin-tree.cxx +++ b/src/app/rpmostree-compose-builtin-tree.cxx @@ -83,6 +83,9 @@ static char **opt_lockfiles; static gboolean opt_lockfile_strict; static char *opt_parent; +static char *opt_extensions_output_dir; +static char *opt_extensions_base_rev; + /* shared by both install & commit */ static GOptionEntry common_option_entries[] = { { "repo", 'r', 0, G_OPTION_ARG_STRING, &opt_repo, "Path to OSTree repository", "REPO" }, @@ -124,6 +127,14 @@ static GOptionEntry commit_option_entries[] = { { NULL } }; +static GOptionEntry extensions_option_entries[] = { + { "output-dir", 0, 0, G_OPTION_ARG_STRING, &opt_extensions_output_dir, "Path to extensions output directory", "PATH" }, + { "base-rev", 0, 0, G_OPTION_ARG_STRING, &opt_extensions_base_rev, "Base OSTree revision", "REV" }, + { "cachedir", 0, 0, G_OPTION_ARG_STRING, &opt_cachedir, "Cached state", "CACHEDIR" }, + { "touch-if-changed", 0, 0, G_OPTION_ARG_STRING, &opt_touch_if_changed, "Update the modification time on FILE if new extensions were downloaded", "FILE" }, + { NULL } +}; + typedef struct { RpmOstreeContext *corectx; GFile *treefile_path; @@ -1430,3 +1441,179 @@ rpmostree_compose_builtin_tree (int argc, return TRUE; } + +gboolean +rpmostree_compose_builtin_extensions (int argc, + char **argv, + RpmOstreeCommandInvocation *invocation, + GCancellable *cancellable, + GError **error) +{ + g_autoptr(GOptionContext) context = g_option_context_new ("TREEFILE EXTYAML"); + g_option_context_add_main_entries (context, common_option_entries, NULL); + g_option_context_add_main_entries (context, extensions_option_entries, NULL); + + if (!rpmostree_option_context_parse (context, + NULL, + &argc, &argv, + invocation, + cancellable, + NULL, NULL, NULL, NULL, NULL, + error)) + return FALSE; + + if (argc < 3) + { + rpmostree_usage_error (context, "TREEFILE and EXTYAML must be specified", error); + return FALSE; + } + if (!opt_repo) + { + rpmostree_usage_error (context, "--repo must be specified", error); + return FALSE; + } + if (!opt_extensions_output_dir) + { + rpmostree_usage_error (context, "--output-dir must be specified", error); + return FALSE; + } + + const char *treefile_path = argv[1]; + const char *extensions_path = argv[2]; + + g_autofree char *basearch = rpm_ostree_get_basearch (); + g_autoptr(RORTreefile) treefile = ror_treefile_new (treefile_path, basearch, -1, error); + if (!treefile) + return glnx_prefix_error (error, "Failed to load treefile"); + + g_autoptr(OstreeRepo) repo = ostree_repo_open_at (AT_FDCWD, opt_repo, cancellable, error); + if (!repo) + return FALSE; + + /* this is a similar construction to what's in rpm_ostree_compose_context_new() */ + g_auto(GLnxTmpDir) cachedir_tmp = { 0, }; + glnx_autofd int cachedir_dfd = -1; + if (opt_cachedir) + { + if (!glnx_opendirat (AT_FDCWD, opt_cachedir, TRUE, &cachedir_dfd, error)) + return glnx_prefix_error (error, "Opening cachedir"); + } + else + { + if (!glnx_mkdtempat (ostree_repo_get_dfd (repo), + "tmp/rpm-ostree-compose.XXXXXX", 0700, + &cachedir_tmp, error)) + return FALSE; + + cachedir_dfd = fcntl (cachedir_tmp.fd, F_DUPFD_CLOEXEC, 3); + if (cachedir_dfd < 0) + return glnx_throw_errno_prefix (error, "fcntl"); + } + + g_autofree char *base_rev = NULL; + if (!ostree_repo_resolve_rev (repo, opt_extensions_base_rev, FALSE, &base_rev, error)) + return FALSE; + + g_autoptr(GVariant) commit = NULL; + if (!ostree_repo_load_commit (repo, base_rev, &commit, NULL, error)) + return FALSE; + + g_autoptr(GPtrArray) packages = + rpm_ostree_db_query_all (repo, opt_extensions_base_rev, cancellable, error); + if (!packages) + return FALSE; + + auto packages_mapping = std::make_unique>(); + for (guint i = 0; i < packages->len; i++) + { + RpmOstreePackage *pkg = (RpmOstreePackage*)packages->pdata[i]; + const char *name = rpm_ostree_package_get_name (pkg); + const char *evr = rpm_ostree_package_get_evr (pkg); + packages_mapping->push_back(rpmostreecxx::StringMapping {k: name, v: evr}); + } + + auto extensions = rpmostreecxx::extensions_load (extensions_path, basearch, *packages_mapping); + + g_autoptr(RpmOstreeContext) ctx = + rpmostree_context_new_tree (cachedir_dfd, repo, cancellable, error); + if (!ctx) + return FALSE; + + { int tf_dfd = ror_treefile_get_dfd (treefile); + g_autofree char *abs_tf_path = glnx_fdrel_abspath (tf_dfd, "."); + dnf_context_set_repo_dir (rpmostree_context_get_dnf (ctx), abs_tf_path); + } + +#define TMP_EXTENSIONS_ROOTFS "rpmostree-extensions.tmp" + + if (!glnx_shutil_rm_rf_at (cachedir_dfd, TMP_EXTENSIONS_ROOTFS, cancellable, error)) + return FALSE; + + g_print ("Checking out %.7s... ", base_rev); + OstreeRepoCheckoutAtOptions opts = { .mode = OSTREE_REPO_CHECKOUT_MODE_USER }; + if (!ostree_repo_checkout_at (repo, &opts, cachedir_dfd, TMP_EXTENSIONS_ROOTFS, + base_rev, cancellable, error)) + return FALSE; + g_print ("done!\n"); + + g_autoptr(RpmOstreeTreespec) spec = NULL; + { g_autoptr(GPtrArray) gpkgs = g_ptr_array_new_with_free_func (g_free); + auto pkgs = extensions->get_packages(); + for (auto pkg : pkgs) + g_ptr_array_add (gpkgs, (gpointer*) g_strdup (pkg.c_str())); + char **repos = ror_treefile_get_repos (treefile); + g_autoptr(GKeyFile) treespec = g_key_file_new (); + g_key_file_set_string_list (treespec, "tree", "packages", + (const char* const*)gpkgs->pdata, gpkgs->len); + g_key_file_set_string_list (treespec, "tree", "repos", + (const char* const*)repos, + g_strv_length (repos)); + spec = rpmostree_treespec_new_from_keyfile (treespec, NULL); + } + + g_autofree char *checkout_path = glnx_fdrel_abspath (cachedir_dfd, TMP_EXTENSIONS_ROOTFS); + if (!rpmostree_context_setup (ctx, checkout_path, checkout_path, spec, cancellable, error)) + return FALSE; + +#undef TMP_EXTENSIONS_ROOTFS + + if (!rpmostree_context_prepare (ctx, cancellable, error)) + return FALSE; + + if (!glnx_shutil_mkdir_p_at (AT_FDCWD, opt_extensions_output_dir, 0755, cancellable, error)) + return FALSE; + + glnx_autofd int output_dfd = -1; + if (!glnx_opendirat (AT_FDCWD, opt_extensions_output_dir, TRUE, &output_dfd, error)) + return glnx_prefix_error (error, "Opening output dir"); + + g_autofree char *state_checksum; + if (!rpmostree_context_get_state_sha512 (ctx, &state_checksum, error)) + return FALSE; + + if (!extensions->state_checksum_changed (state_checksum, opt_extensions_output_dir)) + { + g_print ("No change.\n"); + return TRUE; + } + + if (!rpmostree_context_download (ctx, cancellable, error)) + return FALSE; + + g_autoptr(GPtrArray) extensions_pkgs = rpmostree_context_get_packages (ctx); + for (guint i = 0; i < extensions_pkgs->len; i++) + { + DnfPackage *pkg = (DnfPackage*)extensions_pkgs->pdata[i]; + const char *src = dnf_package_get_filename (pkg); + const char *basename = glnx_basename (src); + if (!glnx_file_copy_at (AT_FDCWD, dnf_package_get_filename (pkg), NULL, output_dfd, + basename, GLNX_FILE_COPY_NOXATTRS, cancellable, error)) + return FALSE; + } + + extensions->update_state_checksum (state_checksum, opt_extensions_output_dir); + if (!process_touch_if_changed (error)) + return FALSE; + + return TRUE; +} diff --git a/src/app/rpmostree-compose-builtins.h b/src/app/rpmostree-compose-builtins.h index 84af5b3a9a..946ee5aa6c 100644 --- a/src/app/rpmostree-compose-builtins.h +++ b/src/app/rpmostree-compose-builtins.h @@ -31,6 +31,7 @@ gboolean rpmostree_compose_builtin_rojig (int argc, char **argv, RpmOstreeComman gboolean rpmostree_compose_builtin_install (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error); gboolean rpmostree_compose_builtin_postprocess (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error); gboolean rpmostree_compose_builtin_commit (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error); +gboolean rpmostree_compose_builtin_extensions (int argc, char **argv, RpmOstreeCommandInvocation *invocation, GCancellable *cancellable, GError **error); G_END_DECLS diff --git a/tests/compose/test-basic-unified.sh b/tests/compose/test-basic-unified.sh index 78e99a6303..7baf949d6b 100755 --- a/tests/compose/test-basic-unified.sh +++ b/tests/compose/test-basic-unified.sh @@ -84,3 +84,36 @@ if ostree rev-parse --repo "${repo}" "${newrev}"^ 2>error.txt; then fi assert_file_has_content_literal error.txt 'has no parent' echo "ok --no-parent" + +build_rpm dodo-base +build_rpm dodo requires dodo-base +build_rpm solitaire + +cat > extensions.yaml << EOF +extensions: + extinct-birds: + packages: + - dodo + - solitaire +EOF + +# we don't actually need root here, but in CI the cache may be in a qcow2 and +# the supermin code is gated behind `runasroot` +runasroot rpm-ostree compose extensions --repo=${repo} \ + --cachedir=${test_tmpdir}/cache --base-rev ${treeref} \ + --output-dir extensions ${treefile} extensions.yaml \ + --touch-if-changed extensions-changed + +ls extensions/{dodo-1.0,dodo-base-1.0,solitaire-1.0}-*.rpm +test -f extensions-changed +echo "ok extensions" + +rm extensions-changed +runasroot rpm-ostree compose extensions --repo=${repo} \ + --cachedir=${test_tmpdir}/cache --base-rev ${treeref} \ + --output-dir extensions ${treefile} extensions.yaml \ + --touch-if-changed extensions-changed +if test -f extensions-changed; then + fatal "found extensions-changed" +fi +echo "ok extensions no change"