From dd9b8700ff296bf4538407011af526078356bf0f Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 1 Oct 2024 17:31:18 -0400 Subject: [PATCH] wip: support for quoting devices This generalizes the whiteout support to handle block/chardev and FIFOs. Signed-off-by: Colin Walters --- man/ostree-commit.xml | 9 ++ src/libostree/ostree-core-private.h | 17 +++ src/libostree/ostree-core.c | 65 +++++++++ src/libostree/ostree-repo-checkout.c | 24 ++-- src/libostree/ostree-repo-commit.c | 204 +++++++++++++++++++++++++++ src/libostree/ostree-repo.h | 7 +- src/ostree/ot-builtin-commit.c | 5 + 7 files changed, 322 insertions(+), 9 deletions(-) diff --git a/man/ostree-commit.xml b/man/ostree-commit.xml index 12f4fd10fa..d612282c7e 100644 --- a/man/ostree-commit.xml +++ b/man/ostree-commit.xml @@ -177,6 +177,15 @@ License along with this library. If not, see . + + + + By default, ostree rejects block and character devices. This option instead "quotes" them + as regular files. In order to be processed back into block and character devices, + the corresponding --unquote-devices must be passed to ostree checkout. + + + diff --git a/src/libostree/ostree-core-private.h b/src/libostree/ostree-core-private.h index 283944b4a9..d70b3eeb8d 100644 --- a/src/libostree/ostree-core-private.h +++ b/src/libostree/ostree-core-private.h @@ -78,6 +78,20 @@ G_BEGIN_DECLS */ #define _OSTREE_ZLIB_FILE_HEADER_GVARIANT_FORMAT G_VARIANT_TYPE ("(tuuuusa(ayay))") +// ostree doesn't have native support for devices. Whiteouts in overlayfs +// are a 0:0 character device, and in some cases people are copying docker/podman +// style overlayfs container storage directly into ostree commits. This +// adds special support for "quoting" the whiteout so it just appears as a regular +// file in the ostree commit, but can be converted back into a character device +// on checkout. +#define OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh." +// Filename prefix to signify a character or block device. This +// is not supported natively by ostree (because there is no reason +// to ship devices in images). But because OCI supports it, and in +// some cases one wants to map OCI to ostree, we have support for +// "quoting" them. +#define OSTREE_QUOTED_DEVICE_PREFIX ".ostree-quoted-device." + GBytes *_ostree_file_header_new (GFileInfo *file_info, GVariant *xattrs); GBytes *_ostree_zlib_file_header_new (GFileInfo *file_info, GVariant *xattrs); @@ -92,6 +106,9 @@ gboolean _ostree_stbuf_equal (struct stat *stbuf_a, struct stat *stbuf_b); GFileInfo *_ostree_mode_uidgid_to_gfileinfo (mode_t mode, uid_t uid, gid_t gid); gboolean _ostree_validate_structureof_xattrs (GVariant *xattrs, GError **error); +gboolean _ostree_parse_quoted_device (const char *name, guint32 src_mode, const char **out_name, guint32 *out_mode, + dev_t *out_dev, GError **error); + static inline void _ostree_checksum_inplace_from_bytes_v (GVariant *csum_v, char *buf) { diff --git a/src/libostree/ostree-core.c b/src/libostree/ostree-core.c index fb11f85bc6..ef7af0e2d9 100644 --- a/src/libostree/ostree-core.c +++ b/src/libostree/ostree-core.c @@ -33,6 +33,7 @@ #include #include #include +#include /* Generic ABI checks */ G_STATIC_ASSERT (OSTREE_REPO_MODE_BARE == 0); @@ -2331,6 +2332,70 @@ ostree_validate_structureof_dirmeta (GVariant *dirmeta, GError **error) return TRUE; } +gboolean +_ostree_parse_quoted_device (const char *name, guint32 src_mode, const char **out_name, guint32 *out_mode, dev_t *out_dev, + GError **error) +{ + // Ensure we start with the quoted device prefix + const char *s = name; + const char *p = strchr (s, '.'); + if (!p) + return glnx_throw (error, "Invalid quoted device: %s", name); + if (strncmp (s, OSTREE_QUOTED_DEVICE_PREFIX, p - name) != 0) + return glnx_throw (error, "Invalid quoted device: %s", name); + s += strlen (OSTREE_QUOTED_DEVICE_PREFIX); + g_assert (out_name); + *out_name = s; + + // The input mode is the same as the source, but without the format bits + guint32 ret_mode = (src_mode & ~S_IFMT); + + // Parse the mode + s++; + switch (*s) + { + case 'b': + ret_mode |= S_IFBLK; + break; + case 'c': + ret_mode |= S_IFCHR; + break; + case 'p': + ret_mode |= S_IFIFO; + break; + default: + return glnx_throw (error, "Invalid quoted device: %s", name); + } + s++; + if (*s != '.') + return glnx_throw (error, "Invalid quoted device: %s", name); + s++; + s = strchr (s, '.'); + if (!s) + return glnx_throw (error, "Invalid quoted device: %s", name); + s++; + char *endptr; + unsigned int major, minor; + major = (unsigned int)g_ascii_strtoull (s, &endptr, 10); + if (errno == ERANGE) + return glnx_throw (error, "Invalid quoted device: %s", name); + s = endptr; + if (*s != '.') + return glnx_throw (error, "Invalid quoted device: %s", name); + s++; + minor = (unsigned int)g_ascii_strtoull (s, &endptr, 10); + if (errno == ERANGE) + return glnx_throw (error, "Invalid quoted device: %s", name); + g_assert (endptr); + if (*endptr != '\0') + return glnx_throw (error, "Invalid quoted device: %s", name); + g_assert (ret_mode); + *out_mode = ret_mode; + g_assert (out_dev); + *out_dev = makedev (major, minor); + return TRUE; +} + /** * ostree_commit_get_parent: * @commit_variant: Variant of type %OSTREE_OBJECT_TYPE_COMMIT diff --git a/src/libostree/ostree-repo-checkout.c b/src/libostree/ostree-repo-checkout.c index e83713d8ce..e8890a04a1 100644 --- a/src/libostree/ostree-repo-checkout.c +++ b/src/libostree/ostree-repo-checkout.c @@ -35,14 +35,6 @@ #define WHITEOUT_PREFIX ".wh." #define OPAQUE_WHITEOUT_NAME ".wh..wh..opq" -// ostree doesn't have native support for devices. Whiteouts in overlayfs -// are a 0:0 character device, and in some cases people are copying docker/podman -// style overlayfs container storage directly into ostree commits. This -// adds special support for "quoting" the whiteout so it just appears as a regular -// file in the ostree commit, but can be converted back into a character device -// on checkout. -#define OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh." - /* Per-checkout call state/caching */ typedef struct { @@ -716,6 +708,9 @@ checkout_one_file_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, Ch const gboolean is_unreadable = (!is_symlink && (source_mode & S_IRUSR) == 0); const gboolean is_whiteout = (!is_symlink && options->process_whiteouts && g_str_has_prefix (destination_name, WHITEOUT_PREFIX)); + const gboolean is_quoted_device + = (!is_symlink && options->unquote_devices + && g_str_has_prefix (destination_name, OSTREE_QUOTED_DEVICE_PREFIX)); const gboolean is_overlayfs_whiteout = (!is_symlink && g_str_has_prefix (destination_name, OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX)); @@ -740,6 +735,16 @@ checkout_one_file_at (OstreeRepo *repo, OstreeRepoCheckoutAtOptions *options, Ch need_copy = FALSE; } + else if (is_quoted_device) + { + const char *devname; + dev_t dev; + guint32 mode; + if (!_ostree_parse_quoted_device (destination_name, source_mode, &devname, &mode, &dev, error)) + return FALSE; + if (mknodat (destination_dfd, devname, (mode_t)mode, dev) < 0) + return glnx_throw_errno_prefix (error, "mknodat"); + } else if (is_overlayfs_whiteout && options->process_passthrough_whiteouts) { const char *name = destination_name + (sizeof (OSTREE_QUOTED_OVERLAYFS_WHITEOUT_PREFIX) - 1); @@ -1437,6 +1442,9 @@ canonicalize_options (OstreeRepo *self, OstreeRepoCheckoutAtOptions *options) /* Force USER mode for BARE_USER_ONLY always - nothing else makes sense */ if (ostree_repo_get_mode (self) == OSTREE_REPO_MODE_BARE_USER_ONLY) options->mode = OSTREE_REPO_CHECKOUT_MODE_USER; + + if (options->unquote_devices) + options->process_whiteouts = TRUE; } /** diff --git a/src/libostree/ostree-repo-commit.c b/src/libostree/ostree-repo-commit.c index 0ee97288d7..35fcdbb4f6 100644 --- a/src/libostree/ostree-repo-commit.c +++ b/src/libostree/ostree-repo-commit.c @@ -3450,6 +3450,202 @@ write_dir_entry_to_mtree_internal (OstreeRepo *self, OstreeRepoFile *repo_dir, return TRUE; } +static gboolean +write_quoted_device (OstreeRepo *self, OstreeRepoFile *repo_dir, + GFileEnumerator *dir_enum, GLnxDirFdIterator *dfd_iter, + WriteDirContentFlags writeflags, GFileInfo *child_info, + OstreeMutableTree *mtree, OstreeRepoCommitModifier *modifier, + GPtrArray *path, GCancellable *cancellable, GError **error) +{ + g_assert (dir_enum != NULL || dfd_iter != NULL); + + GFileType file_type = g_file_info_get_file_type (child_info); + const char *name = g_file_info_get_name (child_info); + + /* Load flags into boolean constants for ease of readability (we also need to + * NULL-check modifier) + */ + const gboolean canonical_permissions + = self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY + || (modifier + && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS)); + const gboolean devino_canonical + = modifier && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL); + /* We currently only honor the CONSUME flag in the dfd_iter case to avoid even + * more complexity in this function, and it'd mostly only be useful when + * operating on local filesystems anyways. + */ + const gboolean delete_after_commit + = dfd_iter && modifier && (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME); + + /* Build the full path which we need for callbacks */ + g_ptr_array_add (path, (char *)name); + g_autofree char *child_relpath = ptrarray_path_join (path); + + /* Call the filter */ + g_autoptr (GFileInfo) modified_info = NULL; + OstreeRepoCommitFilterResult filter_result = _ostree_repo_commit_modifier_apply ( + self, modifier, child_relpath, child_info, &modified_info); + const gboolean child_info_was_modified = !_ostree_gfileinfo_equal (child_info, modified_info); + + if (filter_result != OSTREE_REPO_COMMIT_FILTER_ALLOW) + { + g_ptr_array_remove_index (path, path->len - 1); + if (delete_after_commit) + { + g_assert (dfd_iter); + if (!glnx_shutil_rm_rf_at (dfd_iter->fd, name, cancellable, error)) + return FALSE; + } + /* Note: early return */ + return TRUE; + } + + guint32 src_mode = g_file_info_get_attribute_uint32 (src_info, "unix::mode")';' + switch (file_type) + { + case G_FILE_TYPE_SYMBOLIC_LINK: + case G_FILE_TYPE_REGULAR: + break; + default: + return glnx_throw (error, "Unsupported file type for file: '%s'", child_relpath); + } + + g_autoptr (GFile) child = NULL; + if (dir_enum != NULL) + child = g_file_enumerator_get_child (dir_enum, child_info); + + /* Our filters have passed, etc.; now we prepare to write the content object */ + glnx_autofd int file_input_fd = -1; + + /* Open the file now, since it's better for reading xattrs + * rather than using the /proc/self/fd links. + * + * TODO: Do this lazily, since for e.g. bare-user-only repos + * we don't have xattrs and don't need to open every file + * for things that have devino cache hits. + */ + if (file_type == G_FILE_TYPE_REGULAR && dfd_iter != NULL) + { + if (!glnx_openat_rdonly (dfd_iter->fd, name, FALSE, &file_input_fd, error)) + return FALSE; + } + + g_autoptr (GVariant) xattrs = NULL; + gboolean xattrs_were_modified; + if (dir_enum != NULL) + { + if (!get_final_xattrs (self, modifier, child_relpath, child_info, child, -1, name, + source_xattrs, &xattrs, &xattrs_were_modified, cancellable, error)) + return FALSE; + } + else + { + /* These contortions are basically so we use glnx_fd_get_all_xattrs() + * for regfiles, and glnx_dfd_name_get_all_xattrs() for symlinks. + */ + int xattr_fd_arg = (file_input_fd != -1) ? file_input_fd : dfd_iter->fd; + const char *xattr_path_arg = (file_input_fd != -1) ? NULL : name; + if (!get_final_xattrs (self, modifier, child_relpath, child_info, child, xattr_fd_arg, + xattr_path_arg, source_xattrs, &xattrs, &xattrs_were_modified, + cancellable, error)) + return FALSE; + } + + /* Used below to see whether we can do a fast path commit */ + const gboolean modified_file_meta = child_info_was_modified || xattrs_were_modified; + + /* A big prerequisite list of conditions for whether or not we can + * "adopt", i.e. just checksum and rename() into place + */ + const gboolean can_adopt_basic = file_type == G_FILE_TYPE_REGULAR && dfd_iter != NULL + && delete_after_commit + && ((writeflags & WRITE_DIR_CONTENT_FLAGS_CAN_ADOPT) > 0); + gboolean can_adopt = can_adopt_basic; + /* If basic prerquisites are met, check repo mode specific ones */ + if (can_adopt) + { + /* For bare repos, we could actually chown/reset the xattrs, but let's + * do the basic optimizations here first. + */ + if (self->mode == OSTREE_REPO_MODE_BARE) + can_adopt = !modified_file_meta; + else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY) + can_adopt = canonical_permissions; + else + /* This covers bare-user and archive. See comments in adopt_and_commit_regfile() + * for notes on adding bare-user later here. + */ + can_adopt = FALSE; + } + gboolean did_adopt = FALSE; + + /* The very fast path - we have a devino cache hit, nothing to write */ + if (loose_checksum && !modified_file_meta) + { + if (!ostree_mutable_tree_replace_file (mtree, name, loose_checksum, error)) + return FALSE; + + g_mutex_lock (&self->txn_lock); + self->txn.stats.devino_cache_hits++; + g_mutex_unlock (&self->txn_lock); + } + /* Next fast path - we can "adopt" the file */ + else if (can_adopt) + { + char checksum[OSTREE_SHA256_STRING_LEN + 1]; + if (!adopt_and_commit_regfile (self, dfd_iter->fd, name, modified_info, xattrs, checksum, + cancellable, error)) + return FALSE; + if (!ostree_mutable_tree_replace_file (mtree, name, checksum, error)) + return FALSE; + did_adopt = TRUE; + } + else + { + g_autoptr (GInputStream) file_input = NULL; + + if (file_type == G_FILE_TYPE_REGULAR) + { + if (dir_enum != NULL) + { + g_assert (child != NULL); + file_input = (GInputStream *)g_file_read (child, cancellable, error); + if (!file_input) + return FALSE; + } + else + { + /* We already opened the fd above */ + file_input = g_unix_input_stream_new (file_input_fd, FALSE); + } + } + + g_autofree guchar *child_file_csum = NULL; + if (!write_content_object (self, NULL, file_input, modified_info, xattrs, &child_file_csum, + cancellable, error)) + return FALSE; + + char tmp_checksum[OSTREE_SHA256_STRING_LEN + 1]; + ostree_checksum_inplace_from_bytes (child_file_csum, tmp_checksum); + if (!ostree_mutable_tree_replace_file (mtree, name, tmp_checksum, error)) + return FALSE; + } + + /* Process delete_after_commit. In the adoption case though, we already + * took ownership of the file above, usually via a renameat(). + */ + if (delete_after_commit && !did_adopt) + { + if (!glnx_unlinkat (dfd_iter->fd, name, 0, error)) + return FALSE; + } + + g_ptr_array_remove_index (path, path->len - 1); + + return TRUE; +} + /* Given either a dir_enum or a dfd_iter, writes a non-dir (regfile/symlink) to * the mtree. */ @@ -3889,6 +4085,14 @@ write_dfd_iter_to_mtree_internal (OstreeRepo *self, GLnxDirFdIterator *src_dfd_i error)) return FALSE; } + else if (modifier->flags & OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES) + { + if (!write_quoted_device (self, NULL, NULL, src_dfd_iter, flags, child_info, mtree, + modifier, path, cancellable, error)) + return FALSE; + // Note we skip over the code below + continue; + } else { return glnx_throw (error, "Not a regular file or symlink: %s", dent->d_name); diff --git a/src/libostree/ostree-repo.h b/src/libostree/ostree-repo.h index d38fad9a2b..fc27a5da60 100644 --- a/src/libostree/ostree-repo.h +++ b/src/libostree/ostree-repo.h @@ -519,6 +519,9 @@ typedef OstreeRepoCommitFilterResult (*OstreeRepoCommitFilter) (OstreeRepo *repo * modifier filters (non-directories only); Since: 2017.14 * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SELINUX_LABEL_V1: For SELinux and other systems, label * /usr/etc as if it was /etc. + * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES: Instead of erroring out on block/character + * devices, "quote" them as regular files that can optionally be unpacked back into native devices. + * Since: 2024.9 * * Flags modifying commit behavior. In bare-user-only mode, * @OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CANONICAL_PERMISSIONS and @@ -535,6 +538,7 @@ typedef enum OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME = (1 << 4), OSTREE_REPO_COMMIT_MODIFIER_FLAGS_DEVINO_CANONICAL = (1 << 5), OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SELINUX_LABEL_V1 = (1 << 6), + OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES = (1 << 7), } OstreeRepoCommitModifierFlags; /** @@ -802,12 +806,13 @@ typedef struct gboolean enable_uncompressed_cache; /* Deprecated */ gboolean enable_fsync; /* Deprecated */ gboolean process_whiteouts; + gboolean unquote_devices; /* Since: 2024.9 */ gboolean no_copy_fallback; gboolean force_copy; /* Since: 2017.6 */ gboolean bareuseronly_dirs; /* Since: 2017.7 */ gboolean force_copy_zerosized; /* Since: 2018.9 */ gboolean process_passthrough_whiteouts; - gboolean unused_bools[3]; + gboolean unused_bools[2]; /* 3 byte hole on 64 bit */ const char *subpath; diff --git a/src/ostree/ot-builtin-commit.c b/src/ostree/ot-builtin-commit.c index 7c6d63e4df..5e3a6b9d8b 100644 --- a/src/ostree/ot-builtin-commit.c +++ b/src/ostree/ot-builtin-commit.c @@ -62,6 +62,7 @@ static char *opt_base; static char **opt_trees; static gint opt_owner_uid = -1; static gint opt_owner_gid = -1; +static gboolean opt_quote_devices; static gboolean opt_table_output; #ifndef OSTREE_DISABLE_GPGME static char **opt_gpg_key_ids; @@ -124,6 +125,8 @@ static GOptionEntry options[] = { { "owner-gid", 0, 0, G_OPTION_ARG_INT, &opt_owner_gid, "Set file ownership group id", "GID" }, { "canonical-permissions", 0, 0, G_OPTION_ARG_NONE, &opt_canonical_permissions, "Canonicalize permissions in the same way bare-user does for hardlinked files", NULL }, + { "quote-devices", 0, 0, G_OPTION_ARG_NONE, &opt_quote_devices, + "Instead of erroring out on block/character devices, \"quote\" them as regular files", NULL }, { "bootable", 0, 0, G_OPTION_ARG_NONE, &opt_bootable, "Flag this commit as a bootable OSTree (e.g. contains a Linux kernel)", NULL }, { "mode-ro-executables", 0, 0, G_OPTION_ARG_NONE, &opt_ro_executables, @@ -601,6 +604,8 @@ ostree_builtin_commit (int argc, char **argv, OstreeCommandInvocation *invocatio flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_SKIP_XATTRS; if (opt_consume) flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_CONSUME; + if (opt_quote_devices) + flags |= OSTREE_REPO_COMMIT_MODIFIER_FLAGS_QUOTE_DEVICES; switch (opt_selinux_labeling_epoch) { case 0: