WIP: Support mounting /sysroot (and /boot) read-only

We want to support extending the read-only state to cover `/sysroot`
and `/boot`, since conceptually all of the data there should only
be written via libostree.

This change needs to be opt-in though to avoid breaking anyone.

Add a `sysroot/readonly` key to the repository config which instructs
`ostree-remount.service` to ensure `/sysroot` is read-only.  This
requires a bit of a dance because `/sysroot` is actually the same
filesystem as `/`; so we make `/etc` a writable bind mount in this case.

We also need to handle `/var` in the "OSTree default" case of a bind
mount; the systemd generator now looks at the writability state of
`/sysroot` and uses that to determine whether it should have the
`var.mount` unit happen before or after `ostree-remount.service.`

Also add an API to instruct the libostree shared library
that the caller has created a new mount namespace.  This way
we can freely remount read-write.

This approach extends upon in a much better way previous work
we did to support remounting `/boot` read-write.

Closes: #1265

Makefile-switchroot.am

@@ -55,7 +55,8 @@ ostree_remount_SOURCES = \
     src/switchroot/ostree-mount-util.h \
     src/switchroot/ostree-remount.c \
     $(NULL)
-ostree_remount_CPPFLAGS = $(AM_CPPFLAGS) -Isrc/switchroot
+ostree_remount_CPPFLAGS = $(AM_CPPFLAGS) $(OT_INTERNAL_GIO_UNIX_CFLAGS) -Isrc/switchroot -I$(srcdir)/libglnx
+ostree_remount_LDADD = $(AM_LDFLAGS) $(OT_INTERNAL_GIO_UNIX_LIBS) libglnx.la
 
 if BUILDOPT_SYSTEMD
 ostree_prepare_root_CPPFLAGS += -DHAVE_SYSTEMD=1

apidoc/ostree-sections.txt

@@ -497,6 +497,7 @@ ostree_sepolicy_get_type
 OstreeSysroot
 ostree_sysroot_new
 ostree_sysroot_new_default
+ostree_sysroot_initialize
 ostree_sysroot_get_path
 ostree_sysroot_load
 ostree_sysroot_load_if_changed
@@ -506,6 +507,7 @@ ostree_sysroot_lock_async
 ostree_sysroot_lock_finish
 ostree_sysroot_unlock
 ostree_sysroot_unload
+ostree_sysroot_set_mount_namespace_in_use
 ostree_sysroot_get_fd
 ostree_sysroot_ensure_initialized
 ostree_sysroot_get_bootversion

src/boot/ostree-remount.service

@@ -22,7 +22,7 @@ DefaultDependencies=no
 ConditionKernelCommandLine=ostree
 OnFailure=emergency.target
 Conflicts=umount.target
-After=-.mount
+After=-.mount var.mount
 After=systemd-remount-fs.service
 Before=local-fs.target umount.target
 # Other early boot units that need to write to /var

src/libostree/libostree-devel.sym

@@ -19,6 +19,8 @@
 
 /* Add new symbols here.  Release commits should copy this section into -released.sym. */
 LIBOSTREE_2019.4 {
+  ostree_sysroot_initialize;
+  ostree_sysroot_set_mount_namespace_in_use;
 } LIBOSTREE_2019.3;
 
 /* Stub section for the stable release *after* this development one; don't

src/libostree/ostree-impl-system-generator.c

@@ -28,6 +28,7 @@
 #ifdef HAVE_LIBMOUNT
 #include <libmount.h>
 #endif
+#include <sys/statvfs.h>
 #include <stdbool.h>
 #include "otutil.h"
 
@@ -163,6 +164,32 @@ _ostree_impl_system_generator (const char *ostree_cmdline,
   if (found_var_mnt)
     return TRUE;
 
+  struct statvfs stvfsbuf;
+  if (statvfs ("/sysroot", &stvfsbuf) == -1)
+    return glnx_throw_errno_prefix (error, "statvfs(/sysroot)");
+  const gboolean sysroot_currently_writable = ((stvfsbuf.f_flag & ST_RDONLY) == 0);
+  const char *ordering_value;
+  /* For ostree as originally envisioned, e.g. Fedora Atomic Host, the system starts
+   * with the rootfs mounted ro (kernel default), and then gets remounted by
+   * systemd.
+   * 
+   * However, Fedora CoreOS is an Ignition based system and starts out writable
+   * via rw on the kernel command line.
+   * 
+   * Now, we want to support a read-only /sysroot: https://github.com/ostreedev/ostree/issues/1265
+   * And the way that's currently implemented is in ostree-remount.service.
+   * Ideally systemd would support "Options=bind,rw" to force on a writable bind mount,
+   * but it currently doesn't.  So we handle the ordering here.
+   */
+  if (sysroot_currently_writable)
+    {
+      ordering_value = "Before=ostree-remount.service";
+    }
+  else
+    {
+      ordering_value = "After=ostree-remount.service";
+    }
+
   /* Prepare to write to the output unit dir; we use the "normal" dir
    * that overrides /usr, but not /etc.
    */
@@ -179,6 +206,7 @@ _ostree_impl_system_generator (const char *ostree_cmdline,
     return FALSE;
   g_autoptr(GOutputStream) outstream = g_unix_output_stream_new (tmpf.fd, FALSE);
   gsize bytes_written;
+
   /* This code is inspired by systemd's fstab-generator.c.
    *
    * Note that our unit doesn't run if systemd.volatile is enabled;
@@ -189,14 +217,15 @@ _ostree_impl_system_generator (const char *ostree_cmdline,
                                "[Unit]\n"
                                "Documentation=man:ostree(1)\n"
                                "ConditionKernelCommandLine=!systemd.volatile\n"
-                               /* We need /sysroot mounted writable first */
-                               "After=ostree-remount.service\n"
+                               /* See above for ordering */
+                               "%s\n"
                                "Before=local-fs.target\n"
                                "\n"
                                "[Mount]\n"
                                "Where=%s\n"
                                "What=%s\n"
                                "Options=bind\n",
+                               ordering_value,
                                var_path,
                                stateroot_var_path))
     return FALSE;

src/libostree/ostree-sysroot-cleanup.c

@@ -454,6 +454,9 @@ ostree_sysroot_cleanup_prune_repo (OstreeSysroot          *sysroot,
   OstreeRepo *repo = ostree_sysroot_repo (sysroot);
   const guint depth = 0; /* Historical default */
 
+  if (!_ostree_sysroot_ensure_writable (sysroot, error))
+    return FALSE;
+
   /* Hold an exclusive lock by default across gathering refs and doing
    * the prune.
    */
@@ -534,7 +537,10 @@ _ostree_sysroot_cleanup_internal (OstreeSysroot              *self,
                                   GError                    **error)
 {
   g_return_val_if_fail (OSTREE_IS_SYSROOT (self), FALSE);
-  g_return_val_if_fail (self->loaded, FALSE);
+  g_return_val_if_fail (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED, FALSE);
+
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
 
   if (!cleanup_other_bootversions (self, cancellable, error))
     return glnx_prefix_error (error, "Cleaning bootversions");

src/libostree/ostree-sysroot-deploy.c

@@ -55,6 +55,9 @@
 #define OSTREE_DEPLOYMENT_FINALIZING_ID SD_ID128_MAKE(e8,64,6c,d6,3d,ff,46,25,b7,79,09,a8,e7,a4,09,94)
 #endif
 
+static gboolean
+is_ro_mount (const char *path);
+
 /*
  * Like symlinkat() but overwrites (atomically) an existing
  * symlink.
@@ -806,6 +809,9 @@ write_origin_file_internal (OstreeSysroot         *sysroot,
                             GCancellable          *cancellable,
                             GError               **error)
 {
+  if (!_ostree_sysroot_ensure_writable (sysroot, error))
+    return FALSE;
+
   GLNX_AUTO_PREFIX_ERROR ("Writing out origin file", error);
   GKeyFile *origin =
     new_origin ? new_origin : ostree_deployment_get_origin (deployment);
@@ -2179,7 +2185,10 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot     *self,
                                                GCancellable      *cancellable,
                                                GError           **error)
 {
-  g_assert (self->loaded);
+  g_assert (self->loadstate == OSTREE_SYSROOT_LOAD_STATE_LOADED);
+
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
 
   /* Dealing with the staged deployment is quite tricky here. This function is
    * primarily concerned with writing out "finalized" deployments which have
@@ -2336,7 +2345,6 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot     *self,
 
       if (boot_was_ro_mount)
         {
-          /* TODO: Use new mount namespace.  https://github.com/ostreedev/ostree/issues/1265 */
           if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_SILENT, NULL) < 0)
             return glnx_throw_errno_prefix (error, "Remounting /boot read-write");
         }
@@ -2362,8 +2370,10 @@ ostree_sysroot_write_deployments_with_options (OstreeSysroot     *self,
       /* Note equivalent of try/finally here */
       gboolean success = write_deployments_bootswap (self, new_deployments, opts, bootloader,
                                                      &syncstats, cancellable, error);
-      /* Below here don't set GError until the if (!success) check */
-      if (boot_was_ro_mount)
+      /* Below here don't set GError until the if (!success) check.
+       * Note we only bother remounting if a mount namespace isn't in use.
+       * */
+      if (boot_was_ro_mount && !self->mount_namespace_in_use)
         {
           if (mount ("/boot", "/boot", NULL, MS_REMOUNT | MS_RDONLY | MS_SILENT, NULL) < 0)
             {
@@ -2670,6 +2680,9 @@ ostree_sysroot_deploy_tree (OstreeSysroot     *self,
                             GCancellable      *cancellable,
                             GError           **error)
 {
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
+
   g_autoptr(OstreeDeployment) deployment = NULL;
   if (!sysroot_initialize_deployment (self, osname, revision, origin, override_kernel_argv,
                                       &deployment, cancellable, error))
@@ -2771,6 +2784,9 @@ ostree_sysroot_stage_tree (OstreeSysroot     *self,
                            GCancellable      *cancellable,
                            GError           **error)
 {
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
+
   OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (self);
   if (booted_deployment == NULL)
     return glnx_throw (error, "Cannot stage a deployment when not currently booted into an OSTree system");
@@ -2997,6 +3013,9 @@ ostree_sysroot_deployment_set_kargs (OstreeSysroot     *self,
                                      GCancellable      *cancellable,
                                      GError           **error)
 {
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
+
   /* For now; instead of this do a redeployment */
   g_assert (!ostree_deployment_is_staged (deployment));
 
@@ -3044,6 +3063,8 @@ ostree_sysroot_deployment_set_mutable (OstreeSysroot     *self,
                                        GCancellable      *cancellable,
                                        GError           **error)
 {
+  if (!_ostree_sysroot_ensure_writable (self, error))
+    return FALSE;
 
   if (g_cancellable_set_error_if_cancelled (cancellable, error))
     return FALSE;

src/libostree/ostree-sysroot-private.h

@@ -40,6 +40,12 @@ typedef enum {
   OSTREE_SYSROOT_DEBUG_TEST_STAGED_PATH = 1 << 3,
 } OstreeSysrootDebugFlags;
 
+typedef enum {
+      OSTREE_SYSROOT_LOAD_STATE_NONE,
+      OSTREE_SYSROOT_LOAD_STATE_INIT,
+      OSTREE_SYSROOT_LOAD_STATE_LOADED,
+} OstreeSysrootLoadState;
+
 /**
  * OstreeSysroot:
  * Internal struct
@@ -51,7 +57,8 @@ struct OstreeSysroot {
   int sysroot_fd;
   GLnxLockFile lock;
 
-  gboolean loaded;
+  OstreeSysrootLoadState loadstate;
+  gboolean mount_namespace_in_use; /* TRUE if caller has told us they used CLONE_NEWNS */
   gboolean root_is_ostree_booted; /* TRUE if sysroot is / and we are booted via ostree */
   /* The device/inode for /, used to detect booted deployment */
   dev_t root_device;
@@ -79,6 +86,10 @@ struct OstreeSysroot {
 #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_DIR "/run/ostree/deployment-state/"
 #define _OSTREE_SYSROOT_DEPLOYMENT_RUNSTATE_FLAG_DEVELOPMENT "unlocked-development"
 
+gboolean
+_ostree_sysroot_ensure_writable (OstreeSysroot      *self,
+                                 GError            **error);
+
 void
 _ostree_sysroot_emit_journal_msg (OstreeSysroot  *self,
                                   const char     *msg);