WIP/DNM / Support overlayfs whiteout character files

Related-Issue: ostreedev#2712

src/libostree/ostree-core-private.h

@@ -34,6 +34,11 @@ G_BEGIN_DECLS
 #define DEFAULT_DIRECTORY_MODE 0775
 #define DEFAULT_REGFILE_MODE 0660
 
+/* Macros to identify the file mode of an overlayfs whiteout, or a file stat for a whiteout
+ ∫*/
+#define S_ISWHITEOUT(mode) (S_ISCHR (mode) && ((mode & ~S_IFMT) == 0))
+#define ST_ISWHITEOUT(st) (S_ISWHITEOUT (st.st_mode) && st.st_rdev == 0)
+
 /* This file contains private implementation data format definitions
  * read by multiple implementation .c files.
  */

src/libostree/ostree-core.c

@@ -827,7 +827,7 @@ gboolean ostree_break_hardlink (int               dfd,
     return glnx_file_copy_at (dfd, path, &stbuf, dfd, path,
                               copyflags | GLNX_FILE_COPY_OVERWRITE,
                               cancellable, error);
-  else if (S_ISLNK (stbuf.st_mode))
+  else if (S_ISLNK (stbuf.st_mode) || ST_ISWHITEOUT (stbuf)) //WIP:verify break_symhardlink and glnx can handle a char dev
     return break_symhardlink (dfd, path, &stbuf, copyflags,
                               cancellable, error);
   else
@@ -929,6 +929,8 @@ ostree_checksum_file_from_input (GFileInfo        *file_info,
             return FALSE;
         }
     }
+  // WIP: handle whiteouts in checksum, those have no content?, but we should probably
+  // avoid trying to read from them
 
   *out_csum = g_malloc (OSTREE_SHA256_DIGEST_LEN);
   ot_checksum_get_digest (&checksum, *out_csum, OSTREE_SHA256_DIGEST_LEN);
@@ -1051,6 +1053,8 @@ ostree_checksum_file_at (int               dfd,
         return FALSE;
     }
 
+    // WIP: whiteouts?
+
   const gboolean ignore_xattrs =
     ((flags & OSTREE_CHECKSUM_FLAGS_IGNORE_XATTRS) > 0);
 
@@ -2014,7 +2018,7 @@ file_header_parse (GVariant         *metadata,
   mode = GUINT32_FROM_BE (mode);
   g_autoptr(GFileInfo) ret_file_info = _ostree_mode_uidgid_to_gfileinfo (mode, uid, gid);
 
-  if (S_ISREG (mode))
+  if (S_ISREG (mode) || S_ISWHITEOUT(mode))
     {
       ;
     }
@@ -2065,7 +2069,7 @@ zlib_file_header_parse (GVariant         *metadata,
   g_autoptr(GFileInfo) ret_file_info = _ostree_mode_uidgid_to_gfileinfo (mode, uid, gid);
   g_file_info_set_size (ret_file_info, GUINT64_FROM_BE (size));
 
-  if (S_ISREG (mode))
+  if (S_ISREG (mode) || S_ISWHITEOUT (mode))
     {
       ;
     }
@@ -2370,6 +2374,7 @@ _ostree_validate_bareuseronly_mode (guint32     content_mode,
     }
   else if (S_ISLNK (content_mode))
     ; /* Nothing */
+  // WIP: IS_WHITEOUT does need special handling here?
   else
     g_assert_not_reached ();
 
@@ -2400,7 +2405,7 @@ gboolean
 ostree_validate_structureof_file_mode (guint32            mode,
                                        GError           **error)
 {
-  if (!(S_ISREG (mode) || S_ISLNK (mode)))
+  if (!(S_ISREG (mode) || S_ISLNK (mode) || S_ISWHITEOUT(mode)))
     return glnx_throw (error, "Invalid file metadata mode %u; not a valid file type", mode);
 
   if (!validate_stat_mode_perms (mode, error))

src/libostree/ostree-repo-checkout.c

@@ -398,6 +398,14 @@ create_file_copy_from_input_at (OstreeRepo     *repo,
                                  error))
         return FALSE;
     }
+  else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SPECIAL)
+    {
+       guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
+       g_assert(S_ISWHITEOUT(file_mode));
+       if (mknodat(destination_dfd, destination_name, file_mode, (dev_t)0) < 0) {
+          return glnx_throw_errno_prefix (error, "Creating whiteout char device");
+       }
+    }
   else
     g_assert_not_reached ();
 

src/libostree/ostree-repo-commit.c

@@ -213,6 +213,36 @@ _ostree_repo_commit_tmpf_final (OstreeRepo        *self,
   return TRUE;
 }
 
+gboolean
+_ostree_repo_commit_bare_whiteout (OstreeRepo        *self,
+                                   const char        *checksum,
+                                   guint32            uid,
+                                   guint32            gid,
+                                   GVariant          *xattrs,
+                                   GCancellable      *cancellable,
+                                   GError           **error)
+{
+  g_assert(self->mode == OSTREE_REPO_MODE_BARE);
+
+  char tmpbuf[_OSTREE_LOOSE_PATH_MAX];
+  _ostree_loose_path (tmpbuf, checksum, OSTREE_OBJECT_TYPE_FILE, self->mode);
+
+  int dest_dfd = commit_dest_dfd (self);
+  if (!_ostree_repo_ensure_loose_objdir_at (dest_dfd, tmpbuf, cancellable, error))
+    return FALSE;
+
+  if (mknodat(dest_dfd, tmpbuf, S_IFCHR, (dev_t)0) < 0)
+    return glnx_throw_errno_prefix (error, "Creating whiteout char device");
+
+  if (!glnx_dfd_name_set_all_xattrs(dest_dfd, tmpbuf, xattrs, cancellable, error))
+    return glnx_throw_errno_prefix (error, "Setting xattrs for whiteout char device");
+
+  if (fchownat(dest_dfd, tmpbuf, uid, gid, 0) < 0)
+    return glnx_throw_errno_prefix (error, "fchownat");
+
+  return TRUE;
+}
+
 /* Given a dfd+path combination (may be regular file or symlink),
  * rename it into place.
  */
@@ -301,7 +331,7 @@ commit_loose_regfile_object (OstreeRepo        *self,
             return FALSE;
         }
       else
-        g_assert (S_ISLNK (mode));
+        g_assert (S_ISLNK (mode) || S_ISWHITEOUT(mode));
     }
   else if (self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)
     {
@@ -966,7 +996,9 @@ write_content_object (OstreeRepo         *self,
   else
     file_input = input;
 
+  const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
   gboolean phys_object_is_symlink = FALSE;
+  gboolean phys_object_is_whiteout = FALSE;
   switch (object_file_type)
     {
     case G_FILE_TYPE_REGULAR:
@@ -975,6 +1007,19 @@ write_content_object (OstreeRepo         *self,
       if (self->mode == OSTREE_REPO_MODE_BARE || self->mode == OSTREE_REPO_MODE_BARE_USER_ONLY)
         phys_object_is_symlink = TRUE;
       break;
+    case G_FILE_TYPE_SPECIAL:
+        /* Only overlayfs whiteout char 0:0 files are supported for G_FILE_TYPE_SPECIAL */
+        if (S_ISWHITEOUT(mode))
+          {
+            /* For bare mode repositories where no side file metadata is stored we want to
+             * avoid the creation of an empty tmp file and later setup of permissions because
+             * this won't result in a char device.
+             */
+            if (self->mode == OSTREE_REPO_MODE_BARE)
+              phys_object_is_whiteout = TRUE;
+            break;
+          }
+         return glnx_throw (error, "Unsupported file type %u with mode 0%o", object_file_type, mode);
     default:
       return glnx_throw (error, "Unsupported file type %u", object_file_type);
     }
@@ -1021,7 +1066,8 @@ write_content_object (OstreeRepo         *self,
    * binary with trailing garbage, creating a window on the local
    * system where a malicious setuid binary exists.
    *
-   * We use GLnxTmpfile for regular files, and OtCleanupUnlinkat for symlinks.
+   * We use GLnxTmpfile for regular files, OtCleanupUnlinkat for symlinks,
+   * we use no temporary for whiteout char devices in bare mode.
    */
   g_auto(OtCleanupUnlinkat) tmp_unlinker = { commit_tmp_dfd (self), NULL };
   g_auto(GLnxTmpfile) tmpf = { 0, };
@@ -1037,7 +1083,9 @@ write_content_object (OstreeRepo         *self,
                                               cancellable, error))
         return FALSE;
     }
-  else if (repo_mode != OSTREE_REPO_MODE_ARCHIVE)
+  else if (phys_object_is_whiteout)
+    ;
+  else if (repo_mode != OSTREE_REPO_MODE_ARCHIVE )
     {
       if (!create_regular_tmpfile_linkable_with_content (self, size, file_input,
                                                          &tmpf, cancellable, error))
@@ -1079,11 +1127,16 @@ write_content_object (OstreeRepo         *self,
 
           unpacked_size = g_file_info_get_size (file_info);
         }
-      else
+      else if (g_file_info_get_file_type (file_info) == G_FILE_TYPE_SYMBOLIC_LINK)
         {
           /* For a symlink, the size is the length of the target */
           unpacked_size = strlen (g_file_info_get_symlink_target (file_info));
         }
+      else
+        {
+          /* For char devices 0:0 whiteouts the content size is 0 */
+          unpacked_size = 0;
+        }
 
       if (!g_output_stream_flush (temp_out, cancellable, error))
         return FALSE;
@@ -1151,7 +1204,6 @@ write_content_object (OstreeRepo         *self,
 
   const guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
   const guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
-  const guint32 mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
   /* Is it "physically" a symlink? */
   if (phys_object_is_symlink)
     {
@@ -1189,6 +1241,14 @@ write_content_object (OstreeRepo         *self,
                               &tmp_unlinker, cancellable, error))
         return FALSE;
     }
+  else if (phys_object_is_whiteout)
+    {
+      if (!_ostree_repo_commit_bare_whiteout (self, actual_checksum,
+                                              uid, gid, xattrs,
+                                              cancellable, error))
+        return FALSE;
+        //WIP: gid, uid, xattrs
+    }
   else
     {
       /* Check if a file with the same payload is present in the repository,
@@ -3739,6 +3799,7 @@ write_content_to_mtree_internal (OstreeRepo                  *self,
     {
     case G_FILE_TYPE_SYMBOLIC_LINK:
     case G_FILE_TYPE_REGULAR:
+    case G_FILE_TYPE_SPECIAL:
       break;
     default:
       return glnx_throw (error, "Unsupported file type for file: '%s'", child_relpath);
@@ -4090,7 +4151,7 @@ write_dfd_iter_to_mtree_internal (OstreeRepo                  *self,
           continue;
         }
 
-      if (S_ISREG (stbuf.st_mode))
+      if (S_ISREG (stbuf.st_mode) || S_ISWHITEOUT(stbuf.st_mode))
         ;
       else if (S_ISLNK (stbuf.st_mode))
         {

src/libostree/ostree-repo-private.h

@@ -414,6 +414,15 @@ _ostree_repo_commit_tmpf_final (OstreeRepo        *self,
                                 GCancellable      *cancellable,
                                 GError           **error);
 
+gboolean
+_ostree_repo_commit_bare_whiteout (OstreeRepo        *self,
+                                   const char        *checksum,
+                                   guint32            uid,
+                                   guint32            gid,
+                                   GVariant          *xattrs,
+                                   GCancellable      *cancellable,
+                                   GError           **error);
+
 typedef struct {
   gboolean initialized;
   gpointer opaque0[10];

src/libostree/ostree-repo-static-delta-compilation.c

@@ -514,6 +514,11 @@ process_one_object (OstreeRepo                       *repo,
             g_memory_input_stream_new_from_data (target, strlen (target), NULL);
           content_size = strlen (target);
         }
+      else if (S_ISWHITEOUT(mode))
+        {
+          // WIP: can a whiteout trigger a delta?
+          g_assert(0);
+        }
       else
         {
           g_assert (S_ISREG (mode));

src/libostree/ostree-repo.c

@@ -4322,8 +4322,8 @@ _ostree_repo_load_file_bare (OstreeRepo         *self,
         return glnx_throw_errno_prefix (error, "openat");
     }
 
-  if (!(S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode)))
-    return glnx_throw (error, "Not a regular file or symlink");
+  if (!(S_ISREG (stbuf.st_mode) || S_ISLNK (stbuf.st_mode) || S_ISWHITEOUT (stbuf.st_mode)))
+    return glnx_throw (error, "Not a regular file, symlink or whiteout");
 
   /* In the non-bare-user case, gather symlink info if requested */
   if (self->mode != OSTREE_REPO_MODE_BARE_USER
@@ -4474,7 +4474,7 @@ ostree_repo_load_file (OstreeRepo         *self,
           if (S_ISLNK (stbuf.st_mode))
             g_file_info_set_symlink_target (*out_file_info, symlink_target);
           else
-            g_assert (S_ISREG (stbuf.st_mode));
+            g_assert (S_ISREG (stbuf.st_mode) || S_ISWHITEOUT(stbuf.st_mode));
         }
 
       ot_transfer_out_value (out_xattrs, &ret_xattrs);