feat: default AUFS -> OverlayFS whiteout conversion for squashfs

By default, replace AUFS whiteout markers with OverlayFS whiteout
markers when converting a layer from TAR to SquashFS.

It is expected that SquashFS layers will be used via direct mount,
rather than extraction to disk. Therefore, the whiteout markers must
be in a format that supports assembling multiple mounted layers into a
rootfs, using OverlayFS.

A `.wh.file` marker indicating a whiteout of `file` is replaced by
`file` as a 0:0 character device.

A `dir/.wh..wh..opq` marker indicating an opaque directory is replaced
with the `trusted.overlay.opaque="y"` xattr on `dir`.

A two-pass approach is required. AUFS directory opaque markers are not
required to be adjacent to the directory to which they apply, in the
tar stream. As `tar2sqfs` and `sqfstar` do not support tar files with
two instances of the same dir/file, we cannot append an additional tar
entry if we come across an opaque marker distant from the target
directory entry.

A new option `OptNoConvertWhiteout` will disable whiteout
conversion. This can be employed by callers to avoid unneccessary
processing where the layer is part of a squashed / single-layer image
that will not have any whiteout markers.

Fixes #25

pkg/mutate/squashfs.go

@@ -20,9 +20,10 @@ import (
 const layerMediaType types.MediaType = "application/vnd.sylabs.image.layer.v1.squashfs"
 
 type squashfsConverter struct {
-	converter string   // Path to converter program.
-	args      []string // Arguments required for converter program.
-	dir       string   // Working directory.
+	converter         string   // Path to converter program.
+	args              []string // Arguments required for converter program.
+	dir               string   // Working directory.
+	noConvertWhiteout bool     // Skip default conversion of whiteout markers from AUFS -> OverlayFS
 }
 
 // SquashfsConverterOpt are used to specify squashfs converter options.
@@ -45,13 +46,29 @@ func OptSquashfsLayerConverter(converter string) SquashfsConverterOpt {
 
 var errSquashfsConverterNotSupported = errors.New("squashfs converter not supported")
 
+// OptNoConvertWhiteout is set to skip the default conversion of whiteout /
+// opaque markers from AUFS to OverlayFS format.
+func OptNoConvertWhiteout(b bool) SquashfsConverterOpt {
+	return func(c *squashfsConverter) error {
+		c.noConvertWhiteout = b
+		return nil
+	}
+}
+
 // SquashfsLayer converts the base layer into a layer using the squashfs format. A dir must be
 // specified, which is used as a working directory during conversion. The caller is responsible for
 // cleaning up dir.
 //
 // By default, this will attempt to locate a suitable TAR to SquashFS converter such as 'tar2sqfs'
 // or `sqfstar` via exec.LookPath. To specify a path to a specific converter program, consider
 // using OptSquashfsLayerConverter.
+//
+// By default, AUFS whiteout markers in the base TAR layer will be converted to OverlayFS whiteout
+// markers in the SquashFS layer. This can be disabled, e.g. where it is known that the layer is
+// part of a squashed image that will not have any whiteouts, using OptNoConvertWhiteout.
+//
+// Note - when whiteout conversion is performed the base layer will be read twice. Callers should
+// ensure it is cached, and is not a streaming layer.
 func SquashfsLayer(base v1.Layer, dir string, opts ...SquashfsConverterOpt) (v1.Layer, error) {
 	c := squashfsConverter{
 		dir: dir,
@@ -124,6 +141,53 @@ func (c *squashfsConverter) makeSquashfs(r io.Reader) (string, error) {
 	return path, nil
 }
 
+// convertWhiteout accepts a Layer l and returns:
+//
+//   - A ReadCloser providing the layer tar, passed through a filter that converts
+//     whiteout markers from AUFS -> OverlayFS, if c.noConvertWhiteout is false.
+//     In this case, any error from the filter will propagate via the returned channel.
+//   - A ReadCloser providing the layer tar directly, if c.noConvertWhiteout is true.
+//
+// Note that when conversion is performed, the layer is read twice.
+func (c *squashfsConverter) convertWhiteout(l v1.Layer) (io.ReadCloser, chan error, error) {
+	rc, err := l.Uncompressed()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// No conversion - direct tar stream from the layer.
+	if c.noConvertWhiteout {
+		return rc, nil, nil
+	}
+
+	// Conversion - first, scan for opaque directories and presence of file
+	// whiteout markers.
+	opaquePaths, fileWhiteout, err := scanAUFSWhiteouts(rc)
+	if err != nil {
+		return nil, nil, err
+	}
+	rc.Close()
+
+	rc, err = l.Uncompressed()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Nothing found to filter
+	if len(opaquePaths) == 0 && !fileWhiteout {
+		return rc, nil, nil
+	}
+
+	filterErr := make(chan error, 1)
+	pr, pw := io.Pipe()
+	go func() {
+		defer rc.Close()
+		filterErr <- whiteoutFilter(rc, pw, opaquePaths)
+		close(filterErr)
+	}()
+	return pr, filterErr, nil
+}
+
 type squashfsLayer struct {
 	base      v1.Layer
 	converter *squashfsConverter
@@ -170,7 +234,7 @@ func (l *squashfsLayer) populate() error {
 		return nil
 	}
 
-	rc, err := l.base.Uncompressed()
+	rc, filterErr, err := l.converter.convertWhiteout(l.base)
 	if err != nil {
 		return err
 	}
@@ -181,6 +245,12 @@ func (l *squashfsLayer) populate() error {
 		return err
 	}
 
+	if filterErr != nil {
+		if err = <-filterErr; err != nil {
+			return err
+		}
+	}
+
 	f, err := os.Open(path)
 	if err != nil {
 		return err

pkg/mutate/squashfs_test.go

@@ -49,18 +49,21 @@ func diffSquashFS(tb testing.TB, pathA, pathB string, diffArgs ...string) {
 
 func Test_SquashfsLayer(t *testing.T) {
 	tests := []struct {
-		name      string
-		layer     v1.Layer
-		converter string
-		diffArgs  []string
+		name              string
+		layer             v1.Layer
+		converter         string
+		noConvertWhiteout bool
+		diffArgs          []string
 	}{
+		// HelloWorld layer contains no whiteouts - conversion should have no effect on output.
 		{
 			name: "HelloWorldBlob_sqfstar",
 			layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{
 				Algorithm: "sha256",
 				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
 			}),
-			converter: "sqfstar",
+			converter:         "sqfstar",
+			noConvertWhiteout: false,
 			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
 			// differences in ownership.
 			diffArgs: []string{"--no-owner"},
@@ -71,7 +74,89 @@ func Test_SquashfsLayer(t *testing.T) {
 				Algorithm: "sha256",
 				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
 			}),
-			converter: "tar2sqfs",
+			converter:         "tar2sqfs",
+			noConvertWhiteout: false,
+		},
+		{
+			name: "HelloWorldBlob_sqfstar_noConvertWhiteout",
+			layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
+			}),
+			converter:         "sqfstar",
+			noConvertWhiteout: true,
+			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
+			// differences in ownership.
+			diffArgs: []string{"--no-owner"},
+		},
+		{
+			name: "HelloWorldBlob_tar2sqfs_noConvertWhiteout",
+			layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01",
+			}),
+			converter:         "tar2sqfs",
+			noConvertWhiteout: true,
+		},
+		// AUFS layer contains whiteouts. Should be converted to overlayfs form when noConvertWhiteout = false.
+		//
+		//    Original (AUFS)
+		//		All regular files.
+		//
+		//        [drwxr-xr-x]  .
+		//			├── [drwxr-xr-x]  dir
+		//			│   └── [-rw-r--r--]  .wh..wh..opq
+		//			└── [-rw-r--r--]  .wh.file
+		//
+		//    Converted (OverlayFS)
+		//		.wh.file becomes file as a char 0:0 device
+		// 		dir/.wh..wh..opq becomes trusted.overlay.opaque="y" xattr on dir
+		//
+		//			[drwxr-xr-x]  .
+		//			├── [drwxr-xr-x]  dir
+		//			└── [crw-r--r--]  file
+		//
+		{
+			name: "AUFSBlob_sqfstar",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:         "sqfstar",
+			noConvertWhiteout: false,
+			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
+			// differences in ownership.
+			diffArgs: []string{"--no-owner"},
+		},
+		{
+			name: "AUFSBlob_tar2sqfs",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:         "tar2sqfs",
+			noConvertWhiteout: false,
+		},
+		{
+			name: "AUFSBlob_sqfstar_noConvertWhiteout",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:         "sqfstar",
+			noConvertWhiteout: true,
+			// Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore
+			// differences in ownership.
+			diffArgs: []string{"--no-owner"},
+		},
+		{
+			name: "AUFSBlob_tar2sqfs_noConvertWhiteout",
+			layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf",
+			}),
+			converter:         "tar2sqfs",
+			noConvertWhiteout: true,
 		},
 	}
 	for _, tt := range tests {
@@ -82,7 +167,10 @@ func Test_SquashfsLayer(t *testing.T) {
 				t.Skip(err)
 			}
 
-			l, err := SquashfsLayer(tt.layer, t.TempDir(), OptSquashfsLayerConverter(tt.converter))
+			l, err := SquashfsLayer(tt.layer, t.TempDir(),
+				OptSquashfsLayerConverter(tt.converter),
+				OptNoConvertWhiteout(tt.noConvertWhiteout),
+			)
 			if err != nil {
 				t.Fatal(err)
 			}

pkg/mutate/whiteout.go

@@ -0,0 +1,122 @@
+// Copyright 2023 Sylabs Inc. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package mutate
+
+import (
+	"archive/tar"
+	"errors"
+	"fmt"
+	"io"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	aufsWhiteoutPrefix = ".wh."
+	aufsOpaqueMarker   = ".wh..wh..opq"
+)
+
+var errUnexpectedOpaque = errors.New("unexpected opaque marker")
+
+// scanAUFSWhiteouts reads a TAR stream, returning a map of <path>:true for
+// directories in the tar that contain an AUFS .wh..wh..opq opaque directory
+// marker file, and a boolean indicating the presence of any .wh.<file> markers.
+// Note that paths returned are clean, per filepath.Clean.
+func scanAUFSWhiteouts(in io.Reader) (map[string]bool, bool, error) {
+	opaquePaths := map[string]bool{}
+	fileWhiteout := false
+
+	tr := tar.NewReader(in)
+	for {
+		header, err := tr.Next()
+
+		if err == io.EOF {
+			return opaquePaths, fileWhiteout, nil
+		}
+		if err != nil {
+			return nil, false, err
+		}
+
+		base := filepath.Base(header.Name)
+
+		if base == aufsOpaqueMarker {
+			parent := filepath.Dir(header.Name)
+			opaquePaths[parent] = true
+		}
+
+		if !fileWhiteout && strings.HasPrefix(base, aufsWhiteoutPrefix) {
+			fileWhiteout = true
+		}
+	}
+}
+
+// whiteOutFilter streams a tar file from in to out, replacing AUFS whiteout
+// markers with OverlayFS whiteout markers. Due to unrestricted ordering of
+// markers vs their target, the list of opaquePaths must be obtained prior to
+// filtering and provided to this filter.
+func whiteoutFilter(in io.ReadCloser, out io.WriteCloser, opaquePaths map[string]bool) error {
+	tr := tar.NewReader(in)
+	tw := tar.NewWriter(out)
+	defer out.Close()
+	defer tw.Close()
+
+	for {
+		header, err := tr.Next()
+
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		// Must force to PAX format, to accommodate xattrs
+		header.Format = tar.FormatPAX
+
+		clean := filepath.Clean(header.Name)
+		base := filepath.Base(header.Name)
+		parent := filepath.Dir(header.Name)
+
+		// Don't include .wh..wh..opq opaque directory markers in output.
+		if base == aufsOpaqueMarker {
+			// If we don't know the target should be opaque, then provided opaquePaths is incorrect.
+			if !opaquePaths[parent] {
+				return fmt.Errorf("%q: %w", parent, errUnexpectedOpaque)
+			}
+			continue
+		}
+		// Set overlayfs xattr on a dir that was previously found to contain a .wh..wh..opq marker.
+		if opq := opaquePaths[clean]; opq {
+			if header.PAXRecords == nil {
+				header.PAXRecords = map[string]string{}
+			}
+			header.PAXRecords["SCHILY.xattr."+"trusted.overlay.opaque"] = "y"
+		}
+		// Replace a `.wh.<name>` marker with a char dev 0 at <name>
+		if strings.HasPrefix(base, aufsWhiteoutPrefix) {
+			target := filepath.Join(parent, strings.TrimPrefix(base, aufsWhiteoutPrefix))
+			header.Name = target
+			header.Typeflag = tar.TypeChar
+			header.Devmajor = 0
+			header.Devminor = 0
+			if err := tw.WriteHeader(header); err != nil {
+				return err
+			}
+			continue
+		}
+
+		if err := tw.WriteHeader(header); err != nil {
+			return err
+		}
+
+		// Disable gosec G110: Potential DoS vulnerability via decompression bomb.
+		// We are just filtering a flow directly from tar reader to tar writer - we aren't reading
+		// into memory beyond the stdlib buffering.
+		//nolint:gosec
+		if _, err := io.Copy(tw, tr); err != nil {
+			return err
+		}
+	}
+}