diff --git a/pkg/mutate/squashfs.go b/pkg/mutate/squashfs.go index 8ce9c79..6d3ca3d 100644 --- a/pkg/mutate/squashfs.go +++ b/pkg/mutate/squashfs.go @@ -20,9 +20,10 @@ import ( const layerMediaType types.MediaType = "application/vnd.sylabs.image.layer.v1.squashfs" type squashfsConverter struct { - converter string // Path to converter program. - args []string // Arguments required for converter program. - dir string // Working directory. + converter string // Path to converter program. + args []string // Arguments required for converter program. + dir string // Working directory. + noConvertWhiteout bool // Skip default conversion of whiteout markers from AUFS -> OverlayFS } // SquashfsConverterOpt are used to specify squashfs converter options. @@ -45,6 +46,15 @@ func OptSquashfsLayerConverter(converter string) SquashfsConverterOpt { var errSquashfsConverterNotSupported = errors.New("squashfs converter not supported") +// OptNoConvertWhiteout is set to skip the default conversion of whiteout / +// opaque markers from AUFS to OverlayFS format. +func OptNoConvertWhiteout(b bool) SquashfsConverterOpt { + return func(c *squashfsConverter) error { + c.noConvertWhiteout = b + return nil + } +} + // SquashfsLayer converts the base layer into a layer using the squashfs format. A dir must be // specified, which is used as a working directory during conversion. The caller is responsible for // cleaning up dir. @@ -52,6 +62,13 @@ var errSquashfsConverterNotSupported = errors.New("squashfs converter not suppor // By default, this will attempt to locate a suitable TAR to SquashFS converter such as 'tar2sqfs' // or `sqfstar` via exec.LookPath. To specify a path to a specific converter program, consider // using OptSquashfsLayerConverter. +// +// By default, AUFS whiteout markers in the base TAR layer will be converted to OverlayFS whiteout +// markers in the SquashFS layer. This can be disabled, e.g. where it is known that the layer is +// part of a squashed image that will not have any whiteouts, using OptNoConvertWhiteout. +// +// Note - when whiteout conversion is performed the base layer will be read twice. Callers should +// ensure it is cached, and is not a streaming layer. func SquashfsLayer(base v1.Layer, dir string, opts ...SquashfsConverterOpt) (v1.Layer, error) { c := squashfsConverter{ dir: dir, @@ -124,6 +141,53 @@ func (c *squashfsConverter) makeSquashfs(r io.Reader) (string, error) { return path, nil } +// convertWhiteout accepts a Layer l and returns: +// +// - A ReadCloser providing the layer tar, passed through a filter that converts +// whiteout markers from AUFS -> OverlayFS, if c.noConvertWhiteout is false. +// In this case, any error from the filter will propagate via the returned channel. +// - A ReadCloser providing the layer tar directly, if c.noConvertWhiteout is true. +// +// Note that when conversion is performed, the layer is read twice. +func (c *squashfsConverter) convertWhiteout(l v1.Layer) (io.ReadCloser, chan error, error) { + rc, err := l.Uncompressed() + if err != nil { + return nil, nil, err + } + + // No conversion - direct tar stream from the layer. + if c.noConvertWhiteout { + return rc, nil, nil + } + + // Conversion - first, scan for opaque directories and presence of file + // whiteout markers. + opaquePaths, fileWhiteout, err := scanAUFSWhiteouts(rc) + if err != nil { + return nil, nil, err + } + rc.Close() + + rc, err = l.Uncompressed() + if err != nil { + return nil, nil, err + } + + // Nothing found to filter + if len(opaquePaths) == 0 && !fileWhiteout { + return rc, nil, nil + } + + filterErr := make(chan error, 1) + pr, pw := io.Pipe() + go func() { + defer rc.Close() + filterErr <- whiteoutFilter(rc, pw, opaquePaths) + close(filterErr) + }() + return pr, filterErr, nil +} + type squashfsLayer struct { base v1.Layer converter *squashfsConverter @@ -170,7 +234,7 @@ func (l *squashfsLayer) populate() error { return nil } - rc, err := l.base.Uncompressed() + rc, filterErr, err := l.converter.convertWhiteout(l.base) if err != nil { return err } @@ -181,6 +245,12 @@ func (l *squashfsLayer) populate() error { return err } + if filterErr != nil { + if err = <-filterErr; err != nil { + return err + } + } + f, err := os.Open(path) if err != nil { return err diff --git a/pkg/mutate/squashfs_test.go b/pkg/mutate/squashfs_test.go index 71b385d..dd5f648 100644 --- a/pkg/mutate/squashfs_test.go +++ b/pkg/mutate/squashfs_test.go @@ -49,18 +49,21 @@ func diffSquashFS(tb testing.TB, pathA, pathB string, diffArgs ...string) { func Test_SquashfsLayer(t *testing.T) { tests := []struct { - name string - layer v1.Layer - converter string - diffArgs []string + name string + layer v1.Layer + converter string + noConvertWhiteout bool + diffArgs []string }{ + // HelloWorld layer contains no whiteouts - conversion should have no effect on output. { name: "HelloWorldBlob_sqfstar", layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{ Algorithm: "sha256", Hex: "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01", }), - converter: "sqfstar", + converter: "sqfstar", + noConvertWhiteout: false, // Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore // differences in ownership. diffArgs: []string{"--no-owner"}, @@ -71,7 +74,89 @@ func Test_SquashfsLayer(t *testing.T) { Algorithm: "sha256", Hex: "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01", }), - converter: "tar2sqfs", + converter: "tar2sqfs", + noConvertWhiteout: false, + }, + { + name: "HelloWorldBlob_sqfstar_noConvertWhiteout", + layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01", + }), + converter: "sqfstar", + noConvertWhiteout: true, + // Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore + // differences in ownership. + diffArgs: []string{"--no-owner"}, + }, + { + name: "HelloWorldBlob_tar2sqfs_noConvertWhiteout", + layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01", + }), + converter: "tar2sqfs", + noConvertWhiteout: true, + }, + // AUFS layer contains whiteouts. Should be converted to overlayfs form when noConvertWhiteout = false. + // + // Original (AUFS) + // All regular files. + // + // [drwxr-xr-x] . + // ├── [drwxr-xr-x] dir + // │   └── [-rw-r--r--] .wh..wh..opq + // └── [-rw-r--r--] .wh.file + // + // Converted (OverlayFS) + // .wh.file becomes file as a char 0:0 device + // dir/.wh..wh..opq becomes trusted.overlay.opaque="y" xattr on dir + // + // [drwxr-xr-x] . + // ├── [drwxr-xr-x] dir + // └── [crw-r--r--] file + // + { + name: "AUFSBlob_sqfstar", + layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf", + }), + converter: "sqfstar", + noConvertWhiteout: false, + // Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore + // differences in ownership. + diffArgs: []string{"--no-owner"}, + }, + { + name: "AUFSBlob_tar2sqfs", + layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf", + }), + converter: "tar2sqfs", + noConvertWhiteout: false, + }, + { + name: "AUFSBlob_sqfstar_noConvertWhiteout", + layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf", + }), + converter: "sqfstar", + noConvertWhiteout: true, + // Some versions of squashfs-tools do not implement '-root-uid'/'-root-gid', so ignore + // differences in ownership. + diffArgs: []string{"--no-owner"}, + }, + { + name: "AUFSBlob_tar2sqfs_noConvertWhiteout", + layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf", + }), + converter: "tar2sqfs", + noConvertWhiteout: true, }, } for _, tt := range tests { @@ -82,7 +167,10 @@ func Test_SquashfsLayer(t *testing.T) { t.Skip(err) } - l, err := SquashfsLayer(tt.layer, t.TempDir(), OptSquashfsLayerConverter(tt.converter)) + l, err := SquashfsLayer(tt.layer, t.TempDir(), + OptSquashfsLayerConverter(tt.converter), + OptNoConvertWhiteout(tt.noConvertWhiteout), + ) if err != nil { t.Fatal(err) } diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_sqfstar.golden b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_sqfstar.golden new file mode 100644 index 0000000..109d5c6 Binary files /dev/null and b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_sqfstar.golden differ diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_sqfstar_noConvertWhiteout.golden b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_sqfstar_noConvertWhiteout.golden new file mode 100644 index 0000000..8d9f3f9 Binary files /dev/null and b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_sqfstar_noConvertWhiteout.golden differ diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_tar2sqfs.golden b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_tar2sqfs.golden new file mode 100644 index 0000000..f7679aa Binary files /dev/null and b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_tar2sqfs.golden differ diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_tar2sqfs_noConvertWhiteout.golden b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_tar2sqfs_noConvertWhiteout.golden new file mode 100644 index 0000000..2b424cb Binary files /dev/null and b/pkg/mutate/testdata/Test_SquashfsLayer/AUFSBlob_tar2sqfs_noConvertWhiteout.golden differ diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar.golden b/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar.golden index 64b0609..e74b998 100644 Binary files a/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar.golden and b/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar.golden differ diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar_noConvertWhiteout.golden b/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar_noConvertWhiteout.golden new file mode 100644 index 0000000..64b0609 Binary files /dev/null and b/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_sqfstar_noConvertWhiteout.golden differ diff --git a/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_tar2sqfs_noConvertWhiteout.golden b/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_tar2sqfs_noConvertWhiteout.golden new file mode 100644 index 0000000..ec6f5e4 Binary files /dev/null and b/pkg/mutate/testdata/Test_SquashfsLayer/HelloWorldBlob_tar2sqfs_noConvertWhiteout.golden differ diff --git a/pkg/mutate/whiteout.go b/pkg/mutate/whiteout.go new file mode 100644 index 0000000..2f56d3f --- /dev/null +++ b/pkg/mutate/whiteout.go @@ -0,0 +1,122 @@ +// Copyright 2023 Sylabs Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package mutate + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "path/filepath" + "strings" +) + +const ( + aufsWhiteoutPrefix = ".wh." + aufsOpaqueMarker = ".wh..wh..opq" +) + +var errUnexpectedOpaque = errors.New("unexpected opaque marker") + +// scanAUFSWhiteouts reads a TAR stream, returning a map of :true for +// directories in the tar that contain an AUFS .wh..wh..opq opaque directory +// marker file, and a boolean indicating the presence of any .wh. markers. +// Note that paths returned are clean, per filepath.Clean. +func scanAUFSWhiteouts(in io.Reader) (map[string]bool, bool, error) { + opaquePaths := map[string]bool{} + fileWhiteout := false + + tr := tar.NewReader(in) + for { + header, err := tr.Next() + + if err == io.EOF { + return opaquePaths, fileWhiteout, nil + } + if err != nil { + return nil, false, err + } + + base := filepath.Base(header.Name) + + if base == aufsOpaqueMarker { + parent := filepath.Dir(header.Name) + opaquePaths[parent] = true + } + + if !fileWhiteout && strings.HasPrefix(base, aufsWhiteoutPrefix) { + fileWhiteout = true + } + } +} + +// whiteOutFilter streams a tar file from in to out, replacing AUFS whiteout +// markers with OverlayFS whiteout markers. Due to unrestricted ordering of +// markers vs their target, the list of opaquePaths must be obtained prior to +// filtering and provided to this filter. +func whiteoutFilter(in io.ReadCloser, out io.WriteCloser, opaquePaths map[string]bool) error { + tr := tar.NewReader(in) + tw := tar.NewWriter(out) + defer out.Close() + defer tw.Close() + + for { + header, err := tr.Next() + + if err == io.EOF { + return nil + } + if err != nil { + return err + } + + // Must force to PAX format, to accommodate xattrs + header.Format = tar.FormatPAX + + clean := filepath.Clean(header.Name) + base := filepath.Base(header.Name) + parent := filepath.Dir(header.Name) + + // Don't include .wh..wh..opq opaque directory markers in output. + if base == aufsOpaqueMarker { + // If we don't know the target should be opaque, then provided opaquePaths is incorrect. + if !opaquePaths[parent] { + return fmt.Errorf("%q: %w", parent, errUnexpectedOpaque) + } + continue + } + // Set overlayfs xattr on a dir that was previously found to contain a .wh..wh..opq marker. + if opq := opaquePaths[clean]; opq { + if header.PAXRecords == nil { + header.PAXRecords = map[string]string{} + } + header.PAXRecords["SCHILY.xattr."+"trusted.overlay.opaque"] = "y" + } + // Replace a `.wh.` marker with a char dev 0 at + if strings.HasPrefix(base, aufsWhiteoutPrefix) { + target := filepath.Join(parent, strings.TrimPrefix(base, aufsWhiteoutPrefix)) + header.Name = target + header.Typeflag = tar.TypeChar + header.Devmajor = 0 + header.Devminor = 0 + if err := tw.WriteHeader(header); err != nil { + return err + } + continue + } + + if err := tw.WriteHeader(header); err != nil { + return err + } + + // Disable gosec G110: Potential DoS vulnerability via decompression bomb. + // We are just filtering a flow directly from tar reader to tar writer - we aren't reading + // into memory beyond the stdlib buffering. + //nolint:gosec + if _, err := io.Copy(tw, tr); err != nil { + return err + } + } +} diff --git a/pkg/mutate/whiteout_test.go b/pkg/mutate/whiteout_test.go new file mode 100644 index 0000000..a9fbee6 --- /dev/null +++ b/pkg/mutate/whiteout_test.go @@ -0,0 +1,69 @@ +// Copyright 2023 Sylabs Inc. All rights reserved. +// +// SPDX-License-Identifier: Apache-2.0 + +package mutate + +import ( + "reflect" + "testing" + + v1 "github.com/google/go-containerregistry/pkg/v1" +) + +func Test_scanAUFSOpaque(t *testing.T) { + tests := []struct { + name string + layer v1.Layer + expectOpaque map[string]bool + expectFileWhiteout bool + }{ + // HelloWorld layer contains no opaque markers + { + name: "HelloWorldTar", + layer: testLayer(t, "hello-world-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "7050e35b49f5e348c4809f5eff915842962cb813f32062d3bbdd35c750dd7d01", + }), + expectOpaque: map[string]bool{}, + expectFileWhiteout: false, + }, + // AUFS layer contains a single opaque marker on dir + // [drwxr-xr-x] . + // ├── [drwxr-xr-x] dir + // │   └── [-rw-r--r--] .wh..wh..opq + // └── [-rw-r--r--] .wh.file + { + name: "AUFSTar", + layer: testLayer(t, "aufs-docker-v2-manifest", v1.Hash{ + Algorithm: "sha256", + Hex: "da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf", + }), + expectOpaque: map[string]bool{ + "dir": true, + }, + expectFileWhiteout: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + rc, err := tt.layer.Uncompressed() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { rc.Close() }) + + opaque, fileWhiteout, err := scanAUFSWhiteouts(rc) + if err != nil { + t.Errorf("unexpected error: %v", err) + } + if !reflect.DeepEqual(tt.expectOpaque, opaque) { + t.Errorf("opaque directories - expected: %v, got: %v", tt.expectOpaque, opaque) + } + if fileWhiteout != tt.expectFileWhiteout { + t.Errorf("file whiteout(s) - expected: %v, got: %v", tt.expectFileWhiteout, fileWhiteout) + } + }) + } +} diff --git a/test/images/aufs-docker-v2-manifest/blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a b/test/images/aufs-docker-v2-manifest/blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a new file mode 100644 index 0000000..902a739 --- /dev/null +++ b/test/images/aufs-docker-v2-manifest/blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a @@ -0,0 +1 @@ +{"architecture":"arm64","config":{"Hostname":"","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/hello"],"Image":"sha256:cc0fff24c4ece63ade5d9f549e42c926cf569112c4f5c439a4a57f3f33f5588b","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":null},"container":"b2af51419cbf516f3c99b877a64906b21afedc175bd3cd082eb5798e2f277bb4","container_config":{"Hostname":"b2af51419cbf","Domainname":"","User":"","AttachStdin":false,"AttachStdout":false,"AttachStderr":false,"Tty":false,"OpenStdin":false,"StdinOnce":false,"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","#(nop) ","CMD [\"/hello\"]"],"Image":"sha256:cc0fff24c4ece63ade5d9f549e42c926cf569112c4f5c439a4a57f3f33f5588b","Volumes":null,"WorkingDir":"","Entrypoint":null,"OnBuild":null,"Labels":{}},"created":"2022-03-19T16:12:58.923371954Z","docker_version":"20.10.12","history":[{"created":"2022-03-19T16:12:58.834095198Z","created_by":"/bin/sh -c #(nop) COPY file:a79dd5bda1e77203401956a93401d3aef45221fc750295a4291896f3386f4f54 in / "},{"created":"2022-03-19T16:12:58.923371954Z","created_by":"/bin/sh -c #(nop) CMD [\"/hello\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:efb53921da3394806160641b72a2cbd34ca1a9a8345ac670a85a04ad3d0e3507"]},"variant":"v8"} \ No newline at end of file diff --git a/test/images/aufs-docker-v2-manifest/blobs/sha256/6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9 b/test/images/aufs-docker-v2-manifest/blobs/sha256/6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9 new file mode 100644 index 0000000..aa59ff8 --- /dev/null +++ b/test/images/aufs-docker-v2-manifest/blobs/sha256/6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9 @@ -0,0 +1,16 @@ +{ + "schemaVersion": 2, + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "size": 2, + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "size": 252, + "digest": "sha256:da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf" + } + ] + } \ No newline at end of file diff --git a/test/images/aufs-docker-v2-manifest/blobs/sha256/da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf b/test/images/aufs-docker-v2-manifest/blobs/sha256/da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf new file mode 100644 index 0000000..ccadfc7 Binary files /dev/null and b/test/images/aufs-docker-v2-manifest/blobs/sha256/da55812559dec81445c289c3832cee4a2f725b15aeb258791640185c3126b2bf differ diff --git a/test/images/aufs-docker-v2-manifest/index.json b/test/images/aufs-docker-v2-manifest/index.json new file mode 100644 index 0000000..fe1cd39 --- /dev/null +++ b/test/images/aufs-docker-v2-manifest/index.json @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":525,"digest":"sha256:6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9","platform":{"architecture":"arm64","os":"linux","variant":"v8"}}]} \ No newline at end of file diff --git a/test/images/aufs-docker-v2-manifest/oci-layout b/test/images/aufs-docker-v2-manifest/oci-layout new file mode 100644 index 0000000..224a869 --- /dev/null +++ b/test/images/aufs-docker-v2-manifest/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} \ No newline at end of file