mutate: stream tar output from sqfs2tar

Stream the tar output from sqfs2tar when performing a SquashFS to TAR
layer conversion. This removes the need for one temporary file.

Although intended, we cannot remove the SquashFS temporary file by using
a named pipe for intput to sqfs2tar. Due to the structure of SquashFS
filesystems, sqfs2tar re-reads portions of the input and cannot work
from a named pipe that is streaming input.

Temporary directory is also now passed by option, per comments in review
of #52.

Closes #57

pkg/mutate/tar.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"os"
 	"os/exec"
-	"path/filepath"
 
 	v1 "github.com/google/go-containerregistry/pkg/v1"
 	"github.com/google/go-containerregistry/pkg/v1/tarball"
@@ -49,10 +48,21 @@ func OptTarSkipWhiteoutConversion(b bool) TarConverterOpt {
 	}
 }
 
+// OptTarTempDir sets the directory to use for temporary files. If not set, the
+// directory returned by TempDir is used.
+func OptTarTempDir(d string) TarConverterOpt {
+	return func(c *tarConverter) error {
+		c.dir = d
+		return nil
+	}
+}
+
 // TarFromSquashfsLayer returns an opener that will provide a TAR conversion of
-// the SquashFS format base layer. A dir must be specified, which is used as a
-// working directory during conversion. The caller is responsible for cleaning
-// up dir.
+// the SquashFS format base layer.
+//
+// TarFromSquashfsLayer may create one or more temporary files during the
+// conversion process. By default, the directory returned by TempDir is used. To
+// override this, consider using OptTarTempDir.
 //
 // By default, this will attempt to locate a suitable SquashFS to tar converter,
 // currently only 'sqfs2tar', via exec.LookPath. To specify a path to a specific
@@ -62,7 +72,7 @@ func OptTarSkipWhiteoutConversion(b bool) TarConverterOpt {
 // converted to AUFS whiteout markers in the TAR layer. This can be disabled,
 // e.g. where it is known that the layer is part of a squashed image that will
 // not have any whiteouts, using OptTarSkipWhiteourConversion.
-func TarFromSquashfsLayer(base v1.Layer, dir string, opts ...TarConverterOpt) (tarball.Opener, error) {
+func TarFromSquashfsLayer(base v1.Layer, opts ...TarConverterOpt) (tarball.Opener, error) {
 	mt, err := base.MediaType()
 	if err != nil {
 		return nil, err
@@ -72,7 +82,6 @@ func TarFromSquashfsLayer(base v1.Layer, dir string, opts ...TarConverterOpt) (t
 	}
 
 	c := tarConverter{
-		dir:             dir,
 		convertWhiteout: true,
 	}
 
@@ -93,44 +102,43 @@ func TarFromSquashfsLayer(base v1.Layer, dir string, opts ...TarConverterOpt) (t
 	return c.opener(base), nil
 }
 
-// makeSquashfs returns a the path to a TAR file that contains the contents of
-// the SquashFS stream from r.
-func (c *tarConverter) makeTAR(r io.Reader) (string, error) {
-	dir, err := os.MkdirTemp(c.dir, "")
+// makeTar returns an io.ReadCloser that provides a TAR conversion of the
+// contents of the SquashFS stream from r.
+func (c *tarConverter) makeTAR(r io.Reader) (io.ReadCloser, error) {
+	sqfsFile, err := os.CreateTemp(c.dir, "*.sqfs")
 	if err != nil {
-		return "", err
-	}
-
-	sqfsPath := filepath.Join(dir, "layer.sqfs")
-	sqfsFile, err := os.Create(sqfsPath)
-	if err != nil {
-		return "", err
+		return nil, err
 	}
 	defer sqfsFile.Close()
 
 	_, err = io.Copy(sqfsFile, r)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
-
-	tarPath := filepath.Join(dir, "layer.tar")
-	tarFile, err := os.Create(tarPath)
-	if err != nil {
-		return "", err
+	if err := sqfsFile.Close(); err != nil {
+		return nil, err
 	}
-	defer tarFile.Close()
 
+	pr, pw := io.Pipe()
 	//nolint:gosec // Arguments are created programatically.
-	cmd := exec.Command(c.converter, sqfsPath)
-	cmd.Stdout = tarFile
+	cmd := exec.Command(c.converter, sqfsFile.Name())
+	cmd.Stdout = pw
 	errBuff := bytes.Buffer{}
 	cmd.Stderr = &errBuff
 
-	if err := cmd.Run(); err != nil {
-		return "", fmt.Errorf("%s error: %w %v", c.converter, err, errBuff)
+	convert := func() error {
+		defer os.Remove(sqfsFile.Name())
+		if err := cmd.Run(); err != nil {
+			return fmt.Errorf("%s error: %w %s", c.converter, err, errBuff.String())
+		}
+		return nil
 	}
 
-	return tarPath, nil
+	go func() {
+		pw.CloseWithError(convert())
+	}()
+
+	return pr, nil
 }
 
 // Opener returns a tarball.Opener that will open a TAR file holding the content
@@ -143,12 +151,7 @@ func (c *tarConverter) opener(l v1.Layer) tarball.Opener {
 		}
 		defer rc.Close()
 
-		tarFile, err := c.makeTAR(rc)
-		if err != nil {
-			return nil, err
-		}
-
-		tr, err := os.Open(tarFile)
+		tr, err := c.makeTAR(rc)
 		if err != nil {
 			return nil, err
 		}

pkg/mutate/tar_test.go

@@ -16,10 +16,10 @@ import (
 
 func Test_TarFromSquashfsLayer(t *testing.T) {
 	tests := []struct {
-		name              string
-		layer             v1.Layer
-		noConvertWhiteout bool
-		diffArgs          []string
+		name     string
+		layer    v1.Layer
+		opts     []TarConverterOpt
+		diffArgs []string
 	}{
 		// SquashFS layer contains whiteouts in OverlayFS format. Should be
 		// converted to AUFS form when noConvertWhiteout = false.
@@ -45,15 +45,22 @@ func Test_TarFromSquashfsLayer(t *testing.T) {
 				Algorithm: "sha256",
 				Hex:       "2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887",
 			}),
-			noConvertWhiteout: false,
 		},
 		{
 			name: "OverlayFSBlob_SkipWhiteoutConversion",
 			layer: testLayer(t, "overlayfs-docker-v2-manifest", v1.Hash{
 				Algorithm: "sha256",
 				Hex:       "2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887",
 			}),
-			noConvertWhiteout: true,
+			opts: []TarConverterOpt{OptTarSkipWhiteoutConversion(true)},
+		},
+		{
+			name: "OverlayFSBlobTempDir",
+			layer: testLayer(t, "overlayfs-docker-v2-manifest", v1.Hash{
+				Algorithm: "sha256",
+				Hex:       "2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887",
+			}),
+			opts: []TarConverterOpt{OptTarTempDir(t.TempDir())},
 		},
 	}
 	for _, tt := range tests {
@@ -62,9 +69,7 @@ func Test_TarFromSquashfsLayer(t *testing.T) {
 				t.Skip(err)
 			}
 
-			opener, err := TarFromSquashfsLayer(tt.layer, t.TempDir(),
-				OptTarSkipWhiteoutConversion(tt.noConvertWhiteout),
-			)
+			opener, err := TarFromSquashfsLayer(tt.layer, tt.opts...)
 			if err != nil {
 				t.Fatal(err)
 			}