diff --git a/pkg/mutate/tar.go b/pkg/mutate/tar.go index 7d3d342..a98da24 100644 --- a/pkg/mutate/tar.go +++ b/pkg/mutate/tar.go @@ -14,7 +14,6 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/tarball" - "github.com/google/go-containerregistry/pkg/v1/types" ) type tarConverter struct { @@ -50,9 +49,10 @@ func OptTarSkipWhiteoutConversion(b bool) TarConverterOpt { } } -// TarLayer converts the base layer into a layer using the tar format. A dir -// must be specified, which is used as a working directory during conversion. -// The caller is responsible for cleaning up dir. +// TarFromSquashfsLayer returns an opener that will provide a TAR conversion of +// the SquashFS format base layer. A dir must be specified, which is used as a +// working directory during conversion. The caller is responsible for cleaning +// up dir. // // By default, this will attempt to locate a suitable SquashFS to tar converter, // currently only 'sqfs2tar', via exec.LookPath. To specify a path to a specific @@ -62,10 +62,15 @@ func OptTarSkipWhiteoutConversion(b bool) TarConverterOpt { // converted to AUFS whiteout markers in the TAR layer. This can be disabled, // e.g. where it is known that the layer is part of a squashed image that will // not have any whiteouts, using OptTarSkipWhiteourConversion. -// -// Note - when whiteout conversion is performed the base layer will be read -// twice. Callers should ensure it is cached, and is not a streaming layer. -func TarLayer(base v1.Layer, dir string, opts ...TarConverterOpt) (v1.Layer, error) { +func TarFromSquashfsLayer(base v1.Layer, dir string, opts ...TarConverterOpt) (tarball.Opener, error) { + mt, err := base.MediaType() + if err != nil { + return nil, err + } + if mt != squashfsLayerMediaType { + return nil, fmt.Errorf("%w: %v", errUnsupportedLayerType, mt) + } + c := tarConverter{ dir: dir, convertWhiteout: true, @@ -85,7 +90,7 @@ func TarLayer(base v1.Layer, dir string, opts ...TarConverterOpt) (v1.Layer, err c.converter = path } - return c.layer(base) + return c.opener(base), nil } // makeSquashfs returns a the path to a TAR file that contains the contents of @@ -160,23 +165,3 @@ func (c *tarConverter) opener(l v1.Layer) tarball.Opener { return pr, nil } } - -// layer converts base to TAR format. -func (c *tarConverter) layer(base v1.Layer) (v1.Layer, error) { - mt, err := base.MediaType() - if err != nil { - return nil, err - } - - //nolint:exhaustive // Exhaustive cases not appropriate. - switch mt { - case squashfsLayerMediaType: - return tarball.LayerFromOpener(c.opener(base)) - - case types.DockerLayer, types.DockerUncompressedLayer, types.OCILayer, types.OCIUncompressedLayer: - return base, nil - - default: - return nil, fmt.Errorf("%w: %v", errUnsupportedLayerType, mt) - } -} diff --git a/pkg/mutate/tar_test.go b/pkg/mutate/tar_test.go index f26c65e..fb8c38e 100644 --- a/pkg/mutate/tar_test.go +++ b/pkg/mutate/tar_test.go @@ -14,7 +14,7 @@ import ( "github.com/sebdah/goldie/v2" ) -func Test_TARLayer(t *testing.T) { +func Test_TarFromSquashfsLayer(t *testing.T) { tests := []struct { name string layer v1.Layer @@ -43,7 +43,7 @@ func Test_TARLayer(t *testing.T) { name: "OverlayFSBlob", layer: testLayer(t, "overlayfs-docker-v2-manifest", v1.Hash{ Algorithm: "sha256", - Hex: "2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887", + Hex: "1887579465ca7b9a51295c5c11db2c3b3b383699208576c3a466f25a3ea4d399", }), noConvertWhiteout: false, }, @@ -51,7 +51,7 @@ func Test_TARLayer(t *testing.T) { name: "OverlayFSBlob_SkipWhiteoutConversion", layer: testLayer(t, "overlayfs-docker-v2-manifest", v1.Hash{ Algorithm: "sha256", - Hex: "2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887", + Hex: "1887579465ca7b9a51295c5c11db2c3b3b383699208576c3a466f25a3ea4d399", }), noConvertWhiteout: true, }, @@ -62,14 +62,14 @@ func Test_TARLayer(t *testing.T) { t.Skip(err) } - l, err := TarLayer(tt.layer, t.TempDir(), + opener, err := TarFromSquashfsLayer(tt.layer, t.TempDir(), OptTarSkipWhiteoutConversion(tt.noConvertWhiteout), ) if err != nil { t.Fatal(err) } - rc, err := l.Uncompressed() + rc, err := opener() if err != nil { t.Fatal(err) } diff --git a/pkg/mutate/testdata/Test_TARLayer/OverlayFSBlob.golden b/pkg/mutate/testdata/Test_TarFromSquashfsLayer/OverlayFSBlob.golden similarity index 100% rename from pkg/mutate/testdata/Test_TARLayer/OverlayFSBlob.golden rename to pkg/mutate/testdata/Test_TarFromSquashfsLayer/OverlayFSBlob.golden diff --git a/pkg/mutate/testdata/Test_TARLayer/OverlayFSBlob_SkipWhiteoutConversion.golden b/pkg/mutate/testdata/Test_TarFromSquashfsLayer/OverlayFSBlob_SkipWhiteoutConversion.golden similarity index 100% rename from pkg/mutate/testdata/Test_TARLayer/OverlayFSBlob_SkipWhiteoutConversion.golden rename to pkg/mutate/testdata/Test_TarFromSquashfsLayer/OverlayFSBlob_SkipWhiteoutConversion.golden diff --git a/test/images/gen_images.go b/test/images/gen_images.go index 36579bf..ac29fc3 100644 --- a/test/images/gen_images.go +++ b/test/images/gen_images.go @@ -6,6 +6,7 @@ package main import ( "archive/tar" + "errors" "fmt" "io" "os" @@ -20,6 +21,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/tarball" + ocimutate "github.com/sylabs/oci-tools/pkg/mutate" ) // Use a fixed digest, so that this is repeatable. @@ -465,6 +467,85 @@ func generateManyLayerImage(path string) error { return nil } +var errMultipleImages = errors.New("multiple images found in index") + +func generateSquashFSImages(path string) error { + images := []struct { + source string + destination string + squashLayers bool + }{ + { + source: filepath.Join(path, "aufs-docker-v2-manifest"), + destination: filepath.Join(path, "overlayfs-docker-v2-manifest"), + squashLayers: false, // need to preserve whiteout test files + }, + } + + for _, im := range images { + ii, err := layout.ImageIndexFromPath(im.source) + if err != nil { + return err + } + + ix, err := ii.IndexManifest() + if err != nil { + return err + } + if len(ix.Manifests) != 1 { + return errMultipleImages + } + + ih := ix.Manifests[0].Digest + img, err := ii.Image(ih) + if err != nil { + return err + } + + if im.squashLayers { + img, err = ocimutate.Squash(img) + if err != nil { + return err + } + } + + ms := []ocimutate.Mutation{} + ls, err := img.Layers() + if err != nil { + return err + } + + for i, l := range ls { + squashfsLayer, err := ocimutate.SquashfsLayer(l, os.TempDir()) + if err != nil { + return err + } + ms = append(ms, ocimutate.SetLayer(i, squashfsLayer)) + } + + img, err = ocimutate.Apply(img, ms...) + if err != nil { + return err + } + + desc, err := partial.Descriptor(img) + if err != nil { + return err + } + + ii = mutate.AppendManifests(empty.Index, mutate.IndexAddendum{ + Add: img, + Descriptor: *desc, + }) + + if _, err := layout.Write(im.destination, ii); err != nil { + return err + } + } + + return nil +} + func main() { path := "." if len(os.Args) > 1 { @@ -490,4 +571,9 @@ func main() { fmt.Fprintln(os.Stderr, "Error:", err) os.Exit(1) } + + if err := generateSquashFSImages(path); err != nil { + fmt.Fprintln(os.Stderr, "Error:", err) + os.Exit(1) + } } diff --git a/test/images/overlayfs-docker-v2-manifest/blobs/sha256/853a0e609ae7e88e02720e4c7960c2526d968d30955fd8bc0fbc5dcfb1fbbebd b/test/images/overlayfs-docker-v2-manifest/blobs/sha256/853a0e609ae7e88e02720e4c7960c2526d968d30955fd8bc0fbc5dcfb1fbbebd new file mode 100644 index 0000000..180c6a4 --- /dev/null +++ b/test/images/overlayfs-docker-v2-manifest/blobs/sha256/853a0e609ae7e88e02720e4c7960c2526d968d30955fd8bc0fbc5dcfb1fbbebd @@ -0,0 +1 @@ +{"architecture":"arm64","container":"b2af51419cbf516f3c99b877a64906b21afedc175bd3cd082eb5798e2f277bb4","created":"2022-03-19T16:12:58.923371954Z","docker_version":"20.10.12","history":[{"created":"2022-03-19T16:12:58.834095198Z","created_by":"/bin/sh -c #(nop) COPY file:a79dd5bda1e77203401956a93401d3aef45221fc750295a4291896f3386f4f54 in / "},{"created":"2022-03-19T16:12:58.923371954Z","created_by":"/bin/sh -c #(nop) CMD [\"/hello\"]","empty_layer":true}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887"]},"config":{"Cmd":["/hello"],"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Image":"sha256:cc0fff24c4ece63ade5d9f549e42c926cf569112c4f5c439a4a57f3f33f5588b"},"variant":"v8"} \ No newline at end of file diff --git a/test/images/overlayfs-docker-v2-manifest/blobs/sha256/b9cdf2f5591da561b6a8d7dd7515a4b37b11c72c25bd11327e50da1126f227b8 b/test/images/overlayfs-docker-v2-manifest/blobs/sha256/b9cdf2f5591da561b6a8d7dd7515a4b37b11c72c25bd11327e50da1126f227b8 deleted file mode 100644 index 597f009..0000000 --- a/test/images/overlayfs-docker-v2-manifest/blobs/sha256/b9cdf2f5591da561b6a8d7dd7515a4b37b11c72c25bd11327e50da1126f227b8 +++ /dev/null @@ -1 +0,0 @@ -{"architecture":"arm64","container":"b2af51419cbf516f3c99b877a64906b21afedc175bd3cd082eb5798e2f277bb4","created":"2022-03-19T16:12:58.923371954Z","docker_version":"20.10.12","history":[{"created":"2024-06-03T10:19:53.014399694+01:00","created_by":"Singularity-Ce/4.1.0 (Linux amd64) Go/1.22.2","comment":"oci-sif created from 6c9c1b8d1adba535a40046b52c051cccf33d85f60827bd269c978fd95f05c3c9"}],"os":"linux","rootfs":{"type":"layers","diff_ids":["sha256:d0b43e8fac0b3212439b21cd740e782e81e826f74b3cd69d0fdbb359c7a0a45d"]},"config":{"Cmd":["/hello"],"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Image":"sha256:cc0fff24c4ece63ade5d9f549e42c926cf569112c4f5c439a4a57f3f33f5588b"},"variant":"v8"} \ No newline at end of file diff --git a/test/images/overlayfs-docker-v2-manifest/blobs/sha256/422627f9dff4768fe49c54173be06a1542cba43d4f4df60cf77f0f7251918ae8 b/test/images/overlayfs-docker-v2-manifest/blobs/sha256/c6ec7891ab06148a4097f8d3d638671a2ca03945c8f400b4d50061c2560cb11a similarity index 69% rename from test/images/overlayfs-docker-v2-manifest/blobs/sha256/422627f9dff4768fe49c54173be06a1542cba43d4f4df60cf77f0f7251918ae8 rename to test/images/overlayfs-docker-v2-manifest/blobs/sha256/c6ec7891ab06148a4097f8d3d638671a2ca03945c8f400b4d50061c2560cb11a index 35140c8..c4f13d4 100644 --- a/test/images/overlayfs-docker-v2-manifest/blobs/sha256/422627f9dff4768fe49c54173be06a1542cba43d4f4df60cf77f0f7251918ae8 +++ b/test/images/overlayfs-docker-v2-manifest/blobs/sha256/c6ec7891ab06148a4097f8d3d638671a2ca03945c8f400b4d50061c2560cb11a @@ -1 +1 @@ -{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":722,"digest":"sha256:b9cdf2f5591da561b6a8d7dd7515a4b37b11c72c25bd11327e50da1126f227b8"},"layers":[{"mediaType":"application/vnd.sylabs.image.layer.v1.squashfs","size":4096,"digest":"sha256:2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887"}]} \ No newline at end of file +{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","size":788,"digest":"sha256:853a0e609ae7e88e02720e4c7960c2526d968d30955fd8bc0fbc5dcfb1fbbebd"},"layers":[{"mediaType":"application/vnd.sylabs.image.layer.v1.squashfs","size":4096,"digest":"sha256:2addb7e8ed33f5f080813d437f455a2ae0c6a3cd41f978eaa05fc776d4f7a887"}]} \ No newline at end of file diff --git a/test/images/overlayfs-docker-v2-manifest/index.json b/test/images/overlayfs-docker-v2-manifest/index.json old mode 100644 new mode 100755 index ccf7e92..a943e56 --- a/test/images/overlayfs-docker-v2-manifest/index.json +++ b/test/images/overlayfs-docker-v2-manifest/index.json @@ -1 +1 @@ -{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":421,"digest":"sha256:422627f9dff4768fe49c54173be06a1542cba43d4f4df60cf77f0f7251918ae8"}]} \ No newline at end of file +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.docker.distribution.manifest.v2+json","size":421,"digest":"sha256:c6ec7891ab06148a4097f8d3d638671a2ca03945c8f400b4d50061c2560cb11a"}]} \ No newline at end of file diff --git a/test/images/overlayfs-docker-v2-manifest/oci-layout b/test/images/overlayfs-docker-v2-manifest/oci-layout old mode 100644 new mode 100755 index e69de29..224a869 --- a/test/images/overlayfs-docker-v2-manifest/oci-layout +++ b/test/images/overlayfs-docker-v2-manifest/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} \ No newline at end of file