diff --git a/acceptance/analyzer_test.go b/acceptance/analyzer_test.go index ec1a232f7..37e0345f5 100644 --- a/acceptance/analyzer_test.go +++ b/acceptance/analyzer_test.go @@ -487,7 +487,7 @@ func testAnalyzerFunc(platformAPI string) func(t *testing.T, when spec.G, it spe analyzer := assertAnalyzedMetadata(t, filepath.Join(copyDir, "analyzed.toml")) h.AssertNotNil(t, analyzer.RunImage) analyzedImagePath := filepath.Join(path.RootDir, "layout-repo", "index.docker.io", "library", "busybox", "latest") - reference := fmt.Sprintf("%s@%s", analyzedImagePath, "sha256:834f8848308af7090ed7b2270071d28411afc42078e3ba395b1b0a78e2f8b0e2") + reference := fmt.Sprintf("%s@%s", analyzedImagePath, "sha256:f75f3d1a317fc82c793d567de94fc8df2bece37acd5f2bd364a0d91a0d1f3dab") h.AssertEq(t, analyzer.RunImage.Reference, reference) }) }) diff --git a/acceptance/extender_test.go b/acceptance/extender_test.go index 4ea54a01d..6e322f96a 100644 --- a/acceptance/extender_test.go +++ b/acceptance/extender_test.go @@ -11,9 +11,9 @@ import ( "runtime" "testing" + "github.com/buildpacks/imgutil/layout/sparse" "github.com/google/go-containerregistry/pkg/authn" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/layout" "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/sclevine/spec" @@ -22,7 +22,6 @@ import ( "github.com/buildpacks/lifecycle/api" "github.com/buildpacks/lifecycle/auth" "github.com/buildpacks/lifecycle/cmd" - "github.com/buildpacks/lifecycle/internal/selective" "github.com/buildpacks/lifecycle/platform/files" h "github.com/buildpacks/lifecycle/testhelpers" ) @@ -105,9 +104,10 @@ func testExtenderFunc(platformAPI string) func(t *testing.T, when spec.G, it spe baseCacheDir := filepath.Join(kanikoDir, "cache", "base") h.AssertNil(t, os.MkdirAll(baseCacheDir, 0755)) - layoutPath, err := selective.Write(filepath.Join(baseCacheDir, baseImageDigest), empty.Index) + // write sparse image + layoutImage, err := sparse.NewImage(filepath.Join(baseCacheDir, baseImageDigest), remoteImage) h.AssertNil(t, err) - h.AssertNil(t, layoutPath.AppendImage(remoteImage)) + h.AssertNil(t, layoutImage.Save()) // write image reference in analyzed.toml analyzedMD := files.Analyzed{ diff --git a/cmd/lifecycle/exporter.go b/cmd/lifecycle/exporter.go index f980faab6..7e4bfc504 100644 --- a/cmd/lifecycle/exporter.go +++ b/cmd/lifecycle/exporter.go @@ -504,7 +504,7 @@ func (e *exportCmd) customSourceDateEpoch() time.Time { } func (e *exportCmd) supportsRunImageExtension() bool { - return e.PlatformAPI.AtLeast("0.12") && !e.UseLayout // FIXME: add layout support as part of https://github.com/buildpacks/lifecycle/issues/1057 + return e.PlatformAPI.AtLeast("0.12") && !e.UseLayout // FIXME: add layout support as part of https://github.com/buildpacks/lifecycle/issues/1102 } func (e *exportCmd) supportsHistory() bool { diff --git a/cmd/lifecycle/restorer.go b/cmd/lifecycle/restorer.go index 0fba26723..2a82093ff 100644 --- a/cmd/lifecycle/restorer.go +++ b/cmd/lifecycle/restorer.go @@ -219,7 +219,7 @@ func (r *restoreCmd) supportsBuildImageExtension() bool { } func (r *restoreCmd) supportsRunImageExtension() bool { - return r.PlatformAPI.AtLeast("0.12") && !r.UseLayout // FIXME: add layout support as part of https://github.com/buildpacks/lifecycle/issues/1057 + return r.PlatformAPI.AtLeast("0.12") && !r.UseLayout // FIXME: add layout support as part of https://github.com/buildpacks/lifecycle/issues/1102 } func (r *restoreCmd) supportsTargetData() bool { diff --git a/go.mod b/go.mod index e52125c17..d963dd3c0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/GoogleContainerTools/kaniko v1.20.0 github.com/apex/log v1.9.0 github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20231213181459-b0fcec718dc6 - github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02 + github.com/buildpacks/imgutil v0.0.0-20240206215312-f8d38e1de03d github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 github.com/containerd/containerd v1.7.12 github.com/docker/docker v24.0.7+incompatible diff --git a/go.sum b/go.sum index 49909295e..a6182801e 100644 --- a/go.sum +++ b/go.sum @@ -90,8 +90,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02 h1:Ac/FoFzAhz34zIDvrC3ivShQgoywg/HrA+Kkcb13Mr4= -github.com/buildpacks/imgutil v0.0.0-20230919143643-4ec9360d5f02/go.mod h1:Ade+4Q1OovFw6Zdzd+/UVaqWptZSlpnZ8n/vlkgS7M8= +github.com/buildpacks/imgutil v0.0.0-20240206215312-f8d38e1de03d h1:b/tiReZf9jorbpaOwVw9MX3n99w7Ta4u+SVV7yy6XCE= +github.com/buildpacks/imgutil v0.0.0-20240206215312-f8d38e1de03d/go.mod h1:7zUmt4wkVJNuXCZhQndEd3kvVGmWLVyzRFIQXTaeXlU= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= diff --git a/internal/selective/layoutpath.go b/internal/selective/layoutpath.go deleted file mode 100644 index 111853401..000000000 --- a/internal/selective/layoutpath.go +++ /dev/null @@ -1,7 +0,0 @@ -package selective - -import "github.com/google/go-containerregistry/pkg/v1/layout" - -type Path struct { - layout.Path -} diff --git a/internal/selective/write.go b/internal/selective/write.go deleted file mode 100644 index 7d78764ac..000000000 --- a/internal/selective/write.go +++ /dev/null @@ -1,85 +0,0 @@ -package selective - -import ( - "bytes" - "io" - - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/layout" -) - -// AppendImage mimics GGCR's `layout` AppendImage in that it appends an image to a `layout.Path`, -// but the image appended does not include any layers in the `blobs` directory. -// The returned image will return layers when Layers(), LayerByDiffID(), or LayerByDigest() are called, -// but the returned layer will error when DiffID(), Compressed(), or Uncompressed() are called. -// This is useful when we need to satisfy the v1.Image interface but do not need to access any layers, such as when extending -// base images with kaniko. -func (l Path) AppendImage(img v1.Image) error { // FIXME: add the ability to pass image options - if err := l.writeImage(img); err != nil { - return err - } - - mt, err := img.MediaType() - if err != nil { - return err - } - - d, err := img.Digest() - if err != nil { - return err - } - - manifest, err := img.RawManifest() - if err != nil { - return err - } - - desc := v1.Descriptor{ - MediaType: mt, - Size: int64(len(manifest)), - Digest: d, - } - - return l.AppendDescriptor(desc) -} - -// writeImage mimics GGCR's `layout` writeImage in that it writes an image config and manifest, -// but it does not write any layers in the `blobs` directory. -// The returned image will return layers when Layers(), LayerByDiffID(), or LayerByDigest() are called, -// but the returned layer will error when DiffID(), Compressed(), or Uncompressed() are called. -// This is useful when we need to satisfy the v1.Image interface but do not need to access any layers, -// such as when extending base images with kaniko. -func (l Path) writeImage(img v1.Image) error { - // Write the config. - cfgName, err := img.ConfigName() - if err != nil { - return err - } - cfgBlob, err := img.RawConfigFile() - if err != nil { - return err - } - if err = l.WriteBlob(cfgName, io.NopCloser(bytes.NewReader(cfgBlob))); err != nil { - return err - } - - // Write the img manifest. - d, err := img.Digest() - if err != nil { - return err - } - manifest, err := img.RawManifest() - if err != nil { - return err - } - return l.WriteBlob(d, io.NopCloser(bytes.NewReader(manifest))) -} - -func Write(path string, ii v1.ImageIndex) (Path, error) { - layoutPath, err := layout.Write(path, ii) - if err != nil { - return Path{}, err - } - - return Path{Path: layoutPath}, nil -} diff --git a/internal/selective/write_test.go b/internal/selective/write_test.go deleted file mode 100644 index 2ec5f678d..000000000 --- a/internal/selective/write_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package selective_test - -import ( - "os" - "path/filepath" - "runtime" - "testing" - - "github.com/google/go-containerregistry/pkg/authn" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" - "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/sclevine/spec" - "github.com/sclevine/spec/report" - - "github.com/buildpacks/lifecycle/auth" - "github.com/buildpacks/lifecycle/internal/selective" - h "github.com/buildpacks/lifecycle/testhelpers" -) - -func TestSelective(t *testing.T) { - spec.Run(t, "Selective", testSelective, spec.Report(report.Terminal{})) -} - -func testSelective(t *testing.T, when spec.G, it spec.S) { - when("AppendImage", func() { - var ( - testImage v1.Image - tmpDir string - fileNotFoundMsg string - ) - - it.Before(func() { - testImageName := "busybox" - var opts []remote.Option - fileNotFoundMsg = "no such file or directory" - if runtime.GOOS == "windows" { - testImageName = "mcr.microsoft.com/windows/nanoserver@sha256:8bd4389d56e69bebf6e4666251fba42f7cce3d5b768d28816884fb4370155fee" // mcr.microsoft.com/windows/nanoserver:1809 - - windowsPlatform := v1.Platform{ - Architecture: "amd64", - OS: "windows", - OSVersion: "10.0.17763.3532", - } - opts = append(opts, remote.WithPlatform(windowsPlatform)) - fileNotFoundMsg = "The system cannot find the file specified" - } - - ref, authr, err := auth.ReferenceForRepoName(authn.DefaultKeychain, testImageName) - h.AssertNil(t, err) - opts = append(opts, remote.WithAuth(authr)) - - testImage, err = remote.Image(ref, opts...) - h.AssertNil(t, err) - - tmpDir, err = os.MkdirTemp("", "") - h.AssertNil(t, err) - }) - - it("appends an image to a path without any layers", func() { - digest, err := testImage.Digest() - h.AssertNil(t, err) - layoutPath, err := selective.Write(filepath.Join(tmpDir, "some-image-index"), empty.Index) - h.AssertNil(t, err) - - h.AssertNil(t, layoutPath.AppendImage(testImage)) - - fis, err := os.ReadDir(filepath.Join(tmpDir, "some-image-index", "blobs", "sha256")) - h.AssertNil(t, err) - h.AssertEq(t, len(fis), 2) // manifest, config - foundImage, err := layoutPath.Image(digest) - h.AssertNil(t, err) - - // found image satisfies v1.Image interface - _, err = foundImage.MediaType() - h.AssertNil(t, err) - _, err = foundImage.Size() - h.AssertNil(t, err) - _, err = foundImage.ConfigName() - h.AssertNil(t, err) - configFile, err := foundImage.ConfigFile() - h.AssertNil(t, err) - _, err = foundImage.RawConfigFile() - h.AssertNil(t, err) - foundImageDigest, err := foundImage.Digest() - h.AssertNil(t, err) - h.AssertEq(t, foundImageDigest.String(), digest.String()) - _, err = foundImage.Manifest() - h.AssertNil(t, err) - _, err = foundImage.RawManifest() - h.AssertNil(t, err) - foundLayers, err := foundImage.Layers() - h.AssertNil(t, err) - h.AssertEq(t, len(foundLayers), 1) - foundLayerDigest, err := foundLayers[0].Digest() - h.AssertNil(t, err) - foundLayer, err := foundImage.LayerByDigest(foundLayerDigest) - h.AssertNil(t, err) - h.AssertEq(t, len(configFile.RootFS.DiffIDs), 1) - _, err = foundImage.LayerByDiffID(configFile.RootFS.DiffIDs[0]) - h.AssertNil(t, err) - - // found layers satisfy v1.Layer interface - _, err = foundLayer.DiffID() - h.AssertNotNil(t, err) // the diffID could be obtained from the config, but ggcr tries to open the layer when getting this value - h.AssertStringContains(t, err.Error(), fileNotFoundMsg) - _, err = foundLayer.Compressed() - h.AssertNotNil(t, err) - h.AssertStringContains(t, err.Error(), fileNotFoundMsg) - _, err = foundLayer.Uncompressed() - h.AssertNotNil(t, err) - h.AssertStringContains(t, err.Error(), fileNotFoundMsg) - _, err = foundLayer.Size() - h.AssertNil(t, err) - _, err = foundLayer.MediaType() - h.AssertNil(t, err) - }) - }) -} diff --git a/phase/extender.go b/phase/extender.go index dab1dd56e..58d2f8c0e 100644 --- a/phase/extender.go +++ b/phase/extender.go @@ -12,15 +12,14 @@ import ( "github.com/BurntSushi/toml" "github.com/buildpacks/imgutil" + "github.com/buildpacks/imgutil/layout/sparse" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/uuid" "golang.org/x/sync/errgroup" "github.com/buildpacks/lifecycle/buildpack" "github.com/buildpacks/lifecycle/internal/extend" - "github.com/buildpacks/lifecycle/internal/selective" "github.com/buildpacks/lifecycle/launch" "github.com/buildpacks/lifecycle/layers" "github.com/buildpacks/lifecycle/log" @@ -147,8 +146,8 @@ func (e *Extender) extendRun(logger log.Logger) error { return fmt.Errorf("extending run image: %w", err) } - if err = e.saveSelective(extendedImage, origTopLayer); err != nil { - return fmt.Errorf("copying selective image to output directory: %w", err) + if err = e.saveSparse(extendedImage, origTopLayer); err != nil { + return fmt.Errorf("copying extended image to output directory: %w", err) } return e.DockerfileApplier.Cleanup() } @@ -166,22 +165,23 @@ func topLayer(image v1.Image) (string, error) { return layer.Digest.String(), nil } -func (e *Extender) saveSelective(image v1.Image, origTopLayerHash string) error { +func (e *Extender) saveSparse(image v1.Image, origTopLayerHash string) error { // save sparse image (manifest and config) imageHash, err := image.Digest() if err != nil { return fmt.Errorf("getting image hash: %w", err) } toPath := filepath.Join(e.ExtendedDir, "run", imageHash.String()) - layoutPath, err := selective.Write(toPath, empty.Index) // FIXME: this should use the imgutil layout/sparse package instead, but for some reason sparse.NewImage().Save() fails when the provided base image is already sparse + layoutImage, err := sparse.NewImage(toPath, image) if err != nil { - return fmt.Errorf("initializing selective image: %w", err) + return fmt.Errorf("failed to initialize image: %w", err) } - if err = layoutPath.AppendImage(image); err != nil { - return fmt.Errorf("saving selective image: %w", err) + if err := layoutImage.Save(); err != nil { + return fmt.Errorf("failed to save image: %w", err) } - // get all image layers (we will only copy those following the original top layer) - layers, err := image.Layers() + // copy only the extended layers (those following the original top layer) to the layout path + // FIXME: it would be nice if this were supported natively in imgutil + allLayers, err := image.Layers() if err != nil { return fmt.Errorf("getting image layers: %w", err) } @@ -193,7 +193,7 @@ func (e *Extender) saveSelective(image v1.Image, origTopLayerHash string) error needsCopying = true } group, _ := errgroup.WithContext(context.TODO()) - for _, currentLayer := range layers { + for _, currentLayer := range allLayers { currentHash, err = currentLayer.Digest() if err != nil { return fmt.Errorf("getting layer hash: %w", err) diff --git a/phase/extender_test.go b/phase/extender_test.go index f0c2bd5f0..a5f15a60b 100644 --- a/phase/extender_test.go +++ b/phase/extender_test.go @@ -435,7 +435,7 @@ func testExtender(t *testing.T, when spec.G, it spec.S) { someFakeImage.ConfigFileReturnsOnCall(3, secondConfig, nil) someFakeImage.ConfigFileReturnsOnCall(4, secondConfig, nil) - // save selective + // save without base layers imageHash := v1.Hash{Algorithm: "sha256", Hex: "some-image-hex"} someFakeImage.DigestReturns(imageHash, nil)