Merge branch 'main' into dev-image-index

.github/dependabot.yml

@@ -3,5 +3,14 @@ updates:
 - package-ecosystem: gomod
   directory: "/"
   schedule:
-    interval: daily
-  open-pull-requests-limit: 10
+    interval: weekly
+  groups:
+    # Group all minor/patch go dependencies into a single PR.
+    go-dependencies:
+      update-types:
+        - "minor"
+        - "patch"
+- package-ecosystem: github-actions
+  directory: "/"
+  schedule:
+    interval: weekly

.github/workflows/test.yaml

@@ -12,19 +12,19 @@ jobs:
   test-and-build-linux:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: '1.20'
       - name: Test
         run: make test
   test-and-build-windows:
     runs-on: windows-2019
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: '1.20'
       - name: Test

acceptance/reproducibility_test.go

@@ -164,6 +164,12 @@ func compare(t *testing.T, img1, img2 string) {
 	cfg2, err := v1img2.ConfigFile()
 	h.AssertNil(t, err)
 
+	// images that were created locally may have `DockerVersion` equal to "dev" and missing `Config.Image` if the daemon uses containerd storage
+	cfg1.DockerVersion = ""
+	cfg2.DockerVersion = ""
+	cfg1.Config.Image = ""
+	cfg2.Config.Image = ""
+
 	h.AssertEq(t, cfg1, cfg2)
 
 	h.AssertEq(t, ref1.Identifier(), ref2.Identifier())

go.mod

@@ -1,12 +1,12 @@
 module github.com/buildpacks/imgutil
 
 require (
-	github.com/docker/docker v24.0.2+incompatible
-	github.com/google/go-cmp v0.5.9
-	github.com/google/go-containerregistry v0.15.2
+	github.com/docker/docker v24.0.7+incompatible
+	github.com/google/go-cmp v0.6.0
+	github.com/google/go-containerregistry v0.16.1
 	github.com/pkg/errors v0.9.1
 	github.com/sclevine/spec v1.4.0
-	golang.org/x/sync v0.3.0
+	golang.org/x/sync v0.4.0
 )
 
 require (
@@ -28,9 +28,9 @@ require (
 	github.com/sirupsen/logrus v1.9.2 // indirect
 	github.com/vbatts/tar-split v0.11.3 // indirect
 	golang.org/x/mod v0.10.0 // indirect
-	golang.org/x/net v0.10.0 // indirect
-	golang.org/x/sys v0.8.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/tools v0.9.1 // indirect
 )
 
-go 1.20
+go 1.20

go.sum

@@ -14,8 +14,8 @@ github.com/docker/cli v24.0.2+incompatible h1:QdqR7znue1mtkXIJ+ruQMGQhpw2JzMJLRX
 github.com/docker/cli v24.0.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8=
 github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg=
-github.com/docker/docker v24.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM=
+github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
 github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A=
 github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
@@ -24,10 +24,10 @@ github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4
 github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE=
-github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ=
+github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI=
@@ -76,21 +76,21 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
-golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
+golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ=
+golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
+golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=

layout/layout_test.go

@@ -157,7 +157,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 					it("sets the initial state from a linux/arm base image", func() {
 						existingLayerSha := "sha256:5a0b973aa300cd2650869fd76d8546b361fcd6dfc77bd37b9d4f082cca9874e4"
 
-						img, err := layout.NewImage(imagePath, layout.FromBaseImage(testImage))
+						img, err := layout.NewImage(imagePath, layout.FromBaseImage(testImage), layout.WithMediaTypes(imgutil.OCITypes))
 						h.AssertNil(t, err)
 						h.AssertOCIMediaTypes(t, img)
 
@@ -199,7 +199,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 
 					img, err := layout.NewImage(imagePath, layout.FromBaseImagePath(fullBaseImagePath))
 					h.AssertNil(t, err)
-					h.AssertOCIMediaTypes(t, img)
+					h.AssertDockerMediaTypes(t, img)
 
 					os, err := img.OS()
 					h.AssertNil(t, err)
@@ -225,7 +225,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 
 					img, err := layout.NewImage(imagePath, layout.FromBaseImagePath(sparseBaseImagePath))
 					h.AssertNil(t, err)
-					h.AssertOCIMediaTypes(t, img)
+					h.AssertDockerMediaTypes(t, img)
 
 					os, err := img.OS()
 					h.AssertNil(t, err)

layout/new.go

@@ -42,6 +42,7 @@ func NewImage(path string, ops ...ImageOption) (*Image, error) {
 		}
 	}
 
+	hasBaseImage := imageOpts.baseImagePath != "" || imageOpts.baseImage != nil
 	if imageOpts.baseImagePath != "" {
 		if err := processBaseImageOption(ri, imageOpts.baseImagePath, platform); err != nil {
 			return nil, err
@@ -58,10 +59,10 @@ func NewImage(path string, ops ...ImageOption) (*Image, error) {
 		ri.createdAt = imageOpts.createdAt
 	}
 
-	if imageOpts.mediaTypes == imgutil.MissingTypes {
-		ri.requestedMediaTypes = imgutil.OCITypes
-	} else {
+	if imageOpts.mediaTypes != imgutil.MissingTypes {
 		ri.requestedMediaTypes = imageOpts.mediaTypes
+	} else if !hasBaseImage {
+		ri.requestedMediaTypes = imgutil.OCITypes
 	}
 	if err = ri.setUnderlyingImage(ri.Image); err != nil { // update media types
 		return nil, err

layout/sparse/sparse_test.go

@@ -9,6 +9,7 @@ import (
 	"github.com/sclevine/spec"
 	"github.com/sclevine/spec/report"
 
+	"github.com/buildpacks/imgutil"
 	"github.com/buildpacks/imgutil/layout"
 	"github.com/buildpacks/imgutil/layout/sparse"
 	h "github.com/buildpacks/imgutil/testhelpers"
@@ -107,5 +108,49 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 				h.AssertEqAnnotation(t, index.Manifests[0], layout.ImageRefNameKey, "my-tag")
 			})
 		})
+
+		when("#MediaType", func() {
+			it("returns the base image media type when there are no requested media type changes", func() {
+				image, err := sparse.NewImage(imagePath, testImage)
+				h.AssertNil(t, err)
+
+				err = image.Save()
+				h.AssertNil(t, err)
+
+				expectedMediaType, err := testImage.MediaType()
+				h.AssertNil(t, err)
+
+				actualMediaType, err := image.MediaType()
+				h.AssertNil(t, err)
+				h.AssertEq(t, actualMediaType, expectedMediaType)
+			})
+
+			it("mutates the media type to the specified media type", func() {
+				image, err := sparse.NewImage(imagePath, testImage, layout.WithMediaTypes(imgutil.OCITypes))
+				h.AssertNil(t, err)
+
+				err = image.Save()
+				h.AssertNil(t, err)
+
+				h.AssertOCIMediaTypes(t, image)
+			})
+		})
+
+		when("#Digest", func() {
+			it("returns the original image digest when there are no modifications", func() {
+				image, err := sparse.NewImage(imagePath, testImage)
+				h.AssertNil(t, err)
+
+				err = image.Save()
+				h.AssertNil(t, err)
+
+				expectedDigest, err := testImage.Digest()
+				h.AssertNil(t, err)
+
+				actualDigest, err := image.Digest()
+				h.AssertNil(t, err)
+				h.AssertEq(t, actualDigest, expectedDigest)
+			})
+		})
 	})
 }

local/local.go

@@ -41,6 +41,7 @@ type DockerClient interface {
 	ImageSave(ctx context.Context, images []string) (io.ReadCloser, error)
 	ImageTag(ctx context.Context, image, ref string) error
 	Info(ctx context.Context) (types.Info, error)
+	ServerVersion(ctx context.Context) (types.Version, error)
 }
 
 // getters

local/local_test.go

@@ -43,17 +43,19 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 	var (
 		dockerClient          client.CommonAPIClient
 		daemonOS              string
+		daemonArchitecture    string
 		runnableBaseImageName string
 	)
 
 	it.Before(func() {
 		var err error
 		dockerClient = h.DockerCli(t)
 
-		daemonInfo, err := dockerClient.Info(context.TODO())
+		versionInfo, err := dockerClient.ServerVersion(context.TODO())
 		h.AssertNil(t, err)
 
-		daemonOS = daemonInfo.OSType
+		daemonOS = versionInfo.Os
+		daemonArchitecture = versionInfo.Arch
 		runnableBaseImageName = h.RunnableBaseImage(daemonOS)
 
 		h.PullIfMissing(t, dockerClient, runnableBaseImageName)
@@ -79,11 +81,11 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 				inspect, _, err := dockerClient.ImageInspectWithRaw(context.TODO(), img.Name())
 				h.AssertNil(t, err)
 
-				daemonInfo, err := dockerClient.Info(context.TODO())
+				versionInfo, err := dockerClient.ServerVersion(context.TODO())
 				h.AssertNil(t, err)
 
-				h.AssertEq(t, inspect.Os, daemonInfo.OSType)
-				h.AssertEq(t, inspect.Architecture, "amd64")
+				h.AssertEq(t, inspect.Os, versionInfo.Os)
+				h.AssertEq(t, inspect.Architecture, versionInfo.Arch)
 				h.AssertEq(t, inspect.RootFS.Type, "layers")
 			})
 		})
@@ -210,7 +212,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 						inspect, _, err := dockerClient.ImageInspectWithRaw(context.TODO(), img.Name())
 						h.AssertNil(t, err)
 						h.AssertEq(t, inspect.Os, daemonOS)
-						h.AssertEq(t, inspect.Architecture, "amd64")
+						h.AssertEq(t, inspect.Architecture, daemonArchitecture)
 						h.AssertEq(t, inspect.RootFS.Type, "layers")
 
 						h.AssertEq(t, img.Found(), true)
@@ -1114,15 +1116,15 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 	when("#Rebase", func() {
 		when("image exists", func() {
 			var (
-				repoName                              = newTestImageName()
-				oldBase, oldTopLayer, newBase, origID string
-				oldBaseLayer1DiffID                   string
-				oldBaseLayer2DiffID                   string
-				newBaseLayer1DiffID                   string
-				newBaseLayer2DiffID                   string
-				imgLayer1DiffID                       string
-				imgLayer2DiffID                       string
-				origNumLayers                         int
+				repoName                      = newTestImageName()
+				oldBase, oldTopLayer, newBase string
+				oldBaseLayer1DiffID           string
+				oldBaseLayer2DiffID           string
+				newBaseLayer1DiffID           string
+				newBaseLayer2DiffID           string
+				imgLayer1DiffID               string
+				imgLayer2DiffID               string
+				origNumLayers                 int
 			)
 
 			it.Before(func() {
@@ -1198,11 +1200,10 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 				inspect, _, err = dockerClient.ImageInspectWithRaw(context.TODO(), repoName)
 				h.AssertNil(t, err)
 				origNumLayers = len(inspect.RootFS.Layers)
-				origID = inspect.ID
 			})
 
 			it.After(func() {
-				h.AssertNil(t, h.DockerRmi(dockerClient, repoName, oldBase, newBase, origID))
+				h.AssertNil(t, h.DockerRmi(dockerClient, repoName, oldBase, newBase))
 			})
 
 			it("switches the base", func() {
@@ -1604,6 +1605,8 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 			repoName      = newTestImageName()
 			prevLayer1SHA string
 			prevLayer2SHA string
+			layer1Path    string
+			layer2Path    string
 		)
 
 		it.Before(func() {
@@ -1616,13 +1619,11 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 			)
 			h.AssertNil(t, err)
 
-			layer1Path, err := h.CreateSingleFileLayerTar("/layer-1.txt", "old-layer-1", daemonOS)
+			layer1Path, err = h.CreateSingleFileLayerTar("/layer-1.txt", "old-layer-1", daemonOS)
 			h.AssertNil(t, err)
-			defer os.Remove(layer1Path)
 
-			layer2Path, err := h.CreateSingleFileLayerTar("/layer-2.txt", "old-layer-2", daemonOS)
+			layer2Path, err = h.CreateSingleFileLayerTar("/layer-2.txt", "old-layer-2", daemonOS)
 			h.AssertNil(t, err)
-			defer os.Remove(layer2Path)
 
 			h.AssertNil(t, prevImage.AddLayer(layer1Path))
 			h.AssertNil(t, prevImage.AddLayer(layer2Path))
@@ -1638,6 +1639,8 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 
 		it.After(func() {
 			h.AssertNil(t, h.DockerRmi(dockerClient, repoName, prevImageName))
+			h.AssertNil(t, os.RemoveAll(layer1Path))
+			h.AssertNil(t, os.RemoveAll(layer2Path))
 		})
 
 		it("reuses a layer", func() {
@@ -1869,7 +1872,7 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 
 			it.After(func() {
 				h.AssertNil(t, os.Remove(tarPath))
-				h.AssertNil(t, h.DockerRmi(dockerClient, repoName, origID))
+				h.AssertNil(t, h.DockerRmi(dockerClient, repoName))
 			})
 
 			it("saved image overrides image with new ID", func() {
@@ -1909,7 +1912,6 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 
 				h.AssertEq(t, inspect.Created, imgutil.NormalizedDateTime.Format(time.RFC3339))
 				h.AssertEq(t, inspect.Container, "")
-				h.AssertEq(t, inspect.DockerVersion, "")
 
 				history, err := dockerClient.ImageHistory(context.TODO(), repoName)
 				h.AssertNil(t, err)
@@ -1940,7 +1942,6 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 
 					h.AssertEq(t, inspect.Created, expectedTime.Format(time.RFC3339))
 					h.AssertEq(t, inspect.Container, "")
-					h.AssertEq(t, inspect.DockerVersion, "")
 
 					history, err := dockerClient.ImageHistory(context.TODO(), repoName)
 					h.AssertNil(t, err)