From 5dd2a929c7074258cfcd71a857fe305f10912999 Mon Sep 17 00:00:00 2001 From: Rob Dimsdale-Zucker Date: Wed, 20 Sep 2023 04:54:46 -0700 Subject: [PATCH] Add Source and Source256 fields to dependencies in buildpack.toml. (#196) * Add Source and Source256 fields to dependencies in buildpack.toml. - This is primarily used by third-party provenance tools which leverage these fields to determine what dependencies a buildpack ships. * adds source flags to update-bp-dep fixes test format issues --------- Co-authored-by: David O'Sullivan Co-authored-by: David O'Sullivan <31728678+pivotal-david-osullivan@users.noreply.github.com> --- carton/buildpack_dependency.go | 20 ++- carton/buildpack_dependency_test.go | 224 ++++++++++++++++-------- carton/package_dependency_test.go | 157 ++++++++++------- carton/package_test.go | 2 +- cmd/update-buildpack-dependency/main.go | 2 + 5 files changed, 263 insertions(+), 142 deletions(-) diff --git a/carton/buildpack_dependency.go b/carton/buildpack_dependency.go index 7cb05dd..b2ef05b 100644 --- a/carton/buildpack_dependency.go +++ b/carton/buildpack_dependency.go @@ -43,6 +43,8 @@ type BuildpackDependency struct { CPEPattern string PURL string PURLPattern string + Source string `toml:"source,omitempty"` + SourceSHA256 string `toml:"source-sha256,omitempty"` } func (b BuildpackDependency) Update(options ...Option) { @@ -56,11 +58,13 @@ func (b BuildpackDependency) Update(options ...Option) { logger := bard.NewLogger(os.Stdout) _, _ = fmt.Fprintf(logger.TitleWriter(), "\n%s\n", bard.FormatIdentity(b.ID, b.VersionPattern)) - logger.Headerf("Version: %s", b.Version) - logger.Headerf("PURL: %s", b.PURL) - logger.Headerf("CPEs: %s", b.CPE) - logger.Headerf("URI: %s", b.URI) - logger.Headerf("SHA256: %s", b.SHA256) + logger.Headerf("Version: %s", b.Version) + logger.Headerf("PURL: %s", b.PURL) + logger.Headerf("CPEs: %s", b.CPE) + logger.Headerf("URI: %s", b.URI) + logger.Headerf("SHA256: %s", b.SHA256) + logger.Headerf("Source: %s", b.Source) + logger.Headerf("SourceSHA256: %s", b.SourceSHA256) versionExp, err := regexp.Compile(b.VersionPattern) if err != nil { @@ -151,6 +155,12 @@ func (b BuildpackDependency) Update(options ...Option) { dep["version"] = b.Version dep["uri"] = b.URI dep["sha256"] = b.SHA256 + if b.SourceSHA256 != "" { + dep["source-sha256"] = b.SourceSHA256 + } + if b.Source != "" { + dep["source"] = b.Source + } purlUnwrapped, found := dep["purl"] if found { diff --git a/carton/buildpack_dependency_test.go b/carton/buildpack_dependency_test.go index 0c81e0e..48f0842 100644 --- a/carton/buildpack_dependency_test.go +++ b/carton/buildpack_dependency_test.go @@ -61,12 +61,14 @@ name = "Some Buildpack" version = "1.2.3" [[metadata.dependencies]] -id = "test-id" -name = "Test Name" -version = "test-version-1" -uri = "test-uri-1" -sha256 = "test-sha256-1" -stacks = [ "test-stack" ] +id = "test-id" +name = "Test Name" +version = "test-version-1" +uri = "test-uri-1" +sha256 = "test-sha256-1" +stacks = [ "test-stack" ] +source = "test-source-uri-1" +source-sha256 = "test-source-sha256-1" `), 0644)).To(Succeed()) d := carton.BuildpackDependency{ @@ -76,6 +78,8 @@ stacks = [ "test-stack" ] URI: "test-uri-2", Version: "test-version-2", VersionPattern: `test-version-[\d]`, + Source: "test-source-uri-2", + SourceSHA256: "test-source-sha256-2", } d.Update(carton.WithExitHandler(exitHandler)) @@ -86,12 +90,15 @@ id = "some-buildpack" name = "Some Buildpack" version = "1.2.3" -[[metadata.dependencies]]id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +source = "test-source-uri-2" +source-sha256 = "test-source-sha256-2" `)) }) @@ -134,18 +141,19 @@ id = "some-buildpack" name = "Some Buildpack" version = "1.2.3" -[[metadata.dependencies]]id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] -purl = "pkg:generic/test-jre@different-version-2?arch=amd64" -cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-2?arch=amd64" +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] `)) }) - it("updates multiple dependencies with different versions", func() { + it("updates dependency with source & sourceSha", func() { Expect(os.WriteFile(path, []byte(`api = "0.7" [buildpack] id = "some-buildpack" @@ -161,16 +169,75 @@ sha256 = "test-sha256-1" stacks = [ "test-stack" ] purl = "pkg:generic/test-jre@different-version-1?arch=amd64" cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-1:patch1:*:*:*:*:*:*:*"] +`), 0644)).To(Succeed()) + + d := carton.BuildpackDependency{ + BuildpackPath: path, + ID: "test-id", + SHA256: "test-sha256-2", + URI: "test-uri-2", + Version: "test-version-2", + VersionPattern: `test-version-[\d]`, + PURL: "different-version-2", + PURLPattern: `different-version-[\d]`, + CPE: "test-version-2:patch2", + CPEPattern: `test-version-[\d]:patch[\d]`, + Source: "test-new-source", + SourceSHA256: "test-new-source-sha", + } + + d.Update(carton.WithExitHandler(exitHandler)) + + Expect(os.ReadFile(path)).To(internal.MatchTOML(`api = "0.7" +[buildpack] +id = "some-buildpack" +name = "Some Buildpack" +version = "1.2.3" [[metadata.dependencies]] -id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] -purl = "pkg:generic/test-jre@different-version-2?arch=amd64" -cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-2?arch=amd64" +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +source = "test-new-source" +source-sha256 = "test-new-source-sha" +`)) + }) + + it("updates multiple dependencies with different versions", func() { + Expect(os.WriteFile(path, []byte(`api = "0.7" +[buildpack] +id = "some-buildpack" +name = "Some Buildpack" +version = "1.2.3" + +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-1" +uri = "test-uri-1" +sha256 = "test-sha256-1" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-1?arch=amd64" +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-1:patch1:*:*:*:*:*:*:*"] +source = "test-source-uri-1" +source-sha256 = "test-source-sha256-1" + +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-2?arch=amd64" +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +source = "test-source-uri-2" +source-sha256 = "test-source-sha256-2" `), 0644)).To(Succeed()) d := carton.BuildpackDependency{ @@ -184,6 +251,8 @@ cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:* PURLPattern: `different-version-[\d]`, CPE: "test-version-3:patch3", CPEPattern: `test-version-[\d]:patch[\d]`, + Source: "test-source-uri-3", + SourceSHA256: "test-source-sha256-3", } d.Update(carton.WithExitHandler(exitHandler)) @@ -195,24 +264,28 @@ name = "Some Buildpack" version = "1.2.3" [[metadata.dependencies]] -id = "test-id" -name = "Test Name" -version = "test-version-3" -uri = "test-uri-3" -sha256 = "test-sha256-3" -stacks = [ "test-stack" ] -purl = "pkg:generic/test-jre@different-version-3?arch=amd64" -cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-3:patch3:*:*:*:*:*:*:*"] +id = "test-id" +name = "Test Name" +version = "test-version-3" +uri = "test-uri-3" +sha256 = "test-sha256-3" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-3?arch=amd64" +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-3:patch3:*:*:*:*:*:*:*"] +source = "test-source-uri-3" +source-sha256 = "test-source-sha256-3" [[metadata.dependencies]] -id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] -purl = "pkg:generic/test-jre@different-version-2?arch=amd64" -cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-2?arch=amd64" +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +source = "test-source-uri-2" +source-sha256 = "test-source-sha256-2" `)) }) @@ -254,13 +327,14 @@ id = "some-buildpack" name = "Some Buildpack" version = "1.2.3" -[[metadata.dependencies]]id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] -cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] `)) }) @@ -303,14 +377,15 @@ id = "some-buildpack" name = "Some Buildpack" version = "1.2.3" -[[metadata.dependencies]]id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] -purl = 1234 -cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +purl = 1234 +cpes = ["cpe:2.3:a:test-vendor:test-product:test-version-2:patch2:*:*:*:*:*:*:*"] `)) }) @@ -353,14 +428,15 @@ id = "some-buildpack" name = "Some Buildpack" version = "1.2.3" -[[metadata.dependencies]]id = "test-id" -name = "Test Name" -version = "test-version-2" -uri = "test-uri-2" -sha256 = "test-sha256-2" -stacks = [ "test-stack" ] -purl = "pkg:generic/test-jre@different-version-2?arch=amd64" -cpes = 1234 +[[metadata.dependencies]] +id = "test-id" +name = "Test Name" +version = "test-version-2" +uri = "test-uri-2" +sha256 = "test-sha256-2" +stacks = [ "test-stack" ] +purl = "pkg:generic/test-jre@different-version-2?arch=amd64" +cpes = 1234 `)) }) @@ -409,12 +485,12 @@ name = "Some Buildpack" version = "1.2.3" [[metadata.dependencies]] - id = "test-id" - name = "Test Name" - version = "test-version-2" - uri = "test-uri-2" - sha256 = "test-sha256-2" - stacks = [ "test-stack" ] + id = "test-id" + name = "Test Name" + version = "test-version-2" + uri = "test-uri-2" + sha256 = "test-sha256-2" + stacks = [ "test-stack" ] `)) }) } diff --git a/carton/package_dependency_test.go b/carton/package_dependency_test.go index 1aa63ac..d5e4a9d 100644 --- a/carton/package_dependency_test.go +++ b/carton/package_dependency_test.go @@ -64,10 +64,13 @@ id = "some-id" name = "some-name" [[order]] -group = [ - { id = "paketo-buildpacks/test-1", version="test-version-1" }, - { id = "paketo-buildpacks/test-2", version="test-version-2" }, -] + [[order.group]] + id = "paketo-buildpacks/test-1" + version="test-version-1" + [[order.group]] + id = "paketo-buildpacks/test-2" + version="test-version-2" + [metadata] include-files = [ "LICENSE", @@ -90,31 +93,39 @@ include-files = [ # exactly api = "0.6"`)) - Expect(body).To(internal.MatchTOML(`api = "0.6" -[buildpack] -id = "some-id" -name = "some-name" - -[[order]] -group = [ - { id = "paketo-buildpacks/test-1", version="test-version-3" }, - { id = "paketo-buildpacks/test-2", version="test-version-2" }, -] -[metadata] -include-files = [ - "LICENSE", - "README.md", - "buildpack.toml", -]`)) + Expect(body).To(internal.MatchTOML(`# it should preserve + # these comments + # exactly + + api = "0.6" + [buildpack] + id = "some-id" + name = "some-name" + + [[order]] + [[order.group]] + id = "paketo-buildpacks/test-1" + version="test-version-3" + [[order.group]] + id = "paketo-buildpacks/test-2" + version="test-version-2" + + [metadata] + include-files = [ + "LICENSE", + "README.md", + "buildpack.toml", + ]`)) }) it("updates paketo-buildpacks dependency id partial id", func() { - Expect(os.WriteFile(path, []byte(` -[[order]] -group = [ - { id = "paketo-buildpacks/test-1", version="test-version-1" }, - { id = "paketo-buildpacks/test-2", version="test-version-2" }, -]`), 0644)).To(Succeed()) + Expect(os.WriteFile(path, []byte(`[[order]] + [[order.group]] + id = "paketo-buildpacks/test-1" + version="test-version-1" + [[order.group]] + id = "paketo-buildpacks/test-2" + version="test-version-2"`), 0644)).To(Succeed()) p := carton.PackageDependency{ BuildpackPath: path, @@ -125,18 +136,22 @@ group = [ p.Update(carton.WithExitHandler(exitHandler)) Expect(os.ReadFile(path)).To(internal.MatchTOML(`[[order]] -group = [ - { id = "paketo-buildpacks/test-1", version="test-version-3" }, - { id = "paketo-buildpacks/test-2", version="test-version-2" }, -]`)) + [[order.group]] + id = "paketo-buildpacks/test-1" + version="test-version-3" + [[order.group]] + id = "paketo-buildpacks/test-2" + version="test-version-2"`)) }) it("updates paketocommunity dependency", func() { Expect(os.WriteFile(path, []byte(`[[order]] -group = [ - { id = "paketocommunity/test-1", version="test-version-1" }, - { id = "paketocommunity/test-2", version="test-version-2" }, -]`), 0644)).To(Succeed()) + [[order.group]] + id = "paketocommunity/test-1" + version="test-version-1" + [[order.group]] + id = "paketocommunity/test-2" + version="test-version-2"`), 0644)).To(Succeed()) p := carton.PackageDependency{ BuildpackPath: path, @@ -147,17 +162,22 @@ group = [ p.Update(carton.WithExitHandler(exitHandler)) Expect(os.ReadFile(path)).To(internal.MatchTOML(`[[order]] -group = [ - { id = "paketocommunity/test-1", version="test-version-3" }, - { id = "paketocommunity/test-2", version="test-version-2" }, -]`)) + [[order.group]] + id = "paketocommunity/test-1" + version="test-version-3" + [[order.group]] + id = "paketocommunity/test-2" + version="test-version-2"`)) }) it("updates builder dependency", func() { - Expect(os.WriteFile(path, []byte(`buildpacks = [ - { id = "paketo-buildpacks/test-1", uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-1" }, - { id = "paketo-buildpacks/test-2", uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" }, -]`), 0644)).To(Succeed()) + Expect(os.WriteFile(path, []byte(`[[buildpacks]] + id = "paketo-buildpacks/test-1" + uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-1" + + [[buildpacks]] + id = "paketo-buildpacks/test-2" + uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2"`), 0644)).To(Succeed()) p := carton.PackageDependency{ BuilderPath: path, @@ -167,17 +187,24 @@ group = [ p.Update(carton.WithExitHandler(exitHandler)) - Expect(os.ReadFile(path)).To(internal.MatchTOML(`buildpacks = [ - { id = "paketo-buildpacks/test-1", uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-3" }, - { id = "paketo-buildpacks/test-2", uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" }, -]`)) + Expect(os.ReadFile(path)).To(internal.MatchTOML(` + [[buildpacks]] + id = "paketo-buildpacks/test-1" + uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-3" + + [[buildpacks]] + id = "paketo-buildpacks/test-2" + uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" +`)) }) it("updates paketo-buildpacks package dependency", func() { - Expect(os.WriteFile(path, []byte(`dependencies = [ - { uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-1" }, - { uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" }, -]`), 0644)).To(Succeed()) + Expect(os.WriteFile(path, []byte(` + [[dependencies]] + uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-1" + [[dependencies]] + uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" +`), 0644)).To(Succeed()) p := carton.PackageDependency{ PackagePath: path, @@ -187,17 +214,21 @@ group = [ p.Update(carton.WithExitHandler(exitHandler)) - Expect(os.ReadFile(path)).To(internal.MatchTOML(`dependencies = [ - { uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-3" }, - { uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" }, -]`)) + Expect(os.ReadFile(path)).To(internal.MatchTOML(` + [[dependencies]] + uri = "docker://gcr.io/paketo-buildpacks/test-1:test-version-3" + [[dependencies]] + uri = "docker://gcr.io/paketo-buildpacks/test-2:test-version-2" +`)) }) it("updates paketocommunity package dependency", func() { - Expect(os.WriteFile(path, []byte(`dependencies = [ - { uri = "docker://docker.io/paketocommunity/test-1:test-version-1" }, - { uri = "docker://docker.io/paketocommunity/test-2:test-version-2" }, -]`), 0644)).To(Succeed()) + Expect(os.WriteFile(path, []byte(` + [[dependencies]] + uri = "docker://docker.io/paketocommunity/test-1:test-version-1" + [[dependencies]] + uri = "docker://docker.io/paketocommunity/test-2:test-version-2" +`), 0644)).To(Succeed()) p := carton.PackageDependency{ PackagePath: path, @@ -207,10 +238,12 @@ group = [ p.Update(carton.WithExitHandler(exitHandler)) - Expect(os.ReadFile(path)).To(internal.MatchTOML(`dependencies = [ - { uri = "docker://docker.io/paketocommunity/test-1:test-version-3" }, - { uri = "docker://docker.io/paketocommunity/test-2:test-version-2" }, -]`)) + Expect(os.ReadFile(path)).To(internal.MatchTOML(` + [[dependencies]] + uri = "docker://docker.io/paketocommunity/test-1:test-version-3" + [[dependencies]] + uri = "docker://docker.io/paketocommunity/test-2:test-version-2" +`)) }) } diff --git a/carton/package_test.go b/carton/package_test.go index a138586..0649f26 100644 --- a/carton/package_test.go +++ b/carton/package_test.go @@ -286,7 +286,7 @@ include-files = [ Expect(entryWriter.Calls[5].Arguments[1]).To(Equal(filepath.Join("test-destination", "test-include-files"))) }) - it.Focus("includes filter by version and id", func() { + it("includes filter by version and id", func() { carton.Package{ Source: path, Destination: "test-destination", diff --git a/cmd/update-buildpack-dependency/main.go b/cmd/update-buildpack-dependency/main.go index f8ec7d2..4fcabe5 100644 --- a/cmd/update-buildpack-dependency/main.go +++ b/cmd/update-buildpack-dependency/main.go @@ -40,6 +40,8 @@ func main() { flagSet.StringVar(&b.PURLPattern, "purl-pattern", "", "the purl version pattern of the dependency, if not set defaults to version-pattern") flagSet.StringVar(&b.CPE, "cpe", "", "the new version use in all CPEs, if not set defaults to version") flagSet.StringVar(&b.CPEPattern, "cpe-pattern", "", "the cpe version pattern of the dependency, if not set defaults to version-pattern") + flagSet.StringVar(&b.Source, "source", "", "the new uri of the dependency source") + flagSet.StringVar(&b.SourceSHA256, "source-sha256", "", "the new sha256 of the dependency source") if err := flagSet.Parse(os.Args[1:]); err != nil { log.Fatal(fmt.Errorf("unable to parse flags\n%w", err))