diff --git a/cmd/cnbBuild.go b/cmd/cnbBuild.go index 4ef75a4c07..9c80aa8ab9 100644 --- a/cmd/cnbBuild.go +++ b/cmd/cnbBuild.go @@ -112,10 +112,29 @@ func processConfigs(main cnbBuildOptions, multipleImages []map[string]interface{ return result, nil } -func setCustomBuildpacks(bpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { +func setCustomBuildpacks(bpacks, preBuildpacks, postBuildpacks []string, dockerCreds string, utils cnbutils.BuildUtils) (string, string, error) { buildpacksPath := "/tmp/buildpacks" orderPath := "/tmp/buildpacks/order.toml" - newOrder, err := cnbutils.DownloadBuildpacks(buildpacksPath, bpacks, dockerCreds, utils) + err := cnbutils.DownloadBuildpacks(buildpacksPath, append(bpacks, append(preBuildpacks, postBuildpacks...)...), dockerCreds, utils) + if err != nil { + return "", "", err + } + + if len(bpacks) == 0 && (len(postBuildpacks) > 0 || len(preBuildpacks) > 0) { + matches, err := utils.Glob("/cnb/buildpacks/*") + if err != nil { + return "", "", err + } + + for _, match := range matches { + err = cnbutils.CreateVersionSymlinks(buildpacksPath, match, utils) + if err != nil { + return "", "", err + } + } + } + + newOrder, err := cnbutils.CreateOrder(bpacks, preBuildpacks, postBuildpacks, dockerCreds, utils) if err != nil { return "", "", err } @@ -475,10 +494,18 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils config.mergeEnvVars(descriptor.EnvVars) - if (config.Buildpacks == nil || len(config.Buildpacks) == 0) && len(descriptor.Buildpacks) > 0 { + if len(config.Buildpacks) == 0 { config.Buildpacks = descriptor.Buildpacks } + if len(config.PreBuildpacks) == 0 { + config.PreBuildpacks = descriptor.PreBuildpacks + } + + if len(config.PostBuildpacks) == 0 { + config.PostBuildpacks = descriptor.PostBuildpacks + } + if descriptor.Exclude != nil { exclude = descriptor.Exclude } @@ -563,11 +590,13 @@ func runCnbBuild(config *cnbBuildOptions, cnbTelemetry *cnbBuildTelemetry, utils metadata.WriteProjectMetadata(GeneralConfig.EnvRootPath, utils) var buildpacksPath = "/cnb/buildpacks" - var orderPath = "/cnb/order.toml" + var orderPath = cnbutils.DefaultOrderPath - if config.Buildpacks != nil && len(config.Buildpacks) > 0 { + if len(config.Buildpacks) > 0 || len(config.PreBuildpacks) > 0 || len(config.PostBuildpacks) > 0 { log.Entry().Infof("Setting custom buildpacks: '%v'", config.Buildpacks) - buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.DockerConfigJSON, utils) + log.Entry().Infof("Pre-buildpacks: '%v'", config.PreBuildpacks) + log.Entry().Infof("Post-buildpacks: '%v'", config.PostBuildpacks) + buildpacksPath, orderPath, err = setCustomBuildpacks(config.Buildpacks, config.PreBuildpacks, config.PostBuildpacks, config.DockerConfigJSON, utils) defer func() { _ = utils.RemoveAll(buildpacksPath) }() defer func() { _ = utils.RemoveAll(orderPath) }() if err != nil { diff --git a/cmd/cnbBuild_generated.go b/cmd/cnbBuild_generated.go index 945daa6600..0a150d1607 100644 --- a/cmd/cnbBuild_generated.go +++ b/cmd/cnbBuild_generated.go @@ -27,6 +27,8 @@ type cnbBuildOptions struct { ContainerImageTag string `json:"containerImageTag,omitempty"` ContainerRegistryURL string `json:"containerRegistryUrl,omitempty"` Buildpacks []string `json:"buildpacks,omitempty"` + PreBuildpacks []string `json:"preBuildpacks,omitempty"` + PostBuildpacks []string `json:"postBuildpacks,omitempty"` BuildEnvVars map[string]interface{} `json:"buildEnvVars,omitempty"` Path string `json:"path,omitempty"` ProjectDescriptor string `json:"projectDescriptor,omitempty"` @@ -226,7 +228,9 @@ func addCnbBuildFlags(cmd *cobra.Command, stepConfig *cnbBuildOptions) { cmd.Flags().StringVar(&stepConfig.ContainerImageAlias, "containerImageAlias", os.Getenv("PIPER_containerImageAlias"), "Logical name used for this image.\n") cmd.Flags().StringVar(&stepConfig.ContainerImageTag, "containerImageTag", os.Getenv("PIPER_containerImageTag"), "Tag of the container which will be built") cmd.Flags().StringVar(&stepConfig.ContainerRegistryURL, "containerRegistryUrl", os.Getenv("PIPER_containerRegistryUrl"), "Container registry where the image should be pushed to.\n\n**Note**: `containerRegistryUrl` should include only the domain. If you want to publish an image under `docker.io/example/my-image`, you must set `containerRegistryUrl: \"docker.io\"` and `containerImageName: \"example/my-image\"`.\n") - cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`.") + cmd.Flags().StringSliceVar(&stepConfig.Buildpacks, "buildpacks", []string{}, "List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. When this property is specified, buildpacks which are part of the builder will be ignored.") + cmd.Flags().StringSliceVar(&stepConfig.PreBuildpacks, "preBuildpacks", []string{}, "Buildpacks to prepend to the groups in the builder's order.") + cmd.Flags().StringSliceVar(&stepConfig.PostBuildpacks, "postBuildpacks", []string{}, "Buildpacks to append to the groups in the builder's order.") cmd.Flags().StringVar(&stepConfig.Path, "path", os.Getenv("PIPER_path"), "Glob that should either point to a directory with your sources or one artifact in zip format.\nThis property determines the input to the buildpack.\n") cmd.Flags().StringVar(&stepConfig.ProjectDescriptor, "projectDescriptor", `project.toml`, "Relative path to the project.toml file.\nSee [buildpacks.io](https://buildpacks.io/docs/reference/config/project-descriptor/) for the reference.\nParameters passed to the cnbBuild step will take precedence over the parameters set in the project.toml file, except the `env` block.\nEnvironment variables declared in a project descriptor file, will be merged with the `buildEnvVars` property, with the `buildEnvVars` having a precedence.\n\n*Note*: The project descriptor path should be relative to what is set in the [path](#path) property. If the `path` property is pointing to a zip archive (e.g. jar file), project descriptor path will be relative to the root of the workspace.\n\n*Note*: Inline buildpacks (see [specification](https://buildpacks.io/docs/reference/config/project-descriptor/#build-_table-optional_)) are not supported yet.\n") @@ -325,6 +329,34 @@ func cnbBuildMetadata() config.StepData { Aliases: []config.Alias{}, Default: []string{}, }, + { + Name: "preBuildpacks", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/preBuildpacks", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, + { + Name: "postBuildpacks", + ResourceRef: []config.ResourceReference{ + { + Name: "commonPipelineEnvironment", + Param: "container/postBuildpacks", + }, + }, + Scope: []string{"PARAMETERS", "STAGES", "STEPS"}, + Type: "[]string", + Mandatory: false, + Aliases: []config.Alias{}, + Default: []string{}, + }, { Name: "buildEnvVars", ResourceRef: []config.ResourceReference{}, diff --git a/cmd/cnbBuild_test.go b/cmd/cnbBuild_test.go index 2f1fa8226c..f5c7592a70 100644 --- a/cmd/cnbBuild_test.go +++ b/cmd/cnbBuild_test.go @@ -25,23 +25,56 @@ import ( const imageRegistry = "some-registry" func newCnbBuildTestsUtils() cnbutils.MockUtils { + imageStub := func(imageRef, target string) (v1.Image, error) { + fakeImage := &fake.FakeImage{} + var imageConfig v1.Config + switch imageRef { + case "pre-test": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"pre-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + case "post-test": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"post-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + default: + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + } + + fakeImage.ConfigFileReturns(&v1.ConfigFile{ + Config: imageConfig, + }, nil) + + return fakeImage, nil + } + utils := cnbutils.MockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, FilesMock: &mock.FilesMock{}, - DownloadMock: &mock.DownloadMock{}, - } - - fakeImage := &fake.FakeImage{} - fakeImage.ConfigFileReturns(&v1.ConfigFile{ - Config: v1.Config{ - Labels: map[string]string{ - "io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}", + DownloadMock: &mock.DownloadMock{ + ImageContentStub: imageStub, + ImageInfoStub: func(imageRef string) (v1.Image, error) { + return imageStub(imageRef, "") }, }, - }, nil) + } - utils.RemoteImageInfo = fakeImage - utils.ReturnImage = fakeImage + utils.AddFile("/cnb/order.toml", []byte(`[[order]] + [[order.group]] + id = "buildpacks/java" + version = "1.8.0" +[[order]] + [[order.group]] + id = "buildpacks/nodejs" + version = "1.6.0"`)) utils.AddFile("/layers/report.toml", []byte(`[build] [image] tags = ["localhost:5000/not-found:0.0.1"] @@ -234,6 +267,75 @@ func TestRunCnbBuild(t *testing.T) { assert.True(t, copiedFileExists) }) + t.Run("success case (custom buildpacks, pre and post buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) { + t.Parallel() + config := cnbBuildOptions{ + ContainerImageName: "my-image", + ContainerImageTag: "0.0.1", + ContainerRegistryURL: imageRegistry, + DockerConfigJSON: "/path/to/test.json", + PreBuildpacks: []string{"pre-test"}, + PostBuildpacks: []string{"post-test"}, + Buildpacks: []string{"test"}, + BuildEnvVars: map[string]interface{}{ + "FOO": "BAR", + }, + AdditionalTags: []string{"latest"}, + } + + utils := newCnbBuildTestsUtils() + utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) + addBuilderFiles(&utils) + + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) + + require.NoError(t, err) + runner := utils.ExecMockRunner + assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}") + assert.Equal(t, creatorPath, runner.Calls[0].Exec) + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks") + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks/order.toml") + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag)) + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName)) + + copiedFileExists, _ := utils.FileExists("/tmp/config.json") + assert.True(t, copiedFileExists) + }) + + t.Run("success case (custom pre and post buildpacks and custom env variables, renaming docker conf file, additional tag)", func(t *testing.T) { + t.Parallel() + config := cnbBuildOptions{ + ContainerImageName: "my-image", + ContainerImageTag: "0.0.1", + ContainerRegistryURL: imageRegistry, + DockerConfigJSON: "/path/to/test.json", + PostBuildpacks: []string{"post-test"}, + PreBuildpacks: []string{"pre-test"}, + BuildEnvVars: map[string]interface{}{ + "FOO": "BAR", + }, + AdditionalTags: []string{"latest"}, + } + + utils := newCnbBuildTestsUtils() + utils.FilesMock.AddFile(config.DockerConfigJSON, []byte(`{"auths":{"my-registry":{"auth":"dXNlcjpwYXNz"}}}`)) + addBuilderFiles(&utils) + + err := callCnbBuild(&config, &telemetry.CustomData{}, &utils, &cnbBuildCommonPipelineEnvironment{}, &piperhttp.Client{}) + + require.NoError(t, err) + runner := utils.ExecMockRunner + assert.Contains(t, runner.Env, "CNB_REGISTRY_AUTH={\"my-registry\":\"Basic dXNlcjpwYXNz\"}") + assert.Equal(t, creatorPath, runner.Calls[0].Exec) + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks") + assert.Contains(t, runner.Calls[0].Params, "/tmp/buildpacks/order.toml") + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:%s", config.ContainerRegistryURL, config.ContainerImageName, config.ContainerImageTag)) + assert.Contains(t, runner.Calls[0].Params, fmt.Sprintf("%s/%s:latest", config.ContainerRegistryURL, config.ContainerImageName)) + + copiedFileExists, _ := utils.FileExists("/tmp/config.json") + assert.True(t, copiedFileExists) + }) + t.Run("success case (customTlsCertificates)", func(t *testing.T) { t.Parallel() httpmock.Activate() diff --git a/go.mod b/go.mod index d0057e6bfd..704331cfab 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ replace golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d => golang.org/x/c require ( cloud.google.com/go/storage v1.22.1 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.0 + github.com/BurntSushi/toml v1.1.0 github.com/Jeffail/gabs/v2 v2.6.1 github.com/Masterminds/sprig v2.22.0+incompatible github.com/antchfx/htmlquery v1.2.4 @@ -46,7 +47,6 @@ require ( github.com/mitchellh/mapstructure v1.5.0 github.com/motemen/go-nuts v0.0.0-20210915132349-615a782f2c69 github.com/package-url/packageurl-go v0.1.0 - github.com/pelletier/go-toml v1.9.5 github.com/piper-validation/fortify-client-go v0.0.0-20220126145513-7b3e9a72af01 github.com/pkg/errors v0.9.1 github.com/sabhiram/go-gitignore v0.0.0-20210923224102-525f6e181f06 @@ -87,7 +87,6 @@ require ( github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect - github.com/BurntSushi/toml v1.1.0 // indirect github.com/CycloneDX/cyclonedx-go v0.6.0 github.com/DataDog/datadog-go v3.2.0+incompatible // indirect github.com/Jeffail/gabs v1.1.1 // indirect diff --git a/go.sum b/go.sum index a0f987ce80..2194e632ac 100644 --- a/go.sum +++ b/go.sum @@ -1707,8 +1707,6 @@ github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAv github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= diff --git a/integration/github_actions_integration_test_list.yml b/integration/github_actions_integration_test_list.yml index fc9acd2dff..006f177b2d 100644 --- a/integration/github_actions_integration_test_list.yml +++ b/integration/github_actions_integration_test_list.yml @@ -9,6 +9,7 @@ run: - '"TestCNBIntegrationNPMCustomBuildpacksBuildpacklessProject"' - '"TestCNBIntegrationNPMCustomBuildpacksFullProject"' - '"TestCNBIntegrationProjectDescriptor"' + - '"TestCNBIntegrationPrePostBuildpacks"' - '"TestGolangIntegration"' - '"TestGradleIntegration"' diff --git a/integration/integration_cnb_test.go b/integration/integration_cnb_test.go index 944651cf8e..1319e1bcc1 100644 --- a/integration/integration_cnb_test.go +++ b/integration/integration_cnb_test.go @@ -17,7 +17,7 @@ import ( const ( registryURL = "localhost:5000" - baseBuilder = "paketobuildpacks/builder:0.3.26-base" + baseBuilder = "paketobuildpacks/builder:0.3.280-base" ) func setupDockerRegistry(t *testing.T, ctx context.Context) testcontainers.Container { @@ -135,7 +135,7 @@ func TestCNBIntegrationZipPath(t *testing.T) { container.assertHasOutput(t, "running command: /cnb/lifecycle/creator", "Installing Go", - "Paketo Go Build Buildpack", + "Paketo Buildpack for Go Build", fmt.Sprintf("Saving %s/not-found:0.0.1", registryURL), "*** Images (sha256:", "SUCCESS", @@ -279,9 +279,9 @@ func TestCNBIntegrationMultiImage(t *testing.T) { assert.NoError(t, err) container.assertHasOutput(t, - "Previous image with name \"localhost:5000/io-buildpacks-my-app:latest\" not found", + "Image with name \"localhost:5000/io-buildpacks-my-app:latest\" not found", "Saving localhost:5000/io-buildpacks-my-app:latest...", - "Previous image with name \"localhost:5000/go-app:v1.0.0\" not found", + "Image with name \"localhost:5000/go-app:v1.0.0\" not found", "Saving localhost:5000/go-app:v1.0.0...", "Using cached buildpack", "Saving localhost:5000/my-app2:latest...", @@ -334,3 +334,27 @@ func TestCNBIntegrationPreserveFilesIgnored(t *testing.T) { container.assertHasOutput(t, "skipping preserving files because the source") container.terminate(t) } + +func TestCNBIntegrationPrePostBuildpacks(t *testing.T) { + t.Parallel() + ctx := context.Background() + registryContainer := setupDockerRegistry(t, ctx) + defer registryContainer.Terminate(ctx) + + container := givenThisContainer(t, IntegrationTestDockerExecRunnerBundle{ + Image: baseBuilder, + User: "cnb", + TestDir: []string{"testdata", "TestCnbIntegration"}, + Network: fmt.Sprintf("container:%s", registryContainer.GetContainerID()), + Environment: map[string]string{ + "PIPER_VAULTCREDENTIAL_DYNATRACE_API_KEY": "api-key-content", + }, + }) + + err := container.whenRunningPiperCommand("cnbBuild", "--noTelemetry", "--verbose", "--projectDescriptor", "", "--path", "project", "--customConfig", "config.yml", "--containerImageTag", "0.0.1", "--containerImageName", "not-found", "--containerRegistryUrl", registryURL, "--postBuildpacks", "paketobuildpacks/datadog") + assert.NoError(t, err) + container.assertHasOutput(t, "Setting custom buildpacks: '[]'") + container.assertHasOutput(t, "Pre-buildpacks: '[]'") + container.assertHasOutput(t, "Post-buildpacks: '[paketobuildpacks/datadog]'") + container.terminate(t) +} diff --git a/integration/testdata/TestCnbIntegration/config.yml b/integration/testdata/TestCnbIntegration/config.yml index da73f0510e..78aed906a0 100644 --- a/integration/testdata/TestCnbIntegration/config.yml +++ b/integration/testdata/TestCnbIntegration/config.yml @@ -3,6 +3,9 @@ general: collectTelemetryData: false steps: cnbBuild: + buildEnvVars: + BP_DATADOG_ENABLED: true + BP_EAR_KEY: 74657374 bindings: maven-settings: type: maven diff --git a/integration/testdata/TestCnbIntegration/project/package.json b/integration/testdata/TestCnbIntegration/project/package.json index 2c34dae7ee..be34f94b01 100644 --- a/integration/testdata/TestCnbIntegration/project/package.json +++ b/integration/testdata/TestCnbIntegration/project/package.json @@ -1,6 +1,7 @@ { "name": "test-mta-js", "version": "1.0.0", + "main": "srv/hello.js", "dependencies": { "jest": "^26.0.1", "jest-jenkins-reporter": "^1.0.2" diff --git a/pkg/cnbutils/buildpack.go b/pkg/cnbutils/buildpack.go index 5dc929f09d..8b744206dd 100644 --- a/pkg/cnbutils/buildpack.go +++ b/pkg/cnbutils/buildpack.go @@ -27,36 +27,30 @@ type License struct { URI string `toml:"uri" json:"uri"` } -func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils BuildUtils) (Order, error) { +func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils BuildUtils) error { if dockerCreds != "" { os.Setenv("DOCKER_CONFIG", filepath.Dir(dockerCreds)) } - var orderEntry OrderEntry - order := Order{ - Utils: utils, - } - err := utils.MkdirAll(bpCacheDir, os.ModePerm) if err != nil { - return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache") + return errors.Wrap(err, "failed to create temp directory for buildpack cache") } for _, bpack := range bpacks { - var bpackMeta BuildPackMetadata imageInfo, err := utils.GetRemoteImageInfo(bpack) if err != nil { - return Order{}, errors.Wrap(err, "failed to get remote image info of buildpack") + return errors.Wrap(err, "failed to get remote image info of buildpack") } hash, err := imageInfo.Digest() if err != nil { - return Order{}, errors.Wrap(err, "failed to get image digest") + return errors.Wrap(err, "failed to get image digest") } cacheDir := filepath.Join(bpCacheDir, hash.String()) cacheExists, err := utils.DirExists(cacheDir) if err != nil { - return Order{}, errors.Wrapf(err, "failed to check if cache dir '%s' exists", cacheDir) + return errors.Wrapf(err, "failed to check if cache dir '%s' exists", cacheDir) } if cacheExists { @@ -64,36 +58,83 @@ func DownloadBuildpacks(path string, bpacks []string, dockerCreds string, utils } else { err := utils.MkdirAll(cacheDir, os.ModePerm) if err != nil { - return Order{}, errors.Wrap(err, "failed to create temp directory for buildpack cache") + return errors.Wrap(err, "failed to create temp directory for buildpack cache") } log.Entry().Infof("Downloading buildpack '%s' to %s", bpack, cacheDir) - img, err := utils.DownloadImageContent(bpack, cacheDir) + _, err = utils.DownloadImageContent(bpack, cacheDir) if err != nil { - return Order{}, errors.Wrapf(err, "failed download buildpack image '%s'", bpack) + return errors.Wrapf(err, "failed download buildpack image '%s'", bpack) } - imageInfo = img + } + + matches, err := utils.Glob(filepath.Join(cacheDir, "cnb/buildpacks/*")) + if err != nil { + return err + } + + for _, match := range matches { + err = CreateVersionSymlinks(path, match, utils) + if err != nil { + return err + } + } + } + + return nil +} + +func GetMetadata(bpacks []string, utils BuildUtils) ([]BuildPackMetadata, error) { + var metadata []BuildPackMetadata + + for _, bpack := range bpacks { + var bpackMeta BuildPackMetadata + imageInfo, err := utils.GetRemoteImageInfo(bpack) + if err != nil { + return nil, err } imgConf, err := imageInfo.ConfigFile() if err != nil { - return Order{}, errors.Wrapf(err, "failed to read '%s' image config", bpack) + return nil, errors.Wrapf(err, "failed to read '%s' image config", bpack) } err = json.Unmarshal([]byte(imgConf.Config.Labels["io.buildpacks.buildpackage.metadata"]), &bpackMeta) if err != nil { - return Order{}, errors.Wrapf(err, "failed unmarshal '%s' image label", bpack) + return nil, err } - log.Entry().Debugf("Buildpack metadata: '%v'", bpackMeta) - orderEntry.Group = append(orderEntry.Group, bpackMeta) + metadata = append(metadata, bpackMeta) + } + + return metadata, nil +} + +func CreateVersionSymlinks(basePath, buildpackDir string, utils BuildUtils) error { + newBuildpackPath := filepath.Join(basePath, filepath.Base(buildpackDir)) + err := utils.MkdirAll(newBuildpackPath, os.ModePerm) + if err != nil { + return err + } - err = CopyProject(filepath.Join(cacheDir, "cnb/buildpacks"), path, nil, nil, utils) + versions, err := utils.Glob(filepath.Join(buildpackDir, "*")) + if err != nil { + return err + } + + for _, version := range versions { + newVersionPath := filepath.Join(newBuildpackPath, filepath.Base(version)) + exists, err := utils.DirExists(newVersionPath) if err != nil { - return Order{}, err + return err } - } - order.Order = []OrderEntry{orderEntry} + if !exists { + err = utils.Symlink(version, newVersionPath) + if err != nil { + return err + } + } + } - return order, nil + return nil } diff --git a/pkg/cnbutils/buildpack_test.go b/pkg/cnbutils/buildpack_test.go index a480272716..af9a694b39 100644 --- a/pkg/cnbutils/buildpack_test.go +++ b/pkg/cnbutils/buildpack_test.go @@ -4,11 +4,13 @@ package cnbutils_test import ( + "fmt" "testing" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/mock" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/fake" fakeImage "github.com/google/go-containerregistry/pkg/v1/fake" "github.com/stretchr/testify/assert" ) @@ -20,8 +22,9 @@ func TestBuildpackDownload(t *testing.T) { DownloadMock: &mock.DownloadMock{}, } - t.Run("it creates an order object", func(t *testing.T) { + t.Run("successfully downloads a buildpack", func(t *testing.T) { fakeImg := &fakeImage.FakeImage{} + fakeImg.DigestReturns(v1.NewHash("sha256:2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824")) fakeImg.ConfigFileReturns(&v1.ConfigFile{ Config: v1.Config{ Labels: map[string]string{ @@ -32,9 +35,50 @@ func TestBuildpackDownload(t *testing.T) { mockUtils.ReturnImage = fakeImg mockUtils.RemoteImageInfo = fakeImg - order, err := cnbutils.DownloadBuildpacks("/destination", []string{"buildpack"}, "/tmp/config.json", mockUtils) + err := cnbutils.DownloadBuildpacks("/destination", []string{"buildpack"}, "/tmp/config.json", mockUtils) + assert.NoError(t, err) + }) +} + +func TestGetMetadata(t *testing.T) { + var mockUtils = &cnbutils.MockUtils{ + ExecMockRunner: &mock.ExecMockRunner{}, + FilesMock: &mock.FilesMock{}, + DownloadMock: &mock.DownloadMock{ + ImageInfoStub: func(imageRef string) (v1.Image, error) { + return &fake.FakeImage{ + ConfigFileStub: func() (*v1.ConfigFile, error) { + return &v1.ConfigFile{ + Config: v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": fmt.Sprintf("{\"id\": \"%s\", \"version\": \"0.0.1\"}", imageRef), + }, + }, + }, nil + }, + }, nil + }, + }, + } + t.Run("returns empty metadata", func(t *testing.T) { + meta, err := cnbutils.GetMetadata(nil, mockUtils) assert.NoError(t, err) - assert.Equal(t, 1, len(order.Order)) + assert.Empty(t, meta) + }) + + t.Run("returns metadata of the provided buildpacks", func(t *testing.T) { + meta, err := cnbutils.GetMetadata([]string{"buildpack1", "buildpack2"}, mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.BuildPackMetadata{ + { + ID: "buildpack1", + Version: "0.0.1", + }, + { + ID: "buildpack2", + Version: "0.0.1", + }, + }, meta) }) } diff --git a/pkg/cnbutils/order.go b/pkg/cnbutils/order.go index 8c9213e23c..a7f9d0dd1d 100644 --- a/pkg/cnbutils/order.go +++ b/pkg/cnbutils/order.go @@ -2,10 +2,14 @@ package cnbutils import ( "bytes" + "os" + "path/filepath" - "github.com/pelletier/go-toml" + "github.com/BurntSushi/toml" ) +const DefaultOrderPath = "/cnb/order.toml" + type Order struct { Order []OrderEntry `toml:"order"` Utils BuildUtils `toml:"-"` @@ -30,3 +34,73 @@ func (o Order) Save(path string) error { return nil } + +func loadExistingOrder(utils BuildUtils) (Order, error) { + order := Order{ + Utils: utils, + } + + orderReader, err := utils.Open(DefaultOrderPath) + if err != nil { + return Order{}, err + } + defer orderReader.Close() + + _, err = toml.NewDecoder(orderReader).Decode(&order) + if err != nil { + return Order{}, err + } + + return order, nil +} + +func newOrder(bpacks []string, utils BuildUtils) (Order, error) { + buildpacksMeta, err := GetMetadata(bpacks, utils) + if err != nil { + return Order{}, err + } + + return Order{ + Utils: utils, + Order: []OrderEntry{{ + Group: buildpacksMeta, + }}, + }, nil +} + +func CreateOrder(bpacks, preBpacks, postBpacks []string, dockerCreds string, utils BuildUtils) (Order, error) { + if dockerCreds != "" { + os.Setenv("DOCKER_CONFIG", filepath.Dir(dockerCreds)) + } + + var order Order + var err error + if len(bpacks) == 0 { + order, err = loadExistingOrder(utils) + if err != nil { + return Order{}, err + } + } else { + order, err = newOrder(bpacks, utils) + if err != nil { + return Order{}, err + } + } + + for idx := range order.Order { + preMetadata, err := GetMetadata(preBpacks, utils) + if err != nil { + return Order{}, err + } + + postMetadata, err := GetMetadata(postBpacks, utils) + if err != nil { + return Order{}, err + } + + order.Order[idx].Group = append(preMetadata, order.Order[idx].Group...) + order.Order[idx].Group = append(order.Order[idx].Group, postMetadata...) + } + + return order, nil +} diff --git a/pkg/cnbutils/order_test.go b/pkg/cnbutils/order_test.go index fa75e1170b..3b6968428e 100644 --- a/pkg/cnbutils/order_test.go +++ b/pkg/cnbutils/order_test.go @@ -9,6 +9,8 @@ import ( "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/mock" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/fake" "github.com/stretchr/testify/assert" ) @@ -43,7 +45,7 @@ func TestOrderSave(t *testing.T) { assert.True(t, mockUtils.HasWrittenFile("/tmp/order.toml")) result, err := mockUtils.FileRead("/tmp/order.toml") assert.NoError(t, err) - assert.Equal(t, "\n[[order]]\n\n [[order.group]]\n id = \"paketo-buildpacks/sap-machine\"\n version = \"1.1.1\"\n\n [[order.group]]\n id = \"paketo-buildpacks/java\"\n version = \"2.2.2\"\n", string(result)) + assert.Equal(t, "[[order]]\n\n [[order.group]]\n id = \"paketo-buildpacks/sap-machine\"\n version = \"1.1.1\"\n\n [[order.group]]\n id = \"paketo-buildpacks/java\"\n version = \"2.2.2\"\n", string(result)) }) t.Run("raises an error if unable to write the file", func(t *testing.T) { @@ -64,3 +66,155 @@ func TestOrderSave(t *testing.T) { assert.False(t, mockUtils.HasWrittenFile("/tmp/order.toml")) }) } + +func TestCreateOrder(t *testing.T) { + imageStub := func(imageRef, target string) (v1.Image, error) { + fakeImage := &fake.FakeImage{} + var imageConfig v1.Config + switch imageRef { + case "pre-buildpack": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"pre-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + case "post-buildpack": + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"post-testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + default: + imageConfig = v1.Config{ + Labels: map[string]string{ + "io.buildpacks.buildpackage.metadata": "{\"id\": \"testbuildpack\", \"version\": \"0.0.1\"}", + }, + } + } + + fakeImage.ConfigFileReturns(&v1.ConfigFile{ + Config: imageConfig, + }, nil) + + return fakeImage, nil + } + + mockUtils := &cnbutils.MockUtils{ + FilesMock: &mock.FilesMock{}, + DownloadMock: &mock.DownloadMock{ + ImageContentStub: imageStub, + ImageInfoStub: func(imageRef string) (v1.Image, error) { + return imageStub(imageRef, "") + }, + }, + } + + mockUtils.AddFile(cnbutils.DefaultOrderPath, []byte(`[[order]] + [[order.group]] + id = "buildpacks/java" + version = "1.8.0" +[[order]] + [[order.group]] + id = "buildpacks/nodejs" + version = "1.6.0"`)) + + t.Run("successfully loads baked in order.toml", func(t *testing.T) { + order, err := cnbutils.CreateOrder(nil, nil, nil, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "buildpacks/java", + Version: "1.8.0", + }, + }, + }, + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "buildpacks/nodejs", + Version: "1.6.0", + }, + }, + }, + }, order.Order) + }) + + t.Run("successfully loads baked in order.toml and adds pre/post buildpacks", func(t *testing.T) { + order, err := cnbutils.CreateOrder(nil, []string{"pre-buildpack"}, []string{"post-buildpack"}, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "pre-testbuildpack", + Version: "0.0.1", + }, + { + ID: "buildpacks/java", + Version: "1.8.0", + }, + { + ID: "post-testbuildpack", + Version: "0.0.1", + }, + }, + }, + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "pre-testbuildpack", + Version: "0.0.1", + }, + { + ID: "buildpacks/nodejs", + Version: "1.6.0", + }, + { + ID: "post-testbuildpack", + Version: "0.0.1", + }, + }, + }, + }, order.Order) + }) + + t.Run("successfully creates new order with custom buildpacks", func(t *testing.T) { + order, err := cnbutils.CreateOrder([]string{"testbuildpack"}, nil, nil, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "testbuildpack", + Version: "0.0.1", + }, + }, + }, + }, order.Order) + }) + + t.Run("successfully creates new order with custom buildpacks and adds pre/post buildpacks", func(t *testing.T) { + order, err := cnbutils.CreateOrder([]string{"testbuildpack"}, []string{"pre-buildpack"}, []string{"post-buildpack"}, "", mockUtils) + assert.NoError(t, err) + assert.Equal(t, []cnbutils.OrderEntry{ + { + Group: []cnbutils.BuildPackMetadata{ + { + ID: "pre-testbuildpack", + Version: "0.0.1", + }, + { + ID: "testbuildpack", + Version: "0.0.1", + }, + { + ID: "post-testbuildpack", + Version: "0.0.1", + }, + }, + }, + }, order.Order) + }) +} diff --git a/pkg/cnbutils/project/descriptor.go b/pkg/cnbutils/project/descriptor.go index 71a544d6a6..620b27c79c 100644 --- a/pkg/cnbutils/project/descriptor.go +++ b/pkg/cnbutils/project/descriptor.go @@ -2,56 +2,40 @@ package project import ( - "errors" + "github.com/pkg/errors" + "github.com/BurntSushi/toml" "github.com/SAP/jenkins-library/pkg/cnbutils" + "github.com/SAP/jenkins-library/pkg/cnbutils/project/types" + v01 "github.com/SAP/jenkins-library/pkg/cnbutils/project/v01" + v02 "github.com/SAP/jenkins-library/pkg/cnbutils/project/v02" "github.com/SAP/jenkins-library/pkg/cnbutils/registry" piperhttp "github.com/SAP/jenkins-library/pkg/http" "github.com/SAP/jenkins-library/pkg/log" - "github.com/pelletier/go-toml" ignore "github.com/sabhiram/go-gitignore" ) -type script struct { - API string `toml:"api"` - Inline string `toml:"inline"` - Shell string `toml:"shell"` -} -type buildpack struct { - ID string `toml:"id"` - Version string `toml:"version"` - URI string `toml:"uri"` - Script script `toml:"script"` -} - -type envVar struct { - Name string `toml:"name"` - Value string `toml:"value"` +type project struct { + Version string `toml:"schema-version"` } -type build struct { - Include []string `toml:"include"` - Exclude []string `toml:"exclude"` - Buildpacks []buildpack `toml:"buildpacks"` - Env []envVar `toml:"env"` +type versionDescriptor struct { + Project project `toml:"_"` } -type project struct { - ID string `toml:"id"` -} - -type projectDescriptor struct { - Build build `toml:"build"` - Project project `toml:"project"` - Metadata map[string]interface{} `toml:"metadata"` +var parsers = map[string]func(string) (types.Descriptor, error){ + "0.1": v01.NewDescriptor, + "0.2": v02.NewDescriptor, } type Descriptor struct { - Exclude *ignore.GitIgnore - Include *ignore.GitIgnore - EnvVars map[string]interface{} - Buildpacks []string - ProjectID string + Exclude *ignore.GitIgnore + Include *ignore.GitIgnore + EnvVars map[string]interface{} + Buildpacks []string + PreBuildpacks []string + PostBuildpacks []string + ProjectID string } func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClient piperhttp.Sender) (*Descriptor, error) { @@ -62,23 +46,45 @@ func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClien return nil, err } - rawDescriptor := projectDescriptor{} - err = toml.Unmarshal(descriptorContent, &rawDescriptor) + var versionDescriptor versionDescriptor + _, err = toml.Decode(string(descriptorContent), &versionDescriptor) if err != nil { - return nil, err + return &Descriptor{}, errors.Wrapf(err, "parsing schema version") + } + + version := versionDescriptor.Project.Version + if version == "" { + version = "0.1" + } + + rawDescriptor, err := parsers[version](string(descriptorContent)) + if err != nil { + return &Descriptor{}, err } - if rawDescriptor.Build.Buildpacks != nil && len(rawDescriptor.Build.Buildpacks) > 0 { - buildpacksImg, err := rawDescriptor.Build.searchBuildpacks(httpClient) + if len(rawDescriptor.Build.Buildpacks) > 0 { + descriptor.Buildpacks, err = searchBuildpacks(rawDescriptor.Build.Buildpacks, httpClient) if err != nil { return nil, err } + } - descriptor.Buildpacks = buildpacksImg + if len(rawDescriptor.Build.Pre.Buildpacks) > 0 { + descriptor.PreBuildpacks, err = searchBuildpacks(rawDescriptor.Build.Pre.Buildpacks, httpClient) + if err != nil { + return nil, err + } + } + + if len(rawDescriptor.Build.Post.Buildpacks) > 0 { + descriptor.PostBuildpacks, err = searchBuildpacks(rawDescriptor.Build.Post.Buildpacks, httpClient) + if err != nil { + return nil, err + } } - if rawDescriptor.Build.Env != nil && len(rawDescriptor.Build.Env) > 0 { - descriptor.EnvVars = rawDescriptor.Build.envToMap() + if len(rawDescriptor.Build.Env) > 0 { + descriptor.EnvVars = envToMap(rawDescriptor.Build.Env) } if len(rawDescriptor.Build.Exclude) > 0 && len(rawDescriptor.Build.Include) > 0 { @@ -100,10 +106,10 @@ func ParseDescriptor(descriptorPath string, utils cnbutils.BuildUtils, httpClien return descriptor, nil } -func (b *build) envToMap() map[string]interface{} { +func envToMap(env []types.EnvVar) map[string]interface{} { envMap := map[string]interface{}{} - for _, e := range b.Env { + for _, e := range env { if len(e.Name) == 0 { continue } @@ -114,11 +120,11 @@ func (b *build) envToMap() map[string]interface{} { return envMap } -func (b *build) searchBuildpacks(httpClient piperhttp.Sender) ([]string, error) { +func searchBuildpacks(buildpacks []types.Buildpack, httpClient piperhttp.Sender) ([]string, error) { var bpackImg []string - for _, bpack := range b.Buildpacks { - if bpack.Script != (script{}) { + for _, bpack := range buildpacks { + if bpack.Script != (types.Script{}) { return nil, errors.New("inline buildpacks are not supported") } diff --git a/pkg/cnbutils/project/descriptor_test.go b/pkg/cnbutils/project/descriptor_test.go index 88cdbe17bb..33d03cb137 100644 --- a/pkg/cnbutils/project/descriptor_test.go +++ b/pkg/cnbutils/project/descriptor_test.go @@ -16,7 +16,7 @@ import ( ) func TestParseDescriptor(t *testing.T) { - t.Run("parses the project.toml file", func(t *testing.T) { + t.Run("parses the project.toml file v01", func(t *testing.T) { projectToml := `[project] id = "io.buildpacks.my-app" version = "0.1" @@ -41,6 +41,14 @@ value = "VAL2" name = "EMPTY" value = "" +[[build.pre.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[build.post.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + [[build.buildpacks]] id = "paketo-buildpacks/java" version = "5.9.1" @@ -78,6 +86,94 @@ id = "paketo-buildpacks/nodejs" assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-java@5.9.1") assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-nodejs@1.1.1") + assert.Contains(t, descriptor.PreBuildpacks, "index.docker.io/test-java@5.9.1") + assert.Contains(t, descriptor.PostBuildpacks, "index.docker.io/test-java@5.9.1") + + assert.NotNil(t, descriptor.Include) + + t3 := descriptor.Include.MatchesPath("cmd/cobra.go") + assert.True(t, t3) + + t4 := descriptor.Include.MatchesPath("pkg/test/main.go") + assert.True(t, t4) + + t5 := descriptor.Include.MatchesPath("Makefile") + assert.False(t, t5) + }) + + t.Run("parses the project.toml file v02", func(t *testing.T) { + projectToml := `[_] +id = "io.buildpacks.my-app" +version = "0.1" +schema-version = "0.2" + +[io.buildpacks] +include = [ + "cmd/", + "go.mod", + "go.sum", + "*.go" +] + +[[io.buildpacks.build.env]] +name = "VAR1" +value = "VAL1" + +[[io.buildpacks.build.env]] +name = "VAR2" +value = "VAL2" + +[[io.buildpacks.build.env]] +name = "EMPTY" +value = "" + +[[io.buildpacks.pre.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[io.buildpacks.post.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[io.buildpacks.group]] +id = "paketo-buildpacks/java" +version = "5.9.1" + +[[io.buildpacks.group]] +id = "paketo-buildpacks/nodejs" +` + utils := &cnbutils.MockUtils{ + FilesMock: &mock.FilesMock{}, + } + + fakeJavaResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"5.9.1\",\"_link\":\"https://test-java/5.9.1\"}]}" + fakeNodeJsResponse := "{\"latest\":{\"version\":\"1.1.1\",\"namespace\":\"test\",\"name\":\"test\",\"description\":\"\",\"homepage\":\"\",\"licenses\":null,\"stacks\":[\"test\",\"test\"],\"id\":\"test\"},\"versions\":[{\"version\":\"1.1.1\",\"_link\":\"https://test-nodejs/1.1.1\"}]}" + + utils.AddFile("project.toml", []byte(projectToml)) + httpmock.Activate() + defer httpmock.DeactivateAndReset() + httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/paketo-buildpacks/java", httpmock.NewStringResponder(200, fakeJavaResponse)) + httpmock.RegisterResponder(http.MethodGet, "https://registry.buildpacks.io/api/v1/buildpacks/paketo-buildpacks/nodejs", httpmock.NewStringResponder(200, fakeNodeJsResponse)) + + httpmock.RegisterResponder(http.MethodGet, "https://test-java/5.9.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test-java@5.9.1\"}")) + httpmock.RegisterResponder(http.MethodGet, "https://test-nodejs/1.1.1", httpmock.NewStringResponder(200, "{\"addr\": \"index.docker.io/test-nodejs@1.1.1\"}")) + client := &piperhttp.Client{} + client.SetOptions(piperhttp.ClientOptions{MaxRetries: -1, UseDefaultTransport: true}) + + descriptor, err := ParseDescriptor("project.toml", utils, client) + + assert.NoError(t, err) + assert.Equal(t, "VAL1", descriptor.EnvVars["VAR1"]) + assert.Equal(t, "VAL2", descriptor.EnvVars["VAR2"]) + assert.Equal(t, "", descriptor.EnvVars["EMPTY"]) + + assert.Equal(t, "io.buildpacks.my-app", descriptor.ProjectID) + + assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-java@5.9.1") + assert.Contains(t, descriptor.Buildpacks, "index.docker.io/test-nodejs@1.1.1") + assert.Contains(t, descriptor.PreBuildpacks, "index.docker.io/test-java@5.9.1") + assert.Contains(t, descriptor.PostBuildpacks, "index.docker.io/test-java@5.9.1") + assert.NotNil(t, descriptor.Include) t3 := descriptor.Include.MatchesPath("cmd/cobra.go") @@ -160,6 +256,6 @@ exclude = [ _, err := ParseDescriptor("project.toml", utils, &piperhttp.Client{}) assert.Error(t, err) - assert.Equal(t, "(1, 8): was expecting token =, but got EOF instead", err.Error()) + assert.Equal(t, "parsing schema version: toml: line 0: unexpected EOF; expected key separator '='", err.Error()) }) } diff --git a/pkg/cnbutils/project/metadata/metadata.go b/pkg/cnbutils/project/metadata/metadata.go index 167469adac..abbb3498b4 100644 --- a/pkg/cnbutils/project/metadata/metadata.go +++ b/pkg/cnbutils/project/metadata/metadata.go @@ -5,11 +5,11 @@ import ( "bytes" "path/filepath" + "github.com/BurntSushi/toml" "github.com/SAP/jenkins-library/pkg/cnbutils" "github.com/SAP/jenkins-library/pkg/log" "github.com/SAP/jenkins-library/pkg/piperenv" "github.com/buildpacks/lifecycle/platform" - "github.com/pelletier/go-toml" ) var metadataFilePath = "/layers/project-metadata.toml" diff --git a/pkg/cnbutils/project/metadata/metadata_test.go b/pkg/cnbutils/project/metadata/metadata_test.go index 6310d8067b..cf36c9eaaa 100644 --- a/pkg/cnbutils/project/metadata/metadata_test.go +++ b/pkg/cnbutils/project/metadata/metadata_test.go @@ -15,16 +15,13 @@ import ( ) func TestWriteProjectMetadata(t *testing.T) { - expectedResult := ` -[source] + expectedResult := `[source] type = "git" - - [source.metadata] - refs = ["main"] - [source.version] commit = "012548" describe = "test-commit" + [source.metadata] + refs = ["main"] ` mockUtils := &cnbutils.MockUtils{ ExecMockRunner: &mock.ExecMockRunner{}, diff --git a/pkg/cnbutils/project/types/types.go b/pkg/cnbutils/project/types/types.go new file mode 100644 index 0000000000..a4f9521392 --- /dev/null +++ b/pkg/cnbutils/project/types/types.go @@ -0,0 +1,58 @@ +// Source: https://github.com/buildpacks/pack/blob/main/pkg/project/types/types.go +package types + +import ( + "github.com/buildpacks/lifecycle/api" +) + +type Script struct { + API string `toml:"api"` + Inline string `toml:"inline"` + Shell string `toml:"shell"` +} + +type Buildpack struct { + ID string `toml:"id"` + Version string `toml:"version"` + URI string `toml:"uri"` + Script Script `toml:"script"` +} + +type EnvVar struct { + Name string `toml:"name"` + Value string `toml:"value"` +} + +type Build struct { + Include []string `toml:"include"` + Exclude []string `toml:"exclude"` + Buildpacks []Buildpack `toml:"buildpacks"` + Env []EnvVar `toml:"env"` + Builder string `toml:"builder"` + Pre GroupAddition + Post GroupAddition +} + +type Project struct { + ID string `toml:"id"` + Name string `toml:"name"` + Version string `toml:"version"` + SourceURL string `toml:"source-url"` + Licenses []License `toml:"licenses"` +} + +type License struct { + Type string `toml:"type"` + URI string `toml:"uri"` +} + +type Descriptor struct { + Project Project `toml:"project"` + Build Build `toml:"build"` + Metadata map[string]interface{} `toml:"metadata"` + SchemaVersion *api.Version +} + +type GroupAddition struct { + Buildpacks []Buildpack `toml:"group"` +} diff --git a/pkg/cnbutils/project/v01/project.go b/pkg/cnbutils/project/v01/project.go new file mode 100644 index 0000000000..da41b98ad5 --- /dev/null +++ b/pkg/cnbutils/project/v01/project.go @@ -0,0 +1,30 @@ +// Source: https://github.com/buildpacks/pack/blob/main/pkg/project/v01/project.go +package v01 + +import ( + "github.com/BurntSushi/toml" + "github.com/SAP/jenkins-library/pkg/cnbutils/project/types" + "github.com/buildpacks/lifecycle/api" +) + +type Descriptor struct { + Project types.Project `toml:"project"` + Build types.Build `toml:"build"` + Metadata map[string]interface{} `toml:"metadata"` +} + +func NewDescriptor(projectTomlContents string) (types.Descriptor, error) { + versionedDescriptor := &Descriptor{} + + _, err := toml.Decode(projectTomlContents, versionedDescriptor) + if err != nil { + return types.Descriptor{}, err + } + + return types.Descriptor{ + Project: versionedDescriptor.Project, + Build: versionedDescriptor.Build, + Metadata: versionedDescriptor.Metadata, + SchemaVersion: api.MustParse("0.1"), + }, nil +} diff --git a/pkg/cnbutils/project/v02/project.go b/pkg/cnbutils/project/v02/project.go new file mode 100644 index 0000000000..96bf3c6861 --- /dev/null +++ b/pkg/cnbutils/project/v02/project.go @@ -0,0 +1,78 @@ +// Source: https://github.com/buildpacks/pack/blob/main/pkg/project/v02/project.go +package v02 + +import ( + "github.com/BurntSushi/toml" + "github.com/SAP/jenkins-library/pkg/cnbutils/project/types" + "github.com/buildpacks/lifecycle/api" +) + +type Buildpacks struct { + Include []string `toml:"include"` + Exclude []string `toml:"exclude"` + Group []types.Buildpack `toml:"group"` + Env Env `toml:"env"` + Build Build `toml:"build"` + Builder string `toml:"builder"` + Pre types.GroupAddition `toml:"pre"` + Post types.GroupAddition `toml:"post"` +} + +type Build struct { + Env []types.EnvVar `toml:"env"` +} + +// Env is deprecated: use `[[io.buildpacks.build.env]]` instead. see https://github.com/buildpacks/pack/pull/1479 +type Env struct { + Build []types.EnvVar `toml:"build"` +} + +type Project struct { + ID string `toml:"id"` + Name string `toml:"name"` + Licenses []types.License `toml:"licenses"` + Metadata map[string]interface{} `toml:"metadata"` + SchemaVersion string `toml:"schema-version"` +} + +type IO struct { + Buildpacks Buildpacks `toml:"buildpacks"` +} + +type Descriptor struct { + Project Project `toml:"_"` + IO IO `toml:"io"` +} + +func NewDescriptor(projectTomlContents string) (types.Descriptor, error) { + versionedDescriptor := &Descriptor{} + _, err := toml.Decode(projectTomlContents, &versionedDescriptor) + if err != nil { + return types.Descriptor{}, err + } + + // backward compatibility for incorrect key + env := versionedDescriptor.IO.Buildpacks.Build.Env + if env == nil { + env = versionedDescriptor.IO.Buildpacks.Env.Build + } + + return types.Descriptor{ + Project: types.Project{ + ID: versionedDescriptor.Project.ID, + Name: versionedDescriptor.Project.Name, + Licenses: versionedDescriptor.Project.Licenses, + }, + Build: types.Build{ + Include: versionedDescriptor.IO.Buildpacks.Include, + Exclude: versionedDescriptor.IO.Buildpacks.Exclude, + Buildpacks: versionedDescriptor.IO.Buildpacks.Group, + Env: env, + Builder: versionedDescriptor.IO.Buildpacks.Builder, + Pre: versionedDescriptor.IO.Buildpacks.Pre, + Post: versionedDescriptor.IO.Buildpacks.Post, + }, + Metadata: versionedDescriptor.Project.Metadata, + SchemaVersion: api.MustParse("0.2"), + }, nil +} diff --git a/pkg/cnbutils/report.go b/pkg/cnbutils/report.go index 0f5e99f491..8b0fe37280 100644 --- a/pkg/cnbutils/report.go +++ b/pkg/cnbutils/report.go @@ -3,8 +3,8 @@ package cnbutils import ( "fmt" + "github.com/BurntSushi/toml" "github.com/buildpacks/lifecycle/platform" - "github.com/pelletier/go-toml" ) const reportFile = "/layers/report.toml" diff --git a/pkg/cnbutils/report_test.go b/pkg/cnbutils/report_test.go index c82eee561e..4ac2711099 100644 --- a/pkg/cnbutils/report_test.go +++ b/pkg/cnbutils/report_test.go @@ -46,6 +46,6 @@ digest = "sha256:52eac630560210e5ae13eb10797c4246d6f02d425f32b9430ca00bde697c79e digest, err := cnbutils.DigestFromReport(mockUtils) assert.Empty(t, digest) - assert.EqualError(t, err, "(1, 1): parsing error: keys cannot contain { character") + assert.EqualError(t, err, "toml: line 1: expected '.' or '=', but got '{' instead") }) } diff --git a/pkg/mock/dockerClient.go b/pkg/mock/dockerClient.go index f2fb0689c8..2f1f972715 100644 --- a/pkg/mock/dockerClient.go +++ b/pkg/mock/dockerClient.go @@ -17,7 +17,9 @@ type DownloadMock struct { RemoteImageInfo v1.Image ReturnError string - Stub func(imageRef, targetDir string) (v1.Image, error) + Stub func(imageRef, targetDir string) (v1.Image, error) + ImageContentStub func(imageRef, targetFile string) (v1.Image, error) + ImageInfoStub func(imageRef string) (v1.Image, error) } // DownloadImage . @@ -40,6 +42,10 @@ func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Ima c.ImageRef = imageRef c.FilePath = targetFile + if c.ImageContentStub != nil { + return c.ImageContentStub(imageRef, targetFile) + } + if len(c.ReturnError) > 0 { return nil, fmt.Errorf(c.ReturnError) } @@ -50,6 +56,10 @@ func (c *DownloadMock) DownloadImageContent(imageRef, targetFile string) (v1.Ima func (c *DownloadMock) GetRemoteImageInfo(imageRef string) (v1.Image, error) { c.RemoteImageRef = imageRef + if c.ImageInfoStub != nil { + return c.ImageInfoStub(imageRef) + } + if len(c.ReturnError) > 0 { return nil, fmt.Errorf(c.ReturnError) } diff --git a/resources/metadata/cnbBuild.yaml b/resources/metadata/cnbBuild.yaml index 9fe08524f7..ffd5c44fb8 100644 --- a/resources/metadata/cnbBuild.yaml +++ b/resources/metadata/cnbBuild.yaml @@ -97,7 +97,7 @@ spec: param: container/registryUrl - name: buildpacks type: "[]string" - description: List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. + description: List of custom buildpacks to use in the form of `$HOSTNAME/$REPO[:$TAG]`. When this property is specified, buildpacks which are part of the builder will be ignored. scope: - PARAMETERS - STAGES @@ -105,6 +105,26 @@ spec: resourceRef: - name: commonPipelineEnvironment param: container/buildpacks + - name: preBuildpacks + type: "[]string" + description: Buildpacks to prepend to the groups in the builder's order. + scope: + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/preBuildpacks + - name: postBuildpacks + type: "[]string" + description: Buildpacks to append to the groups in the builder's order. + scope: + - PARAMETERS + - STAGES + - STEPS + resourceRef: + - name: commonPipelineEnvironment + param: container/postBuildpacks - name: buildEnvVars type: "map[string]interface{}" description: |