From f24d3bc1aed9d433f9a34d783d5e4120e99959ac Mon Sep 17 00:00:00 2001 From: Juan Bustamante Date: Tue, 6 Feb 2024 11:08:18 -0500 Subject: [PATCH 1/2] Fixing an parsing error with the buildpacks to be flattened Using StringArrayVar Signed-off-by: Juan Bustamante --- acceptance/acceptance_test.go | 201 ++++++++++++++++++++++++++++ acceptance/invoke/pack.go | 11 ++ internal/commands/builder_create.go | 2 +- 3 files changed, 213 insertions(+), 1 deletion(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6990926988..2895142c32 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -2929,6 +2929,56 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ] }) }) }) + + when("builder create", func() { + when("--flatten=", func() { + it("should flatten together all specified buildpacks", func() { + h.SkipIf(t, createBuilderPack.SupportsFeature(invoke.SkipFlattenBuilderCreation), "pack version <= 0.33.0 fails with this test") + h.SkipIf(t, imageManager.HostOS() == "windows", "These tests are not yet compatible with Windows-based containers") + + // create a task, handled by a 'task manager' which executes our pack commands during tests. + // looks like this is used to de-dup tasks + key := taskKey( + "create-complex-flattened-builder", + append( + []string{runImageMirror, createBuilderPackConfig.Path(), lifecycle.Identifier()}, + createBuilderPackConfig.FixturePaths()..., + )..., + ) + + builderName, err := suiteManager.RunTaskOnceString(key, func() (string, error) { + return createFlattenBuilder(t, + assert, + buildpackManager, + lifecycle, + createBuilderPack, + runImageMirror) + }) + assert.Nil(err) + + // register task to be run to 'clean up' a task + suiteManager.RegisterCleanUp("clean-"+key, func() error { + imageManager.CleanupImages(builderName) + return nil + }) + + assertImage.ExistsLocally(builderName) + + // 3 layers for runtime OS + // 1 layer setting cnb, platform, layers folders + // 1 layer for lifecycle binaries + // 1 layer for order.toml + // 1 layer for run.toml + // 1 layer for stack.toml + // 1 layer status file changed + // Base Layers = 9 + + // 1 layer for 3 flattened builpacks + // 3 layers for single buildpacks not flattened + assertImage.HasLengthLayers(builderName, 13) + }) + }) + }) }) } @@ -3315,6 +3365,157 @@ func createStackImage(dockerCli client.CommonAPIClient, repoName string, dir str })) } +func createFlattenBuilder( + t *testing.T, + assert h.AssertionManager, + buildpackManager buildpacks.BuildModuleManager, + lifecycle config.LifecycleAsset, + pack *invoke.PackInvoker, + runImageMirror string, +) (string, error) { + t.Helper() + t.Log("creating flattened builder image...") + + // CREATE TEMP WORKING DIR + tmpDir, err := os.MkdirTemp("", "create-complex-test-flattened-builder") + if err != nil { + return "", err + } + defer os.RemoveAll(tmpDir) + + // ARCHIVE BUILDPACKS + builderBuildpacks := []buildpacks.TestBuildModule{ + buildpacks.BpNoop, + buildpacks.BpNoop2, + buildpacks.BpOtherStack, + buildpacks.BpReadEnv, + } + + templateMapping := map[string]interface{}{ + "run_image_mirror": runImageMirror, + } + + packageImageName := registryConfig.RepoName("nested-level-1-buildpack-" + h.RandString(8)) + nestedLevelTwoBuildpackName := registryConfig.RepoName("nested-level-2-buildpack-" + h.RandString(8)) + simpleLayersBuildpackName := registryConfig.RepoName("simple-layers-buildpack-" + h.RandString(8)) + simpleLayersBuildpackDifferentShaName := registryConfig.RepoName("simple-layers-buildpack-different-name-" + h.RandString(8)) + + templateMapping["package_id"] = "simple/nested-level-1" + templateMapping["package_image_name"] = packageImageName + templateMapping["nested_level_1_buildpack"] = packageImageName + templateMapping["nested_level_2_buildpack"] = nestedLevelTwoBuildpackName + templateMapping["simple_layers_buildpack"] = simpleLayersBuildpackName + templateMapping["simple_layers_buildpack_different_sha"] = simpleLayersBuildpackDifferentShaName + + fixtureManager := pack.FixtureManager() + + nestedLevelOneConfigFile, err := os.CreateTemp(tmpDir, "nested-level-1-package.toml") + assert.Nil(err) + fixtureManager.TemplateFixtureToFile( + "nested-level-1-buildpack_package.toml", + nestedLevelOneConfigFile, + templateMapping, + ) + err = nestedLevelOneConfigFile.Close() + assert.Nil(err) + + nestedLevelTwoConfigFile, err := os.CreateTemp(tmpDir, "nested-level-2-package.toml") + assert.Nil(err) + fixtureManager.TemplateFixtureToFile( + "nested-level-2-buildpack_package.toml", + nestedLevelTwoConfigFile, + templateMapping, + ) + + err = nestedLevelTwoConfigFile.Close() + assert.Nil(err) + + packageImageBuildpack := buildpacks.NewPackageImage( + t, + pack, + packageImageName, + nestedLevelOneConfigFile.Name(), + buildpacks.WithRequiredBuildpacks( + buildpacks.BpNestedLevelOne, + buildpacks.NewPackageImage( + t, + pack, + nestedLevelTwoBuildpackName, + nestedLevelTwoConfigFile.Name(), + buildpacks.WithRequiredBuildpacks( + buildpacks.BpNestedLevelTwo, + buildpacks.NewPackageImage( + t, + pack, + simpleLayersBuildpackName, + fixtureManager.FixtureLocation("simple-layers-buildpack_package.toml"), + buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayers), + ), + ), + ), + ), + ) + + simpleLayersDifferentShaBuildpack := buildpacks.NewPackageImage( + t, + pack, + simpleLayersBuildpackDifferentShaName, + fixtureManager.FixtureLocation("simple-layers-buildpack-different-sha_package.toml"), + buildpacks.WithRequiredBuildpacks(buildpacks.BpSimpleLayersDifferentSha), + ) + + defer imageManager.CleanupImages(packageImageName, nestedLevelTwoBuildpackName, simpleLayersBuildpackName, simpleLayersBuildpackDifferentShaName) + + builderBuildpacks = append( + builderBuildpacks, + packageImageBuildpack, + simpleLayersDifferentShaBuildpack, + ) + + buildpackManager.PrepareBuildModules(tmpDir, builderBuildpacks...) + + // ADD lifecycle + if lifecycle.HasLocation() { + lifecycleURI := lifecycle.EscapedPath() + t.Logf("adding lifecycle path '%s' to builder config", lifecycleURI) + templateMapping["lifecycle_uri"] = lifecycleURI + } else { + lifecycleVersion := lifecycle.Version() + t.Logf("adding lifecycle version '%s' to builder config", lifecycleVersion) + templateMapping["lifecycle_version"] = lifecycleVersion + } + + // RENDER builder.toml + builderConfigFile, err := os.CreateTemp(tmpDir, "nested_builder.toml") + if err != nil { + return "", err + } + + pack.FixtureManager().TemplateFixtureToFile("nested_builder.toml", builderConfigFile, templateMapping) + + err = builderConfigFile.Close() + if err != nil { + return "", err + } + + // NAME BUILDER + bldr := registryConfig.RepoName("test/flatten-builder-" + h.RandString(10)) + + // CREATE BUILDER + output := pack.RunSuccessfully( + "builder", "create", bldr, + "-c", builderConfigFile.Name(), + "--no-color", + "--verbose", + "--flatten", "read/env@read-env-version,noop.buildpack@noop.buildpack.version,noop.buildpack@noop.buildpack.later-version", + ) + + assert.Contains(output, fmt.Sprintf("Successfully created builder image '%s'", bldr)) + assert.Succeeds(h.PushImage(dockerCli, bldr, registryConfig)) + + return bldr, nil +} + // taskKey creates a key from the prefix and all arguments to be unique func taskKey(prefix string, args ...string) string { hash := sha256.New() diff --git a/acceptance/invoke/pack.go b/acceptance/invoke/pack.go index caa60f136e..b43ef2a3b5 100644 --- a/acceptance/invoke/pack.go +++ b/acceptance/invoke/pack.go @@ -236,6 +236,7 @@ const ( BuildpackFlatten MetaBuildpackFolder PlatformRetries + SkipFlattenBuilderCreation ) var featureTests = map[Feature]func(i *PackInvoker) bool{ @@ -266,6 +267,9 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{ PlatformRetries: func(i *PackInvoker) bool { return i.atLeast("v0.32.1") }, + SkipFlattenBuilderCreation: func(i *PackInvoker) bool { + return i.equal("v0.33.0") + }, } func (i *PackInvoker) SupportsFeature(f Feature) bool { @@ -294,6 +298,13 @@ func (i *PackInvoker) atLeast(version string) bool { return ver.Equal(minimalVersion) || ver.GreaterThan(minimalVersion) || ver.Equal(semver.MustParse("0.0.0")) } +// equal returns true if pack version is the equal to the provided version +func (i *PackInvoker) equal(version string) bool { + minimalVersion := semver.MustParse(version) + ver := i.semanticVersion() + return ver.Equal(minimalVersion) +} + func (i *PackInvoker) ConfigFileContents() string { i.testObject.Helper() diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index 0a591bfed1..56ac071fe2 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -114,7 +114,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha cmd.Flags().StringVarP(&flags.BuilderTomlPath, "config", "c", "", "Path to builder TOML file (required)") cmd.Flags().BoolVar(&flags.Publish, "publish", false, "Publish the builder directly to the container registry specified in , instead of the daemon.") cmd.Flags().StringVar(&flags.Policy, "pull-policy", "", "Pull policy to use. Accepted values are always, never, and if-not-present. The default is always") - cmd.Flags().StringSliceVar(&flags.Flatten, "flatten", nil, "List of buildpacks to flatten together into a single layer (format: '@,@'") + cmd.Flags().StringArrayVar(&flags.Flatten, "flatten", nil, "List of buildpacks to flatten together into a single layer (format: '@,@'") cmd.Flags().StringToStringVarP(&flags.Label, "label", "l", nil, "Labels to add to the builder image, in the form of '='") AddHelpFlag(cmd, "create") From 572f8696f383f0e959775776b25a73633d4a0263 Mon Sep 17 00:00:00 2001 From: Juan Bustamante Date: Tue, 6 Feb 2024 19:05:53 -0500 Subject: [PATCH 2/2] renaming the methods Signed-off-by: Juan Bustamante --- acceptance/acceptance_test.go | 2 +- acceptance/invoke/pack.go | 13 +++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 2895142c32..05ceea2e2f 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -2933,7 +2933,7 @@ include = [ "*.jar", "media/mountain.jpg", "/media/person.png", ] when("builder create", func() { when("--flatten=", func() { it("should flatten together all specified buildpacks", func() { - h.SkipIf(t, createBuilderPack.SupportsFeature(invoke.SkipFlattenBuilderCreation), "pack version <= 0.33.0 fails with this test") + h.SkipIf(t, !createBuilderPack.SupportsFeature(invoke.FlattenBuilderCreationV2), "pack version <= 0.33.0 fails with this test") h.SkipIf(t, imageManager.HostOS() == "windows", "These tests are not yet compatible with Windows-based containers") // create a task, handled by a 'task manager' which executes our pack commands during tests. diff --git a/acceptance/invoke/pack.go b/acceptance/invoke/pack.go index b43ef2a3b5..e10bcdfe26 100644 --- a/acceptance/invoke/pack.go +++ b/acceptance/invoke/pack.go @@ -236,7 +236,7 @@ const ( BuildpackFlatten MetaBuildpackFolder PlatformRetries - SkipFlattenBuilderCreation + FlattenBuilderCreationV2 ) var featureTests = map[Feature]func(i *PackInvoker) bool{ @@ -267,8 +267,8 @@ var featureTests = map[Feature]func(i *PackInvoker) bool{ PlatformRetries: func(i *PackInvoker) bool { return i.atLeast("v0.32.1") }, - SkipFlattenBuilderCreation: func(i *PackInvoker) bool { - return i.equal("v0.33.0") + FlattenBuilderCreationV2: func(i *PackInvoker) bool { + return i.atLeast("v0.33.1") }, } @@ -298,13 +298,6 @@ func (i *PackInvoker) atLeast(version string) bool { return ver.Equal(minimalVersion) || ver.GreaterThan(minimalVersion) || ver.Equal(semver.MustParse("0.0.0")) } -// equal returns true if pack version is the equal to the provided version -func (i *PackInvoker) equal(version string) bool { - minimalVersion := semver.MustParse(version) - ver := i.semanticVersion() - return ver.Equal(minimalVersion) -} - func (i *PackInvoker) ConfigFileContents() string { i.testObject.Helper()