Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

pack build command to export to OCI layout format on disk #1596

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
4d6d39d
WIP - first version
jjbustamante Dec 16, 2022
c9e1e0c
WIP - running code formatting
jjbustamante Jan 17, 2023
99ad545
WIP - fixing lint error
jjbustamante Jan 17, 2023
c6cef8a
WIP - procesing previous image
jjbustamante Jan 18, 2023
a204c11
WIP - fixing linter error
jjbustamante Jan 18, 2023
edc2bde
WIP - I removed the layout-repo volume and now every input (run-image…
jjbustamante Feb 3, 2023
7e5829e
WIP - Pointing to the imgutil version with name.ref annotation
jjbustamante Feb 3, 2023
abc51fe
WIP - fixing linting error
jjbustamante Feb 3, 2023
114e8b1
WIP - Pointing to the imgutil version with name.ref annotation
jjbustamante Feb 3, 2023
87cc8d3
WIP - tagging the image
jjbustamante Feb 6, 2023
256b0dc
root path for win or linux
jjbustamante Feb 17, 2023
4fe9225
adding layout-dir flag according to latest change in the lifecycle
jjbustamante Feb 20, 2023
8f83efc
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Feb 20, 2023
1185676
fixing lint error
jjbustamante Feb 20, 2023
79c7e54
adding unit test coverage for lifecycle_execution.go
jjbustamante Feb 21, 2023
64ea6b7
adding test coverage for build.go
jjbustamante Feb 21, 2023
70c5416
adding test coverage to the configuration files
jjbustamante Feb 22, 2023
d21e0c8
adding test coverage for input image reference
jjbustamante Feb 22, 2023
d3a9532
adding coverage for fetcher
jjbustamante Feb 22, 2023
fe63a00
adding test coverage for build
jjbustamante Feb 23, 2023
ead6d12
renaming some variables as it was before
jjbustamante Feb 23, 2023
2c636a7
fixing validation on previous image
jjbustamante Feb 23, 2023
6918797
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jkutner Feb 25, 2023
8cfab73
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Feb 27, 2023
3c1a801
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Mar 1, 2023
a3aa7fd
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Mar 4, 2023
7761d79
Fixing broken tests
jjbustamante Mar 6, 2023
4bca03f
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Mar 7, 2023
d2f1615
adding support for platform 0.11 and 0.12, fixing acceptance tests
jjbustamante Mar 7, 2023
18e758c
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Mar 7, 2023
5a02b07
Fixing test on windows
jjbustamante Mar 7, 2023
e038d46
Format issue
jjbustamante Mar 7, 2023
4f8cc8a
Merge branch 'main' into enhancement/issue-1548-imagee-in-oci-layout-…
jjbustamante Mar 8, 2023
12c370a
fixing windows tests, some of the tests are skip for now
jjbustamante Mar 8, 2023
fd9dc9e
fixing issue on windows
jjbustamante Mar 8, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions acceptance/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -579,16 +579,21 @@ func testWithoutSpecificBuilderRequirement(
pack.RunSuccessfully("config", "default-builder", "paketobuildpacks/builder:base")

output := pack.RunSuccessfully("report")

version := pack.Version()

layoutRepoDir := filepath.Join(pack.Home(), "layout-repo")
if runtime.GOOS == "windows" {
layoutRepoDir = strings.ReplaceAll(layoutRepoDir, `\`, `\\`)
}

expectedOutput := pack.FixtureManager().TemplateFixture(
"report_output.txt",
map[string]interface{}{
"DefaultBuilder": "[REDACTED]",
"Version": version,
"OS": runtime.GOOS,
"Arch": runtime.GOARCH,
"LayoutRepoDir": layoutRepoDir,
},
)
assert.Equal(output, expectedOutput)
Expand All @@ -598,16 +603,21 @@ func testWithoutSpecificBuilderRequirement(
pack.RunSuccessfully("config", "default-builder", "paketobuildpacks/builder:base")

output := pack.RunSuccessfully("report", "--explicit")

version := pack.Version()

layoutRepoDir := filepath.Join(pack.Home(), "layout-repo")
if runtime.GOOS == "windows" {
layoutRepoDir = strings.ReplaceAll(layoutRepoDir, `\`, `\\`)
}

expectedOutput := pack.FixtureManager().TemplateFixture(
"report_output.txt",
map[string]interface{}{
"DefaultBuilder": "paketobuildpacks/builder:base",
"Version": version,
"OS": runtime.GOOS,
"Arch": runtime.GOARCH,
"LayoutRepoDir": layoutRepoDir,
},
)
assert.Equal(output, expectedOutput)
Expand Down
4 changes: 4 additions & 0 deletions acceptance/invoke/pack.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,10 @@ func (i *PackInvoker) StartWithWriter(combinedOutput *bytes.Buffer, name string,
}
}

func (i *PackInvoker) Home() string {
return i.home
}

type InterruptCmd struct {
testObject *testing.T
assert h.AssertionManager
Expand Down
5 changes: 3 additions & 2 deletions acceptance/testdata/pack_fixtures/report_output.txt
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,11 @@ Pack:
Version: {{ .Version }}
OS/Arch: {{ .OS }}/{{ .Arch }}

Default Lifecycle Version: 0.15.2
Default Lifecycle Version: 0.16.0

Supported Platform APIs: 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10
Supported Platform APIs: 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12

Config:
default-builder-image = "{{ .DefaultBuilder }}"
experimental = true
layout-repo-dir = "{{ .LayoutRepoDir }}"
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ require (
github.com/Masterminds/semver v1.5.0
github.com/Microsoft/go-winio v0.6.0
github.com/apex/log v1.9.0
github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215
github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2
github.com/buildpacks/lifecycle v0.16.0
github.com/docker/cli v23.0.1+incompatible
github.com/docker/docker v20.10.23+incompatible
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -264,8 +264,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7
github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215 h1:V/fmMFCX0jA73zqnKxmHnYq2dsWefpdTkytsorowbG0=
github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215/go.mod h1:zL5lZzgFuv9l36n52FjomVrUHpyuZf6r1UHKaZ4LeSQ=
github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2 h1:UjLEI78jFKLQwpFI2rpgKOZyXKW1cQdy7Wf+8Z6Lu1M=
github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2/go.mod h1:zL5lZzgFuv9l36n52FjomVrUHpyuZf6r1UHKaZ4LeSQ=
github.com/buildpacks/lifecycle v0.16.0 h1:Q80RNP1JImJbkOXY/z/rWD9spqgEkTe/5/JypkOxJZ8=
github.com/buildpacks/lifecycle v0.16.0/go.mod h1:fiM5EwiDImyWA5kZ2fTNy0+bC4izwiCMR9rNsXbQnFc=
github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
Expand Down
17 changes: 16 additions & 1 deletion internal/build/lifecycle_execution.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"math/rand"
"path/filepath"
"strconv"

"github.com/buildpacks/lifecycle/api"
Expand Down Expand Up @@ -333,7 +334,15 @@ func (l *LifecycleExecution) Create(ctx context.Context, buildCache, launchCache
withEnv,
}

if l.opts.Publish {
if l.opts.Layout {
var err error
opts, err = l.appendLayoutOperations(opts)
if err != nil {
return err
}
}

if l.opts.Publish || l.opts.Layout {
authConfig, err := auth.BuildEnvVar(authn.DefaultKeychain, l.opts.Image.String(), l.opts.RunImage, l.opts.CacheImage, l.opts.PreviousImage)
if err != nil {
return err
Expand Down Expand Up @@ -737,6 +746,12 @@ func (l *LifecycleExecution) hasExtensions() bool {
return len(l.opts.Builder.OrderExtensions()) > 0
}

func (l *LifecycleExecution) appendLayoutOperations(opts []PhaseConfigProviderOperation) ([]PhaseConfigProviderOperation, error) {
layoutDir := filepath.Join(paths.RootDir, "layout-repo")
opts = append(opts, WithEnv("CNB_USE_LAYOUT=true", "CNB_LAYOUT_DIR="+layoutDir, "CNB_EXPERIMENTAL_MODE=warn"))
return opts, nil
}

func prependArg(arg string, args []string) []string {
return append([]string{arg}, args...)
}
Expand Down
16 changes: 16 additions & 0 deletions internal/build/lifecycle_execution_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ import (
"github.com/sclevine/spec"
"github.com/sclevine/spec/report"

"github.com/buildpacks/pack/internal/paths"

"github.com/buildpacks/pack/internal/build"
"github.com/buildpacks/pack/internal/build/fakes"
"github.com/buildpacks/pack/internal/cache"
Expand All @@ -48,6 +50,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
providedClearCache bool
providedPublish bool
providedUseCreator bool
providedLayout bool
providedDockerHost string
providedNetworkMode = "some-network-mode"
providedRunImage = "some-run-image"
Expand Down Expand Up @@ -81,6 +84,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
opts.RunImage = providedRunImage
opts.UseCreator = providedUseCreator
opts.Volumes = providedVolumes
opts.Layout = providedLayout

targetImageRef, err := name.ParseReference(providedTargetImage)
h.AssertNil(t, err)
Expand Down Expand Up @@ -1106,6 +1110,18 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
})
})
})

when("layout", func() {
providedLayout = true
layoutRepo := filepath.Join(paths.RootDir, "layout-repo")
platformAPI = api.MustParse("0.12")

it("configures the phase with oci layout environment variables", func() {
h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_USE_LAYOUT=true")
h.AssertSliceContains(t, configProvider.ContainerConfig().Env, fmt.Sprintf("CNB_LAYOUT_DIR=%s", layoutRepo))
h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_EXPERIMENTAL_MODE=warn")
})
})
})

when("#Detect", func() {
Expand Down
3 changes: 3 additions & 0 deletions internal/build/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ var (
api.MustParse("0.8"),
api.MustParse("0.9"),
api.MustParse("0.10"),
api.MustParse("0.11"),
api.MustParse("0.12"),
}
)

Expand Down Expand Up @@ -79,6 +81,7 @@ type LifecycleOptions struct {
TrustBuilder bool
UseCreator bool
Interactive bool
Layout bool
Termui Termui
DockerHost string
Cache cache.CacheOpts
Expand Down
2 changes: 1 addition & 1 deletion internal/builder/lifecycle.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ import (

// A snapshot of the latest tested lifecycle version values
const (
DefaultLifecycleVersion = "0.15.2"
DefaultLifecycleVersion = "0.16.0"
DefaultBuildpackAPIVersion = "0.2"
)

Expand Down
26 changes: 20 additions & 6 deletions internal/commands/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ type BuildFlags struct {
ClearCache bool
TrustBuilder bool
Interactive bool
Sparse bool
DockerHost string
CacheImage string
Cache cache.CacheOpts
Expand Down Expand Up @@ -68,11 +69,12 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
"be provided directly to build using `--builder`, or can be set using the `set-default-builder` command. For more " +
"on how to use `pack build`, see: https://buildpacks.io/docs/app-developer-guide/build-an-app/.",
RunE: logError(logger, func(cmd *cobra.Command, args []string) error {
if err := validateBuildFlags(&flags, cfg, packClient, logger); err != nil {
inputImageName := client.ParseInputImageReference(args[0])
if err := validateBuildFlags(&flags, cfg, inputImageName, logger); err != nil {
return err
}

imageName := args[0]
inputPreviousImage := client.ParseInputImageReference(flags.PreviousImage)

descriptor, actualDescriptorPath, err := parseProjectToml(flags.AppPath, flags.DescriptorPath)
if err != nil {
Expand Down Expand Up @@ -150,7 +152,7 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
AdditionalTags: flags.AdditionalTags,
RunImage: flags.RunImage,
Env: env,
Image: imageName,
Image: inputImageName.Name(),
Publish: flags.Publish,
DockerHost: flags.DockerHost,
PullPolicy: pullPolicy,
Expand All @@ -171,17 +173,23 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob
Workspace: flags.Workspace,
LifecycleImage: lifecycleImage,
GroupID: gid,
PreviousImage: flags.PreviousImage,
PreviousImage: inputPreviousImage.Name(),
Interactive: flags.Interactive,
SBOMDestinationDir: flags.SBOMDestinationDir,
ReportDestinationDir: flags.ReportDestinationDir,
CreationTime: dateTime,
PreBuildpacks: flags.PreBuildpacks,
PostBuildpacks: flags.PostBuildpacks,
LayoutConfig: &client.LayoutConfig{
Sparse: flags.Sparse,
InputImage: inputImageName,
PreviousInputImage: inputPreviousImage,
LayoutRepoDir: cfg.LayoutRepositoryDir,
},
}); err != nil {
return errors.Wrap(err, "failed to build")
}
logger.Infof("Successfully built image %s", style.Symbol(imageName))
logger.Infof("Successfully built image %s", style.Symbol(inputImageName.Name()))
return nil
}),
}
Expand Down Expand Up @@ -248,12 +256,14 @@ This option may set DOCKER_HOST environment variable for the build container if
cmd.Flags().StringVar(&buildFlags.SBOMDestinationDir, "sbom-output-dir", "", "Path to export SBoM contents.\nOmitting the flag will yield no SBoM content.")
cmd.Flags().StringVar(&buildFlags.ReportDestinationDir, "report-output-dir", "", "Path to export build report.toml.\nOmitting the flag yield no report file.")
cmd.Flags().BoolVar(&buildFlags.Interactive, "interactive", false, "Launch a terminal UI to depict the build process")
cmd.Flags().BoolVar(&buildFlags.Sparse, "sparse", false, "Use this flag to avoid saving on disk the run-image layers when the application image is exported to OCI layout format")
if !cfg.Experimental {
cmd.Flags().MarkHidden("interactive")
cmd.Flags().MarkHidden("sparse")
}
}

func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackClient, logger logging.Logger) error {
func validateBuildFlags(flags *BuildFlags, cfg config.Config, inputImageRef client.InputImageReference, logger logging.Logger) error {
if flags.Registry != "" && !cfg.Experimental {
return client.NewExperimentError("Support for buildpack registries is currently experimental.")
}
Expand Down Expand Up @@ -282,6 +292,10 @@ func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackCli
return client.NewExperimentError("Interactive mode is currently experimental.")
}

if inputImageRef.Layout() && !cfg.Experimental {
return client.NewExperimentError("Exporting to OCI layout is currently experimental.")
}

return nil
}

Expand Down
85 changes: 85 additions & 0 deletions internal/commands/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ import (
"github.com/sclevine/spec/report"
"github.com/spf13/cobra"

"github.com/buildpacks/pack/internal/paths"

"github.com/buildpacks/pack/internal/commands"
"github.com/buildpacks/pack/internal/commands/testmocks"
"github.com/buildpacks/pack/internal/config"
Expand Down Expand Up @@ -866,6 +868,73 @@ builder = "my-builder"
})
})
})

when("export to OCI layout is expected but experimental isn't set in the config", func() {
it("errors with a descriptive message", func() {
command.SetArgs([]string{"oci:image", "--builder", "my-builder"})
err := command.Execute()
h.AssertNotNil(t, err)
h.AssertError(t, err, "Exporting to OCI layout is currently experimental.")
})
})
})

when("export to OCI layout is expected", func() {
var (
sparse bool
previousImage string
layoutDir string
)

it.Before(func() {
layoutDir = filepath.Join(paths.RootDir, "local", "repo")
previousImage = ""
cfg = config.Config{
Experimental: true,
LayoutRepositoryDir: layoutDir,
}
command = commands.Build(logger, cfg, mockClient)
})

when("path to save the image is provided", func() {
it("build is called with oci layout configuration", func() {
sparse = false
mockClient.EXPECT().
Build(gomock.Any(), EqBuildOptionsWithLayoutConfig("image", previousImage, sparse, layoutDir)).
Return(nil)

command.SetArgs([]string{"oci:image", "--builder", "my-builder"})
err := command.Execute()
h.AssertNil(t, err)
})
})

when("previous-image flag is provided", func() {
it("build is called with oci layout configuration", func() {
sparse = false
previousImage = "my-previous-image"
mockClient.EXPECT().
Build(gomock.Any(), EqBuildOptionsWithLayoutConfig("image", previousImage, sparse, layoutDir)).
Return(nil)

command.SetArgs([]string{"oci:image", "--previous-image", "oci:my-previous-image", "--builder", "my-builder"})
err := command.Execute()
h.AssertNil(t, err)
})
})

when("-sparse flag is provided", func() {
it("build is called with oci layout configuration and sparse true", func() {
sparse = true
mockClient.EXPECT().
Build(gomock.Any(), EqBuildOptionsWithLayoutConfig("image", previousImage, sparse, layoutDir)).
Return(nil)

command.SetArgs([]string{"oci:image", "--sparse", "--builder", "my-builder"})
err := command.Execute()
h.AssertNil(t, err)
})
})
})
}

Expand Down Expand Up @@ -1035,6 +1104,22 @@ func EqBuildOptionsWithDateTime(t *time.Time) interface{} {
}
}

func EqBuildOptionsWithLayoutConfig(image, previousImage string, sparse bool, layoutDir string) interface{} {
return buildOptionsMatcher{
description: fmt.Sprintf("image=%s, previous-image=%s, sparse=%t, layout-dir=%s", image, previousImage, sparse, layoutDir),
equals: func(o client.BuildOptions) bool {
if o.Layout() {
result := o.Image == image
if previousImage != "" {
result = result && previousImage == o.PreviousImage
}
return result && o.LayoutConfig.Sparse == sparse && o.LayoutConfig.LayoutRepoDir == layoutDir
}
return false
},
}
}

type buildOptionsMatcher struct {
equals func(client.BuildOptions) bool
description string
Expand Down
Loading