diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e55becc1ab..b94ff20afb 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -579,9 +579,13 @@ func testWithoutSpecificBuilderRequirement( pack.RunSuccessfully("config", "default-builder", "paketobuildpacks/builder:base") output := pack.RunSuccessfully("report") - version := pack.Version() + layoutRepoDir := filepath.Join(pack.Home(), "layout-repo") + if runtime.GOOS == "windows" { + layoutRepoDir = strings.ReplaceAll(layoutRepoDir, `\`, `\\`) + } + expectedOutput := pack.FixtureManager().TemplateFixture( "report_output.txt", map[string]interface{}{ @@ -589,6 +593,7 @@ func testWithoutSpecificBuilderRequirement( "Version": version, "OS": runtime.GOOS, "Arch": runtime.GOARCH, + "LayoutRepoDir": layoutRepoDir, }, ) assert.Equal(output, expectedOutput) @@ -598,9 +603,13 @@ func testWithoutSpecificBuilderRequirement( pack.RunSuccessfully("config", "default-builder", "paketobuildpacks/builder:base") output := pack.RunSuccessfully("report", "--explicit") - version := pack.Version() + layoutRepoDir := filepath.Join(pack.Home(), "layout-repo") + if runtime.GOOS == "windows" { + layoutRepoDir = strings.ReplaceAll(layoutRepoDir, `\`, `\\`) + } + expectedOutput := pack.FixtureManager().TemplateFixture( "report_output.txt", map[string]interface{}{ @@ -608,6 +617,7 @@ func testWithoutSpecificBuilderRequirement( "Version": version, "OS": runtime.GOOS, "Arch": runtime.GOARCH, + "LayoutRepoDir": layoutRepoDir, }, ) assert.Equal(output, expectedOutput) diff --git a/acceptance/invoke/pack.go b/acceptance/invoke/pack.go index 7b2eb72ece..facde02ec0 100644 --- a/acceptance/invoke/pack.go +++ b/acceptance/invoke/pack.go @@ -140,6 +140,10 @@ func (i *PackInvoker) StartWithWriter(combinedOutput *bytes.Buffer, name string, } } +func (i *PackInvoker) Home() string { + return i.home +} + type InterruptCmd struct { testObject *testing.T assert h.AssertionManager diff --git a/acceptance/testdata/pack_fixtures/report_output.txt b/acceptance/testdata/pack_fixtures/report_output.txt index 544c3ea3ae..cbf0fddb29 100644 --- a/acceptance/testdata/pack_fixtures/report_output.txt +++ b/acceptance/testdata/pack_fixtures/report_output.txt @@ -2,10 +2,11 @@ Pack: Version: {{ .Version }} OS/Arch: {{ .OS }}/{{ .Arch }} -Default Lifecycle Version: 0.15.2 +Default Lifecycle Version: 0.16.0 -Supported Platform APIs: 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10 +Supported Platform APIs: 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12 Config: default-builder-image = "{{ .DefaultBuilder }}" experimental = true + layout-repo-dir = "{{ .LayoutRepoDir }}" diff --git a/go.mod b/go.mod index a0bfa8f9a2..1aa9d0059b 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/Microsoft/go-winio v0.6.0 github.com/apex/log v1.9.0 - github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215 + github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2 github.com/buildpacks/lifecycle v0.16.0 github.com/docker/cli v23.0.1+incompatible github.com/docker/docker v20.10.23+incompatible diff --git a/go.sum b/go.sum index e99340fb00..9c15a83644 100644 --- a/go.sum +++ b/go.sum @@ -264,8 +264,8 @@ github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= -github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215 h1:V/fmMFCX0jA73zqnKxmHnYq2dsWefpdTkytsorowbG0= -github.com/buildpacks/imgutil v0.0.0-20230120191822-4d50b9a7e215/go.mod h1:zL5lZzgFuv9l36n52FjomVrUHpyuZf6r1UHKaZ4LeSQ= +github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2 h1:UjLEI78jFKLQwpFI2rpgKOZyXKW1cQdy7Wf+8Z6Lu1M= +github.com/buildpacks/imgutil v0.0.0-20230221152838-4cf98dd677d2/go.mod h1:zL5lZzgFuv9l36n52FjomVrUHpyuZf6r1UHKaZ4LeSQ= github.com/buildpacks/lifecycle v0.16.0 h1:Q80RNP1JImJbkOXY/z/rWD9spqgEkTe/5/JypkOxJZ8= github.com/buildpacks/lifecycle v0.16.0/go.mod h1:fiM5EwiDImyWA5kZ2fTNy0+bC4izwiCMR9rNsXbQnFc= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= diff --git a/internal/build/lifecycle_execution.go b/internal/build/lifecycle_execution.go index e2b56bd88b..34711ea179 100644 --- a/internal/build/lifecycle_execution.go +++ b/internal/build/lifecycle_execution.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "math/rand" + "path/filepath" "strconv" "github.com/buildpacks/lifecycle/api" @@ -333,7 +334,15 @@ func (l *LifecycleExecution) Create(ctx context.Context, buildCache, launchCache withEnv, } - if l.opts.Publish { + if l.opts.Layout { + var err error + opts, err = l.appendLayoutOperations(opts) + if err != nil { + return err + } + } + + if l.opts.Publish || l.opts.Layout { authConfig, err := auth.BuildEnvVar(authn.DefaultKeychain, l.opts.Image.String(), l.opts.RunImage, l.opts.CacheImage, l.opts.PreviousImage) if err != nil { return err @@ -737,6 +746,12 @@ func (l *LifecycleExecution) hasExtensions() bool { return len(l.opts.Builder.OrderExtensions()) > 0 } +func (l *LifecycleExecution) appendLayoutOperations(opts []PhaseConfigProviderOperation) ([]PhaseConfigProviderOperation, error) { + layoutDir := filepath.Join(paths.RootDir, "layout-repo") + opts = append(opts, WithEnv("CNB_USE_LAYOUT=true", "CNB_LAYOUT_DIR="+layoutDir, "CNB_EXPERIMENTAL_MODE=warn")) + return opts, nil +} + func prependArg(arg string, args []string) []string { return append([]string{arg}, args...) } diff --git a/internal/build/lifecycle_execution_test.go b/internal/build/lifecycle_execution_test.go index a57def9c44..996c5b91f3 100644 --- a/internal/build/lifecycle_execution_test.go +++ b/internal/build/lifecycle_execution_test.go @@ -22,6 +22,8 @@ import ( "github.com/sclevine/spec" "github.com/sclevine/spec/report" + "github.com/buildpacks/pack/internal/paths" + "github.com/buildpacks/pack/internal/build" "github.com/buildpacks/pack/internal/build/fakes" "github.com/buildpacks/pack/internal/cache" @@ -48,6 +50,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) { providedClearCache bool providedPublish bool providedUseCreator bool + providedLayout bool providedDockerHost string providedNetworkMode = "some-network-mode" providedRunImage = "some-run-image" @@ -81,6 +84,7 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) { opts.RunImage = providedRunImage opts.UseCreator = providedUseCreator opts.Volumes = providedVolumes + opts.Layout = providedLayout targetImageRef, err := name.ParseReference(providedTargetImage) h.AssertNil(t, err) @@ -1106,6 +1110,18 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) { }) }) }) + + when("layout", func() { + providedLayout = true + layoutRepo := filepath.Join(paths.RootDir, "layout-repo") + platformAPI = api.MustParse("0.12") + + it("configures the phase with oci layout environment variables", func() { + h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_USE_LAYOUT=true") + h.AssertSliceContains(t, configProvider.ContainerConfig().Env, fmt.Sprintf("CNB_LAYOUT_DIR=%s", layoutRepo)) + h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_EXPERIMENTAL_MODE=warn") + }) + }) }) when("#Detect", func() { diff --git a/internal/build/lifecycle_executor.go b/internal/build/lifecycle_executor.go index 54cb6df5aa..08e1c9856c 100644 --- a/internal/build/lifecycle_executor.go +++ b/internal/build/lifecycle_executor.go @@ -29,6 +29,8 @@ var ( api.MustParse("0.8"), api.MustParse("0.9"), api.MustParse("0.10"), + api.MustParse("0.11"), + api.MustParse("0.12"), } ) @@ -79,6 +81,7 @@ type LifecycleOptions struct { TrustBuilder bool UseCreator bool Interactive bool + Layout bool Termui Termui DockerHost string Cache cache.CacheOpts diff --git a/internal/builder/lifecycle.go b/internal/builder/lifecycle.go index e16b998624..012c42fc18 100644 --- a/internal/builder/lifecycle.go +++ b/internal/builder/lifecycle.go @@ -14,7 +14,7 @@ import ( // A snapshot of the latest tested lifecycle version values const ( - DefaultLifecycleVersion = "0.15.2" + DefaultLifecycleVersion = "0.16.0" DefaultBuildpackAPIVersion = "0.2" ) diff --git a/internal/commands/build.go b/internal/commands/build.go index 46f54e5b99..548b9c58bb 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -26,6 +26,7 @@ type BuildFlags struct { ClearCache bool TrustBuilder bool Interactive bool + Sparse bool DockerHost string CacheImage string Cache cache.CacheOpts @@ -68,11 +69,12 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob "be provided directly to build using `--builder`, or can be set using the `set-default-builder` command. For more " + "on how to use `pack build`, see: https://buildpacks.io/docs/app-developer-guide/build-an-app/.", RunE: logError(logger, func(cmd *cobra.Command, args []string) error { - if err := validateBuildFlags(&flags, cfg, packClient, logger); err != nil { + inputImageName := client.ParseInputImageReference(args[0]) + if err := validateBuildFlags(&flags, cfg, inputImageName, logger); err != nil { return err } - imageName := args[0] + inputPreviousImage := client.ParseInputImageReference(flags.PreviousImage) descriptor, actualDescriptorPath, err := parseProjectToml(flags.AppPath, flags.DescriptorPath) if err != nil { @@ -150,7 +152,7 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob AdditionalTags: flags.AdditionalTags, RunImage: flags.RunImage, Env: env, - Image: imageName, + Image: inputImageName.Name(), Publish: flags.Publish, DockerHost: flags.DockerHost, PullPolicy: pullPolicy, @@ -171,17 +173,23 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob Workspace: flags.Workspace, LifecycleImage: lifecycleImage, GroupID: gid, - PreviousImage: flags.PreviousImage, + PreviousImage: inputPreviousImage.Name(), Interactive: flags.Interactive, SBOMDestinationDir: flags.SBOMDestinationDir, ReportDestinationDir: flags.ReportDestinationDir, CreationTime: dateTime, PreBuildpacks: flags.PreBuildpacks, PostBuildpacks: flags.PostBuildpacks, + LayoutConfig: &client.LayoutConfig{ + Sparse: flags.Sparse, + InputImage: inputImageName, + PreviousInputImage: inputPreviousImage, + LayoutRepoDir: cfg.LayoutRepositoryDir, + }, }); err != nil { return errors.Wrap(err, "failed to build") } - logger.Infof("Successfully built image %s", style.Symbol(imageName)) + logger.Infof("Successfully built image %s", style.Symbol(inputImageName.Name())) return nil }), } @@ -248,12 +256,14 @@ This option may set DOCKER_HOST environment variable for the build container if cmd.Flags().StringVar(&buildFlags.SBOMDestinationDir, "sbom-output-dir", "", "Path to export SBoM contents.\nOmitting the flag will yield no SBoM content.") cmd.Flags().StringVar(&buildFlags.ReportDestinationDir, "report-output-dir", "", "Path to export build report.toml.\nOmitting the flag yield no report file.") cmd.Flags().BoolVar(&buildFlags.Interactive, "interactive", false, "Launch a terminal UI to depict the build process") + cmd.Flags().BoolVar(&buildFlags.Sparse, "sparse", false, "Use this flag to avoid saving on disk the run-image layers when the application image is exported to OCI layout format") if !cfg.Experimental { cmd.Flags().MarkHidden("interactive") + cmd.Flags().MarkHidden("sparse") } } -func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackClient, logger logging.Logger) error { +func validateBuildFlags(flags *BuildFlags, cfg config.Config, inputImageRef client.InputImageReference, logger logging.Logger) error { if flags.Registry != "" && !cfg.Experimental { return client.NewExperimentError("Support for buildpack registries is currently experimental.") } @@ -282,6 +292,10 @@ func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackCli return client.NewExperimentError("Interactive mode is currently experimental.") } + if inputImageRef.Layout() && !cfg.Experimental { + return client.NewExperimentError("Exporting to OCI layout is currently experimental.") + } + return nil } diff --git a/internal/commands/build_test.go b/internal/commands/build_test.go index dd0f2624bd..aeebdbea2e 100644 --- a/internal/commands/build_test.go +++ b/internal/commands/build_test.go @@ -17,6 +17,8 @@ import ( "github.com/sclevine/spec/report" "github.com/spf13/cobra" + "github.com/buildpacks/pack/internal/paths" + "github.com/buildpacks/pack/internal/commands" "github.com/buildpacks/pack/internal/commands/testmocks" "github.com/buildpacks/pack/internal/config" @@ -866,6 +868,73 @@ builder = "my-builder" }) }) }) + + when("export to OCI layout is expected but experimental isn't set in the config", func() { + it("errors with a descriptive message", func() { + command.SetArgs([]string{"oci:image", "--builder", "my-builder"}) + err := command.Execute() + h.AssertNotNil(t, err) + h.AssertError(t, err, "Exporting to OCI layout is currently experimental.") + }) + }) + }) + + when("export to OCI layout is expected", func() { + var ( + sparse bool + previousImage string + layoutDir string + ) + + it.Before(func() { + layoutDir = filepath.Join(paths.RootDir, "local", "repo") + previousImage = "" + cfg = config.Config{ + Experimental: true, + LayoutRepositoryDir: layoutDir, + } + command = commands.Build(logger, cfg, mockClient) + }) + + when("path to save the image is provided", func() { + it("build is called with oci layout configuration", func() { + sparse = false + mockClient.EXPECT(). + Build(gomock.Any(), EqBuildOptionsWithLayoutConfig("image", previousImage, sparse, layoutDir)). + Return(nil) + + command.SetArgs([]string{"oci:image", "--builder", "my-builder"}) + err := command.Execute() + h.AssertNil(t, err) + }) + }) + + when("previous-image flag is provided", func() { + it("build is called with oci layout configuration", func() { + sparse = false + previousImage = "my-previous-image" + mockClient.EXPECT(). + Build(gomock.Any(), EqBuildOptionsWithLayoutConfig("image", previousImage, sparse, layoutDir)). + Return(nil) + + command.SetArgs([]string{"oci:image", "--previous-image", "oci:my-previous-image", "--builder", "my-builder"}) + err := command.Execute() + h.AssertNil(t, err) + }) + }) + + when("-sparse flag is provided", func() { + it("build is called with oci layout configuration and sparse true", func() { + sparse = true + mockClient.EXPECT(). + Build(gomock.Any(), EqBuildOptionsWithLayoutConfig("image", previousImage, sparse, layoutDir)). + Return(nil) + + command.SetArgs([]string{"oci:image", "--sparse", "--builder", "my-builder"}) + err := command.Execute() + h.AssertNil(t, err) + }) + }) }) } @@ -1035,6 +1104,22 @@ func EqBuildOptionsWithDateTime(t *time.Time) interface{} { } } +func EqBuildOptionsWithLayoutConfig(image, previousImage string, sparse bool, layoutDir string) interface{} { + return buildOptionsMatcher{ + description: fmt.Sprintf("image=%s, previous-image=%s, sparse=%t, layout-dir=%s", image, previousImage, sparse, layoutDir), + equals: func(o client.BuildOptions) bool { + if o.Layout() { + result := o.Image == image + if previousImage != "" { + result = result && previousImage == o.PreviousImage + } + return result && o.LayoutConfig.Sparse == sparse && o.LayoutConfig.LayoutRepoDir == layoutDir + } + return false + }, + } +} + type buildOptionsMatcher struct { equals func(client.BuildOptions) bool description string diff --git a/internal/commands/config_experimental.go b/internal/commands/config_experimental.go index 0476b515e2..67008c4e3f 100644 --- a/internal/commands/config_experimental.go +++ b/internal/commands/config_experimental.go @@ -1,6 +1,7 @@ package commands import ( + "path/filepath" "strconv" "github.com/pkg/errors" @@ -33,6 +34,11 @@ func ConfigExperimental(logger logging.Logger, cfg config.Config, cfgPath string return errors.Wrapf(err, "invalid value %s provided", style.Symbol(args[0])) } cfg.Experimental = val + if cfg.Experimental { + cfg.LayoutRepositoryDir = filepath.Join(filepath.Dir(cfgPath), "layout-repo") + } else { + cfg.LayoutRepositoryDir = "" + } if err = config.Write(cfg, cfgPath); err != nil { return errors.Wrap(err, "writing to config") diff --git a/internal/commands/config_experimental_test.go b/internal/commands/config_experimental_test.go index db5b424c3e..2a1f269d45 100644 --- a/internal/commands/config_experimental_test.go +++ b/internal/commands/config_experimental_test.go @@ -77,6 +77,10 @@ func testConfigExperimental(t *testing.T, when spec.G, it spec.S) { cfg, err := config.Read(configPath) h.AssertNil(t, err) h.AssertEq(t, cfg.Experimental, true) + + // oci layout repo is configured + layoutDir := filepath.Join(filepath.Dir(configPath), "layout-repo") + h.AssertEq(t, cfg.LayoutRepositoryDir, layoutDir) }) it("sets false if provided", func() { @@ -87,6 +91,9 @@ func testConfigExperimental(t *testing.T, when spec.G, it spec.S) { cfg, err := config.Read(configPath) h.AssertNil(t, err) h.AssertEq(t, cfg.Experimental, false) + + // oci layout repo is cleaned + h.AssertEq(t, cfg.LayoutRepositoryDir, "") }) it("returns error if invalid value provided", func() { diff --git a/internal/config/config.go b/internal/config/config.go index 5af064f5f9..de370ef727 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -22,6 +22,7 @@ type Config struct { Registries []Registry `toml:"registries,omitempty"` LifecycleImage string `toml:"lifecycle-image,omitempty"` RegistryMirrors map[string]string `toml:"registry-mirrors,omitempty"` + LayoutRepositoryDir string `toml:"layout-repo-dir,omitempty"` } type Registry struct { @@ -75,7 +76,6 @@ func Read(path string) (Config, error) { if err != nil && !os.IsNotExist(err) { return Config{}, errors.Wrapf(err, "failed to read config file at path %s", path) } - return cfg, nil } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 9c4249dd82..25b5b4d0a5 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -46,6 +46,7 @@ func testConfig(t *testing.T, when spec.G, it spec.S) { h.AssertEq(t, len(subject.RunImages), 0) h.AssertEq(t, subject.Experimental, false) h.AssertEq(t, len(subject.RegistryMirrors), 0) + h.AssertEq(t, subject.LayoutRepositoryDir, "") }) }) }) diff --git a/internal/fakes/fake_image_fetcher.go b/internal/fakes/fake_image_fetcher.go index c0a331f616..3ae30f9610 100644 --- a/internal/fakes/fake_image_fetcher.go +++ b/internal/fakes/fake_image_fetcher.go @@ -10,9 +10,10 @@ import ( ) type FetchArgs struct { - Daemon bool - PullPolicy image.PullPolicy - Platform string + Daemon bool + PullPolicy image.PullPolicy + Platform string + LayoutOption image.LayoutOption } type FakeImageFetcher struct { @@ -30,7 +31,7 @@ func NewFakeImageFetcher() *FakeImageFetcher { } func (f *FakeImageFetcher) Fetch(ctx context.Context, name string, options image.FetchOptions) (imgutil.Image, error) { - f.FetchCalls[name] = &FetchArgs{Daemon: options.Daemon, PullPolicy: options.PullPolicy, Platform: options.Platform} + f.FetchCalls[name] = &FetchArgs{Daemon: options.Daemon, PullPolicy: options.PullPolicy, Platform: options.Platform, LayoutOption: options.LayoutOption} ri, remoteFound := f.RemoteImages[name] diff --git a/internal/paths/defaults_unix.go b/internal/paths/defaults_unix.go new file mode 100644 index 0000000000..a570814207 --- /dev/null +++ b/internal/paths/defaults_unix.go @@ -0,0 +1,8 @@ +//go:build linux || darwin +// +build linux darwin + +package paths + +const ( + RootDir = `/` +) diff --git a/internal/paths/defaults_windows.go b/internal/paths/defaults_windows.go new file mode 100644 index 0000000000..a5f0846643 --- /dev/null +++ b/internal/paths/defaults_windows.go @@ -0,0 +1,5 @@ +package paths + +const ( + RootDir = `c:\` +) diff --git a/pkg/client/build.go b/pkg/client/build.go index 6b53ed3676..2ad7f187d6 100644 --- a/pkg/client/build.go +++ b/pkg/client/build.go @@ -13,10 +13,13 @@ import ( "github.com/Masterminds/semver" "github.com/buildpacks/imgutil" + "github.com/buildpacks/imgutil/layout" "github.com/buildpacks/imgutil/local" "github.com/buildpacks/imgutil/remote" "github.com/buildpacks/lifecycle/platform" + "github.com/buildpacks/pack/internal/paths" + "github.com/docker/docker/api/types" "github.com/docker/docker/volume/mounts" "github.com/google/go-containerregistry/pkg/name" @@ -190,6 +193,16 @@ type BuildOptions struct { // Desired create time in the output image config CreationTime *time.Time + + // Configuration to export to OCI layout format + LayoutConfig *LayoutConfig +} + +func (b *BuildOptions) Layout() bool { + if b.LayoutConfig != nil { + return b.LayoutConfig.Enable() + } + return false } // ProxyConfig specifies proxy setting to be set as environment variables in a container. @@ -221,6 +234,33 @@ type ContainerConfig struct { Volumes []string } +type LayoutConfig struct { + // Application image reference provided by the user + InputImage InputImageReference + + // Previous image reference provided by the user + PreviousInputImage InputImageReference + + // Local root path to save the run-image in OCI layout format + LayoutRepoDir string + + // Configure the OCI layout fetch mode to avoid saving layers on disk + Sparse bool +} + +func (l *LayoutConfig) Enable() bool { + return l.InputImage.Layout() +} + +type layoutPathConfig struct { + hostImagePath string + hostPreviousImagePath string + hostRunImagePath string + targetImagePath string + targetPreviousImagePath string + targetRunImagePath string +} + var IsSuggestedBuilderFunc = func(b string) bool { for _, suggestedBuilder := range builder.SuggestedBuilders { if b == suggestedBuilder.Image { @@ -235,10 +275,25 @@ var IsSuggestedBuilderFunc = func(b string) bool { // If any configuration is deemed invalid, or if any lifecycle phases fail, // an error will be returned and no image produced. func (c *Client) Build(ctx context.Context, opts BuildOptions) error { - imageRef, err := c.parseTagReference(opts.Image) + var pathsConfig layoutPathConfig + + imageRef, err := c.parseReference(opts) if err != nil { return errors.Wrapf(err, "invalid image name '%s'", opts.Image) } + imgRegistry := imageRef.Context().RegistryStr() + imageName := imageRef.Name() + + if opts.Layout() { + pathsConfig, err = c.processLayoutPath(opts.LayoutConfig.InputImage, opts.LayoutConfig.PreviousInputImage) + if err != nil { + if opts.LayoutConfig.PreviousInputImage != nil { + return errors.Wrapf(err, "invalid layout paths image name '%s' or previous-image name '%s'", opts.LayoutConfig.InputImage.Name(), + opts.LayoutConfig.PreviousInputImage.Name()) + } + return errors.Wrapf(err, "invalid layout paths image name '%s'", opts.LayoutConfig.InputImage.Name()) + } + } appPath, err := c.processAppPath(opts.AppPath) if err != nil { @@ -262,8 +317,25 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { return errors.Wrapf(err, "invalid builder %s", style.Symbol(opts.Builder)) } - runImageName := c.resolveRunImage(opts.RunImage, imageRef.Context().RegistryStr(), builderRef.Context().RegistryStr(), bldr.Stack(), opts.AdditionalMirrors, opts.Publish) - runImage, err := c.validateRunImage(ctx, runImageName, opts.PullPolicy, opts.Publish, bldr.StackID) + runImageName := c.resolveRunImage(opts.RunImage, imgRegistry, builderRef.Context().RegistryStr(), bldr.Stack(), opts.AdditionalMirrors, opts.Publish) + + fetchOptions := image.FetchOptions{Daemon: !opts.Publish, PullPolicy: opts.PullPolicy} + if opts.Layout() { + targetRunImagePath, err := layout.ParseRefToPath(runImageName) + if err != nil { + return err + } + hostRunImagePath := filepath.Join(opts.LayoutConfig.LayoutRepoDir, targetRunImagePath) + targetRunImagePath = filepath.Join(paths.RootDir, "layout-repo", targetRunImagePath) + fetchOptions.LayoutOption = image.LayoutOption{ + Path: hostRunImagePath, + Sparse: opts.LayoutConfig.Sparse, + } + fetchOptions.Daemon = false + pathsConfig.targetRunImagePath = targetRunImagePath + pathsConfig.hostRunImagePath = hostRunImagePath + } + runImage, err := c.validateRunImage(ctx, runImageName, fetchOptions, bldr.StackID) if err != nil { return errors.Wrapf(err, "invalid run-image '%s'", runImageName) } @@ -324,6 +396,10 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { } } + if opts.Layout() { + opts.ContainerConfig.Volumes = appendLayoutVolumes(opts.ContainerConfig.Volumes, pathsConfig) + } + processedVolumes, warnings, err := processVolumes(imgOS, opts.ContainerConfig.Volumes) if err != nil { return err @@ -390,10 +466,11 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { GID: opts.GroupID, PreviousImage: opts.PreviousImage, Interactive: opts.Interactive, - Termui: termui.NewTermui(imageRef.Name(), ephemeralBuilder, runImageName), + Termui: termui.NewTermui(imageName, ephemeralBuilder, runImageName), ReportDestinationDir: opts.ReportDestinationDir, SBOMDestinationDir: opts.SBOMDestinationDir, CreationTime: opts.CreationTime, + Layout: opts.Layout(), } lifecycleVersion := ephemeralBuilder.LifecycleDescriptor().Info.Version @@ -540,11 +617,11 @@ func (c *Client) getBuilder(img imgutil.Image) (*builder.Builder, error) { return bldr, nil } -func (c *Client) validateRunImage(context context.Context, name string, pullPolicy image.PullPolicy, publish bool, expectedStack string) (imgutil.Image, error) { +func (c *Client) validateRunImage(context context.Context, name string, opts image.FetchOptions, expectedStack string) (imgutil.Image, error) { if name == "" { return nil, errors.New("run image must be specified") } - img, err := c.imageFetcher.Fetch(context, name, image.FetchOptions{Daemon: !publish, PullPolicy: pullPolicy}) + img, err := c.imageFetcher.Fetch(context, name, opts) if err != nil { return nil, err } @@ -680,6 +757,53 @@ func (c *Client) processAppPath(appPath string) (string, error) { return resolvedAppPath, nil } +// processLayoutPath given an image reference and a previous image reference this method calculates the +// local full path and the expected path in the lifecycle container for both images provides. Those values +// can be used to mount the correct volumes +func (c *Client) processLayoutPath(inputImageRef, previousImageRef InputImageReference) (layoutPathConfig, error) { + var ( + hostImagePath, hostPreviousImagePath, targetImagePath, targetPreviousImagePath string + err error + ) + hostImagePath, err = fullImagePath(inputImageRef, true) + if err != nil { + return layoutPathConfig{}, err + } + targetImagePath, err = layout.ParseRefToPath(inputImageRef.Name()) + if err != nil { + return layoutPathConfig{}, err + } + targetImagePath = filepath.Join(paths.RootDir, "layout-repo", targetImagePath) + c.logger.Debugf("local image path %s will be mounted into the container at path %s", hostImagePath, targetImagePath) + + if previousImageRef != nil && previousImageRef.Name() != "" { + hostPreviousImagePath, err = fullImagePath(previousImageRef, false) + if err != nil { + return layoutPathConfig{}, err + } + targetPreviousImagePath, err = layout.ParseRefToPath(previousImageRef.Name()) + if err != nil { + return layoutPathConfig{}, err + } + targetPreviousImagePath = filepath.Join(paths.RootDir, "layout-repo", targetPreviousImagePath) + c.logger.Debugf("local previous image path %s will be mounted into the container at path %s", hostPreviousImagePath, targetPreviousImagePath) + } + return layoutPathConfig{ + hostImagePath: hostImagePath, + targetImagePath: targetImagePath, + hostPreviousImagePath: hostPreviousImagePath, + targetPreviousImagePath: targetPreviousImagePath, + }, nil +} + +func (c *Client) parseReference(opts BuildOptions) (name.Reference, error) { + if !opts.Layout() { + return c.parseTagReference(opts.Image) + } + base := filepath.Base(opts.Image) + return c.parseTagReference(base) +} + func (c *Client) processProxyConfig(config *ProxyConfig) ProxyConfig { var ( httpProxy, httpsProxy, noProxy string @@ -1096,3 +1220,53 @@ exit 0 return pathToInlineBuilpack, nil } + +// fullImagePath parses the inputImageReference provided by the user and creates the directory +// structure if create value is true +func fullImagePath(inputImageRef InputImageReference, create bool) (string, error) { + imagePath, err := inputImageRef.FullName() + if err != nil { + return "", errors.Wrapf(err, "evaluating image %s destination path", inputImageRef.Name()) + } + + if create { + if err := os.MkdirAll(imagePath, os.ModePerm); err != nil { + return "", errors.Wrapf(err, "creating %s layout application destination", imagePath) + } + } + + return imagePath, nil +} + +// appendLayoutVolumes mount host volume into the build container, in the form ':[:]' +// the volumes mounted are: +// - The path where the user wants the image to be exported in OCI layout format +// - The previous image path if it exits +// - The run-image path +func appendLayoutVolumes(volumes []string, config layoutPathConfig) []string { + if config.hostPreviousImagePath != "" { + volumes = append(volumes, readOnlyVolume(config.hostPreviousImagePath, config.targetPreviousImagePath), + readOnlyVolume(config.hostRunImagePath, config.targetRunImagePath), + writableVolume(config.hostImagePath, config.targetImagePath)) + } else { + volumes = append(volumes, readOnlyVolume(config.hostRunImagePath, config.targetRunImagePath), + writableVolume(config.hostImagePath, config.targetImagePath)) + } + return volumes +} + +func writableVolume(hostPath, targetPath string) string { + tp := targetPath + if !filepath.IsAbs(targetPath) { + tp = filepath.Join(string(filepath.Separator), targetPath) + } + return fmt.Sprintf("%s:%s:rw", hostPath, tp) +} + +func readOnlyVolume(hostPath, targetPath string) string { + tp := targetPath + if !filepath.IsAbs(targetPath) { + tp = filepath.Join(string(filepath.Separator), targetPath) + } + return fmt.Sprintf("%s:%s", hostPath, tp) +} diff --git a/pkg/client/build_test.go b/pkg/client/build_test.go index 241b451b7a..f554d2797b 100644 --- a/pkg/client/build_test.go +++ b/pkg/client/build_test.go @@ -2785,6 +2785,93 @@ func testBuild(t *testing.T, when spec.G, it spec.S) { }) }) }) + + when("export to OCI layout", func() { + var ( + inputImageReference, inputPreviousImageReference InputImageReference + layoutConfig *LayoutConfig + hostImagePath, hostPreviousImagePath, hostRunImagePath string + ) + + it.Before(func() { + h.SkipIf(t, runtime.GOOS == "windows", "skip on windows") + + remoteRunImage := fakes.NewImage("default/run", "", nil) + h.AssertNil(t, remoteRunImage.SetLabel("io.buildpacks.stack.id", defaultBuilderStackID)) + h.AssertNil(t, remoteRunImage.SetLabel("io.buildpacks.stack.mixins", `["mixinA", "mixinX", "run:mixinZ"]`)) + fakeImageFetcher.RemoteImages[remoteRunImage.Name()] = remoteRunImage + + hostImagePath = filepath.Join(tmpDir, "my-app") + inputImageReference = ParseInputImageReference(fmt.Sprintf("oci:%s", hostImagePath)) + layoutConfig = &LayoutConfig{ + InputImage: inputImageReference, + LayoutRepoDir: filepath.Join(tmpDir, "local-repo"), + } + }) + + when("previous image is not provided", func() { + when("sparse is false", func() { + it("saves run-image locally in oci layout and mount volumes", func() { + h.AssertNil(t, subject.Build(context.TODO(), BuildOptions{ + Image: inputImageReference.Name(), + Builder: defaultBuilderName, + LayoutConfig: layoutConfig, + })) + + args := fakeImageFetcher.FetchCalls["default/run"] + h.AssertEq(t, args.LayoutOption.Sparse, false) + h.AssertContains(t, args.LayoutOption.Path, layoutConfig.LayoutRepoDir) + + h.AssertEq(t, fakeLifecycle.Opts.Layout, true) + // verify the host path are mounted as volumes + h.AssertSliceContainsMatch(t, fakeLifecycle.Opts.Volumes, hostImagePath, hostRunImagePath) + }) + }) + + when("sparse is true", func() { + it.Before(func() { + layoutConfig.Sparse = true + }) + + it("saves run-image locally (no layers) in oci layout and mount volumes", func() { + h.AssertNil(t, subject.Build(context.TODO(), BuildOptions{ + Image: inputImageReference.Name(), + Builder: defaultBuilderName, + LayoutConfig: layoutConfig, + })) + + args := fakeImageFetcher.FetchCalls["default/run"] + h.AssertEq(t, args.LayoutOption.Sparse, true) + h.AssertContains(t, args.LayoutOption.Path, layoutConfig.LayoutRepoDir) + + h.AssertEq(t, fakeLifecycle.Opts.Layout, true) + // verify the host path are mounted as volumes + h.AssertSliceContainsMatch(t, fakeLifecycle.Opts.Volumes, hostImagePath, hostRunImagePath) + }) + }) + }) + + when("previous image is provided", func() { + it.Before(func() { + hostPreviousImagePath = filepath.Join(tmpDir, "my-previous-app") + inputPreviousImageReference = ParseInputImageReference(fmt.Sprintf("oci:%s", hostPreviousImagePath)) + layoutConfig.PreviousInputImage = inputPreviousImageReference + }) + + it("mount previous image volume", func() { + h.AssertNil(t, subject.Build(context.TODO(), BuildOptions{ + Image: inputImageReference.Name(), + PreviousImage: inputPreviousImageReference.Name(), + Builder: defaultBuilderName, + LayoutConfig: layoutConfig, + })) + + h.AssertEq(t, fakeLifecycle.Opts.Layout, true) + // verify the host path are mounted as volumes + h.AssertSliceContainsMatch(t, fakeLifecycle.Opts.Volumes, hostImagePath, hostPreviousImagePath, hostRunImagePath) + }) + }) + }) }) } diff --git a/pkg/client/input_image_reference.go b/pkg/client/input_image_reference.go new file mode 100644 index 0000000000..d276a4bc85 --- /dev/null +++ b/pkg/client/input_image_reference.go @@ -0,0 +1,101 @@ +package client + +import ( + "os" + "path/filepath" + "runtime" + "strings" + + "github.com/pkg/errors" +) + +type InputImageReference interface { + Name() string + Layout() bool + FullName() (string, error) +} + +type defaultInputImageReference struct { + name string +} + +type layoutInputImageReference struct { + name string +} + +func ParseInputImageReference(input string) InputImageReference { + if strings.HasPrefix(input, "oci:") { + imageNameParsed := strings.SplitN(input, ":", 2) + return &layoutInputImageReference{ + name: imageNameParsed[1], + } + } + return &defaultInputImageReference{ + name: input, + } +} + +func (d *defaultInputImageReference) Name() string { + return d.name +} + +func (d *defaultInputImageReference) Layout() bool { + return false +} + +func (d *defaultInputImageReference) FullName() (string, error) { + return d.name, nil +} + +func (l *layoutInputImageReference) Name() string { + return filepath.Base(l.name) +} + +func (l *layoutInputImageReference) Layout() bool { + return true +} + +func (l *layoutInputImageReference) FullName() (string, error) { + var ( + fullImagePath string + err error + ) + + path := parsePath(l.name) + + if fullImagePath, err = filepath.EvalSymlinks(path); err != nil { + if !os.IsNotExist(err) { + return "", errors.Wrap(err, "evaluate symlink") + } else { + fullImagePath = path + } + } + + if fullImagePath, err = filepath.Abs(fullImagePath); err != nil { + return "", errors.Wrap(err, "resolve absolute path") + } + + return fullImagePath, nil +} + +func parsePath(path string) string { + var result string + if filepath.IsAbs(path) && runtime.GOOS == "windows" { + dir, fileWithTag := filepath.Split(path) + file := removeTag(fileWithTag) + result = filepath.Join(dir, file) + } else { + result = removeTag(path) + } + return result +} + +func removeTag(path string) string { + result := path + if strings.Contains(path, ":") { + split := strings.SplitN(path, ":", 2) + // do not include the tag in the path + result = split[0] + } + return result +} diff --git a/pkg/client/input_image_reference_test.go b/pkg/client/input_image_reference_test.go new file mode 100644 index 0000000000..0b0857cff0 --- /dev/null +++ b/pkg/client/input_image_reference_test.go @@ -0,0 +1,91 @@ +package client + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + h "github.com/buildpacks/pack/testhelpers" +) + +func TestInputImageReference(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + spec.Run(t, "InputImageReference", testInputImageReference, spec.Parallel(), spec.Report(report.Terminal{})) +} + +func testInputImageReference(t *testing.T, when spec.G, it spec.S) { + var defaultImageReference, layoutImageReference InputImageReference + + it.Before(func() { + defaultImageReference = ParseInputImageReference("busybox") + layoutImageReference = ParseInputImageReference("oci:my-app") + }) + + when("#ParseInputImageReference", func() { + when("oci layout image reference is not provided", func() { + it("default implementation is returned", func() { + h.AssertEq(t, defaultImageReference.Layout(), false) + h.AssertEq(t, defaultImageReference.Name(), "busybox") + + fullName, err := defaultImageReference.FullName() + h.AssertNil(t, err) + h.AssertEq(t, fullName, "busybox") + }) + }) + + when("oci layout image reference is provided", func() { + it("layout implementation is returned", func() { + h.AssertTrue(t, layoutImageReference.Layout()) + h.AssertEq(t, layoutImageReference.Name(), "my-app") + }) + }) + }) + + when("#FullName", func() { + when("oci layout image reference is provided", func() { + when("not absolute path provided", func() { + it("it will be joined with the current working directory", func() { + fullPath, err := layoutImageReference.FullName() + h.AssertNil(t, err) + + currentWorkingDir, err := os.Getwd() + h.AssertNil(t, err) + + expectedPath := filepath.Join(currentWorkingDir, layoutImageReference.Name()) + h.AssertEq(t, fullPath, expectedPath) + }) + }) + + when("absolute path provided", func() { + var ( + fullPath, expectedFullPath, tmpDir string + err error + ) + + it.Before(func() { + tmpDir, err = os.MkdirTemp("", "pack.input.image.reference.test") + h.AssertNil(t, err) + expectedFullPath = filepath.Join(tmpDir, "my-app") + layoutImageReference = ParseInputImageReference(fmt.Sprintf("oci:%s", expectedFullPath)) + }) + + it.After(func() { + err = os.RemoveAll(tmpDir) + h.AssertNil(t, err) + }) + + it("it must returned the path provided", func() { + fullPath, err = layoutImageReference.FullName() + h.AssertNil(t, err) + h.AssertEq(t, fullPath, expectedFullPath) + }) + }) + }) + }) +} diff --git a/pkg/image/fetcher.go b/pkg/image/fetcher.go index a0f47f9e42..a014dc8b9d 100644 --- a/pkg/image/fetcher.go +++ b/pkg/image/fetcher.go @@ -7,6 +7,9 @@ import ( "io" "strings" + "github.com/buildpacks/imgutil/layout" + "github.com/buildpacks/imgutil/layout/sparse" + "github.com/buildpacks/imgutil" "github.com/buildpacks/imgutil/local" "github.com/buildpacks/imgutil/remote" @@ -27,6 +30,11 @@ import ( // Values in these functions are set through currying. type FetcherOption func(c *Fetcher) +type LayoutOption struct { + Path string + Sparse bool +} + // WithRegistryMirrors supply your own mirrors for registry. func WithRegistryMirrors(registryMirrors map[string]string) FetcherOption { return func(c *Fetcher) { @@ -53,9 +61,10 @@ type Fetcher struct { } type FetchOptions struct { - Daemon bool - Platform string - PullPolicy PullPolicy + Daemon bool + Platform string + PullPolicy PullPolicy + LayoutOption LayoutOption } func NewFetcher(logger logging.Logger, docker DockerClient, opts ...FetcherOption) *Fetcher { @@ -80,6 +89,10 @@ func (f *Fetcher) Fetch(ctx context.Context, name string, options FetchOptions) return nil, err } + if (options.LayoutOption != LayoutOption{}) { + return f.fetchLayoutImage(name, options.LayoutOption) + } + if !options.Daemon { return f.fetchRemoteImage(name) } @@ -130,6 +143,35 @@ func (f *Fetcher) fetchRemoteImage(name string) (imgutil.Image, error) { return image, nil } +func (f *Fetcher) fetchLayoutImage(name string, options LayoutOption) (imgutil.Image, error) { + var ( + image imgutil.Image + err error + ) + + v1Image, err := remote.NewV1Image(name, f.keychain) + if err != nil { + return nil, err + } + + if options.Sparse { + image, err = sparse.NewImage(options.Path, v1Image) + } else { + image, err = layout.NewImage(options.Path, layout.FromBaseImage(v1Image)) + } + + if err != nil { + return nil, err + } + + err = image.Save() + if err != nil { + return nil, err + } + + return image, nil +} + func (f *Fetcher) pullImage(ctx context.Context, imageID string, platform string) error { regAuth, err := f.registryAuth(imageID) if err != nil { diff --git a/pkg/image/fetcher_test.go b/pkg/image/fetcher_test.go index 4d4d4c0a34..475f8f4ff0 100644 --- a/pkg/image/fetcher_test.go +++ b/pkg/image/fetcher_test.go @@ -6,6 +6,7 @@ import ( "fmt" "math/rand" "os" + "path/filepath" "testing" "time" @@ -325,5 +326,65 @@ func testFetcher(t *testing.T, when spec.G, it spec.S) { }) }) }) + + when("layout option is provided", func() { + var ( + layoutOption image.LayoutOption + imagePath string + tmpDir string + err error + ) + + it.Before(func() { + // set up local layout repo + tmpDir, err = os.MkdirTemp("", "pack.fetcher.test") + h.AssertNil(t, err) + + // dummy layer to validate sparse behavior + tarDir := filepath.Join(tmpDir, "layer") + err = os.MkdirAll(tarDir, os.ModePerm) + h.AssertNil(t, err) + layerPath := h.CreateTAR(t, tarDir, ".", -1) + + // set up the remote image to be used + img, err := remote.NewImage(repoName, authn.DefaultKeychain) + img.AddLayer(layerPath) + h.AssertNil(t, err) + h.AssertNil(t, img.Save()) + + // set up layout options for the tests + imagePath = filepath.Join(tmpDir, repo) + layoutOption = image.LayoutOption{ + Path: imagePath, + Sparse: false, + } + }) + + it.After(func() { + err = os.RemoveAll(tmpDir) + h.AssertNil(t, err) + }) + + when("sparse is false", func() { + it("returns and layout image on disk", func() { + _, err := imageFetcher.Fetch(context.TODO(), repoName, image.FetchOptions{LayoutOption: layoutOption}) + h.AssertNil(t, err) + + // all layers were written + h.AssertBlobsLen(t, imagePath, 3) + }) + }) + + when("sparse is true", func() { + it("returns and layout image on disk", func() { + layoutOption.Sparse = true + _, err := imageFetcher.Fetch(context.TODO(), repoName, image.FetchOptions{LayoutOption: layoutOption}) + h.AssertNil(t, err) + + // only manifest and config was written + h.AssertBlobsLen(t, imagePath, 2) + }) + }) + }) }) } diff --git a/testhelpers/testhelpers.go b/testhelpers/testhelpers.go index 4d8ec2622f..c18921c1e6 100644 --- a/testhelpers/testhelpers.go +++ b/testhelpers/testhelpers.go @@ -834,6 +834,13 @@ func AssertGitHeadEq(t *testing.T, path1, path2 string) { AssertEq(t, h1.Hash().String(), h2.Hash().String()) } +func AssertBlobsLen(t *testing.T, path string, expected int) { + t.Helper() + fis, err := os.ReadDir(filepath.Join(path, "blobs", "sha256")) + AssertNil(t, err) + AssertEq(t, len(fis), expected) +} + func MockWriterAndOutput() (*color.Console, func() string) { r, w, _ := os.Pipe() console := color.NewConsole(w)