WIP - OCI Layout feature

Signed-off-by: Juan Bustamante <[email protected]>

internal/build/container_ops.go

@@ -24,6 +24,24 @@ import (
 
 type ContainerOperation func(ctrClient client.CommonAPIClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error
 
+// CopyOutDirs copies container directories to a handler function. The handler is responsible for closing the Reader.
+func CopyOutDirs(handler func(closer io.ReadCloser) error, srcs ...string) ContainerOperation {
+	return func(ctrClient client.CommonAPIClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error {
+		for _, src := range srcs {
+			reader, _, err := ctrClient.CopyFromContainer(ctx, containerID, src)
+			if err != nil {
+				return err
+			}
+
+			err = handler(reader)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+}
+
 // CopyDir copies a local directory (src) to the destination on the container while filtering files and changing it's UID/GID.
 // if includeRoot is set the UID/GID will be set on the dst directory.
 func CopyDir(src, dst string, uid, gid int, os string, includeRoot bool, fileFilter func(string) bool) ContainerOperation {

internal/build/lifecycle_execution.go

@@ -3,9 +3,13 @@ package build
 import (
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
+	"path/filepath"
 	"strconv"
 
+	"github.com/docker/docker/pkg/archive"
+
 	"github.com/buildpacks/lifecycle/api"
 	"github.com/buildpacks/lifecycle/auth"
 	"github.com/docker/docker/client"
@@ -174,6 +178,48 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
 	return l.Create(ctx, l.opts.Publish, l.opts.DockerHost, l.opts.ClearCache, l.opts.RunImage, l.opts.Image.String(), l.opts.Network, buildCache, launchCache, l.opts.AdditionalTags, l.opts.Volumes, phaseFactory)
 }
 
+func (l *LifecycleExecution) FetchVolumes(ctx context.Context, phaseFactory PhaseFactory, operation ContainerOperation) error {
+	configProvider := NewPhaseConfigProvider(
+		"no-op",
+		l,
+		WithLogPrefix("no-op"),
+		WithArgs(
+			l.withLogLevel()...,
+		),
+		WithContainerOperations(
+			EnsureVolumeAccess(l.opts.Builder.UID(), l.opts.Builder.GID(), l.os, l.layersVolume, l.appVolume),
+			operation,
+		),
+	)
+
+	fetch := phaseFactory.New(configProvider)
+	defer fetch.Cleanup()
+	return fetch.Run(ctx)
+}
+
+func (l *LifecycleExecution) CopyOCI() error {
+	var reterr error
+	if l.opts.OCIPath != "" {
+		if err := l.FetchVolumes(context.Background(), NewDefaultPhaseFactory(l),
+			CopyOutDirs(l.copyOCI, filepath.Join(l.mountPaths.ociDir()))); err != nil {
+			reterr = errors.Wrapf(err, "failed to extract volumes to file-system")
+		}
+	}
+	return reterr
+}
+
+func (l *LifecycleExecution) copyOCI(reader io.ReadCloser) error {
+	defer reader.Close()
+	if l.opts.OCIPath != "" {
+		srcInfo := archive.CopyInfo{
+			Path:  filepath.Join(l.mountPaths.ociDir()),
+			IsDir: true,
+		}
+		archive.CopyTo(reader, srcInfo, l.opts.OCIPath)
+	}
+	return nil
+}
+
 func (l *LifecycleExecution) Cleanup() error {
 	var reterr error
 	if err := l.docker.VolumeRemove(context.Background(), l.layersVolume, true); err != nil {
@@ -239,15 +285,19 @@ func (l *LifecycleExecution) Create(ctx context.Context, publish bool, dockerHos
 		cacheOpts = WithBinds(append(volumes, fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.cacheDir()))...)
 	}
 
-	flags, layoutOpt, layoutEnv := l.configureLayout(flags)
+	layoutEnv := NullOp()
+	if l.opts.OCIPath != "" {
+		flags = append(flags, "-layout")
+		layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
+	}
+
 	opts := []PhaseConfigProviderOperation{
 		WithFlags(l.withLogLevel(flags...)...),
 		WithArgs(repoName),
 		WithNetwork(networkMode),
 		cacheOpts,
 		WithContainerOperations(WriteProjectMetadata(l.mountPaths.projectPath(), l.opts.ProjectMetadata, l.os)),
 		WithContainerOperations(CopyDir(l.opts.AppPath, l.mountPaths.appDir(), l.opts.Builder.UID(), l.opts.Builder.GID(), l.os, true, l.opts.FileFilter)),
-		layoutOpt,
 		layoutEnv,
 	}
 
@@ -388,10 +438,11 @@ func (l *LifecycleExecution) newAnalyze(repoName, networkMode string, publish bo
 		l.opts.Image = prevImage
 	}
 
+	layoutEnv := NullOp()
 	if l.opts.OCIPath != "" {
 		flagsOpt = WithFlags("-layout")
+		layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
 	}
-	_, layoutOpt, layoutEnv := l.configureLayout([]string{})
 
 	if publish {
 		authConfig, err := auth.BuildEnvVar(authn.DefaultKeychain, repoName)
@@ -438,7 +489,6 @@ func (l *LifecycleExecution) newAnalyze(repoName, networkMode string, publish bo
 		flagsOpt,
 		WithNetwork(networkMode),
 		cacheOpt,
-		layoutOpt,
 		layoutEnv,
 	)
 
@@ -495,7 +545,13 @@ func (l *LifecycleExecution) newExport(repoName, runImage string, publish bool,
 	case cache.Volume:
 		cacheOpt = WithBinds(fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.cacheDir()))
 	}
-	flags, layoutOpt, layoutEnv := l.configureLayout(flags)
+
+	layoutEnv := NullOp()
+	if l.opts.OCIPath != "" {
+		flags = append(flags, "-layout")
+		layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
+	}
+
 	opts := []PhaseConfigProviderOperation{
 		WithLogPrefix("exporter"),
 		WithImage(l.opts.LifecycleImage),
@@ -512,7 +568,6 @@ func (l *LifecycleExecution) newExport(repoName, runImage string, publish bool,
 		cacheOpt,
 		WithContainerOperations(WriteStackToml(l.mountPaths.stackPath(), l.opts.Builder.Stack(), l.os)),
 		WithContainerOperations(WriteProjectMetadata(l.mountPaths.projectPath(), l.opts.ProjectMetadata, l.os)),
-		layoutOpt,
 		layoutEnv,
 	}
 
@@ -544,6 +599,7 @@ func (l *LifecycleExecution) Export(ctx context.Context, repoName, runImage stri
 	if err != nil {
 		return err
 	}
+	defer l.CopyOCI()
 	defer export.Cleanup()
 	return export.Run(ctx)
 }
@@ -565,15 +621,3 @@ func addTags(flags, additionalTags []string) []string {
 	}
 	return flags
 }
-
-func (l *LifecycleExecution) configureLayout(flags []string) ([]string, PhaseConfigProviderOperation, PhaseConfigProviderOperation) {
-	layoutOpt := NullOp()
-	layoutEnv := NullOp()
-
-	if l.opts.OCIPath != "" {
-		flags = append(flags, "-layout")
-		layoutOpt = WithBinds(fmt.Sprintf("%s:%s:%s", l.opts.OCIPath, l.mountPaths.ociDir(), "rw"))
-		layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
-	}
-	return flags, layoutOpt, layoutEnv
-}

internal/build/lifecycle_execution_test.go

@@ -932,6 +932,56 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
 				})
 			})
 		})
+
+		when("oci-dir", func() {
+			var (
+				lifecycle        *build.LifecycleExecution
+				fakePhaseFactory *fakes.FakePhaseFactory
+			)
+			fakePhase := &fakes.FakePhase{}
+			fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))
+
+			when("OCIPath is provided", func() {
+				it.Before(func() {
+					lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
+						options.OCIPath = "/path/to/oci"
+					})
+				})
+
+				it("configures the phase with the expected arguments", func() {
+					err := lifecycle.Create(context.Background(), false, "", true, "test", "test", "test", fakeBuildCache, fakeLaunchCache, []string{}, []string{}, fakePhaseFactory)
+					h.AssertNil(t, err)
+
+					lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
+					h.AssertNotEq(t, lastCallIndex, -1)
+
+					configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
+					h.AssertEq(t, configProvider.Name(), "creator")
+					assertValidOCIConfiguration(t, configProvider, "/path/to/oci")
+				})
+			})
+
+			when("OCIPath is not provided", func() {
+				it.Before(func() {
+					lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
+						options.OCIPath = ""
+					})
+				})
+
+				it("layout is not added to the expected arguments", func() {
+					err := lifecycle.Create(context.Background(), false, "", true, "test", "test", "test", fakeBuildCache, fakeLaunchCache, []string{}, []string{}, fakePhaseFactory)
+					h.AssertNil(t, err)
+
+					lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
+					h.AssertNotEq(t, lastCallIndex, -1)
+
+					configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
+					h.AssertEq(t, configProvider.Name(), "creator")
+					h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-layout")
+					h.AssertSliceNotContains(t, configProvider.ContainerConfig().Env, "CNB_LAYOUT_DIR=/oci")
+				})
+			})
+		})
 	})
 
 	when("#Detect", func() {
@@ -1499,6 +1549,56 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
 				})
 			})
 		})
+
+		when("oci-dir", func() {
+			var (
+				lifecycle        *build.LifecycleExecution
+				fakePhaseFactory *fakes.FakePhaseFactory
+			)
+			fakePhase := &fakes.FakePhase{}
+			fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))
+
+			when("OCIPath is provided", func() {
+				it.Before(func() {
+					lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
+						options.OCIPath = "/path/to/oci"
+					})
+				})
+
+				it("configures the phase with the expected arguments", func() {
+					err := lifecycle.Analyze(context.Background(), "test", "test", false, "", false, fakeCache, fakePhaseFactory)
+					h.AssertNil(t, err)
+
+					lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
+					h.AssertNotEq(t, lastCallIndex, -1)
+
+					configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
+					h.AssertEq(t, configProvider.Name(), "analyzer")
+					assertValidOCIConfiguration(t, configProvider, "/path/to/oci")
+				})
+			})
+
+			when("OCIPath is not provided", func() {
+				it.Before(func() {
+					lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
+						options.OCIPath = ""
+					})
+				})
+
+				it("layout is not added to the expected arguments", func() {
+					err := lifecycle.Analyze(context.Background(), "test", "test", false, "", false, fakeCache, fakePhaseFactory)
+					h.AssertNil(t, err)
+
+					lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
+					h.AssertNotEq(t, lastCallIndex, -1)
+
+					configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
+					h.AssertEq(t, configProvider.Name(), "analyzer")
+					h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-layout")
+					h.AssertSliceNotContains(t, configProvider.ContainerConfig().Env, "CNB_LAYOUT_DIR=/oci")
+				})
+			})
+		})
 	})
 
 	when("#Restore", func() {
@@ -2324,6 +2424,56 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
 				})
 			})
 		})
+
+		when("oci-dir", func() {
+			var (
+				lifecycle        *build.LifecycleExecution
+				fakePhaseFactory *fakes.FakePhaseFactory
+			)
+			fakePhase := &fakes.FakePhase{}
+			fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))
+
+			when("OCIPath is provided", func() {
+				it.Before(func() {
+					lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
+						options.OCIPath = "/path/to/oci"
+					})
+				})
+
+				it("configures the phase with the expected arguments", func() {
+					err := lifecycle.Export(context.Background(), "test", "test", false, "", "test", fakeBuildCache, fakeLaunchCache, []string{}, fakePhaseFactory)
+					h.AssertNil(t, err)
+
+					lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
+					h.AssertNotEq(t, lastCallIndex, -1)
+
+					configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
+					h.AssertEq(t, configProvider.Name(), "exporter")
+					assertValidOCIConfiguration(t, configProvider, "/path/to/oci")
+				})
+			})
+
+			when("OCIPath is not provided", func() {
+				it.Before(func() {
+					lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
+						options.OCIPath = ""
+					})
+				})
+
+				it("layout is not added to the expected arguments", func() {
+					err := lifecycle.Export(context.Background(), "test", "test", false, "", "test", fakeBuildCache, fakeLaunchCache, []string{}, fakePhaseFactory)
+					h.AssertNil(t, err)
+
+					lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
+					h.AssertNotEq(t, lastCallIndex, -1)
+
+					configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
+					h.AssertEq(t, configProvider.Name(), "exporter")
+					h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-layout")
+					h.AssertSliceNotContains(t, configProvider.ContainerConfig().Env, "CNB_LAYOUT_DIR=/oci")
+				})
+			})
+		})
 	})
 }
 
@@ -2362,3 +2512,14 @@ func newTestLifecycleExec(t *testing.T, logVerbose bool, ops ...func(*build.Life
 	h.AssertNil(t, err)
 	return lifecycleExec
 }
+
+func assertValidOCIConfiguration(t *testing.T, configProvider *build.PhaseConfigProvider, path string) {
+	h.AssertIncludeAllExpectedPatterns(t,
+		configProvider.ContainerConfig().Cmd,
+		[]string{"-layout"},
+	)
+	h.AssertIncludeAllExpectedPatterns(t,
+		configProvider.ContainerConfig().Env,
+		[]string{"CNB_LAYOUT_DIR=/layers/oci"},
+	)
+}

internal/build/mount_paths.go

@@ -59,5 +59,5 @@ func (m mountPaths) launchCacheDir() string {
 }
 
 func (m mountPaths) ociDir() string {
-	return m.join(m.volume, "oci")
+	return m.join(m.layersDir(), "oci")
 }

internal/builder/builder.go

@@ -44,8 +44,8 @@ const (
 	metadataLabel = "io.buildpacks.builder.metadata"
 	stackLabel    = "io.buildpacks.stack.id"
 
-	EnvUID = "CNB_USER_ID"
-	EnvGID = "CNB_GROUP_ID"
+	EnvUID       = "CNB_USER_ID"
+	EnvGID       = "CNB_GROUP_ID"
 	EnvLayoutDir = "CNB_LAYOUT_DIR"
 
 	BuildpackOnBuilderMessage = `buildpack %s already exists on builder and will be overwritten

internal/commands/build.go

@@ -225,7 +225,7 @@ func validateBuildFlags(flags *BuildFlags, cfg config.Config, packClient PackCli
 	}
 
 	if flags.OCIPath != "" && !cfg.Experimental {
-		return pack.NewExperimentError("Exporting to OCI layout is currently experimental.")
+		return client.NewExperimentError("Exporting to OCI layout is currently experimental.")
 	}
 	return nil
 }