Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add flag to export image to OCI layout format - (Experimental) #1314

Closed
wants to merge 10 commits into from
18 changes: 18 additions & 0 deletions internal/build/container_ops.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,24 @@ import (

type ContainerOperation func(ctrClient client.CommonAPIClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error

// CopyOutDirs copies container directories to a handler function. The handler is responsible for closing the Reader.
func CopyOutDirs(handler func(closer io.ReadCloser) error, srcs ...string) ContainerOperation {
return func(ctrClient client.CommonAPIClient, ctx context.Context, containerID string, stdout, stderr io.Writer) error {
for _, src := range srcs {
reader, _, err := ctrClient.CopyFromContainer(ctx, containerID, src)
if err != nil {
return err
}

err = handler(reader)
if err != nil {
return err
}
}
return nil
}
}

// CopyDir copies a local directory (src) to the destination on the container while filtering files and changing it's UID/GID.
// if includeRoot is set the UID/GID will be set on the dst directory.
func CopyDir(src, dst string, uid, gid int, os string, includeRoot bool, fileFilter func(string) bool) ContainerOperation {
Expand Down
68 changes: 68 additions & 0 deletions internal/build/lifecycle_execution.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,13 @@ package build
import (
"context"
"fmt"
"io"
"math/rand"
"path/filepath"
"strconv"

"github.com/docker/docker/pkg/archive"

"github.com/buildpacks/lifecycle/api"
"github.com/buildpacks/lifecycle/auth"
"github.com/docker/docker/client"
Expand Down Expand Up @@ -174,6 +178,48 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF
return l.Create(ctx, l.opts.Publish, l.opts.DockerHost, l.opts.ClearCache, l.opts.RunImage, l.opts.Image.String(), l.opts.Network, buildCache, launchCache, l.opts.AdditionalTags, l.opts.Volumes, phaseFactory)
}

func (l *LifecycleExecution) FetchVolumes(ctx context.Context, phaseFactory PhaseFactory, operation ContainerOperation) error {
configProvider := NewPhaseConfigProvider(
"no-op",
l,
WithLogPrefix("no-op"),
WithArgs(
l.withLogLevel()...,
),
WithContainerOperations(
EnsureVolumeAccess(l.opts.Builder.UID(), l.opts.Builder.GID(), l.os, l.layersVolume, l.appVolume),
operation,
),
)

fetch := phaseFactory.New(configProvider)
defer fetch.Cleanup()
return fetch.Run(ctx)
}

func (l *LifecycleExecution) CopyOCI() error {
var reterr error
if l.opts.OCIPath != "" {
if err := l.FetchVolumes(context.Background(), NewDefaultPhaseFactory(l),
CopyOutDirs(l.copyOCI, filepath.Join(l.mountPaths.ociDir()))); err != nil {
reterr = errors.Wrapf(err, "failed to extract volumes to file-system")
}
}
return reterr
}

func (l *LifecycleExecution) copyOCI(reader io.ReadCloser) error {
defer reader.Close()
if l.opts.OCIPath != "" {
srcInfo := archive.CopyInfo{
Path: filepath.Join(l.mountPaths.ociDir()),
IsDir: true,
}
archive.CopyTo(reader, srcInfo, l.opts.OCIPath)
}
return nil
}

func (l *LifecycleExecution) Cleanup() error {
var reterr error
if err := l.docker.VolumeRemove(context.Background(), l.layersVolume, true); err != nil {
Expand Down Expand Up @@ -239,13 +285,20 @@ func (l *LifecycleExecution) Create(ctx context.Context, publish bool, dockerHos
cacheOpts = WithBinds(append(volumes, fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.cacheDir()))...)
}

layoutEnv := NullOp()
if l.opts.OCIPath != "" {
flags = append(flags, "-layout")
layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
}

opts := []PhaseConfigProviderOperation{
WithFlags(l.withLogLevel(flags...)...),
WithArgs(repoName),
WithNetwork(networkMode),
cacheOpts,
WithContainerOperations(WriteProjectMetadata(l.mountPaths.projectPath(), l.opts.ProjectMetadata, l.os)),
WithContainerOperations(CopyDir(l.opts.AppPath, l.mountPaths.appDir(), l.opts.Builder.UID(), l.opts.Builder.GID(), l.os, true, l.opts.FileFilter)),
layoutEnv,
}

if publish {
Expand Down Expand Up @@ -385,6 +438,12 @@ func (l *LifecycleExecution) newAnalyze(repoName, networkMode string, publish bo
l.opts.Image = prevImage
}

layoutEnv := NullOp()
if l.opts.OCIPath != "" {
flagsOpt = WithFlags("-layout")
layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
}

if publish {
authConfig, err := auth.BuildEnvVar(authn.DefaultKeychain, repoName)
if err != nil {
Expand Down Expand Up @@ -430,6 +489,7 @@ func (l *LifecycleExecution) newAnalyze(repoName, networkMode string, publish bo
flagsOpt,
WithNetwork(networkMode),
cacheOpt,
layoutEnv,
)

return phaseFactory.New(configProvider), nil
Expand Down Expand Up @@ -486,6 +546,12 @@ func (l *LifecycleExecution) newExport(repoName, runImage string, publish bool,
cacheOpt = WithBinds(fmt.Sprintf("%s:%s", buildCache.Name(), l.mountPaths.cacheDir()))
}

layoutEnv := NullOp()
if l.opts.OCIPath != "" {
flags = append(flags, "-layout")
layoutEnv = WithEnv(fmt.Sprintf("%s=%s", builder.EnvLayoutDir, l.mountPaths.ociDir()))
}

opts := []PhaseConfigProviderOperation{
WithLogPrefix("exporter"),
WithImage(l.opts.LifecycleImage),
Expand All @@ -502,6 +568,7 @@ func (l *LifecycleExecution) newExport(repoName, runImage string, publish bool,
cacheOpt,
WithContainerOperations(WriteStackToml(l.mountPaths.stackPath(), l.opts.Builder.Stack(), l.os)),
WithContainerOperations(WriteProjectMetadata(l.mountPaths.projectPath(), l.opts.ProjectMetadata, l.os)),
layoutEnv,
}

if publish {
Expand Down Expand Up @@ -532,6 +599,7 @@ func (l *LifecycleExecution) Export(ctx context.Context, repoName, runImage stri
if err != nil {
return err
}
defer l.CopyOCI()
defer export.Cleanup()
return export.Run(ctx)
}
Expand Down
161 changes: 161 additions & 0 deletions internal/build/lifecycle_execution_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -932,6 +932,56 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
})
})
})

when("oci-dir", func() {
var (
lifecycle *build.LifecycleExecution
fakePhaseFactory *fakes.FakePhaseFactory
)
fakePhase := &fakes.FakePhase{}
fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))

when("OCIPath is provided", func() {
it.Before(func() {
lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
options.OCIPath = "/path/to/oci"
})
})

it("configures the phase with the expected arguments", func() {
err := lifecycle.Create(context.Background(), false, "", true, "test", "test", "test", fakeBuildCache, fakeLaunchCache, []string{}, []string{}, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
h.AssertNotEq(t, lastCallIndex, -1)

configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
h.AssertEq(t, configProvider.Name(), "creator")
assertValidOCIConfiguration(t, configProvider, "/path/to/oci")
})
})

when("OCIPath is not provided", func() {
it.Before(func() {
lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
options.OCIPath = ""
})
})

it("layout is not added to the expected arguments", func() {
err := lifecycle.Create(context.Background(), false, "", true, "test", "test", "test", fakeBuildCache, fakeLaunchCache, []string{}, []string{}, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
h.AssertNotEq(t, lastCallIndex, -1)

configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
h.AssertEq(t, configProvider.Name(), "creator")
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-layout")
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Env, "CNB_LAYOUT_DIR=/oci")
})
})
})
})

when("#Detect", func() {
Expand Down Expand Up @@ -1499,6 +1549,56 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
})
})
})

when("oci-dir", func() {
var (
lifecycle *build.LifecycleExecution
fakePhaseFactory *fakes.FakePhaseFactory
)
fakePhase := &fakes.FakePhase{}
fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))

when("OCIPath is provided", func() {
it.Before(func() {
lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
options.OCIPath = "/path/to/oci"
})
})

it("configures the phase with the expected arguments", func() {
err := lifecycle.Analyze(context.Background(), "test", "test", false, "", false, fakeCache, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
h.AssertNotEq(t, lastCallIndex, -1)

configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
h.AssertEq(t, configProvider.Name(), "analyzer")
assertValidOCIConfiguration(t, configProvider, "/path/to/oci")
})
})

when("OCIPath is not provided", func() {
it.Before(func() {
lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
options.OCIPath = ""
})
})

it("layout is not added to the expected arguments", func() {
err := lifecycle.Analyze(context.Background(), "test", "test", false, "", false, fakeCache, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
h.AssertNotEq(t, lastCallIndex, -1)

configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
h.AssertEq(t, configProvider.Name(), "analyzer")
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-layout")
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Env, "CNB_LAYOUT_DIR=/oci")
})
})
})
})

when("#Restore", func() {
Expand Down Expand Up @@ -2324,6 +2424,56 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) {
})
})
})

when("oci-dir", func() {
var (
lifecycle *build.LifecycleExecution
fakePhaseFactory *fakes.FakePhaseFactory
)
fakePhase := &fakes.FakePhase{}
fakePhaseFactory = fakes.NewFakePhaseFactory(fakes.WhichReturnsForNew(fakePhase))

when("OCIPath is provided", func() {
it.Before(func() {
lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
options.OCIPath = "/path/to/oci"
})
})

it("configures the phase with the expected arguments", func() {
err := lifecycle.Export(context.Background(), "test", "test", false, "", "test", fakeBuildCache, fakeLaunchCache, []string{}, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
h.AssertNotEq(t, lastCallIndex, -1)

configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
h.AssertEq(t, configProvider.Name(), "exporter")
assertValidOCIConfiguration(t, configProvider, "/path/to/oci")
})
})

when("OCIPath is not provided", func() {
it.Before(func() {
lifecycle = newTestLifecycleExec(t, true, func(options *build.LifecycleOptions) {
options.OCIPath = ""
})
})

it("layout is not added to the expected arguments", func() {
err := lifecycle.Export(context.Background(), "test", "test", false, "", "test", fakeBuildCache, fakeLaunchCache, []string{}, fakePhaseFactory)
h.AssertNil(t, err)

lastCallIndex := len(fakePhaseFactory.NewCalledWithProvider) - 1
h.AssertNotEq(t, lastCallIndex, -1)

configProvider := fakePhaseFactory.NewCalledWithProvider[lastCallIndex]
h.AssertEq(t, configProvider.Name(), "exporter")
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Cmd, "-layout")
h.AssertSliceNotContains(t, configProvider.ContainerConfig().Env, "CNB_LAYOUT_DIR=/oci")
})
})
})
})
}

Expand Down Expand Up @@ -2362,3 +2512,14 @@ func newTestLifecycleExec(t *testing.T, logVerbose bool, ops ...func(*build.Life
h.AssertNil(t, err)
return lifecycleExec
}

func assertValidOCIConfiguration(t *testing.T, configProvider *build.PhaseConfigProvider, path string) {
h.AssertIncludeAllExpectedPatterns(t,
configProvider.ContainerConfig().Cmd,
[]string{"-layout"},
)
h.AssertIncludeAllExpectedPatterns(t,
configProvider.ContainerConfig().Env,
[]string{"CNB_LAYOUT_DIR=/layers/oci"},
)
}
1 change: 1 addition & 0 deletions internal/build/lifecycle_executor.go
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ type LifecycleOptions struct {
LifecycleImage string
RunImage string
ProjectMetadata platform.ProjectMetadata
OCIPath string
ClearCache bool
Publish bool
TrustBuilder bool
Expand Down
4 changes: 4 additions & 0 deletions internal/build/mount_paths.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,7 @@ func (m mountPaths) cacheDir() string {
func (m mountPaths) launchCacheDir() string {
return m.join(m.volume, "launch-cache")
}

func (m mountPaths) ociDir() string {
return m.join(m.layersDir(), "oci")
}
5 changes: 3 additions & 2 deletions internal/builder/builder.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ const (
metadataLabel = "io.buildpacks.builder.metadata"
stackLabel = "io.buildpacks.stack.id"

EnvUID = "CNB_USER_ID"
EnvGID = "CNB_GROUP_ID"
EnvUID = "CNB_USER_ID"
EnvGID = "CNB_GROUP_ID"
EnvLayoutDir = "CNB_LAYOUT_DIR"

BuildpackOnBuilderMessage = `buildpack %s already exists on builder and will be overwritten
- existing diffID: %s
Expand Down
Loading