diff --git a/internal/build/docker.go b/internal/build/docker.go index 800dd2c805..8423a1d717 100644 --- a/internal/build/docker.go +++ b/internal/build/docker.go @@ -12,6 +12,8 @@ import ( specs "github.com/opencontainers/image-spec/specs-go/v1" ) +//go:generate mockgen -package mockdocker -destination ./mockdocker/mockDockerClient.go github.com/buildpacks/pack/internal/build DockerClient + type DockerClient interface { ImageRemove(ctx context.Context, image string, options types.ImageRemoveOptions) ([]image.DeleteResponse, error) VolumeRemove(ctx context.Context, volumeID string, force bool) error @@ -23,6 +25,9 @@ type DockerClient interface { ContainerInspect(ctx context.Context, container string) (types.ContainerJSON, error) ContainerRemove(ctx context.Context, container string, options containertypes.RemoveOptions) error CopyToContainer(ctx context.Context, container, path string, content io.Reader, options types.CopyToContainerOptions) error + ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) + ImageInspectWithRaw(ctx context.Context, image string) (types.ImageInspect, []byte, error) + ImageSave(ctx context.Context, imageIDs []string) (io.ReadCloser, error) } var _ DockerClient = dockerClient.CommonAPIClient(nil) diff --git a/internal/build/extend_build_test.go b/internal/build/extend_build_test.go new file mode 100644 index 0000000000..8d0c833b37 --- /dev/null +++ b/internal/build/extend_build_test.go @@ -0,0 +1,174 @@ +package build_test + +import ( + "bytes" + "context" + "io" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" + + "github.com/apex/log" + "github.com/buildpacks/lifecycle/buildpack" + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + "github.com/buildpacks/pack/internal/build" + "github.com/buildpacks/pack/internal/build/fakes" + mockdocker "github.com/buildpacks/pack/internal/build/mockdocker" + "github.com/buildpacks/pack/pkg/logging" + h "github.com/buildpacks/pack/testhelpers" +) + +func TestBuildDockerfiles(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + spec.Run(t, "buildExtendByDocker", testBuildDockerfiles, spec.Report(report.Terminal{}), spec.Sequential()) +} + +const ( + argUserID = "user_id" + argGroupID = "group_id" +) + +func testBuildDockerfiles(t *testing.T, when spec.G, it spec.S) { + var ( + mockDockerClient *mockdocker.MockDockerClient + mockController *gomock.Controller + lifecycle *build.LifecycleExecution + tmpDir string + + // lifecycle options + providedClearCache bool + providedPublish bool + providedBuilderImage = "some-registry.com/some-namespace/some-builder-name" + extendedBuilderImage = "some-registry.com/some-namespace/some-builder-name-extended" + configureDefaultTestLifecycle func(opts *build.LifecycleOptions) + lifecycleOps []func(*build.LifecycleOptions) + ) + + it.Before(func() { + h.SkipIf(t, runtime.GOOS == "windows", "extensions not supported on windows") + var err error + mockController = gomock.NewController(t) + mockDockerClient = mockdocker.NewMockDockerClient(mockController) + h.AssertNil(t, err) + + configureDefaultTestLifecycle = func(opts *build.LifecycleOptions) { + opts.BuilderImage = providedBuilderImage + opts.ClearCache = providedClearCache + opts.Publish = providedPublish + } + + lifecycleOps = []func(*build.LifecycleOptions){configureDefaultTestLifecycle} + }) + + when("Extend Build Image By Docker", func() { + it("should extend build image using 1 extension", func() { + // set tmp directory + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single") + lifecycle = getTestLifecycleExec(t, true, tmpDir, mockDockerClient, lifecycleOps...) + dockerfile := build.DockerfileInfo{ + Info: &buildpack.DockerfileInfo{ + Path: filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single", "build", "samples_test", "Dockerfile"), + }, + } + expectedBuilder := lifecycle.Builder() + expectedBuildContext, _ := dockerfile. + CreateBuildContext(lifecycle.AppPath(), lifecycle.GetLogger()) + // Set up expected Build Args + UID := strconv.Itoa(expectedBuilder.UID()) + GID := strconv.Itoa(expectedBuilder.GID()) + expectedbuildArguments := map[string]*string{} + expectedbuildArguments["base_image"] = &providedBuilderImage + expectedbuildArguments[argUserID] = &UID + expectedbuildArguments[argGroupID] = &GID + expectedBuildOptions := types.ImageBuildOptions{ + Context: expectedBuildContext, + Dockerfile: "Dockerfile", + Tags: []string{extendedBuilderImage}, + Remove: true, + BuildArgs: expectedbuildArguments, + } + mockResponse := types.ImageBuildResponse{ + Body: io.NopCloser(strings.NewReader("mock-build-response-body")), + OSType: "linux", + } + mockDockerClient.EXPECT().ImageBuild(gomock.Any(), gomock.Any(), gomock.Any()).Do(func(_ context.Context, buildContext io.Reader, buildOptions types.ImageBuildOptions) { + compBuildOptions(t, expectedBuildOptions, buildOptions) + }).Return(mockResponse, nil).Times(1) + mockDockerClient.EXPECT().ImageInspectWithRaw(gomock.Any(), gomock.Any()).Return(types.ImageInspect{ + Config: &container.Config{ + User: "root", + }, + }, nil, nil).Times(1) + err := lifecycle.ExtendBuildByDaemon(context.Background()) + h.AssertNil(t, err) + }) + + it("should extend build image using multiple extension", func() { + // set tmp directory + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "multi") + lifecycle = getTestLifecycleExec(t, true, tmpDir, mockDockerClient, lifecycleOps...) + mockResponse := types.ImageBuildResponse{ + Body: io.NopCloser(strings.NewReader("mock-build-response-body")), + OSType: "linux", + } + mockDockerClient.EXPECT().ImageBuild(gomock.Any(), gomock.Any(), gomock.Any()).Return(mockResponse, nil).Times(2) + mockDockerClient.EXPECT().ImageInspectWithRaw(gomock.Any(), gomock.Any()).Return(types.ImageInspect{ + Config: &container.Config{ + User: "root", + }}, nil, nil).Times(2) + err := lifecycle.ExtendBuildByDaemon(context.Background()) + h.AssertNil(t, err) + }) + }) +} +func GetTestLifecycleExecErr(t *testing.T, logVerbose bool, tmpDir string, mockDockerClient *mockdocker.MockDockerClient, ops ...func(*build.LifecycleOptions)) (*build.LifecycleExecution, error) { + var outBuf bytes.Buffer + logger := logging.NewLogWithWriters(&outBuf, &outBuf) + if logVerbose { + logger.Level = log.DebugLevel + } + defaultBuilder, err := fakes.NewFakeBuilder() + h.AssertNil(t, err) + + opts := build.LifecycleOptions{ + AppPath: "some-app-path", + Builder: defaultBuilder, + HTTPProxy: "some-http-proxy", + HTTPSProxy: "some-https-proxy", + NoProxy: "some-no-proxy", + Termui: &fakes.FakeTermui{}, + } + + for _, op := range ops { + op(&opts) + } + + return build.NewLifecycleExecution(logger, mockDockerClient, tmpDir, opts) +} + +func getTestLifecycleExec(t *testing.T, logVerbose bool, tmpDir string, mockDockerClient *mockdocker.MockDockerClient, ops ...func(*build.LifecycleOptions)) *build.LifecycleExecution { + t.Helper() + + lifecycleExec, err := GetTestLifecycleExecErr(t, logVerbose, tmpDir, mockDockerClient, ops...) + h.AssertNil(t, err) + return lifecycleExec +} + +func compBuildOptions(t *testing.T, expectedBuildOptions types.ImageBuildOptions, actualBuildOptions types.ImageBuildOptions) { + t.Helper() + h.AssertEq(t, expectedBuildOptions.Dockerfile, actualBuildOptions.Dockerfile) + h.AssertEq(t, expectedBuildOptions.Tags, actualBuildOptions.Tags) + h.AssertEq(t, expectedBuildOptions.Remove, actualBuildOptions.Remove) + h.AssertEq(t, expectedBuildOptions.BuildArgs["base_image"], actualBuildOptions.BuildArgs["base_image"]) + h.AssertEq(t, expectedBuildOptions.BuildArgs[argUserID], actualBuildOptions.BuildArgs[argUserID]) + h.AssertEq(t, expectedBuildOptions.BuildArgs[argGroupID], actualBuildOptions.BuildArgs[argGroupID]) +} diff --git a/internal/build/helper.go b/internal/build/helper.go new file mode 100644 index 0000000000..ecb3bcdfa0 --- /dev/null +++ b/internal/build/helper.go @@ -0,0 +1,170 @@ +package build + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/BurntSushi/toml" + "github.com/buildpacks/lifecycle/buildpack" + "github.com/buildpacks/lifecycle/cmd" + "github.com/docker/docker/api/types" + + "github.com/buildpacks/pack/pkg/archive" + "github.com/buildpacks/pack/pkg/logging" +) + +const ( + DockerfileKindBuild = "build" + DockerfileKindRun = "run" +) + +type Extensions struct { + Extensions []buildpack.GroupElement +} + +type DockerfileInfo struct { + Info *buildpack.DockerfileInfo + Args []Arg +} + +type Arg struct { + Name string `toml:"name"` + Value string `toml:"value"` +} + +type Config struct { + Build BuildConfig `toml:"build"` + Run BuildConfig `toml:"run"` +} + +type BuildConfig struct { + Args []Arg `toml:"args"` +} + +func (extensions *Extensions) DockerFiles(kind string, path string, logger logging.Logger) ([]DockerfileInfo, error) { + var dockerfiles []DockerfileInfo + for _, ext := range extensions.Extensions { + dockerfile, err := extensions.ReadDockerFile(path, kind, ext.ID) + if err != nil { + return nil, err + } + if dockerfile != nil { + logger.Debugf("Found %s Dockerfile for extension '%s'", kind, ext.ID) + switch kind { + case DockerfileKindBuild: + break + case DockerfileKindRun: + buildpack.ValidateRunDockerfile(dockerfile.Info, logger) + default: + return nil, fmt.Errorf("unknown dockerfile kind: %s", kind) + } + dockerfiles = append(dockerfiles, *dockerfile) + } + } + return dockerfiles, nil +} + +func (extensions *Extensions) ReadDockerFile(path string, kind string, extID string) (*DockerfileInfo, error) { + dockerfilePath := filepath.Join(path, kind, escapeID(extID), "Dockerfile") + if _, err := os.Stat(dockerfilePath); err != nil { + return nil, nil + } + configPath := filepath.Join(path, kind, escapeID(extID), "extend-config.toml") + var config Config + _, err := toml.DecodeFile(configPath, &config) + if err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } + + var args []Arg + if kind == buildpack.DockerfileKindBuild { + args = config.Build.Args + } else { + args = config.Run.Args + } + return &DockerfileInfo{ + Info: &buildpack.DockerfileInfo{ + ExtensionID: extID, + Kind: kind, + Path: dockerfilePath, + }, + Args: args, + }, nil +} + +func (extensions *Extensions) SetExtensions(path string, logger logging.Logger) error { + groupExt, err := readExtensionsGroup(path) + if err != nil { + return fmt.Errorf("reading group: %w", err) + } + for i := range groupExt { + groupExt[i].Extension = true + } + for _, groupEl := range groupExt { + if err = cmd.VerifyBuildpackAPI(groupEl.Kind(), groupEl.String(), groupEl.API, logger); err != nil { + return err + } + } + extensions.Extensions = groupExt + return nil +} + +func readExtensionsGroup(path string) ([]buildpack.GroupElement, error) { + var group buildpack.Group + _, err := toml.DecodeFile(filepath.Join(path, "group.toml"), &group) + for e := range group.GroupExtensions { + group.GroupExtensions[e].Extension = true + group.GroupExtensions[e].Optional = true + } + return group.GroupExtensions, err +} + +func escapeID(id string) string { + return strings.ReplaceAll(id, "/", "_") +} + +func (dockerfile *DockerfileInfo) CreateBuildContext(path string, logger logging.Logger) (io.Reader, error) { + defaultFilterFunc := func(file string) bool { return true } + buf := new(bytes.Buffer) + tarWriter := tar.NewWriter(buf) + var completeErr error + + defer func() { + if err := tarWriter.Close(); err != nil { + logger.Errorf("Error closing tar writer: %s", err) + completeErr = archive.AggregateError(completeErr, err) + } + }() + if err := archive.WriteDirToTar(tarWriter, path, "/workspace", 0, 0, -1, true, false, defaultFilterFunc); err != nil { + tarWriter.Close() + logger.Errorf("Error adding workspace: %s", err) + completeErr = archive.AggregateError(completeErr, err) + } + + if err := archive.WriteFileToTar(tarWriter, dockerfile.Info.Path, filepath.Join(".", "Dockerfile"), 0, 0, -1, true); err != nil { + tarWriter.Close() + logger.Errorf("Error adding dockerfile: %s", err) + completeErr = archive.AggregateError(completeErr, err) + } + + return buf, completeErr +} + +func userFrom(imageInfo types.ImageInspect) (string, string) { + user := strings.Split(imageInfo.Config.User, ":") + if len(user) < 2 { + return imageInfo.Config.User, "" + } + return user[0], user[1] +} + +func isRoot(userID string) bool { + return userID == "0" || userID == "root" +} diff --git a/internal/build/helper_test.go b/internal/build/helper_test.go new file mode 100644 index 0000000000..b684654086 --- /dev/null +++ b/internal/build/helper_test.go @@ -0,0 +1,174 @@ +package build_test + +import ( + "archive/tar" + "bytes" + "io" + "path/filepath" + "testing" + + "github.com/buildpacks/lifecycle/buildpack" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + + "github.com/buildpacks/pack/internal/build" + "github.com/buildpacks/pack/pkg/logging" + h "github.com/buildpacks/pack/testhelpers" +) + +func TestHelper(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + spec.Run(t, "helperForBuildExtension", testHelper, spec.Report(report.Terminal{}), spec.Sequential()) +} + +func testHelper(t *testing.T, when spec.G, it spec.S) { + var ( + tmpDir string + extensions build.Extensions + logger *logging.LogWithWriters + ) + it.Before(func() { + var outBuf bytes.Buffer + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + }) + when("set-extensions", func() { + it("should set single extension", func() { + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single") + expectedExtension := build.Extensions{ + Extensions: []buildpack.GroupElement{ + { + ID: "samples/test", + Version: "0.0.1", + API: "0.9", + Homepage: "https://github.com/buildpacks/samples/test/main/extensions/test", + }, + }, + } + extensions.SetExtensions(tmpDir, logger) + h.AssertEq(t, extensions.Extensions[0].ID, expectedExtension.Extensions[0].ID) + h.AssertEq(t, extensions.Extensions[0].Version, expectedExtension.Extensions[0].Version) + h.AssertEq(t, extensions.Extensions[0].API, expectedExtension.Extensions[0].API) + h.AssertEq(t, extensions.Extensions[0].Homepage, expectedExtension.Extensions[0].Homepage) + }) + it("should set multiple extensions", func() { + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "multi") + expectedExtension := build.Extensions{ + Extensions: []buildpack.GroupElement{ + { + ID: "samples/tree", + Version: "0.0.1", + API: "0.9", + Homepage: "https://github.com/buildpacks/samples/tree/main/extensions/tree", + }, + { + ID: "samples/test", + Version: "0.0.1", + API: "0.9", + Homepage: "https://github.com/buildpacks/samples/test/main/extensions/test", + }, + }} + extensions.SetExtensions(tmpDir, logger) + h.AssertEq(t, extensions.Extensions[0].ID, expectedExtension.Extensions[0].ID) + h.AssertEq(t, extensions.Extensions[0].Version, expectedExtension.Extensions[0].Version) + h.AssertEq(t, extensions.Extensions[0].API, expectedExtension.Extensions[0].API) + h.AssertEq(t, extensions.Extensions[0].Homepage, expectedExtension.Extensions[0].Homepage) + h.AssertEq(t, extensions.Extensions[1].ID, expectedExtension.Extensions[1].ID) + h.AssertEq(t, extensions.Extensions[1].Version, expectedExtension.Extensions[1].Version) + h.AssertEq(t, extensions.Extensions[1].API, expectedExtension.Extensions[1].API) + h.AssertEq(t, extensions.Extensions[1].Homepage, expectedExtension.Extensions[1].Homepage) + }) + }) + + when("set dockerfiles", func() { + it("should set dockerfiles for single extension", func() { + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single") + expectedDockerfile := build.DockerfileInfo{ + Info: &buildpack.DockerfileInfo{ + ExtensionID: "samples/test", + Kind: build.DockerfileKindBuild, + Path: filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single", "build", "samples_test", "Dockerfile"), + }, + } + extensions.SetExtensions(tmpDir, logger) + dockerfiles, err := extensions.DockerFiles(build.DockerfileKindBuild, tmpDir, logger) + h.AssertNil(t, err) + h.AssertEq(t, dockerfiles[0].Info.ExtensionID, expectedDockerfile.Info.ExtensionID) + h.AssertEq(t, dockerfiles[0].Info.Kind, expectedDockerfile.Info.Kind) + h.AssertEq(t, dockerfiles[0].Info.Path, expectedDockerfile.Info.Path) + }) + it("should set dockerfiles for multiple extensions", func() { + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "multi") + expectedDockerfiles := []build.DockerfileInfo{ + { + Info: &buildpack.DockerfileInfo{ + ExtensionID: "samples/tree", + Kind: build.DockerfileKindBuild, + Path: filepath.Join(".", "testdata", "fake-tmp", "build-extension", "multi", "build", "samples_tree", "Dockerfile"), + }, + }, + { + Info: &buildpack.DockerfileInfo{ + ExtensionID: "samples/test", + Kind: build.DockerfileKindBuild, + Path: filepath.Join(".", "testdata", "fake-tmp", "build-extension", "multi", "build", "samples_test", "Dockerfile"), + }, + }, + } + extensions.SetExtensions(tmpDir, logger) + dockerfiles, err := extensions.DockerFiles(build.DockerfileKindBuild, tmpDir, logger) + h.AssertNil(t, err) + h.AssertEq(t, dockerfiles[0].Info.ExtensionID, expectedDockerfiles[0].Info.ExtensionID) + h.AssertEq(t, dockerfiles[0].Info.Kind, expectedDockerfiles[0].Info.Kind) + h.AssertEq(t, dockerfiles[0].Info.Path, expectedDockerfiles[0].Info.Path) + h.AssertEq(t, dockerfiles[1].Info.ExtensionID, expectedDockerfiles[1].Info.ExtensionID) + h.AssertEq(t, dockerfiles[1].Info.Kind, expectedDockerfiles[1].Info.Kind) + h.AssertEq(t, dockerfiles[1].Info.Path, expectedDockerfiles[1].Info.Path) + }) + }) + + when("create build context", func() { + it("should create build context", func() { + tmpDir = filepath.Join(".", "testdata", "fake-tmp", "build-extension", "single") + extensions.SetExtensions(tmpDir, logger) + dockerfiles, err := extensions.DockerFiles(build.DockerfileKindBuild, tmpDir, logger) + h.AssertNil(t, err) + buildContext, err := dockerfiles[0].CreateBuildContext(tmpDir, logger) + h.AssertNil(t, err) + tr := tar.NewReader(buildContext) + checkDirectoryInTar(t, tr, "/workspace/build") + checkFileInTar(t, tr, "Dockerfile") + }) + }) +} + +func checkDirectoryInTar(t *testing.T, tr *tar.Reader, directoryName string) { + for { + header, err := tr.Next() + if err == io.EOF { + t.Fatalf("directory %s not found", directoryName) + } + if err != nil { + t.Fatal(err) + } + if header.Name == directoryName && header.Typeflag == tar.TypeDir { + return + } + } +} + +func checkFileInTar(t *testing.T, tr *tar.Reader, fileName string) { + for { + header, err := tr.Next() + if err == io.EOF { + t.Fatalf("file %s not found", fileName) + } + if err != nil { + t.Fatal(err) + } + if header.Name == fileName && header.Typeflag == tar.TypeReg { + return + } + } +} diff --git a/internal/build/lifecycle_execution.go b/internal/build/lifecycle_execution.go index 0f5c9a49a4..82fa941c45 100644 --- a/internal/build/lifecycle_execution.go +++ b/internal/build/lifecycle_execution.go @@ -3,6 +3,7 @@ package build import ( "context" "fmt" + "io" "math/rand" "os" "path/filepath" @@ -12,7 +13,9 @@ import ( "github.com/buildpacks/lifecycle/api" "github.com/buildpacks/lifecycle/auth" "github.com/buildpacks/lifecycle/platform/files" + "github.com/docker/docker/api/types" "github.com/google/go-containerregistry/pkg/name" + "github.com/google/uuid" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -263,9 +266,18 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF group, _ := errgroup.WithContext(context.TODO()) if l.platformAPI.AtLeast("0.10") && l.hasExtensionsForBuild() { + /* + [RFC #0105] - As decided, Pack should support build image extension with Docker #1623. We removed the previous implementation that was using kaniko in the extend lifecycle phase and shifted the implementation to use docker daemon to extend the build Image. As pack already has access to a daemon, it can apply the dockerfiles directly, saving the extended build base image in the daemon. Thus it will not need to use the extender phase of lifecycle. Additionally it dropped the requirement that the image being extended must be published to a registry. This implementation resulted us to have build Extension Improved by 87.3578% wrt kaniko implementation with caching and 20.5567% wrt kaniko implementation without caching. + + */ group.Go(func() error { - l.logger.Info(style.Step("EXTENDING (BUILD)")) - return l.ExtendBuild(ctx, kanikoCache, phaseFactory) + l.logger.Info(style.Step("EXTENDING (BUILD) BY DAEMON")) + l.logger.Info(style.Warn("WARNING: Extended build image is saved in the docker daemon as -extended")) + if err := l.ExtendBuildByDaemon(ctx); err != nil { + return err + } + l.Build(ctx, phaseFactory) + return nil }) } else { group.Go(func() error { @@ -280,7 +292,6 @@ func (l *LifecycleExecution) Run(ctx context.Context, phaseFactoryCreator PhaseF return l.ExtendRun(ctx, kanikoCache, phaseFactory, ephemeralRunImage) }) } - if err := group.Wait(); err != nil { return err } @@ -445,6 +456,8 @@ func (l *LifecycleExecution) Detect(ctx context.Context, phaseFactory PhaseFacto CopyOutToMaybe(filepath.Join(l.mountPaths.layersDir(), "analyzed.toml"), l.tmpDir))), If(l.hasExtensions(), WithPostContainerRunOperations( CopyOutToMaybe(filepath.Join(l.mountPaths.layersDir(), "generated", "build"), l.tmpDir))), + If(l.hasExtensions(), WithPostContainerRunOperations( + CopyOutToMaybe(filepath.Join(l.mountPaths.layersDir(), "group.toml"), l.tmpDir))), envOp, ) @@ -483,16 +496,11 @@ func (l *LifecycleExecution) Restore(ctx context.Context, buildCache Cache, kani // for kaniko kanikoCacheBindOp := NullOp() - if (l.platformAPI.AtLeast("0.10") && l.hasExtensionsForBuild()) || - l.platformAPI.AtLeast("0.12") { - if l.hasExtensionsForBuild() { - flags = append(flags, "-build-image", l.opts.BuilderImage) - registryImages = append(registryImages, l.opts.BuilderImage) - } + if l.platformAPI.AtLeast("0.12") { if l.runImageChanged() || l.hasExtensionsForRun() { registryImages = append(registryImages, l.runImageAfterExtensions()) } - if l.hasExtensionsForBuild() || l.hasExtensionsForRun() { + if l.hasExtensionsForRun() { kanikoCacheBindOp = WithBinds(fmt.Sprintf("%s:%s", kanikoCache.Name(), l.mountPaths.kanikoCacheDir())) } } @@ -702,6 +710,8 @@ func (l *LifecycleExecution) Build(ctx context.Context, phaseFactory PhaseFactor WithNetwork(l.opts.Network), WithBinds(l.opts.Volumes...), WithFlags(flags...), + If((l.hasExtensionsForBuild()), WithImage(l.opts.BuilderImage+"-extended")), + If((l.hasExtensionsForBuild() && l.opts.isExtendedBuilderImageRoot), WithUser(l.opts.Builder.UID(), l.opts.Builder.GID())), ) build := phaseFactory.New(configProvider) @@ -709,6 +719,100 @@ func (l *LifecycleExecution) Build(ctx context.Context, phaseFactory PhaseFactor return build.Run(ctx) } +const ( + argBuildID = "build_id" + argUserID = "user_id" + argGroupID = "group_id" +) + +/* + This implementation of ExtendBuildByDaemon is based on the RFC #0105 which uses docker daemon to extend the build Image instead of kaniko. + * Parsing the `group.toml` from the temp directory of buildpack and set the extensions. + * Reading the dockerfiles that were generated during the `generate` phase and also parsing the Arguments given by the user from + `extend-config.toml`. + * Using ImageBuild method of docker API client to extend the Image and save it to the daemon. + * Invoking Build phase of lifecycle by creating a container from the extended Image and dropping the privileges. + +*/ + +func (l *LifecycleExecution) ExtendBuildByDaemon(ctx context.Context) error { + builderImageName := l.opts.BuilderImage + extendedBuilderImageName := l.opts.BuilderImage + "-extended" + var extensions Extensions + extensions.SetExtensions(l.tmpDir, l.logger) + origuserID := strconv.Itoa(l.opts.Builder.UID()) + origgroupID := strconv.Itoa(l.opts.Builder.GID()) + intermediateUserID := origuserID + intermediateGroupID := origgroupID + var extendedBuilderImageInfo types.ImageInspect + dockerfiles, err := extensions.DockerFiles(DockerfileKindBuild, l.tmpDir, l.logger) + if err != nil { + return fmt.Errorf("getting %s.Dockerfiles: %w", DockerfileKindBuild, err) + } + for _, dockerfile := range dockerfiles { + dockerfile.Args = append([]Arg{ + {Name: argBuildID, Value: uuid.New().String()}, + {Name: argUserID, Value: intermediateUserID}, + {Name: argGroupID, Value: intermediateGroupID}, + }, dockerfile.Args...) + buildArguments := map[string]*string{} + buildArguments["base_image"] = &builderImageName + for i := range dockerfile.Args { + arg := &dockerfile.Args[i] + buildArguments[arg.Name] = &arg.Value + } + buildContext, err := dockerfile.CreateBuildContext(l.opts.AppPath, l.logger) + if err != nil { + return err + } + buildOptions := types.ImageBuildOptions{ + Context: buildContext, + Dockerfile: "Dockerfile", + Tags: []string{extendedBuilderImageName}, + Remove: true, + BuildArgs: buildArguments, + } + response, err := l.docker.ImageBuild(ctx, buildContext, buildOptions) + if err != nil { + return err + } + defer response.Body.Close() + _, err = io.Copy(logging.NewPrefixWriter(logging.GetWriterForLevel(l.logger, logging.InfoLevel), "extender (build)"), response.Body) + if err != nil { + return err + } + builderImageName = l.opts.BuilderImage + "-extended" + extendedBuilderImageInfo, _, err = l.docker.ImageInspectWithRaw(ctx, extendedBuilderImageName) + if err != nil { + return fmt.Errorf("inspecting extended builder image: %w", err) + } + userID, groupID := userFrom(extendedBuilderImageInfo) + if isRoot(userID) { + l.logger.Warnf("Extension from %s changed the user ID from %s to %s; this must not be the final user ID (a following extension must reset the user).", dockerfile.Info.Path, intermediateUserID, userID) + } + intermediateUserID = userID + if groupID != "" { + intermediateGroupID = groupID + } + } + userID, groupID := userFrom(extendedBuilderImageInfo) + if userID != origuserID { + l.logger.Warnf("Final User ID changed from %s to %s", origuserID, userID) + } + if groupID != origgroupID && groupID != "" { + l.logger.Warnf("Final Group ID changed from %s to %s", origgroupID, groupID) + } + if isRoot(userID) { + l.logger.Warnf("Final extension left user as root thus forcing the user to be the original user %s and original group %s", origuserID, origgroupID) + l.opts.isExtendedBuilderImageRoot = true + } + return nil +} + +/* + Deprecated: Check RFC #0105 for the new implementation of ExtendBuild using docker daemon #1623. +*/ + func (l *LifecycleExecution) ExtendBuild(ctx context.Context, kanikoCache Cache, phaseFactory PhaseFactory) error { flags := []string{"-app", l.mountPaths.appDir()} @@ -730,6 +834,10 @@ func (l *LifecycleExecution) ExtendBuild(ctx context.Context, kanikoCache Cache, return extend.Run(ctx) } +/* + Note: - Run Image Extension by docker daemon was much worse than kaniko because of saving layers on disk. +*/ + func (l *LifecycleExecution) ExtendRun(ctx context.Context, kanikoCache Cache, phaseFactory PhaseFactory, runImageName string) error { flags := []string{"-app", l.mountPaths.appDir(), "-kind", "run"} @@ -942,6 +1050,10 @@ func (l *LifecycleExecution) appendLayoutOperations(opts []PhaseConfigProviderOp return opts, nil } +func (l *LifecycleExecution) GetLogger() logging.Logger { + return l.logger +} + func withLayoutOperation() PhaseConfigProviderOperation { layoutDir := filepath.Join(paths.RootDir, "layout-repo") return WithEnv("CNB_USE_LAYOUT=true", "CNB_LAYOUT_DIR="+layoutDir, "CNB_EXPERIMENTAL_MODE=warn") diff --git a/internal/build/lifecycle_execution_test.go b/internal/build/lifecycle_execution_test.go index ed12a2e5b1..e2cc824e07 100644 --- a/internal/build/lifecycle_execution_test.go +++ b/internal/build/lifecycle_execution_test.go @@ -607,20 +607,8 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) { platformAPI = api.MustParse("0.10") it("runs the extender (build)", func() { - err := lifecycle.Run(context.Background(), func(execution *build.LifecycleExecution) build.PhaseFactory { - return fakePhaseFactory - }) - h.AssertNil(t, err) - - h.AssertEq(t, len(fakePhaseFactory.NewCalledWithProvider), 5) - - var found bool - for _, entry := range fakePhaseFactory.NewCalledWithProvider { - if entry.Name() == "extender" { - found = true - } - } - h.AssertEq(t, found, true) + // Add tests from mock docker daemon + testBuildDockerfiles(t, when, it) }) }) }) @@ -1790,14 +1778,6 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) { platformAPI = api.MustParse("0.12") providedOrderExt = dist.Order{dist.OrderEntry{Group: []dist.ModuleRef{ /* don't care */ }}} - when("for build", func() { - extensionsForBuild = true - - it("configures the phase with registry access", func() { - h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_REGISTRY_AUTH={}") - }) - }) - when("for run", func() { extensionsForRun = true @@ -1897,16 +1877,6 @@ func testLifecycleExecution(t *testing.T, when spec.G, it spec.S) { h.AssertSliceNotContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko") }) }) - - when("platform >= 0.10", func() { - platformAPI = api.MustParse("0.10") - - it("provides -build-image and /kaniko bind", func() { - h.AssertSliceContainsInOrder(t, configProvider.ContainerConfig().Cmd, "-build-image", providedBuilderImage) - h.AssertSliceContains(t, configProvider.ContainerConfig().Env, "CNB_REGISTRY_AUTH={}") - h.AssertSliceContains(t, configProvider.HostConfig().Binds, "some-kaniko-cache:/kaniko") - }) - }) }) when("not present in /generated/build", func() { diff --git a/internal/build/lifecycle_executor.go b/internal/build/lifecycle_executor.go index 996235a204..89de72a0f3 100644 --- a/internal/build/lifecycle_executor.go +++ b/internal/build/lifecycle_executor.go @@ -70,6 +70,7 @@ type LifecycleOptions struct { Image name.Reference Builder Builder BuilderImage string // differs from Builder.Name() and Builder.Image().Name() in that it includes the registry context + isExtendedBuilderImageRoot bool LifecycleImage string LifecycleApis []string // optional - populated only if custom lifecycle image is downloaded, from that lifecycle's container's Labels. RunImage string diff --git a/internal/build/mockdocker/mockDockerClient.go b/internal/build/mockdocker/mockDockerClient.go new file mode 100644 index 0000000000..55313e7677 --- /dev/null +++ b/internal/build/mockdocker/mockDockerClient.go @@ -0,0 +1,233 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/buildpacks/pack/internal/build (interfaces: DockerClient) + +// Package mockdocker is a generated GoMock package. +package mockdocker + +import ( + context "context" + io "io" + reflect "reflect" + + types "github.com/docker/docker/api/types" + container "github.com/docker/docker/api/types/container" + network "github.com/docker/docker/api/types/network" + gomock "github.com/golang/mock/gomock" + v1 "github.com/opencontainers/image-spec/specs-go/v1" +) + +// MockDockerClient is a mock of DockerClient interface. +type MockDockerClient struct { + ctrl *gomock.Controller + recorder *MockDockerClientMockRecorder +} + +// MockDockerClientMockRecorder is the mock recorder for MockDockerClient. +type MockDockerClientMockRecorder struct { + mock *MockDockerClient +} + +// NewMockDockerClient creates a new mock instance. +func NewMockDockerClient(ctrl *gomock.Controller) *MockDockerClient { + mock := &MockDockerClient{ctrl: ctrl} + mock.recorder = &MockDockerClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockDockerClient) EXPECT() *MockDockerClientMockRecorder { + return m.recorder +} + +// ContainerAttach mocks base method. +func (m *MockDockerClient) ContainerAttach(arg0 context.Context, arg1 string, arg2 types.ContainerAttachOptions) (types.HijackedResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ContainerAttach", arg0, arg1, arg2) + ret0, _ := ret[0].(types.HijackedResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ContainerAttach indicates an expected call of ContainerAttach. +func (mr *MockDockerClientMockRecorder) ContainerAttach(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainerAttach", reflect.TypeOf((*MockDockerClient)(nil).ContainerAttach), arg0, arg1, arg2) +} + +// ContainerCreate mocks base method. +func (m *MockDockerClient) ContainerCreate(arg0 context.Context, arg1 *container.Config, arg2 *container.HostConfig, arg3 *network.NetworkingConfig, arg4 *v1.Platform, arg5 string) (container.CreateResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ContainerCreate", arg0, arg1, arg2, arg3, arg4, arg5) + ret0, _ := ret[0].(container.CreateResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ContainerCreate indicates an expected call of ContainerCreate. +func (mr *MockDockerClientMockRecorder) ContainerCreate(arg0, arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainerCreate", reflect.TypeOf((*MockDockerClient)(nil).ContainerCreate), arg0, arg1, arg2, arg3, arg4, arg5) +} + +// ContainerInspect mocks base method. +func (m *MockDockerClient) ContainerInspect(arg0 context.Context, arg1 string) (types.ContainerJSON, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ContainerInspect", arg0, arg1) + ret0, _ := ret[0].(types.ContainerJSON) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ContainerInspect indicates an expected call of ContainerInspect. +func (mr *MockDockerClientMockRecorder) ContainerInspect(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainerInspect", reflect.TypeOf((*MockDockerClient)(nil).ContainerInspect), arg0, arg1) +} + +// ContainerRemove mocks base method. +func (m *MockDockerClient) ContainerRemove(arg0 context.Context, arg1 string, arg2 types.ContainerRemoveOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ContainerRemove", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ContainerRemove indicates an expected call of ContainerRemove. +func (mr *MockDockerClientMockRecorder) ContainerRemove(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainerRemove", reflect.TypeOf((*MockDockerClient)(nil).ContainerRemove), arg0, arg1, arg2) +} + +// ContainerStart mocks base method. +func (m *MockDockerClient) ContainerStart(arg0 context.Context, arg1 string, arg2 types.ContainerStartOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ContainerStart", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// ContainerStart indicates an expected call of ContainerStart. +func (mr *MockDockerClientMockRecorder) ContainerStart(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainerStart", reflect.TypeOf((*MockDockerClient)(nil).ContainerStart), arg0, arg1, arg2) +} + +// ContainerWait mocks base method. +func (m *MockDockerClient) ContainerWait(arg0 context.Context, arg1 string, arg2 container.WaitCondition) (<-chan container.WaitResponse, <-chan error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ContainerWait", arg0, arg1, arg2) + ret0, _ := ret[0].(<-chan container.WaitResponse) + ret1, _ := ret[1].(<-chan error) + return ret0, ret1 +} + +// ContainerWait indicates an expected call of ContainerWait. +func (mr *MockDockerClientMockRecorder) ContainerWait(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ContainerWait", reflect.TypeOf((*MockDockerClient)(nil).ContainerWait), arg0, arg1, arg2) +} + +// CopyFromContainer mocks base method. +func (m *MockDockerClient) CopyFromContainer(arg0 context.Context, arg1, arg2 string) (io.ReadCloser, types.ContainerPathStat, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CopyFromContainer", arg0, arg1, arg2) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(types.ContainerPathStat) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// CopyFromContainer indicates an expected call of CopyFromContainer. +func (mr *MockDockerClientMockRecorder) CopyFromContainer(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyFromContainer", reflect.TypeOf((*MockDockerClient)(nil).CopyFromContainer), arg0, arg1, arg2) +} + +// CopyToContainer mocks base method. +func (m *MockDockerClient) CopyToContainer(arg0 context.Context, arg1, arg2 string, arg3 io.Reader, arg4 types.CopyToContainerOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CopyToContainer", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(error) + return ret0 +} + +// CopyToContainer indicates an expected call of CopyToContainer. +func (mr *MockDockerClientMockRecorder) CopyToContainer(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CopyToContainer", reflect.TypeOf((*MockDockerClient)(nil).CopyToContainer), arg0, arg1, arg2, arg3, arg4) +} + +// ImageBuild mocks base method. +func (m *MockDockerClient) ImageBuild(arg0 context.Context, arg1 io.Reader, arg2 types.ImageBuildOptions) (types.ImageBuildResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageBuild", arg0, arg1, arg2) + ret0, _ := ret[0].(types.ImageBuildResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageBuild indicates an expected call of ImageBuild. +func (mr *MockDockerClientMockRecorder) ImageBuild(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageBuild", reflect.TypeOf((*MockDockerClient)(nil).ImageBuild), arg0, arg1, arg2) +} + +// ImageInspectWithRaw mocks base method. +func (m *MockDockerClient) ImageInspectWithRaw(arg0 context.Context, arg1 string) (types.ImageInspect, []byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageInspectWithRaw", arg0, arg1) + ret0, _ := ret[0].(types.ImageInspect) + ret1, _ := ret[1].([]byte) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ImageInspectWithRaw indicates an expected call of ImageInspectWithRaw. +func (mr *MockDockerClientMockRecorder) ImageInspectWithRaw(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageInspectWithRaw", reflect.TypeOf((*MockDockerClient)(nil).ImageInspectWithRaw), arg0, arg1) +} + +// ImageRemove mocks base method. +func (m *MockDockerClient) ImageRemove(arg0 context.Context, arg1 string, arg2 types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageRemove", arg0, arg1, arg2) + ret0, _ := ret[0].([]types.ImageDeleteResponseItem) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageRemove indicates an expected call of ImageRemove. +func (mr *MockDockerClientMockRecorder) ImageRemove(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageRemove", reflect.TypeOf((*MockDockerClient)(nil).ImageRemove), arg0, arg1, arg2) +} + +// ImageSave mocks base method. +func (m *MockDockerClient) ImageSave(arg0 context.Context, arg1 []string) (io.ReadCloser, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ImageSave", arg0, arg1) + ret0, _ := ret[0].(io.ReadCloser) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ImageSave indicates an expected call of ImageSave. +func (mr *MockDockerClientMockRecorder) ImageSave(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ImageSave", reflect.TypeOf((*MockDockerClient)(nil).ImageSave), arg0, arg1) +} + +// VolumeRemove mocks base method. +func (m *MockDockerClient) VolumeRemove(arg0 context.Context, arg1 string, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "VolumeRemove", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// VolumeRemove indicates an expected call of VolumeRemove. +func (mr *MockDockerClientMockRecorder) VolumeRemove(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "VolumeRemove", reflect.TypeOf((*MockDockerClient)(nil).VolumeRemove), arg0, arg1, arg2) +} diff --git a/internal/build/phase_config_provider.go b/internal/build/phase_config_provider.go index f38641110d..b32747bf6a 100644 --- a/internal/build/phase_config_provider.go +++ b/internal/build/phase_config_provider.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "os" + "strconv" "strings" "github.com/docker/docker/api/types/container" @@ -251,6 +252,12 @@ func WithRoot() PhaseConfigProviderOperation { } } +func WithUser(uid int, gid int) PhaseConfigProviderOperation { + return func(provider *PhaseConfigProvider) { + provider.ctrConf.User = strconv.Itoa(uid) + ":" + strconv.Itoa(gid) + } +} + func WithContainerOperations(operations ...ContainerOperation) PhaseConfigProviderOperation { return func(provider *PhaseConfigProvider) { provider.containerOps = append(provider.containerOps, operations...) diff --git a/internal/build/testdata/fake-tmp/build-extension/multi/build/samples_test/Dockerfile b/internal/build/testdata/fake-tmp/build-extension/multi/build/samples_test/Dockerfile new file mode 100644 index 0000000000..f2e3fd88b5 --- /dev/null +++ b/internal/build/testdata/fake-tmp/build-extension/multi/build/samples_test/Dockerfile @@ -0,0 +1,24 @@ +ARG base_image +FROM ${base_image} + +USER root +# Update and upgrade the packages +RUN apk update && apk upgrade + +# Install packages +RUN apk add --no-cache build-base openssl libffi-dev python3 tree + +# Print package versions +RUN gcc --version && openssl version && python3 --version + +# Remove packages +RUN apk del tree + +# Install additional packages +RUN apk add --no-cache postgresql-client git curl + +# Print package versions +RUN psql --version && git --version && curl --version + +# ENV variables +ENV ENV_VARIABLE=value diff --git a/internal/build/testdata/fake-tmp/build-extension/multi/build/samples_tree/Dockerfile b/internal/build/testdata/fake-tmp/build-extension/multi/build/samples_tree/Dockerfile new file mode 100644 index 0000000000..3554737318 --- /dev/null +++ b/internal/build/testdata/fake-tmp/build-extension/multi/build/samples_tree/Dockerfile @@ -0,0 +1,5 @@ +ARG base_image +FROM ${base_image} + +USER root +RUN apk update && apk add tree diff --git a/internal/build/testdata/fake-tmp/build-extension/multi/group.toml b/internal/build/testdata/fake-tmp/build-extension/multi/group.toml new file mode 100644 index 0000000000..77bed0fb9e --- /dev/null +++ b/internal/build/testdata/fake-tmp/build-extension/multi/group.toml @@ -0,0 +1,17 @@ +[[group]] + id = "samples/hello-extensions" + version = "0.0.1" + api = "0.9" + homepage = "https://github.com/buildpacks/samples/tree/main/buildpacks/hello-extensions" + +[[group-extensions]] + id = "samples/tree" + version = "0.0.1" + api = "0.9" + homepage = "https://github.com/buildpacks/samples/tree/main/extensions/tree" + +[[group-extensions]] + id = "samples/test" + version = "0.0.1" + api = "0.9" + homepage = "https://github.com/buildpacks/samples/test/main/extensions/test" diff --git a/internal/build/testdata/fake-tmp/build-extension/single/build/samples_test/Dockerfile b/internal/build/testdata/fake-tmp/build-extension/single/build/samples_test/Dockerfile new file mode 100644 index 0000000000..21e4d6eb47 --- /dev/null +++ b/internal/build/testdata/fake-tmp/build-extension/single/build/samples_test/Dockerfile @@ -0,0 +1,24 @@ +ARG base_image +FROM ${base_image} + +USER root +# Update and upgrade the Alpine packages +RUN apk update && apk upgrade + +# Install packages +RUN apk add --no-cache build-base openssl libffi-dev python3 tree + +# Print package versions +RUN gcc --version && openssl version && python3 --version + +# Remove packages +RUN apk del tree + +# Install additional packages +RUN apk add --no-cache postgresql-client git curl + +# Print package versions +RUN psql --version && git --version && curl --version + +# ENV variables +ENV ENV_VARIABLE=value diff --git a/internal/build/testdata/fake-tmp/build-extension/single/group.toml b/internal/build/testdata/fake-tmp/build-extension/single/group.toml new file mode 100644 index 0000000000..167a798713 --- /dev/null +++ b/internal/build/testdata/fake-tmp/build-extension/single/group.toml @@ -0,0 +1,11 @@ +[[group]] + id = "samples/hello-extensions" + version = "0.0.1" + api = "0.9" + homepage = "https://github.com/buildpacks/samples/tree/main/buildpacks/hello-extensions" + +[[group-extensions]] + id = "samples/test" + version = "0.0.1" + api = "0.9" + homepage = "https://github.com/buildpacks/samples/test/main/extensions/test" diff --git a/internal/commands/build.go b/internal/commands/build.go index 3dcf6cc43b..de70be419c 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -74,7 +74,7 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob "requires an image name, which will be generated from the source code. Build defaults to the current directory, " + "but you can use `--path` to specify another source code directory. Build requires a `builder`, which can either " + "be provided directly to build using `--builder`, or can be set using the `set-default-builder` command. For more " + - "on how to use `pack build`, see: https://buildpacks.io/docs/app-developer-guide/build-an-app/.", + "on how to use `pack build`, see: https://buildpacks.io/docs/app-developer-guide/build-an-app/.\n\nNote: Pack uses the nomeclature of -extended to refer to a builder image that has been extended using an extension.", RunE: logError(logger, func(cmd *cobra.Command, args []string) error { inputImageName := client.ParseInputImageReference(args[0]) if err := validateBuildFlags(&flags, cfg, inputImageName, logger); err != nil { diff --git a/pkg/archive/archive.go b/pkg/archive/archive.go index 871dc5fd00..25bb7fa163 100644 --- a/pkg/archive/archive.go +++ b/pkg/archive/archive.go @@ -78,7 +78,7 @@ func GenerateTarWithWriter(genFn func(TarWriter) error, twf TarWriterFactory) io err := genFn(tw) closeErr := tw.Close() - closeErr = aggregateError(closeErr, pw.CloseWithError(err)) + closeErr = AggregateError(closeErr, pw.CloseWithError(err)) errChan <- closeErr }() @@ -89,19 +89,19 @@ func GenerateTarWithWriter(genFn func(TarWriter) error, twf TarWriterFactory) io // closing the reader ensures that if anything attempts // further reading it doesn't block waiting for content if err := pr.Close(); err != nil { - completeErr = aggregateError(completeErr, err) + completeErr = AggregateError(completeErr, err) } // wait until everything closes properly if err := <-errChan; err != nil { - completeErr = aggregateError(completeErr, err) + completeErr = AggregateError(completeErr, err) } return completeErr }) } -func aggregateError(base, addition error) error { +func AggregateError(base, addition error) error { if addition == nil { return base } @@ -347,6 +347,33 @@ func WriteZipToTar(tw TarWriter, srcZip, basePath string, uid, gid int, mode int return nil } +func WriteFileToTar(tw TarWriter, srcFile, destPath string, uid, gid int, mode int64, normalizeModTime bool) error { + f, err := os.Open(srcFile) + if err != nil { + return err + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(fi, "") + if err != nil { + return err + } + + header.Name = destPath + err = writeHeader(header, uid, gid, mode, normalizeModTime, tw) + if err != nil { + return err + } + + _, err = io.Copy(tw, f) + return err +} + // NormalizeHeader normalizes a tar.Header // // Normalizes the following: diff --git a/pkg/archive/archive_test.go b/pkg/archive/archive_test.go index 05b864dd73..6a34a8dc00 100644 --- a/pkg/archive/archive_test.go +++ b/pkg/archive/archive_test.go @@ -9,9 +9,8 @@ import ( "strings" "testing" - "github.com/pkg/errors" - "github.com/heroku/color" + "github.com/pkg/errors" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -618,6 +617,53 @@ func testArchive(t *testing.T, when spec.G, it spec.S) { }) }) + when("#WrtieFileToTar", func() { + var src string + it.Before(func() { + src = filepath.Join("testdata", "file-to-tar.txt") + }) + + when("mode is set to 0777", func() { + it("writes a tar to the dest dir with 0777", func() { + fh, err := os.Create(filepath.Join(tmpDir, "some.tar")) + h.AssertNil(t, err) + + tw := tar.NewWriter(fh) + + err = archive.WriteFileToTar(tw, src, "/nested/dir/dir-in-archive/file-to-tar.txt", 1234, 2345, 0777, true) + h.AssertNil(t, err) + h.AssertNil(t, tw.Close()) + h.AssertNil(t, fh.Close()) + + file, err := os.Open(filepath.Join(tmpDir, "some.tar")) + h.AssertNil(t, err) + defer file.Close() + + tr := tar.NewReader(file) + + verify := h.NewTarVerifier(t, tr, 1234, 2345) + verify.NextFile("/nested/dir/dir-in-archive/file-to-tar.txt", "Hi I love CNB!", 0777) + }) + }) + + when("normalize mod time is false", func() { + it("does not normalize mod times", func() { + tarFile := filepath.Join(tmpDir, "some.tar") + fh, err := os.Create(tarFile) + h.AssertNil(t, err) + + tw := tar.NewWriter(fh) + + err = archive.WriteFileToTar(tw, src, "/foo/file-to-tar.txt", 1234, 2345, 0777, false) + h.AssertNil(t, err) + h.AssertNil(t, tw.Close()) + h.AssertNil(t, fh.Close()) + + h.AssertOnTarEntry(t, tarFile, "/foo/file-to-tar.txt", h.DoesNotHaveModTime(archive.NormalizedDateTime)) + }) + }) + }) + when("#IsZip", func() { when("file is a zip file", func() { it("returns true", func() { diff --git a/pkg/archive/testdata/file-to-tar.txt b/pkg/archive/testdata/file-to-tar.txt new file mode 100644 index 0000000000..4145eed435 --- /dev/null +++ b/pkg/archive/testdata/file-to-tar.txt @@ -0,0 +1 @@ +Hi I love CNB! \ No newline at end of file diff --git a/pkg/client/docker.go b/pkg/client/docker.go index bb81dc487a..1a7ca533e9 100644 --- a/pkg/client/docker.go +++ b/pkg/client/docker.go @@ -32,4 +32,5 @@ type DockerClient interface { ContainerWait(ctx context.Context, container string, condition containertypes.WaitCondition) (<-chan containertypes.WaitResponse, <-chan error) ContainerAttach(ctx context.Context, container string, options containertypes.AttachOptions) (types.HijackedResponse, error) ContainerStart(ctx context.Context, container string, options containertypes.StartOptions) error + ImageBuild(ctx context.Context, buildContext io.Reader, options types.ImageBuildOptions) (types.ImageBuildResponse, error) }