diff --git a/pkg/commands/add.go b/pkg/commands/add.go index b0f07a5996..27cf0c968a 100644 --- a/pkg/commands/add.go +++ b/pkg/commands/add.go @@ -19,12 +19,13 @@ package commands import ( "path/filepath" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/google/go-containerregistry/pkg/v1" "github.com/GoogleContainerTools/kaniko/pkg/util" - "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/sirupsen/logrus" ) @@ -44,18 +45,13 @@ type AddCommand struct { // 2. If is a local tar archive: // -If is a local tar archive, it is unpacked at the dest, as 'tar -x' would func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.BuildArgs) error { - // First, resolve any environment replacement replacementEnvs := buildArgs.ReplacementEnvs(config.Env) - resolvedEnvs, err := util.ResolveEnvironmentReplacementList(a.cmd.SourcesAndDest, replacementEnvs, true) - if err != nil { - return err - } - dest := resolvedEnvs[len(resolvedEnvs)-1] - // Resolve wildcards and get a list of resolved sources - srcs, err := util.ResolveSources(resolvedEnvs, a.buildcontext) + + srcs, dest, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs) if err != nil { return err } + var unresolvedSrcs []string // If any of the sources are local tar archives: // 1. Unpack them to the specified destination @@ -94,6 +90,7 @@ func (a *AddCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bui }, buildcontext: a.buildcontext, } + if err := copyCmd.ExecuteCommand(config, buildArgs); err != nil { return err } @@ -111,6 +108,26 @@ func (a *AddCommand) String() string { return a.cmd.String() } -func (a *AddCommand) UsesContext() bool { - return true +func (a *AddCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) { + replacementEnvs := buildArgs.ReplacementEnvs(config.Env) + + srcs, _, err := resolveEnvAndWildcards(a.cmd.SourcesAndDest, a.buildcontext, replacementEnvs) + if err != nil { + return nil, err + } + + files := []string{} + for _, src := range srcs { + if util.IsSrcRemoteFileURL(src) { + continue + } + if util.IsFileLocalTarArchive(src) { + continue + } + fullPath := filepath.Join(a.buildcontext, src) + files = append(files, fullPath) + } + + logrus.Infof("Using files from context: %v", files) + return files, nil } diff --git a/pkg/commands/base_command.go b/pkg/commands/base_command.go index bcb6448c5f..d2a746ee6f 100644 --- a/pkg/commands/base_command.go +++ b/pkg/commands/base_command.go @@ -16,19 +16,23 @@ limitations under the License. package commands +import ( + "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" + "github.com/google/go-containerregistry/pkg/v1" +) + type BaseCommand struct { - cache bool - usesContext bool + cache bool } func (b *BaseCommand) CacheCommand() bool { return b.cache } -func (b *BaseCommand) UsesContext() bool { - return b.usesContext -} - func (b *BaseCommand) FilesToSnapshot() []string { return []string{} } + +func (b *BaseCommand) FilesUsedFromContext(_ *v1.Config, _ *dockerfile.BuildArgs) ([]string, error) { + return []string{}, nil +} diff --git a/pkg/commands/commands.go b/pkg/commands/commands.go index 4222727652..cbb960e684 100644 --- a/pkg/commands/commands.go +++ b/pkg/commands/commands.go @@ -39,7 +39,7 @@ type DockerCommand interface { CacheCommand() bool // Return true if this command depends on the build context. - UsesContext() bool + FilesUsedFromContext(*v1.Config, *dockerfile.BuildArgs) ([]string, error) } func GetCommand(cmd instructions.Command, buildcontext string) (DockerCommand, error) { diff --git a/pkg/commands/copy.go b/pkg/commands/copy.go index 7bb582ea63..d70d98483b 100644 --- a/pkg/commands/copy.go +++ b/pkg/commands/copy.go @@ -20,12 +20,14 @@ import ( "os" "path/filepath" + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/sirupsen/logrus" + "github.com/GoogleContainerTools/kaniko/pkg/constants" "github.com/GoogleContainerTools/kaniko/pkg/dockerfile" "github.com/GoogleContainerTools/kaniko/pkg/util" "github.com/google/go-containerregistry/pkg/v1" - "github.com/moby/buildkit/frontend/dockerfile/instructions" ) type CopyCommand struct { @@ -40,18 +42,14 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu if c.cmd.From != "" { c.buildcontext = filepath.Join(constants.KanikoDir, c.cmd.From) } + replacementEnvs := buildArgs.ReplacementEnvs(config.Env) - // First, resolve any environment replacement - resolvedEnvs, err := util.ResolveEnvironmentReplacementList(c.cmd.SourcesAndDest, replacementEnvs, true) - if err != nil { - return err - } - dest := resolvedEnvs[len(resolvedEnvs)-1] - // Resolve wildcards and get a list of resolved sources - srcs, err := util.ResolveSources(resolvedEnvs, c.buildcontext) + + srcs, dest, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs) if err != nil { return err } + // For each source, iterate through and copy it over for _, src := range srcs { fullPath := filepath.Join(c.buildcontext, src) @@ -94,6 +92,18 @@ func (c *CopyCommand) ExecuteCommand(config *v1.Config, buildArgs *dockerfile.Bu return nil } +func resolveEnvAndWildcards(sd instructions.SourcesAndDest, buildcontext string, envs []string) ([]string, string, error) { + // First, resolve any environment replacement + resolvedEnvs, err := util.ResolveEnvironmentReplacementList(sd, envs, true) + if err != nil { + return nil, "", err + } + dest := resolvedEnvs[len(resolvedEnvs)-1] + // Resolve wildcards and get a list of resolved sources + srcs, err := util.ResolveSources(resolvedEnvs, buildcontext) + return srcs, dest, err +} + // FilesToSnapshot should return an empty array if still nil; no files were changed func (c *CopyCommand) FilesToSnapshot() []string { return c.snapshotFiles @@ -104,6 +114,23 @@ func (c *CopyCommand) String() string { return c.cmd.String() } -func (c *CopyCommand) UsesContext() bool { - return true +func (c *CopyCommand) FilesUsedFromContext(config *v1.Config, buildArgs *dockerfile.BuildArgs) ([]string, error) { + // We don't use the context if we're performing a copy --from. + if c.cmd.From != "" { + return nil, nil + } + + replacementEnvs := buildArgs.ReplacementEnvs(config.Env) + srcs, _, err := resolveEnvAndWildcards(c.cmd.SourcesAndDest, c.buildcontext, replacementEnvs) + if err != nil { + return nil, err + } + + files := []string{} + for _, src := range srcs { + fullPath := filepath.Join(c.buildcontext, src) + files = append(files, fullPath) + } + logrus.Infof("Using files from context: %v", files) + return files, nil } diff --git a/pkg/executor/build.go b/pkg/executor/build.go index 7d991069d6..29a1cf314a 100644 --- a/pkg/executor/build.go +++ b/pkg/executor/build.go @@ -125,10 +125,6 @@ func (s *stageBuilder) build() error { // Set the initial cache key to be the base image digest, the build args and the SrcContext. compositeKey := NewCompositeCache(s.baseImageDigest) - contextHash, err := HashDir(s.opts.SrcContext) - if err != nil { - return err - } compositeKey.AddKey(s.opts.BuildArgs...) cmds := []commands.DockerCommand{} @@ -148,8 +144,16 @@ func (s *stageBuilder) build() error { // Add the next command to the cache key. compositeKey.AddKey(command.String()) - if command.UsesContext() { - compositeKey.AddKey(contextHash) + + // If the command uses files from the context, add them. + files, err := command.FilesUsedFromContext(&s.cf.Config, args) + if err != nil { + return err + } + for _, f := range files { + if err := compositeKey.AddPath(f); err != nil { + return err + } } logrus.Info(command.String()) @@ -172,7 +176,7 @@ func (s *stageBuilder) build() error { if err := command.ExecuteCommand(&s.cf.Config, args); err != nil { return err } - files := command.FilesToSnapshot() + files = command.FilesToSnapshot() var contents []byte if !s.shouldTakeSnapshot(index, files) { diff --git a/pkg/executor/composite_cache.go b/pkg/executor/composite_cache.go index 0a832e6369..15a88a40b1 100644 --- a/pkg/executor/composite_cache.go +++ b/pkg/executor/composite_cache.go @@ -53,6 +53,32 @@ func (s *CompositeCache) Hash() (string, error) { return util.SHA256(strings.NewReader(s.Key())) } +func (s *CompositeCache) AddPath(p string) error { + sha := sha256.New() + fi, err := os.Lstat(p) + if err != nil { + return err + } + if fi.Mode().IsDir() { + k, err := HashDir(p) + if err != nil { + return err + } + s.keys = append(s.keys, k) + return nil + } + fh, err := util.CacheHasher()(p) + if err != nil { + return err + } + if _, err := sha.Write([]byte(fh)); err != nil { + return err + } + + s.keys = append(s.keys, string(sha.Sum(nil))) + return nil +} + // HashDir returns a hash of the directory. func HashDir(p string) (string, error) { sha := sha256.New()