Merge pull request #32 from avirshup/custom_ignore

Use custom .dockerignores for any build step

dockermake/builds.py

@@ -15,7 +15,7 @@
 
 import os
 from builtins import object
-from termcolor import cprint
+from termcolor import cprint, colored
 
 from dockermake.step import FileCopyStep
 from . import utils
@@ -33,7 +33,7 @@ class BuildTarget(object):
         targetname (str): name to assign the final built image
         steps (List[BuildStep]): list of steps required to build this image
         stagedfiles (List[StagedFile]): list of files to stage into this image from other images
-        from_iamge (str): External base image name
+        from_image (str): External base image name
     """
     def __init__(self, imagename, targetname, steps, sourcebuilds, from_image):
         self.imagename = imagename
@@ -87,8 +87,10 @@ def build(self, client,
         cprint(_centered(line, width), color='blue', attrs=['bold'])
 
         for istep, step in enumerate(self.steps):
-            cprint('* Building image "%s", Step %d/%d:' % (self.imagename, istep+1, len(self.steps)),
-                   color='blue')
+            print(colored('* Step','blue'),
+                  colored('%d/%d' % (istep+1, len(self.steps)), 'blue', attrs=['bold']),
+                  colored('for image', color='blue'),
+                  colored(self.imagename, color='blue', attrs=['bold']))
 
             if not nobuild:
                 if step.bust_cache:
@@ -97,8 +99,9 @@ def build(self, client,
                         step.bust_cache = False
 
                 step.build(client, usecache=usecache)
-                cprint("* Created intermediate image \"%s\"\n" % step.buildname,
-                       'green')
+                print(colored("* Created intermediate image", 'green'),
+                      colored(step.buildname, 'green', attrs=['bold']),
+                      end='\n\n')
 
                 if step.bust_cache:
                     _rebuilt.add(stackkey)

dockermake/cli.py

@@ -109,9 +109,9 @@ def print_yaml_help():
                         'piece of functionality in the build field, and "inherit" all other'
                         ' functionality from a list of other components that your image requires. '
                         'If you need to add files with the ADD and COPY commands, specify the root'
-                        ' directory for those files with build_directory. Your tree of '
+                        ' directory for those files with `build_directory`. Your tree of '
                         '"requires" must have exactly one unique named base image '
-                        'in the FROM field.'))
+                        'in the FROM or FROM_DOCKERFILE field.'))
 
     print('\n\nAN EXAMPLE:')
     print(printable_code("""devbase:

dockermake/errors.py

@@ -64,3 +64,7 @@ class InvalidRequiresList(UserException):
 
 class ParsingFailure(UserException):
     CODE = 50
+
+
+class MultipleIgnoreError(UserException):
+    CODE = 51

dockermake/imagedefs.py

@@ -28,7 +28,7 @@
 from . import errors
 
 RECOGNIZED_KEYS = set(('requires build_directory build copy_from FROM description _sourcefile'
-                       ' FROM_DOCKERFILE')
+                       ' FROM_DOCKERFILE ignore ignorefile')
                       .split())
 SPECIAL_FIELDS = set('_ALL_ _SOURCES_'.split())
 
@@ -62,7 +62,7 @@ def parse_yaml(self, filename):
         with open(fname, 'r') as yaml_file:
             yamldefs = yaml.load(yaml_file)
 
-        self._fix_file_paths(filename, yamldefs)
+        self._check_yaml_and_paths(filename, yamldefs)
 
         sourcedefs = {}
         for s in yamldefs.get('_SOURCES_', []):
@@ -73,35 +73,39 @@ def parse_yaml(self, filename):
         return sourcedefs
 
     @staticmethod
-    def _fix_file_paths(ymlfilepath, yamldefs):
-        """ Interpret all paths relative the the current yaml file
-
-        Note: this also checks the validity of all keys
+    def _check_yaml_and_paths(ymlfilepath, yamldefs):
+        """ Checks YAML for errors and resolves all paths
         """
         relpath = os.path.relpath(ymlfilepath)
         if '/' not in relpath:
             relpath = './%s' % relpath
         pathroot = os.path.abspath(os.path.dirname(ymlfilepath))
 
-        for field, item in iteritems(yamldefs):
-            if field == '_SOURCES_':
+        for imagename, defn in iteritems(yamldefs):
+            if imagename == '_SOURCES_':
                 yamldefs['_SOURCES_'] = [os.path.relpath(_get_abspath(pathroot, p))
                                          for p in yamldefs['_SOURCES_']]
                 continue
-            elif field in SPECIAL_FIELDS:
+            elif imagename in SPECIAL_FIELDS:
                 continue
 
-            for key in ('build_directory', 'FROM_DOCKERFILE'):
-                if key in item:
-                    item[key] = _get_abspath(pathroot, item[key])
+            for key in ('build_directory', 'FROM_DOCKERFILE', 'ignorefile'):
+                if key in defn:
+                    defn[key] = _get_abspath(pathroot, defn[key])
 
             # save the file path for logging
-            item['_sourcefile'] = relpath
+            defn['_sourcefile'] = relpath
+
+            if 'ignore' in defn and 'ignorefile' in defn:
+                raise errors.MultipleIgnoreError(
+                        'Image "%s" has both "ignore" AND "ignorefile" fields.' % imagename +
+                        ' At most ONE of these should be defined')
 
-            for key in item:
+            for key in defn:
                 if key not in RECOGNIZED_KEYS:
-                    raise errors.UnrecognizedKeyError('Field "%s" in image "%s" not recognized' %
-                                                      (key, field))
+                    raise errors.UnrecognizedKeyError(
+                            'Field "%s" in image "%s" in file "%s" not recognized' %
+                            (key, imagename, relpath))
 
     def generate_build(self, image, targetname, rebuilds=None):
         """

dockermake/step.py

@@ -16,9 +16,10 @@
 import os
 import pprint
 from io import StringIO, BytesIO
-
 import sys
-from termcolor import cprint
+
+from termcolor import cprint, colored
+import docker.utils
 
 from . import utils
 from . import staging
@@ -41,13 +42,30 @@ def __init__(self, imagename, baseimage, img_def, buildname,
                  build_first=None, bust_cache=False):
         self.imagename = imagename
         self.baseimage = baseimage
-        self.dockerfile_lines = ['FROM %s\n' % baseimage,
-                                 img_def.get('build', '')]
+        self.dockerfile_lines = ['FROM %s\n' % baseimage, img_def.get('build', '')]
         self.buildname = buildname
         self.build_dir = img_def.get('build_directory', None)
         self.bust_cache = bust_cache
         self.sourcefile = img_def['_sourcefile']
         self.build_first = build_first
+        self.custom_exclude = self._get_ignorefile(img_def)
+        self.ignoredefs_file = img_def.get('ignorefile', img_def['_sourcefile'])
+
+    @staticmethod
+    def _get_ignorefile(img_def):
+        if img_def.get('ignore', None) is not None:
+            assert 'ignorefile' not in img_def
+            lines = img_def['ignore'].splitlines()
+        elif img_def.get('ignorefile', None) is not None:
+            assert 'ignore' not in img_def
+            with open(img_def['ignorefile'], 'r') as igfile:
+                lines = list(igfile)
+        else:
+            return None
+
+        lines.append('_docker_make_tmp')
+
+        return list(filter(bool, lines))
 
     def build(self, client, pull=False, usecache=True):
         """
@@ -60,10 +78,10 @@ def build(self, client, pull=False, usecache=True):
             pull (bool): whether to pull dependent layers from remote repositories
             usecache (bool): whether to use cached layers or rebuild from scratch
         """
-        from .imagedefs import ExternalDockerfile
-
-        cprint('  Image definition "%s" from file %s' % (self.imagename, self.sourcefile),
-               'blue')
+        print(colored('  Building step', 'blue'),
+              colored(self.imagename, 'blue', attrs=['bold']),
+              colored('defined in', 'blue'),
+              colored(self.sourcefile, 'blue', attrs=['bold']))
 
         if self.build_first and not self.build_first.built:
             self.build_external_dockerfile(client, self.build_first)
@@ -83,9 +101,24 @@ def build(self, client, pull=False, usecache=True):
 
         if self.build_dir is not None:
             tempdir = self.write_dockerfile(dockerfile)
+            context_path = os.path.abspath(os.path.expanduser(self.build_dir))
             build_args.update(fileobj=None,
-                              path=os.path.abspath(os.path.expanduser(self.build_dir)),
                               dockerfile=os.path.join(DOCKER_TMPDIR, 'Dockerfile'))
+            print(colored('  Build context:', 'blue'),
+                  colored(os.path.relpath(context_path), 'blue', attrs=['bold']))
+
+            if not self.custom_exclude:
+                build_args.update(path=context_path)
+            else:
+                print(colored('  Custom .dockerignore from:','blue'),
+                      colored(os.path.relpath(self.ignoredefs_file),  'blue', attrs=['bold']))
+                context = docker.utils.tar(self.build_dir,
+                                           exclude=self.custom_exclude,
+                                           dockerfile=os.path.join(DOCKER_TMPDIR, 'Dockerfile'),
+                                           gzip=False)
+                build_args.update(fileobj=context,
+                                  custom_context=True)
+
         else:
             if sys.version_info.major == 2:
                 build_args.update(fileobj=StringIO(dockerfile),

test/data/ignorefile.yml

@@ -0,0 +1,5 @@
+target:
+  FROM: alpine
+  ignorefile: test_build/custom_ignore_file.txt
+  build_directory: ./test_build
+  build: ADD . /opt

test/data/ignores.yml

@@ -0,0 +1,8 @@
+target:
+  FROM: alpine
+  build_directory: ./test_build
+  ignore: |
+    b
+  build: |
+    ADD . /opt
+

test/data/multi_ignore.yml

@@ -0,0 +1,4 @@
+target:
+  description: should error because you can't have both of these
+  ignore: a
+  ignorefile: test_build/.dockerignore

test/data/test_build/.dockerignore

@@ -0,0 +1,2 @@
+a
+b

test/data/test_build/Dockerfile

@@ -0,0 +1,3 @@
+FROM centos
+WORKDIR /opt
+ADD c .

test/data/test_build/a

@@ -0,0 +1 @@
+a

test/data/test_build/b

@@ -0,0 +1 @@
+b

test/data/test_build/c

@@ -0,0 +1 @@
+c

test/data/test_build/custom_ignore_file.txt

@@ -0,0 +1 @@
+c

test/test_errors.py

@@ -13,7 +13,8 @@
     'data/missingfile.yml': errors.MissingFileError,
     'data/baddockerfile.yml': errors.ExternalBuildError,
     'data/invalid_requires.yml': errors.InvalidRequiresList,
-    'data/invalid_yaml.yml': errors.ParsingFailure
+    'data/invalid_yaml.yml': errors.ParsingFailure,
+    'data/multi_ignore.yml': errors.MultipleIgnoreError,
 }
 
 

test/test_features.py

@@ -2,8 +2,13 @@
 
 
 def test_multiple_bases():
-        subprocess.check_call(['docker-make', '-f', 'data/multibase.yml', 'target2', 'target3'])
+    subprocess.check_call(['docker-make', '-f', 'data/multibase.yml', 'target2', 'target3'])
 
 
 def test_paths_relative_interpreted_relative_to_definition_file():
-        subprocess.check_call(['docker-make', '-f', 'data/include.yml', 'target'])
+    subprocess.check_call(['docker-make', '-f', 'data/include.yml', 'target'])
+
+
+def test_ignore_string():
+    subprocess.check_call(['docker-make', '-f', 'data/ignores.yml', 'target'])
+