diff --git a/dockermake/builds.py b/dockermake/builds.py index e17a492..67019ff 100644 --- a/dockermake/builds.py +++ b/dockermake/builds.py @@ -15,7 +15,7 @@ import os from builtins import object -from termcolor import cprint +from termcolor import cprint, colored from dockermake.step import FileCopyStep from . import utils @@ -33,7 +33,7 @@ class BuildTarget(object): targetname (str): name to assign the final built image steps (List[BuildStep]): list of steps required to build this image stagedfiles (List[StagedFile]): list of files to stage into this image from other images - from_iamge (str): External base image name + from_image (str): External base image name """ def __init__(self, imagename, targetname, steps, sourcebuilds, from_image): self.imagename = imagename @@ -87,8 +87,10 @@ def build(self, client, cprint(_centered(line, width), color='blue', attrs=['bold']) for istep, step in enumerate(self.steps): - cprint('* Building image "%s", Step %d/%d:' % (self.imagename, istep+1, len(self.steps)), - color='blue') + print(colored('* Step','blue'), + colored('%d/%d' % (istep+1, len(self.steps)), 'blue', attrs=['bold']), + colored('for image', color='blue'), + colored(self.imagename, color='blue', attrs=['bold'])) if not nobuild: if step.bust_cache: @@ -97,8 +99,9 @@ def build(self, client, step.bust_cache = False step.build(client, usecache=usecache) - cprint("* Created intermediate image \"%s\"\n" % step.buildname, - 'green') + print(colored("* Created intermediate image", 'green'), + colored(step.buildname, 'green', attrs=['bold']), + end='\n\n') if step.bust_cache: _rebuilt.add(stackkey) diff --git a/dockermake/cli.py b/dockermake/cli.py index 88a5861..7430d61 100644 --- a/dockermake/cli.py +++ b/dockermake/cli.py @@ -109,9 +109,9 @@ def print_yaml_help(): 'piece of functionality in the build field, and "inherit" all other' ' functionality from a list of other components that your image requires. ' 'If you need to add files with the ADD and COPY commands, specify the root' - ' directory for those files with build_directory. Your tree of ' + ' directory for those files with `build_directory`. Your tree of ' '"requires" must have exactly one unique named base image ' - 'in the FROM field.')) + 'in the FROM or FROM_DOCKERFILE field.')) print('\n\nAN EXAMPLE:') print(printable_code("""devbase: diff --git a/dockermake/errors.py b/dockermake/errors.py index f8a2c76..fe97f61 100644 --- a/dockermake/errors.py +++ b/dockermake/errors.py @@ -64,3 +64,7 @@ class InvalidRequiresList(UserException): class ParsingFailure(UserException): CODE = 50 + + +class MultipleIgnoreError(UserException): + CODE = 51 diff --git a/dockermake/imagedefs.py b/dockermake/imagedefs.py index 89a6fc4..efa3c58 100644 --- a/dockermake/imagedefs.py +++ b/dockermake/imagedefs.py @@ -28,7 +28,7 @@ from . import errors RECOGNIZED_KEYS = set(('requires build_directory build copy_from FROM description _sourcefile' - ' FROM_DOCKERFILE') + ' FROM_DOCKERFILE ignore ignorefile') .split()) SPECIAL_FIELDS = set('_ALL_ _SOURCES_'.split()) @@ -62,7 +62,7 @@ def parse_yaml(self, filename): with open(fname, 'r') as yaml_file: yamldefs = yaml.load(yaml_file) - self._fix_file_paths(filename, yamldefs) + self._check_yaml_and_paths(filename, yamldefs) sourcedefs = {} for s in yamldefs.get('_SOURCES_', []): @@ -73,35 +73,39 @@ def parse_yaml(self, filename): return sourcedefs @staticmethod - def _fix_file_paths(ymlfilepath, yamldefs): - """ Interpret all paths relative the the current yaml file - - Note: this also checks the validity of all keys + def _check_yaml_and_paths(ymlfilepath, yamldefs): + """ Checks YAML for errors and resolves all paths """ relpath = os.path.relpath(ymlfilepath) if '/' not in relpath: relpath = './%s' % relpath pathroot = os.path.abspath(os.path.dirname(ymlfilepath)) - for field, item in iteritems(yamldefs): - if field == '_SOURCES_': + for imagename, defn in iteritems(yamldefs): + if imagename == '_SOURCES_': yamldefs['_SOURCES_'] = [os.path.relpath(_get_abspath(pathroot, p)) for p in yamldefs['_SOURCES_']] continue - elif field in SPECIAL_FIELDS: + elif imagename in SPECIAL_FIELDS: continue - for key in ('build_directory', 'FROM_DOCKERFILE'): - if key in item: - item[key] = _get_abspath(pathroot, item[key]) + for key in ('build_directory', 'FROM_DOCKERFILE', 'ignorefile'): + if key in defn: + defn[key] = _get_abspath(pathroot, defn[key]) # save the file path for logging - item['_sourcefile'] = relpath + defn['_sourcefile'] = relpath + + if 'ignore' in defn and 'ignorefile' in defn: + raise errors.MultipleIgnoreError( + 'Image "%s" has both "ignore" AND "ignorefile" fields.' % imagename + + ' At most ONE of these should be defined') - for key in item: + for key in defn: if key not in RECOGNIZED_KEYS: - raise errors.UnrecognizedKeyError('Field "%s" in image "%s" not recognized' % - (key, field)) + raise errors.UnrecognizedKeyError( + 'Field "%s" in image "%s" in file "%s" not recognized' % + (key, imagename, relpath)) def generate_build(self, image, targetname, rebuilds=None): """ diff --git a/dockermake/step.py b/dockermake/step.py index 47e4a53..67bb5ed 100644 --- a/dockermake/step.py +++ b/dockermake/step.py @@ -16,9 +16,10 @@ import os import pprint from io import StringIO, BytesIO - import sys -from termcolor import cprint + +from termcolor import cprint, colored +import docker.utils from . import utils from . import staging @@ -41,13 +42,30 @@ def __init__(self, imagename, baseimage, img_def, buildname, build_first=None, bust_cache=False): self.imagename = imagename self.baseimage = baseimage - self.dockerfile_lines = ['FROM %s\n' % baseimage, - img_def.get('build', '')] + self.dockerfile_lines = ['FROM %s\n' % baseimage, img_def.get('build', '')] self.buildname = buildname self.build_dir = img_def.get('build_directory', None) self.bust_cache = bust_cache self.sourcefile = img_def['_sourcefile'] self.build_first = build_first + self.custom_exclude = self._get_ignorefile(img_def) + self.ignoredefs_file = img_def.get('ignorefile', img_def['_sourcefile']) + + @staticmethod + def _get_ignorefile(img_def): + if img_def.get('ignore', None) is not None: + assert 'ignorefile' not in img_def + lines = img_def['ignore'].splitlines() + elif img_def.get('ignorefile', None) is not None: + assert 'ignore' not in img_def + with open(img_def['ignorefile'], 'r') as igfile: + lines = list(igfile) + else: + return None + + lines.append('_docker_make_tmp') + + return list(filter(bool, lines)) def build(self, client, pull=False, usecache=True): """ @@ -60,10 +78,10 @@ def build(self, client, pull=False, usecache=True): pull (bool): whether to pull dependent layers from remote repositories usecache (bool): whether to use cached layers or rebuild from scratch """ - from .imagedefs import ExternalDockerfile - - cprint(' Image definition "%s" from file %s' % (self.imagename, self.sourcefile), - 'blue') + print(colored(' Building step', 'blue'), + colored(self.imagename, 'blue', attrs=['bold']), + colored('defined in', 'blue'), + colored(self.sourcefile, 'blue', attrs=['bold'])) if self.build_first and not self.build_first.built: self.build_external_dockerfile(client, self.build_first) @@ -83,9 +101,24 @@ def build(self, client, pull=False, usecache=True): if self.build_dir is not None: tempdir = self.write_dockerfile(dockerfile) + context_path = os.path.abspath(os.path.expanduser(self.build_dir)) build_args.update(fileobj=None, - path=os.path.abspath(os.path.expanduser(self.build_dir)), dockerfile=os.path.join(DOCKER_TMPDIR, 'Dockerfile')) + print(colored(' Build context:', 'blue'), + colored(os.path.relpath(context_path), 'blue', attrs=['bold'])) + + if not self.custom_exclude: + build_args.update(path=context_path) + else: + print(colored(' Custom .dockerignore from:','blue'), + colored(os.path.relpath(self.ignoredefs_file), 'blue', attrs=['bold'])) + context = docker.utils.tar(self.build_dir, + exclude=self.custom_exclude, + dockerfile=os.path.join(DOCKER_TMPDIR, 'Dockerfile'), + gzip=False) + build_args.update(fileobj=context, + custom_context=True) + else: if sys.version_info.major == 2: build_args.update(fileobj=StringIO(dockerfile), diff --git a/test/data/ignorefile.yml b/test/data/ignorefile.yml new file mode 100644 index 0000000..32c198b --- /dev/null +++ b/test/data/ignorefile.yml @@ -0,0 +1,5 @@ +target: + FROM: alpine + ignorefile: test_build/custom_ignore_file.txt + build_directory: ./test_build + build: ADD . /opt diff --git a/test/data/ignores.yml b/test/data/ignores.yml new file mode 100644 index 0000000..7a4fa83 --- /dev/null +++ b/test/data/ignores.yml @@ -0,0 +1,8 @@ +target: + FROM: alpine + build_directory: ./test_build + ignore: | + b + build: | + ADD . /opt + diff --git a/test/data/multi_ignore.yml b/test/data/multi_ignore.yml new file mode 100644 index 0000000..d04f037 --- /dev/null +++ b/test/data/multi_ignore.yml @@ -0,0 +1,4 @@ +target: + description: should error because you can't have both of these + ignore: a + ignorefile: test_build/.dockerignore diff --git a/test/data/test_build/.dockerignore b/test/data/test_build/.dockerignore new file mode 100644 index 0000000..422c2b7 --- /dev/null +++ b/test/data/test_build/.dockerignore @@ -0,0 +1,2 @@ +a +b diff --git a/test/data/test_build/Dockerfile b/test/data/test_build/Dockerfile new file mode 100644 index 0000000..7d4989c --- /dev/null +++ b/test/data/test_build/Dockerfile @@ -0,0 +1,3 @@ +FROM centos +WORKDIR /opt +ADD c . \ No newline at end of file diff --git a/test/data/test_build/a b/test/data/test_build/a new file mode 100644 index 0000000..2e65efe --- /dev/null +++ b/test/data/test_build/a @@ -0,0 +1 @@ +a \ No newline at end of file diff --git a/test/data/test_build/b b/test/data/test_build/b new file mode 100644 index 0000000..6178079 --- /dev/null +++ b/test/data/test_build/b @@ -0,0 +1 @@ +b diff --git a/test/data/test_build/c b/test/data/test_build/c new file mode 100644 index 0000000..f2ad6c7 --- /dev/null +++ b/test/data/test_build/c @@ -0,0 +1 @@ +c diff --git a/test/data/test_build/custom_ignore_file.txt b/test/data/test_build/custom_ignore_file.txt new file mode 100644 index 0000000..3410062 --- /dev/null +++ b/test/data/test_build/custom_ignore_file.txt @@ -0,0 +1 @@ +c \ No newline at end of file diff --git a/test/test_errors.py b/test/test_errors.py index d587d6d..aabbee0 100644 --- a/test/test_errors.py +++ b/test/test_errors.py @@ -13,7 +13,8 @@ 'data/missingfile.yml': errors.MissingFileError, 'data/baddockerfile.yml': errors.ExternalBuildError, 'data/invalid_requires.yml': errors.InvalidRequiresList, - 'data/invalid_yaml.yml': errors.ParsingFailure + 'data/invalid_yaml.yml': errors.ParsingFailure, + 'data/multi_ignore.yml': errors.MultipleIgnoreError, } diff --git a/test/test_features.py b/test/test_features.py index 8d83cdc..21c730a 100644 --- a/test/test_features.py +++ b/test/test_features.py @@ -2,8 +2,13 @@ def test_multiple_bases(): - subprocess.check_call(['docker-make', '-f', 'data/multibase.yml', 'target2', 'target3']) + subprocess.check_call(['docker-make', '-f', 'data/multibase.yml', 'target2', 'target3']) def test_paths_relative_interpreted_relative_to_definition_file(): - subprocess.check_call(['docker-make', '-f', 'data/include.yml', 'target']) + subprocess.check_call(['docker-make', '-f', 'data/include.yml', 'target']) + + +def test_ignore_string(): + subprocess.check_call(['docker-make', '-f', 'data/ignores.yml', 'target']) +