From b4ec0e0659d8f376042d4fc391616bf235996cf5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 13 May 2016 15:57:45 -0500 Subject: [PATCH] fix extraneous output when outputting build path --- conda_build/build.py | 11 ++++- conda_build/main_build.py | 3 +- conda_build/main_render.py | 47 ++---------------- conda_build/render.py | 57 +++++++++++++++++++--- conda_build/source.py | 87 +++++++++++++++++++++++---------- tests/test_build_recipes.py | 97 +++++++++++++++++++++++++++++++++++++ tests/test_render.py | 22 +++++++++ 7 files changed, 247 insertions(+), 77 deletions(-) create mode 100644 tests/test_render.py diff --git a/conda_build/build.py b/conda_build/build.py index 2316a7d31c..9223da4d68 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -183,6 +183,12 @@ def create_info_files(m, files, include_recipe=True): shutil.copytree(src_path, dst_path) else: shutil.copy(src_path, dst_path) + # move the potentially unrendered meta.yaml file + os.rename(join(recipe_dir, "meta.yaml"), join(recipe_dir, "meta.yaml.template")) + # store the rendered meta.yaml file, plus information about where it came from + # and what version of conda-build created it + _render_recipe() + license_file = m.get_value('about/license_file') if license_file: @@ -368,7 +374,7 @@ def bldpkg_path(m): return join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist()) -def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_download=False): +def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_download=True): ''' Build the package with the specified metadata. @@ -436,7 +442,8 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_do try: os.environ['PATH'] = prepend_bin_path({'PATH': _old_path}, config.build_prefix)['PATH'] - m, need_source_download = parse_or_try_download(m, no_download_source=False) + m, need_source_download = parse_or_try_download(m, no_download_source=False, + hide_output=False) assert not need_source_download, "Source download failed. Please investigate." finally: os.environ['PATH'] = _old_path diff --git a/conda_build/main_build.py b/conda_build/main_build.py index bb7e62ff6d..9652613b2b 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -270,7 +270,8 @@ def execute(args, parser): sys.exit("Error: no such directory: %s" % recipe_dir) # this fully renders any jinja templating, throwing an error if any data is missing - m, need_source_download = render_recipe(recipe_dir, no_download_source=False) + m, need_source_download = render_recipe(recipe_dir, no_download_source=False, + hide_download_output=args.output) if m.get_value('build/noarch_python'): config.noarch = True diff --git a/conda_build/main_render.py b/conda_build/main_render.py index 8e6c072325..d4d64ad25b 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -41,7 +41,7 @@ def get_render_parser(): version='conda-build %s' % __version__, ) p.add_argument( - '-n', "--no_source", + '-n', "--no-source", action="store_true", help="When templating can't be completed, do not obtain the \ source to try fill in related template variables.", @@ -100,37 +100,6 @@ def get_render_parser(): return p -# Next bit of stuff is to support YAML output in the order we expect. -# http://stackoverflow.com/a/17310199/1170370 -class MetaYaml(dict): - fields = ["package", "source", "build", "requirements", "test", "about", "extra"] - - def to_omap(self): - return [(field, self[field]) for field in MetaYaml.fields if field in self] - - -def represent_omap(dumper, data): - return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.to_omap()) - - -def unicode_representer(dumper, uni): - node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni) - return node - - -class IndentDumper(yaml.Dumper): - def increase_indent(self, flow=False, indentless=False): - return super(IndentDumper, self).increase_indent(flow, False) - - -yaml.add_representer(MetaYaml, represent_omap) -if PY3: - yaml.add_representer(str, unicode_representer) - unicode = None # silence pyflakes about unicode not existing in py3 -else: - yaml.add_representer(unicode, unicode_representer) - - def main(): p = get_render_parser() p.add_argument( @@ -151,18 +120,12 @@ def main(): args = p.parse_args() set_language_env_vars(args, p) - metadata = render_recipe(find_recipe(args.recipe), no_download_source=args.no_source) + metadata, _ = _render_recipe(find_recipe(args.recipe), no_download_source=args.no_source, + hide_download_output=args.output) if args.output: - print(bldpkg_path(metadata)) + print(_get_output_path(metadata)) else: - output = yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper, - default_flow_style=False, indent=4) - if args.file: - with open(args.file, "w") as f: - f.write(output) - else: - print(output) - + print(_output_yaml(metadata, args.file) if __name__ == '__main__': main() diff --git a/conda_build/render.py b/conda_build/render.py index d5581f9266..2031500ebe 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -18,6 +18,7 @@ from conda.compat import PY3 from conda.lock import Locked +from conda_build.build import bldpkg_path from conda_build import exceptions from conda_build.config import config from conda_build.metadata import MetaData @@ -65,7 +66,10 @@ def set_language_env_vars(args, parser, execute=None): os.environ[var] = str(getattr(config, var)) -def parse_or_try_download(metadata, no_download_source): +def get_output_path(metadata): + return bldpkg_path(metadata) + +def parse_or_try_download(metadata, no_download_source, hide_output): if not no_download_source: # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. @@ -75,15 +79,15 @@ def parse_or_try_download(metadata, no_download_source): need_source_download = False except subprocess.CalledProcessError: print("Warning: failed to download source. If building, will try " - "again after downloading recipe dependencies.") + "again after downloading recipe dependencies.") need_source_download = True - else: - metadata.parse_again(permit_undefined_jinja=False) - need_source_download = no_download_source + else: + need_source_download = no_download_source + metadata.parse_again(permit_undefined_jinja=False) return metadata, need_source_download -def render_recipe(recipe_path, no_download_source): +def render_recipe(recipe_path, no_download_source, hide_download_output): with Locked(config.croot): arg = recipe_path # Don't use byte literals for paths in Python 2 @@ -112,9 +116,48 @@ def render_recipe(recipe_path, no_download_source): sys.stderr.write(e.error_msg()) sys.exit(1) - m = parse_or_try_download(m, no_download_source=no_download_source) + m = parse_or_try_download(m, no_download_source=no_download_source, + hide_output=hide_download_output) if need_cleanup: shutil.rmtree(recipe_dir) return m + + +# Next bit of stuff is to support YAML output in the order we expect. +# http://stackoverflow.com/a/17310199/1170370 +class _MetaYaml(dict): + fields = ["package", "source", "build", "requirements", "test", "about", "extra"] + + def to_omap(self): + return [(field, self[field]) for field in MetaYaml.fields if field in self] + +def _represent_omap(dumper, data): + return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.to_omap()) + +def _unicode_representer(dumper, uni): + node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni) + return node + +class _IndentDumper(yaml.Dumper): + def increase_indent(self, flow=False, indentless=False): + return super(_IndentDumper, self).increase_indent(flow, False) + +yaml.add_representer(_MetaYaml, _represent_omap) +if PY3: + yaml.add_representer(str, _unicode_representer) + unicode = None # silence pyflakes about unicode not existing in py3 +else: + yaml.add_representer(unicode, _unicode_representer) + + +def output_yaml(metdata, filename=None): + output = yaml.dump(_MetaYaml(metadata.meta), Dumper=_IndentDumper, + default_flow_style=False, indent=4) + if filename: + with open(filename, "w") as f: + f.write(output) + return("Wrote yaml to %s" % filename) + else: + return(output) diff --git a/conda_build/source.py b/conda_build/source.py index 81f5e35252..1c9537dedc 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -7,6 +7,7 @@ from subprocess import check_call, Popen, PIPE, CalledProcessError import locale +from conda.compat import StringIO from conda.fetch import download from conda.install import move_to_trash from conda.utils import hashsum_file @@ -69,12 +70,13 @@ def download_to_cache(meta): return path -def unpack(meta): +def unpack(meta, hide_output=False): ''' Uncompress a downloaded source. ''' src_path = download_to_cache(meta) os.makedirs(WORK_DIR) - print("Extracting download") + if not hide_output: + print("Extracting download") if src_path.lower().endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz', '.tar', 'tar.z')): tar_xf(src_path, WORK_DIR) @@ -82,12 +84,20 @@ def unpack(meta): unzip(src_path, WORK_DIR) else: # In this case, the build script will need to deal with unpacking the source - print("Warning: Unrecognized source format. Source file will be copied to the SRC_DIR") + if not hide_output: + print("Warning: Unrecognized source format. Source file will be copied to the SRC_DIR") copy2(src_path, WORK_DIR) -def git_source(meta, recipe_dir): +def git_source(meta, recipe_dir, hide_output=False): ''' Download a source from Git repo. ''' + if hide_output: + stdout=StringIO() + stderr=StringIO() + else: + stdout=None + stderr=None + if not isdir(GIT_CACHE): os.makedirs(GIT_CACHE) @@ -113,7 +123,7 @@ def git_source(meta, recipe_dir): # update (or create) the cache repo if isdir(cache_repo): if meta.get('git_rev', 'HEAD') != 'HEAD': - check_call([git, 'fetch'], cwd=cache_repo) + check_call([git, 'fetch'], cwd=cache_repo, stdout=stdout, stderr=stderr) else: # Unlike 'git clone', fetch doesn't automatically update the cache's HEAD, # So here we explicitly store the remote HEAD in the cache's local refs/heads, @@ -122,15 +132,15 @@ def git_source(meta, recipe_dir): # but the user is working with a branch other than 'master' without # explicitly providing git_rev. check_call([git, 'fetch', 'origin', '+HEAD:_conda_cache_origin_head'], - cwd=cache_repo) + cwd=cache_repo, stdout=stdout, stderr=stderr) check_call([git, 'symbolic-ref', 'HEAD', 'refs/heads/_conda_cache_origin_head'], - cwd=cache_repo) + cwd=cache_repo, stdout=stdout, stderr=stderr) else: args = [git, 'clone', '--mirror'] if git_depth > 0: args += ['--depth', str(git_depth)] - check_call(args + [git_url, cache_repo_arg], cwd=recipe_dir) + check_call(args + [git_url, cache_repo_arg], cwd=recipe_dir, stdout=stdout, stderr=stderr) assert isdir(cache_repo) # now clone into the work directory @@ -146,9 +156,9 @@ def git_source(meta, recipe_dir): if checkout: print('checkout: %r' % checkout) - check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR]) + check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR], stdout=stdout, stderr=stderr) if checkout: - check_call([git, 'checkout', checkout], cwd=WORK_DIR) + check_call([git, 'checkout', checkout], cwd=WORK_DIR, stdout=stdout, stderr=stderr) git_info() return WORK_DIR @@ -185,8 +195,15 @@ def git_info(fo=None): safe_print_unicode(stdout + u'\n') -def hg_source(meta): +def hg_source(meta, hide_output=False): ''' Download a source from Mercurial repo. ''' + if hide_output: + stdout=StringIO() + stderr=StringIO() + else: + stdout=None + stderr=None + hg = external.find_executable('hg') if not hg: sys.exit('Error: hg not installed') @@ -196,22 +213,29 @@ def hg_source(meta): hg_dn = hg_url.split(':')[-1].replace('/', '_') cache_repo = join(HG_CACHE, hg_dn) if isdir(cache_repo): - check_call([hg, 'pull'], cwd=cache_repo) + check_call([hg, 'pull'], cwd=cache_repo, stdout=stdout, stderr=stderr) else: - check_call([hg, 'clone', hg_url, cache_repo]) + check_call([hg, 'clone', hg_url, cache_repo], stdout=stdout, stderr=stderr) assert isdir(cache_repo) # now clone in to work directory update = meta.get('hg_tag') or 'tip' print('checkout: %r' % update) - check_call([hg, 'clone', cache_repo, WORK_DIR]) - check_call([hg, 'update', '-C', update], cwd=WORK_DIR) + check_call([hg, 'clone', cache_repo, WORK_DIR], stdout=stdout, stderr=stderr) + check_call([hg, 'update', '-C', update], cwd=WORK_DIR, stdout=stdout, stderr=stderr) return WORK_DIR -def svn_source(meta): +def svn_source(meta, hide_output=False): ''' Download a source from SVN repo. ''' + if hide_output: + stdout=StringIO() + stderr=StringIO() + else: + stdout=None + stderr=None + def parse_bool(s): return str(s).lower().strip() in ('yes', 'true', '1', 'on') @@ -230,10 +254,11 @@ def parse_bool(s): else: extra_args = [] if isdir(cache_repo): - check_call([svn, 'up', '-r', svn_revision] + extra_args, cwd=cache_repo) + check_call([svn, 'up', '-r', svn_revision] + extra_args, cwd=cache_repo, + stdout=stdout, stderr=stderr) else: - check_call([svn, 'co', '-r', svn_revision] + extra_args + [svn_url, - cache_repo]) + check_call([svn, 'co', '-r', svn_revision] + extra_args + [svn_url, cache_repo], + stdout=stdout, stderr=stderr) assert isdir(cache_repo) # now copy into work directory @@ -275,14 +300,20 @@ def apply_patch(src_dir, path): os.remove(patch_args[-1]) # clean up .patch_unix file -def provide(recipe_dir, meta, patch=True): +def provide(recipe_dir, meta, hide_output=False, patch=True): """ given a recipe_dir: - download (if necessary) - unpack - apply patches (if any) """ - print("Removing old work directory") + # temporarily catch output to stdout and stderr + if hide_output: + stdout = sys.stdout + stderr = sys.stderr + sys.stdout = StringIO() + sys.stderr = StringIO() + if sys.platform == 'win32': if isdir(WORK_DIR): move_to_trash(WORK_DIR, '') @@ -292,13 +323,14 @@ def provide(recipe_dir, meta, patch=True): if any(k in meta for k in ('fn', 'url')): unpack(meta) elif 'git_url' in meta: - git_source(meta, recipe_dir) + git_source(meta, recipe_dir, hide_output=hide_output) elif 'hg_url' in meta: - hg_source(meta) + hg_source(meta, hide_output=hide_output) elif 'svn_url' in meta: - svn_source(meta) + svn_source(meta, hide_output=hide_output) elif 'path' in meta: - print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) + if not hide_output: + print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) else: # no source os.makedirs(WORK_DIR) @@ -308,6 +340,11 @@ def provide(recipe_dir, meta, patch=True): for patch in meta.get('patches', []): apply_patch(src_dir, join(recipe_dir, patch)) + # restore outputs + if hide_output: + sys.stdout = stdout + sys.stderr = stderr + if __name__ == '__main__': print(provide('.', diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index bb94226f54..899f55044d 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -6,6 +6,11 @@ import pytest +from conda.compat import PY3 +from conda.config import subdir +from conda.fetch import download + + thisdir = os.path.dirname(os.path.realpath(__file__)) metadata_dir = os.path.join(thisdir, "test-recipes/metadata") fail_dir = os.path.join(thisdir, "test-recipes/fail") @@ -18,6 +23,98 @@ def is_valid_dir(parent_dir, dirname): return valid +# TODO: this does not currently take into account post-build versioning changes with __conda_? files +def test_output_build_path(): + cmd = 'conda build --output {}'.format(os.path.join(metadata_dir, "python_run")) + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + test_path = os.path.join(sys.prefix, "conda-bld", subdir, + "conda-build-test-python-run-1.0-py{}{}_0.tar.bz2".format( + sys.version_info.major, sys.version_info.minor)) + if PY3: + output = output.decode("UTF-8") + assert output.rstrip() == test_path + + +def _get_describe_tag(): + process = subprocess.Popen(['git', 'describe', '--tag'], + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + if PY3: + output = output.decode("UTF-8") + if error: + print(error) + return output.rstrip() + + +# TODO: need much simpler repo with just a very simple git jinja recipe, and 2 or 3 tags. +# should be created on conda org, probably +def test_output_build_path_local_git(): + subprocess.check_call(['git', 'clone', 'https://github.com/numba/numba']) + os.chdir('numba') + + try: + subprocess.check_call(['git', 'checkout', '0.24.0']) + cmd = 'conda build --output buildscripts/condarecipe.local --numpy=1.10' + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + tag = _get_describe_tag() + test_path = os.path.join(sys.prefix, "conda-bld", subdir, + "numba-{}-np110py{}{}_0.tar.bz2".format( + tag, sys.version_info.major, sys.version_info.minor)) + # be carefuly here: output should only be path, NOT git output or anything else + assert output.rstrip() == test_path + + subprocess.check_call(['git', 'checkout', '0.23.0']) + cmd = 'conda build --output buildscripts/condarecipe.local --numpy=1.10' + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + tag = _get_describe_tag() + test_path = os.path.join(sys.prefix, "conda-bld", subdir, + "numba-{}-np110py{}{}_0.tar.bz2".format( + tag, sys.version_info.major, sys.version_info.minor)) + assert output.rstrip() == test_path + + subprocess.check_call(['git', 'checkout', '0.24.0']) + cmd = 'conda build --output buildscripts/condarecipe.local --numpy=1.10' + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + tag = _get_describe_tag() + test_path = os.path.join(sys.prefix, "conda-bld", subdir, + "numba-{}-np110py{}{}_0.tar.bz2".format( + tag, sys.version_info.major, sys.version_info.minor)) + assert output.rstrip() == test_path + + finally: + os.chdir('..') + shutil.rmtree('numba') + + +def test_local_source_mismatch_with_downloaded_package(): + subprocess.check_call(['git', 'clone', 'https://github.com/numba/numba']) + os.chdir('numba') + + filename = 'numba-0.24.0-np110py{}{}_0.tar.bz2'.format(sys.version_info.major, + sys.version_info.minor) + downloaded_file = os.path.join(sys.prefix, 'conda-bld', subdir, filename) + download('https://repo.continuum.io/pkgs/free/{}/{}'.format(subdir, filename), downloaded_file) + + cmd = 'conda build --output numba/buildscripts/condarecipe.local --numpy=1.10' + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + cmd = 'conda build --test {} --numpy=1.10'.format(downloaded_file) + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + + + + @pytest.fixture(params=[dirname for dirname in os.listdir(metadata_dir) if is_valid_dir(metadata_dir, dirname)]) def recipe(request): diff --git a/tests/test_render.py b/tests/test_render.py new file mode 100644 index 0000000000..e58bba9bc0 --- /dev/null +++ b/tests/test_render.py @@ -0,0 +1,22 @@ +import os +import subprocess +import sys + +from conda.compat import PY3 +from conda.config import subdir + +thisdir = os.path.dirname(os.path.realpath(__file__)) +metadata_dir = os.path.join(thisdir, "test-recipes/metadata") + + +def test_output_build_path(): + cmd = 'conda render --output {}'.format(os.path.join(metadata_dir, "python_run")) + process = subprocess.Popen(cmd.split(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = process.communicate() + test_path = os.path.join(sys.prefix, "conda-bld", subdir, + "conda-build-test-python-run-1.0-py{}{}_0.tar.bz2".format( + sys.version_info.major, sys.version_info.minor)) + if PY3: + output = output.decode("UTF-8") + assert output.rstrip() == test_path