diff --git a/bdist_conda.py b/bdist_conda.py index 436d101ab7..2ffed3aa25 100644 --- a/bdist_conda.py +++ b/bdist_conda.py @@ -16,7 +16,7 @@ import conda.config from conda.cli.common import spec_from_line from conda_build.metadata import MetaData -from conda_build import build, pypi +from conda_build import build, pypi, render from conda_build.config import config from conda_build.main_build import handle_binstar_upload @@ -267,13 +267,13 @@ def run(self): if self.binstar_upload: class args: binstar_upload = self.binstar_upload - handle_binstar_upload(build.bldpkg_path(m), args) + handle_binstar_upload(render.bldpkg_path(m), args) else: no_upload_message = """\ # If you want to upload this package to anaconda.org later, type: # # $ anaconda upload %s -""" % build.bldpkg_path(m) +""" % render.bldpkg_path(m) print(no_upload_message) diff --git a/conda_build/build.py b/conda_build/build.py index 9223da4d68..6f734a8fe0 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -27,9 +27,10 @@ from conda.utils import url_path from conda.resolve import Resolve, MatchSpec, NoPackagesFound +from conda_build import __version__ from conda_build import environ, source, tarcheck from conda_build.config import config -from conda_build.render import parse_or_try_download +from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path from conda_build.scripts import create_entry_points, prepend_bin_path from conda_build.post import (post_process, post_build, fix_permissions, get_build_metadata) @@ -183,12 +184,17 @@ def create_info_files(m, files, include_recipe=True): shutil.copytree(src_path, dst_path) else: shutil.copy(src_path, dst_path) - # move the potentially unrendered meta.yaml file os.rename(join(recipe_dir, "meta.yaml"), join(recipe_dir, "meta.yaml.template")) - # store the rendered meta.yaml file, plus information about where it came from - # and what version of conda-build created it - _render_recipe() + # store the rendered meta.yaml file, plus information about where it came from + # and what version of conda-build created it + metayaml = output_yaml(m) + with open(join(config.info_dir, "meta.yaml"), 'w') as f: + f.write("# This file created by conda-build {}\n".format(__version__)) + f.write("# meta.yaml template originally from:\n") + f.write("# " + source.get_repository_info(m.path) + "\n") + f.write("# ------------------------------------------------\n\n") + f.write(metayaml) license_file = m.get_value('about/license_file') if license_file: @@ -235,7 +241,7 @@ def create_info_files(m, files, include_recipe=True): if sys.platform == 'win32': # make sure we use '/' path separators in metadata - files = [f.replace('\\', '/') for f in files] + files = [_f.replace('\\', '/') for _f in files] with open(join(config.info_dir, 'files'), 'w') as fo: if m.get_value('build/noarch_python'): @@ -367,14 +373,8 @@ def rm_pkgs_cache(dist): plan.execute_plan(rmplan) -def bldpkg_path(m): - ''' - Returns path to built package's tarball given its ``Metadata``. - ''' - return join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist()) - - -def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_download=True): +def build(m, post=None, include_recipe=True, keep_old_work=False, + need_source_download=True, verbose=True): ''' Build the package with the specified metadata. @@ -442,8 +442,10 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_do try: os.environ['PATH'] = prepend_bin_path({'PATH': _old_path}, config.build_prefix)['PATH'] - m, need_source_download = parse_or_try_download(m, no_download_source=False, - hide_output=False) + m, need_source_download = parse_or_try_download(m, + no_download_source=False, + force_download=True, + verbose=verbose) assert not need_source_download, "Source download failed. Please investigate." finally: os.environ['PATH'] = _old_path diff --git a/conda_build/main_build.py b/conda_build/main_build.py index 9652613b2b..4adb3ff7e8 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -70,7 +70,9 @@ def main(): p.add_argument( '-t', "--test", action="store_true", - help="Test package (assumes package is already build).", + help="Test package (assumes package is already built). RECIPE_DIR argument can be either " + "recipe directory, in which case source download may be necessary to resolve package" + "version, or path to built package .tar.bz2 file, in which case no source is necessary.", ) p.add_argument( '--no-test', @@ -271,7 +273,7 @@ def execute(args, parser): # this fully renders any jinja templating, throwing an error if any data is missing m, need_source_download = render_recipe(recipe_dir, no_download_source=False, - hide_download_output=args.output) + verbose=build.verbose) if m.get_value('build/noarch_python'): config.noarch = True diff --git a/conda_build/main_render.py b/conda_build/main_render.py index d4d64ad25b..01c0c96487 100644 --- a/conda_build/main_render.py +++ b/conda_build/main_render.py @@ -8,15 +8,11 @@ import sys -import yaml - -from conda.compat import PY3 from conda.cli.common import add_parser_channels from conda.cli.conda_argparse import ArgumentParser from conda_build import __version__ -from conda_build.build import bldpkg_path -from conda_build.render import render_recipe, set_language_env_vars +from conda_build.render import render_recipe, set_language_env_vars, bldpkg_path, output_yaml from conda_build.utils import find_recipe from conda_build.completers import (RecipeCompleter, PythonVersionCompleter, RVersionsCompleter, LuaVersionsCompleter, NumPyVersionCompleter) @@ -116,16 +112,21 @@ def main(): choices=RecipeCompleter(), help="Path to recipe directory.", ) - + # this is here because we have a different default than build + p.add_argument( + '--verbose', + action='store_true', + help='Enable verbose output from download tools and progress updates', + ) args = p.parse_args() set_language_env_vars(args, p) - metadata, _ = _render_recipe(find_recipe(args.recipe), no_download_source=args.no_source, - hide_download_output=args.output) + metadata, _ = render_recipe(find_recipe(args.recipe), no_download_source=args.no_source, + verbose=args.verbose) if args.output: - print(_get_output_path(metadata)) + print(bldpkg_path(metadata)) else: - print(_output_yaml(metadata, args.file) + print(output_yaml(metadata, args.file)) if __name__ == '__main__': main() diff --git a/conda_build/post.py b/conda_build/post.py index 1513fc7287..03de7d72ec 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -376,26 +376,26 @@ def check_symlinks(files): def get_build_metadata(m): src_dir = source.get_dir() if exists(join(src_dir, '__conda_version__.txt')): - print("Deprecation warning: support for __conda_version__ will be removed in Conda build 2.0." + print("Deprecation warning: support for __conda_version__ will be removed in Conda build 2.0." # noqa "Try Jinja templates instead: " - "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") + "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa with open(join(src_dir, '__conda_version__.txt')) as f: version = f.read().strip() print("Setting version from __conda_version__.txt: %s" % version) m.meta['package']['version'] = version if exists(join(src_dir, '__conda_buildnum__.txt')): - print("Deprecation warning: support for __conda_buildnum__ will be removed in Conda build 2.0." + print("Deprecation warning: support for __conda_buildnum__ will be removed in Conda build 2.0." # noqa "Try Jinja templates instead: " - "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") + "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa with open(join(src_dir, '__conda_buildnum__.txt')) as f: build_number = f.read().strip() print("Setting build number from __conda_buildnum__.txt: %s" % build_number) m.meta['build']['number'] = build_number if exists(join(src_dir, '__conda_buildstr__.txt')): - print("Deprecation warning: support for __conda_buildstr__ will be removed in Conda build 2.0." + print("Deprecation warning: support for __conda_buildstr__ will be removed in Conda build 2.0." # noqa "Try Jinja templates instead: " - "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") + "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa with open(join(src_dir, '__conda_buildstr__.txt')) as f: buildstr = f.read().strip() print("Setting version from __conda_buildstr__.txt: %s" % buildstr) diff --git a/conda_build/render.py b/conda_build/render.py index 2031500ebe..066e5915fb 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -15,10 +15,11 @@ from locale import getpreferredencoding import subprocess +import yaml + from conda.compat import PY3 from conda.lock import Locked -from conda_build.build import bldpkg_path from conda_build import exceptions from conda_build.config import config from conda_build.metadata import MetaData @@ -66,15 +67,21 @@ def set_language_env_vars(args, parser, execute=None): os.environ[var] = str(getattr(config, var)) -def get_output_path(metadata): - return bldpkg_path(metadata) +def bldpkg_path(m): + ''' + Returns path to built package's tarball given its ``Metadata``. + ''' + return os.path.join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist()) + -def parse_or_try_download(metadata, no_download_source, hide_output): - if not no_download_source: +def parse_or_try_download(metadata, no_download_source, verbose, force_download=False): + if (("version" not in metadata.meta["package"] or + not metadata.meta["package"]["version"]) and + not no_download_source) or force_download: # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. try: - source.provide(metadata.path, metadata.get_section('source')) + source.provide(metadata.path, metadata.get_section('source'), verbose=verbose) metadata.parse_again(permit_undefined_jinja=False) need_source_download = False except subprocess.CalledProcessError: @@ -83,11 +90,15 @@ def parse_or_try_download(metadata, no_download_source, hide_output): need_source_download = True else: need_source_download = no_download_source + else: + # we have not downloaded source in the render phase. Download it in + # the build phase + need_source_download = True metadata.parse_again(permit_undefined_jinja=False) return metadata, need_source_download -def render_recipe(recipe_path, no_download_source, hide_download_output): +def render_recipe(recipe_path, no_download_source, verbose): with Locked(config.croot): arg = recipe_path # Don't use byte literals for paths in Python 2 @@ -117,7 +128,7 @@ def render_recipe(recipe_path, no_download_source, hide_download_output): sys.exit(1) m = parse_or_try_download(m, no_download_source=no_download_source, - hide_output=hide_download_output) + verbose=verbose) if need_cleanup: shutil.rmtree(recipe_dir) @@ -131,15 +142,18 @@ class _MetaYaml(dict): fields = ["package", "source", "build", "requirements", "test", "about", "extra"] def to_omap(self): - return [(field, self[field]) for field in MetaYaml.fields if field in self] + return [(field, self[field]) for field in _MetaYaml.fields if field in self] + def _represent_omap(dumper, data): return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.to_omap()) + def _unicode_representer(dumper, uni): node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni) return node + class _IndentDumper(yaml.Dumper): def increase_indent(self, flow=False, indentless=False): return super(_IndentDumper, self).increase_indent(flow, False) @@ -152,7 +166,7 @@ def increase_indent(self, flow=False, indentless=False): yaml.add_representer(unicode, _unicode_representer) -def output_yaml(metdata, filename=None): +def output_yaml(metadata, filename=None): output = yaml.dump(_MetaYaml(metadata.meta), Dumper=_IndentDumper, default_flow_style=False, indent=4) if filename: diff --git a/conda_build/source.py b/conda_build/source.py index 1c9537dedc..48e938a2c3 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -1,11 +1,13 @@ from __future__ import absolute_import, division, print_function import os +import re import sys from os.path import join, isdir, isfile, abspath, expanduser, basename from shutil import copytree, copy2 -from subprocess import check_call, Popen, PIPE, CalledProcessError +from subprocess import check_call, Popen, PIPE, CalledProcessError, check_output import locale +import time from conda.compat import StringIO from conda.fetch import download @@ -70,12 +72,12 @@ def download_to_cache(meta): return path -def unpack(meta, hide_output=False): +def unpack(meta, verbose=False): ''' Uncompress a downloaded source. ''' src_path = download_to_cache(meta) os.makedirs(WORK_DIR) - if not hide_output: + if verbose: print("Extracting download") if src_path.lower().endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz', '.tar', 'tar.z')): @@ -84,19 +86,18 @@ def unpack(meta, hide_output=False): unzip(src_path, WORK_DIR) else: # In this case, the build script will need to deal with unpacking the source - if not hide_output: - print("Warning: Unrecognized source format. Source file will be copied to the SRC_DIR") + print("Warning: Unrecognized source format. Source file will be copied to the SRC_DIR") copy2(src_path, WORK_DIR) -def git_source(meta, recipe_dir, hide_output=False): +def git_source(meta, recipe_dir, verbose=False): ''' Download a source from Git repo. ''' - if hide_output: - stdout=StringIO() - stderr=StringIO() + if verbose: + stdout = None + stderr = None else: - stdout=None - stderr=None + stdout = StringIO() + stderr = StringIO() if not isdir(GIT_CACHE): os.makedirs(GIT_CACHE) @@ -156,7 +157,8 @@ def git_source(meta, recipe_dir, hide_output=False): if checkout: print('checkout: %r' % checkout) - check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR], stdout=stdout, stderr=stderr) + check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR], + stdout=stdout, stderr=stderr) if checkout: check_call([git, 'checkout', checkout], cwd=WORK_DIR, stdout=stdout, stderr=stderr) @@ -195,14 +197,14 @@ def git_info(fo=None): safe_print_unicode(stdout + u'\n') -def hg_source(meta, hide_output=False): +def hg_source(meta, verbose=False): ''' Download a source from Mercurial repo. ''' - if hide_output: - stdout=StringIO() - stderr=StringIO() + if verbose: + stdout = None + stderr = None else: - stdout=None - stderr=None + stdout = StringIO() + stderr = StringIO() hg = external.find_executable('hg') if not hg: @@ -227,14 +229,14 @@ def hg_source(meta, hide_output=False): return WORK_DIR -def svn_source(meta, hide_output=False): +def svn_source(meta, verbose=False): ''' Download a source from SVN repo. ''' - if hide_output: - stdout=StringIO() - stderr=StringIO() + if verbose: + stdout = None + stderr = None else: - stdout=None - stderr=None + stdout = StringIO() + stderr = StringIO() def parse_bool(s): return str(s).lower().strip() in ('yes', 'true', '1', 'on') @@ -266,6 +268,28 @@ def parse_bool(s): return WORK_DIR +def get_repository_info(recipe_path): + """This tries to get information about where a recipe came from. This is different + from the source - you can have a recipe in svn that gets source via git.""" + if isdir(join(recipe_path, ".git")): + origin = check_output(["git", "config", "--get", "remote.origin.url"]) + rev = check_output(["git", "rev-parse", "HEAD"]) + return "Origin {}, commit {}".format(origin, rev) + elif isdir(join(recipe_path, ".hg")): + origin = check_output(["hg", "paths", "default"]) + rev = check_output(["hg", "id"]).split()[0] + return "Origin {}, commit {}".format(origin, rev) + elif isdir(join(recipe_path, ".svn")): + info = check_output(["svn", "info"]) + server = re.search("Repository Root: (.*)$", info, flags=re.M).group(1) + revision = re.search("Revision: (.*)$", info, flags=re.M).group(1) + return "{}, Revision {}".format(server, revision) + else: + return "{}, last modified {}".format(recipe_path, + time.ctime(os.path.getmtime( + join(recipe_path, "meta.yaml")))) + + def _ensure_unix_line_endings(path): """Replace windows line endings with Unix. Return path to modified file.""" out_path = path + "_unix" @@ -300,7 +324,7 @@ def apply_patch(src_dir, path): os.remove(patch_args[-1]) # clean up .patch_unix file -def provide(recipe_dir, meta, hide_output=False, patch=True): +def provide(recipe_dir, meta, verbose=False, patch=True): """ given a recipe_dir: - download (if necessary) @@ -308,7 +332,7 @@ def provide(recipe_dir, meta, hide_output=False, patch=True): - apply patches (if any) """ # temporarily catch output to stdout and stderr - if hide_output: + if not verbose: stdout = sys.stdout stderr = sys.stderr sys.stdout = StringIO() @@ -323,13 +347,17 @@ def provide(recipe_dir, meta, hide_output=False, patch=True): if any(k in meta for k in ('fn', 'url')): unpack(meta) elif 'git_url' in meta: - git_source(meta, recipe_dir, hide_output=hide_output) + git_source(meta, recipe_dir, verbose=verbose) + # build to make sure we have a work directory with source in it. We want to make sure that + # build to make sure we have a work directory with source in it. We want to make sure that + # whatever version that is does not interfere with the test we run next. + # whatever version that is does not interfere with the test we run next. elif 'hg_url' in meta: - hg_source(meta, hide_output=hide_output) + hg_source(meta, verbose=verbose) elif 'svn_url' in meta: - svn_source(meta, hide_output=hide_output) + svn_source(meta, verbose=verbose) elif 'path' in meta: - if not hide_output: + if verbose: print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) else: # no source @@ -341,7 +369,7 @@ def provide(recipe_dir, meta, hide_output=False, patch=True): apply_patch(src_dir, join(recipe_dir, patch)) # restore outputs - if hide_output: + if not verbose: sys.stdout = stdout sys.stderr = stderr diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index 899f55044d..b71d6dad56 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -10,7 +10,6 @@ from conda.config import subdir from conda.fetch import download - thisdir = os.path.dirname(os.path.realpath(__file__)) metadata_dir = os.path.join(thisdir, "test-recipes/metadata") fail_dir = os.path.join(thisdir, "test-recipes/fail") @@ -37,82 +36,39 @@ def test_output_build_path(): assert output.rstrip() == test_path -def _get_describe_tag(): - process = subprocess.Popen(['git', 'describe', '--tag'], - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - if PY3: - output = output.decode("UTF-8") - if error: - print(error) - return output.rstrip() - - -# TODO: need much simpler repo with just a very simple git jinja recipe, and 2 or 3 tags. -# should be created on conda org, probably -def test_output_build_path_local_git(): - subprocess.check_call(['git', 'clone', 'https://github.com/numba/numba']) - os.chdir('numba') - +def test_cached_source_not_interfere_with_versioning(): + """Test that work dir does not cache and cause inaccurate test target""" + subprocess.check_call(['git', 'clone', 'https://github.com/conda/conda_build_test_recipe']) try: - subprocess.check_call(['git', 'checkout', '0.24.0']) - cmd = 'conda build --output buildscripts/condarecipe.local --numpy=1.10' - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - tag = _get_describe_tag() - test_path = os.path.join(sys.prefix, "conda-bld", subdir, - "numba-{}-np110py{}{}_0.tar.bz2".format( - tag, sys.version_info.major, sys.version_info.minor)) - # be carefuly here: output should only be path, NOT git output or anything else - assert output.rstrip() == test_path - - subprocess.check_call(['git', 'checkout', '0.23.0']) - cmd = 'conda build --output buildscripts/condarecipe.local --numpy=1.10' - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - tag = _get_describe_tag() - test_path = os.path.join(sys.prefix, "conda-bld", subdir, - "numba-{}-np110py{}{}_0.tar.bz2".format( - tag, sys.version_info.major, sys.version_info.minor)) - assert output.rstrip() == test_path - - subprocess.check_call(['git', 'checkout', '0.24.0']) - cmd = 'conda build --output buildscripts/condarecipe.local --numpy=1.10' - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - tag = _get_describe_tag() - test_path = os.path.join(sys.prefix, "conda-bld", subdir, - "numba-{}-np110py{}{}_0.tar.bz2".format( - tag, sys.version_info.major, sys.version_info.minor)) - assert output.rstrip() == test_path - - finally: + # build to make sure we have a work directory with source in it. We want to make sure that + # whatever version that is does not interfere with the test we run next. + subprocess.check_call(['conda', 'build', '--no-test', + '--no-anaconda-upload', + 'conda_build_test_recipe']) + + os.chdir('conda_build_test_recipe') + subprocess.check_call(['git', 'checkout', '1.20.0']) os.chdir('..') - shutil.rmtree('numba') + # this should fail, because we have not built v1.0, so there should be nothing to test. + # if it succeeds, it means that it used the cached master checkout for determining which + # version to test. + cmd = 'conda build --output conda_build_test_recipe' + output = subprocess.check_output(cmd.split()) + if PY3: + output = output.decode("UTF-8") + assert ("conda-build-test-source-git-jinja2-1.20.0" in output) + finally: + shutil.rmtree('conda_build_test_recipe') -def test_local_source_mismatch_with_downloaded_package(): - subprocess.check_call(['git', 'clone', 'https://github.com/numba/numba']) - os.chdir('numba') - filename = 'numba-0.24.0-np110py{}{}_0.tar.bz2'.format(sys.version_info.major, - sys.version_info.minor) +def test_package_test(): + """Test calling conda build -t - rather than """ + filename = "jinja2-2.8-py{}{}_0.tar.bz2".format(sys.version_info.major, sys.version_info.minor) downloaded_file = os.path.join(sys.prefix, 'conda-bld', subdir, filename) - download('https://repo.continuum.io/pkgs/free/{}/{}'.format(subdir, filename), downloaded_file) - - cmd = 'conda build --output numba/buildscripts/condarecipe.local --numpy=1.10' - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - cmd = 'conda build --test {} --numpy=1.10'.format(downloaded_file) - process = subprocess.Popen(cmd.split(), - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - output, error = process.communicate() - - + download('https://anaconda.org/conda-forge/jinja2/2.8/download/{}/{}'.format(subdir, filename), + downloaded_file) + subprocess.check_call(["conda", "build", "--test", downloaded_file]) @pytest.fixture(params=[dirname for dirname in os.listdir(metadata_dir)