From 5482880e12b9c4309d8cba1f71efae11717b9f72 Mon Sep 17 00:00:00 2001 From: heather999 Date: Mon, 11 Apr 2016 12:02:50 -0400 Subject: [PATCH 01/45] Update config.py Recently hit this 80 character placeholder limit and was hoping to at least increase it to 128. From this discussion, it sounds like >128 is not deemed useful on Linux systems. https://groups.google.com/a/continuum.io/forum/#!topic/anaconda/fgDBJ2YwETI --- conda_build/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/config.py b/conda_build/config.py index ada6743ab4..f12ab5bdea 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -57,7 +57,7 @@ def get_conda_py(self): croot = abspath(expanduser('~/conda-bld')) short_build_prefix = join(cc.envs_dirs[0], '_build') - long_build_prefix = max(short_build_prefix, (short_build_prefix + 8 * '_placehold')[:80]) + long_build_prefix = max(short_build_prefix, (short_build_prefix + 8 * '_placehold')[:128]) # XXX: Make this None to be more rigorous about requiring the build_prefix # to be known before it is used. use_long_build_prefix = False From 1f2c4d920348d499133b892b7189c56204d96624 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Wed, 11 May 2016 22:17:29 -0500 Subject: [PATCH 02/45] simplify file writing --- conda_build/build.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 9d13054302..4f2fa5c79c 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -20,7 +20,6 @@ import conda.config as cc import conda.plan as plan from conda.api import get_index -from conda.compat import PY3 from conda.fetch import fetch_index from conda.install import prefix_placeholder, linked, move_to_trash from conda.lock import Locked @@ -218,13 +217,11 @@ def create_info_files(m, files, include_recipe=True): info_index['depends'] = [' '.join(dist.rsplit('-', 2)) for dist in dists] - # Deal with Python 2 and 3's different json module type reqs - mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} - with open(join(config.info_dir, 'index.json'), **mode_dict) as fo: + with open(join(config.info_dir, 'index.json'), 'w') as fo: json.dump(info_index, fo, indent=2, sort_keys=True) if include_recipe: - with open(join(config.info_dir, 'recipe.json'), **mode_dict) as fo: + with open(join(config.info_dir, 'recipe.json'), 'w') as fo: json.dump(m.meta, fo, indent=2, sort_keys=True) if sys.platform == 'win32': From c1de28872c16fae5642d348fd7a0f683ea64a209 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Wed, 11 May 2016 22:24:19 -0500 Subject: [PATCH 03/45] add info/about.json --- conda_build/build.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/conda_build/build.py b/conda_build/build.py index 4f2fa5c79c..fea7f384ed 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -220,6 +220,15 @@ def create_info_files(m, files, include_recipe=True): with open(join(config.info_dir, 'index.json'), 'w') as fo: json.dump(info_index, fo, indent=2, sort_keys=True) + with open(join(config.info_dir, 'about.json'), 'w') as fo: + d = {} + for key in ('home', 'dev_url', 'doc_url', 'license_url', + 'license', 'summary', 'description', 'license_family'): + value = m.get_value('about/%s' % key) + if value: + d[key] = value + json.dump(d, fo, indent=2, sort_keys=True) + if include_recipe: with open(join(config.info_dir, 'recipe.json'), 'w') as fo: json.dump(m.meta, fo, indent=2, sort_keys=True) From a26c88fac0911f18850342805951ded3c3e07377 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Mon, 16 May 2016 22:49:27 -0500 Subject: [PATCH 04/45] allow setting selector --- conda_build/metadata.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 9aadf8bbd9..76fa77d096 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -65,6 +65,15 @@ def ns_cfg(): for machine in cc.non_x86_linux_machines: d[machine] = bool(plat == 'linux-%s' % machine) + sel_pat = re.compile('SELECTOR_(\w+)') + for key, value in iteritems(os.environ): + m = sel_pat.match(key) + if m: + if value not in ('0', '1'): + sys.exit("Error: did not expect environment variable '%s' " + "being set to '%s' (not '0' or '1')" % (key, value)) + d[m.group(1)] = bool(int(value)) + d.update(os.environ) return d From a5cccc7274073f268fc3a3483adcbe90f164715c Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 18 May 2016 08:37:53 -0500 Subject: [PATCH 05/45] search for double backslashes in prefix replacement --- conda_build/build.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/conda_build/build.py b/conda_build/build.py index 6f734a8fe0..94a8a8d1cc 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -133,6 +133,9 @@ def have_prefix_files(files): if (sys.platform == 'win32') and mm.find(alt_prefix_bytes) != -1: # some windows libraries use unix-style path separators yield (alt_prefix, mode, f) + elif (sys.platform == 'win32') and mm.find(prefix_bytes.replace("\\", "\\\\")) != -1: + # some windows libraries have double backslashes as escaping + yield (prefix.replace("\\", "\\\\"), mode, f) if mm.find(prefix_placeholder_bytes) != -1: yield (prefix_placeholder, mode, f) mm.close() and fi.close() From e3da205d20e6931524a4a0ee55cbb5378180c2b5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 18 May 2016 11:14:09 -0500 Subject: [PATCH 06/45] refine double backslash finding; add test --- conda_build/build.py | 16 ++++++++++------ tests/test_build.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 6 deletions(-) create mode 100644 tests/test_build.py diff --git a/conda_build/build.py b/conda_build/build.py index 94a8a8d1cc..5a63ee7ff5 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -97,9 +97,13 @@ def have_prefix_files(files): ''' prefix = config.build_prefix prefix_bytes = prefix.encode('utf-8') - alt_prefix = prefix.replace('\\', '/') - alt_prefix_bytes = alt_prefix.encode('utf-8') prefix_placeholder_bytes = prefix_placeholder.encode('utf-8') + if on_win: + forward_slash_prefix = prefix.replace('\\', '/') + forward_slash_prefix_bytes = forward_slash_prefix.encode('utf-8') + double_backslash_prefix = prefix.replace('\\', '\\\\') + double_backslash_prefix_bytes = double_backslash_prefix.encode('utf-8') + for f in files: if f.endswith(('.pyc', '.pyo', '.a')): continue @@ -130,12 +134,12 @@ def have_prefix_files(files): mm = mmap.mmap(fi.fileno(), 0) if mm.find(prefix_bytes) != -1: yield (prefix, mode, f) - if (sys.platform == 'win32') and mm.find(alt_prefix_bytes) != -1: + if on_win and mm.find(forward_slash_prefix_bytes) != -1: # some windows libraries use unix-style path separators - yield (alt_prefix, mode, f) - elif (sys.platform == 'win32') and mm.find(prefix_bytes.replace("\\", "\\\\")) != -1: + yield (forward_slash_prefix, mode, f) + elif on_win and mm.find(double_backslash_prefix_bytes) != -1: # some windows libraries have double backslashes as escaping - yield (prefix.replace("\\", "\\\\"), mode, f) + yield (double_backslash_prefix, mode, f) if mm.find(prefix_placeholder_bytes) != -1: yield (prefix_placeholder, mode, f) mm.close() and fi.close() diff --git a/tests/test_build.py b/tests/test_build.py new file mode 100644 index 0000000000..c364853b70 --- /dev/null +++ b/tests/test_build.py @@ -0,0 +1,32 @@ +import os +import sys + +from conda_build import build +from conda.compat import TemporaryDirectory + +prefix_tests = {"normal": os.path.sep} +if sys.platform == "win32": + prefix_tests.update({"double_backslash": "\\\\", + "forward_slash": "/"}) + + +def _write_prefix(filename, prefix, replacement): + with open(filename, "w") as f: + f.write(prefix.replace(os.path.sep, replacement)) + f.write("\n") + + +def test_find_prefix_files(): + # create a temporary folder + prefix = os.path.join(sys.prefix, "envs", "_build") + if not os.path.isdir(prefix): + os.makedirs(prefix) + with TemporaryDirectory(prefix=prefix + os.path.sep) as tmpdir: + # create text files to be replaced + files = [] + for slash_style in prefix_tests: + filename = os.path.join(tmpdir, "%s.txt" % slash_style) + _write_prefix(filename, prefix, prefix_tests[slash_style]) + files.append(filename) + + assert len(list(build.have_prefix_files(files))) == len(files) From 15cca590954cbfb6cf1dfd2b911c2a4d26f112fc Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Fri, 20 May 2016 23:36:31 -0500 Subject: [PATCH 07/45] update work on feature setting --- conda_build/environ.py | 5 +++++ conda_build/metadata.py | 40 ++++++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index e08a3ec30f..10e1be51d7 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -14,6 +14,7 @@ from conda_build import external from conda_build import source from conda_build.config import config +from conda_build.metadata import get_features from conda_build.scripts import prepend_bin_path @@ -161,6 +162,10 @@ def get_dict(m=None, prefix=None): # system d.update(system_vars(d, prefix)) + # features + d.update({ket.upper(): str(int(value)) for key, value in + iteritems(get_features)}) + return d diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 76fa77d096..00d80490d4 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -11,7 +11,7 @@ from conda.resolve import MatchSpec from conda.cli.common import specs_from_url -from . import exceptions +from conda_build import exceptions try: import yaml @@ -29,6 +29,26 @@ from conda_build.utils import comma_join + +@memoized +def get_features(): + """ + Returns a dict mapping feature names to booleans, corresponding to + environment variables starting with 'FEATURE_'. For example having + FEATURE_A=1 and FEATURE_B=0 will return {'a': True, 'b': False} + """ + sel_pat = re.compile('FEATURE_(\w+)') + res = {} + for key, value in iteritems(os.environ): + m = sel_pat.match(key) + if m: + if value not in ('0', '1'): + sys.exit("Error: did not expect environment variable '%s' " + "being set to '%s' (not '0' or '1')" % (key, value)) + res[m.group(1).lower()] = bool(int(value)) + return res + + def ns_cfg(): # Remember to update the docs of any of this changes plat = cc.subdir @@ -65,15 +85,7 @@ def ns_cfg(): for machine in cc.non_x86_linux_machines: d[machine] = bool(plat == 'linux-%s' % machine) - sel_pat = re.compile('SELECTOR_(\w+)') - for key, value in iteritems(os.environ): - m = sel_pat.match(key) - if m: - if value not in ('0', '1'): - sys.exit("Error: did not expect environment variable '%s' " - "being set to '%s' (not '0' or '1')" % (key, value)) - d[m.group(1)] = bool(int(value)) - + d.update(get_features()) d.update(os.environ) return d @@ -715,11 +727,3 @@ def __repr__(self): String representation of the MetaData. ''' return self.__str__() - - -if __name__ == '__main__': - from pprint import pprint - from os.path import expanduser - - m = MetaData(expanduser('~/conda-recipes/pycosat')) - pprint(m.info_index()) From ec8671d730efba17f553258cd8637beb67da3230 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Fri, 20 May 2016 23:48:58 -0500 Subject: [PATCH 08/45] fix import errors and typos --- conda_build/environ.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 10e1be51d7..d786733a2d 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -9,7 +9,7 @@ from subprocess import STDOUT, check_output, CalledProcessError, Popen, PIPE import conda.config as cc -from conda.compat import text_type +from conda.compat import iteritems, text_type from conda_build import external from conda_build import source @@ -163,8 +163,8 @@ def get_dict(m=None, prefix=None): d.update(system_vars(d, prefix)) # features - d.update({ket.upper(): str(int(value)) for key, value in - iteritems(get_features)}) + d.update({key.upper(): str(int(value)) for key, value in + iteritems(get_features())}) return d From b9439f842e3f69840145b047f26e894a0c6c2d61 Mon Sep 17 00:00:00 2001 From: Stuart Berg Date: Sun, 22 May 2016 19:50:26 -0400 Subject: [PATCH 09/45] jinja_context: Make sure all undefined variables have names (not None). --- conda_build/jinja_context.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/conda_build/jinja_context.py b/conda_build/jinja_context.py index 2b42241249..2a3da2dfdb 100644 --- a/conda_build/jinja_context.py +++ b/conda_build/jinja_context.py @@ -39,7 +39,10 @@ def __init__(self, hint=None, obj=jinja2.runtime.missing, name=None, __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = \ __complex__ = __pow__ = __rpow__ = \ - lambda *args, **kwargs: UndefinedNeverFail() + lambda self, *args, **kwargs: UndefinedNeverFail(hint=self._undefined_hint, + obj=self._undefined_obj, + name=self._undefined_name, + exc=self._undefined_exception) __str__ = __repr__ = \ lambda *args, **kwargs: u'' @@ -51,10 +54,10 @@ def __getattr__(self, k): try: return object.__getattr__(self, k) except AttributeError: - return UndefinedNeverFail() - - def __setattr__(self, k, v): - pass + return UndefinedNeverFail(hint=self._undefined_hint, + obj=self._undefined_obj, + name=self._undefined_name + '.' + k, + exc=self._undefined_exception) class FilteredLoader(jinja2.BaseLoader): From 37509f7cf61e4015025e7530ed0ffeb134e4bba2 Mon Sep 17 00:00:00 2001 From: Stuart Berg Date: Sun, 22 May 2016 19:51:11 -0400 Subject: [PATCH 10/45] environ: In verify_git_repo(), verification of remote url is case-INsensitive --- conda_build/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index e08a3ec30f..8e9a51e0cf 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -93,7 +93,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. - if remote_url != git_url: + if remote_url.lower() != git_url.lower(): return False except CalledProcessError: return False From 09e88a32ddc0522c52fae104835a97258f6ff90d Mon Sep 17 00:00:00 2001 From: Julien Schueller Date: Mon, 23 May 2016 15:39:09 +0200 Subject: [PATCH 11/45] Handle -p1 patch level fixes https://github.com/conda/conda-build/issues/968 --- conda_build/source.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/conda_build/source.py b/conda_build/source.py index c89bfb027d..e1b8daaba7 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -336,7 +336,12 @@ def apply_patch(src_dir, path): try: check_call([patch] + patch_args, cwd=src_dir) except CalledProcessError: - sys.exit(1) + # fallback to -p1, the git default + patch_args[0] = '-p1' + try: + check_call([patch] + patch_args, cwd=src_dir) + except CalledProcessError: + sys.exit(1) if sys.platform == 'win32' and os.path.exists(patch_args[-1]): os.remove(patch_args[-1]) # clean up .patch_unix file From 106c6757f168ca3756bfe2bfeb28b2c6b427245e Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Mon, 23 May 2016 18:10:39 -0500 Subject: [PATCH 12/45] add SUBDIR environment variable in build process --- conda_build/environ.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conda_build/environ.py b/conda_build/environ.py index d786733a2d..3cb2b6c642 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -178,6 +178,7 @@ def conda_build_vars(prefix): 'PREFIX': prefix, 'SYS_PREFIX': sys.prefix, 'SYS_PYTHON': sys.executable, + 'SUBDIR': cc.subdir, 'SRC_DIR': source.get_dir(), 'HTTPS_PROXY': os.getenv('HTTPS_PROXY', ''), 'HTTP_PROXY': os.getenv('HTTP_PROXY', ''), From 81722f6d8ed57cd7e167db5403a58c821bf6ce49 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Tue, 24 May 2016 14:30:21 -0500 Subject: [PATCH 13/45] track features --- conda_build/build.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 2316a7d31c..b2c3042344 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -20,7 +20,7 @@ import conda.config as cc import conda.plan as plan from conda.api import get_index -from conda.compat import PY3 +from conda.compat import PY3, iteritems from conda.fetch import fetch_index from conda.install import prefix_placeholder, linked, move_to_trash from conda.lock import Locked @@ -38,6 +38,7 @@ from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) from conda_build.exceptions import indent +from conda_build.metadata import get_features on_win = (sys.platform == 'win32') @@ -309,6 +310,11 @@ def create_env(prefix, specs, clear_cache=True): ''' Create a conda envrionment for the given prefix and specs. ''' + specs = list(specs) + for feature, value in iteritems(get_features()): + if value: + specs.append('%s@' % feature) + for d in config.bldpkgs_dirs: if not isdir(d): os.makedirs(d) From 80a594d91d0cfd82de21bb0598678730a789ef8a Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Tue, 24 May 2016 15:37:12 -0500 Subject: [PATCH 14/45] prefix environment variable --- conda_build/environ.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index fc9ebd2e81..c01245b470 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -163,7 +163,7 @@ def get_dict(m=None, prefix=None): d.update(system_vars(d, prefix)) # features - d.update({key.upper(): str(int(value)) for key, value in + d.update({'FEATURE_%s' % key.upper(): str(int(value)) for key, value in iteritems(get_features())}) return d From 6813aec465ab971aed9bdff24f03eb409642b0f2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 24 May 2016 16:46:52 -0500 Subject: [PATCH 15/45] add --dirty flag to skip download/extraction step --- conda_build/build.py | 22 +++++++------- conda_build/main_build.py | 10 ++++++- conda_build/render.py | 6 ++-- conda_build/source.py | 62 ++++++++++++++++++++------------------- 4 files changed, 57 insertions(+), 43 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 6f734a8fe0..a298e590c0 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -374,7 +374,7 @@ def rm_pkgs_cache(dist): def build(m, post=None, include_recipe=True, keep_old_work=False, - need_source_download=True, verbose=True): + need_source_download=True, verbose=True, dirty=False): ''' Build the package with the specified metadata. @@ -419,14 +419,15 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, if post in [False, None]: print("Removing old build environment") - if on_win: - if isdir(config.short_build_prefix): - move_to_trash(config.short_build_prefix, '') - if isdir(config.long_build_prefix): - move_to_trash(config.long_build_prefix, '') - else: - rm_rf(config.short_build_prefix) - rm_rf(config.long_build_prefix) + if not dirty: + if on_win: + if isdir(config.short_build_prefix): + move_to_trash(config.short_build_prefix, '') + if isdir(config.long_build_prefix): + move_to_trash(config.long_build_prefix, '') + else: + rm_rf(config.short_build_prefix) + rm_rf(config.long_build_prefix) # Display the name only # Version number could be missing due to dependency on source info. @@ -445,7 +446,8 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, m, need_source_download = parse_or_try_download(m, no_download_source=False, force_download=True, - verbose=verbose) + verbose=verbose, + dirty=dirty) assert not need_source_download, "Source download failed. Please investigate." finally: os.environ['PATH'] = _old_path diff --git a/conda_build/main_build.py b/conda_build/main_build.py index bba41e3872..2f9a4bf659 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -111,6 +111,12 @@ def main(): help="""Keep any existing, old work directory. Useful if debugging across callstacks involving multiple packages/recipes. """ ) + p.add_argument( + '--dirty', + action='store_true', + help='Do not remove work directory or _build environment, ' + 'to speed up debugging. Does not apply patches or download source.' + ) p.add_argument( '-q', "--quiet", action="store_true", @@ -317,7 +323,9 @@ def execute(args, parser): build.build(m, post=post, include_recipe=args.include_recipe, keep_old_work=args.keep_old_work, - need_source_download=need_source_download) + need_source_download=need_source_download, + dirty=args.dirty + ) except (NoPackagesFound, Unsatisfiable) as e: error_str = str(e) # Typically if a conflict is with one of these diff --git a/conda_build/render.py b/conda_build/render.py index 3fa7fb5a40..dcef854f8e 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -74,13 +74,15 @@ def bldpkg_path(m): return os.path.join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist()) -def parse_or_try_download(metadata, no_download_source, verbose, force_download=False): +def parse_or_try_download(metadata, no_download_source, verbose, + force_download=False, dirty=False): if (force_download or (not no_download_source and any(var.startswith('GIT_') for var in metadata.undefined_jinja_vars))): # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. try: - source.provide(metadata.path, metadata.get_section('source'), verbose=verbose) + source.provide(metadata.path, metadata.get_section('source'), + verbose=verbose, dirty=dirty) metadata.parse_again(permit_undefined_jinja=False) need_source_download = False except subprocess.CalledProcessError: diff --git a/conda_build/source.py b/conda_build/source.py index e1b8daaba7..4ab54ebe72 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -346,7 +346,7 @@ def apply_patch(src_dir, path): os.remove(patch_args[-1]) # clean up .patch_unix file -def provide(recipe_dir, meta, verbose=False, patch=True): +def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False): """ given a recipe_dir: - download (if necessary) @@ -354,35 +354,37 @@ def provide(recipe_dir, meta, verbose=False, patch=True): - apply patches (if any) """ - if sys.platform == 'win32': - if isdir(WORK_DIR): - move_to_trash(WORK_DIR, '') - else: - rm_rf(WORK_DIR) - - if any(k in meta for k in ('fn', 'url')): - unpack(meta, verbose=verbose) - elif 'git_url' in meta: - git_source(meta, recipe_dir, verbose=verbose) - # build to make sure we have a work directory with source in it. We want to make sure that - # build to make sure we have a work directory with source in it. We want to make sure that - # whatever version that is does not interfere with the test we run next. - # whatever version that is does not interfere with the test we run next. - elif 'hg_url' in meta: - hg_source(meta, verbose=verbose) - elif 'svn_url' in meta: - svn_source(meta, verbose=verbose) - elif 'path' in meta: - if verbose: - print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) - copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) - else: # no source - os.makedirs(WORK_DIR) - - if patch: - src_dir = get_dir() - for patch in meta.get('patches', []): - apply_patch(src_dir, join(recipe_dir, patch)) + if not dirty: + if sys.platform == 'win32': + if isdir(WORK_DIR): + move_to_trash(WORK_DIR, '') + else: + rm_rf(WORK_DIR) + + if not os.path.exists(WORK_DIR): + if any(k in meta for k in ('fn', 'url')): + unpack(meta, verbose=verbose) + elif 'git_url' in meta: + git_source(meta, recipe_dir, verbose=verbose) + # build to make sure we have a work directory with source in it. We want to make sure that + # build to make sure we have a work directory with source in it. We want to make sure that + # whatever version that is does not interfere with the test we run next. + # whatever version that is does not interfere with the test we run next. + elif 'hg_url' in meta: + hg_source(meta, verbose=verbose) + elif 'svn_url' in meta: + svn_source(meta, verbose=verbose) + elif 'path' in meta: + if verbose: + print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) + copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) + else: # no source + os.makedirs(WORK_DIR) + + if patch: + src_dir = get_dir() + for patch in meta.get('patches', []): + apply_patch(src_dir, join(recipe_dir, patch)) if __name__ == '__main__': From 536ddb9530b29a51e9ae14aa4c9b0563b58ea48b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 25 May 2016 08:45:10 -0500 Subject: [PATCH 16/45] set DIRTY env variable in build scripts. Pass dirty to render_recipe --- conda_build/build.py | 12 ++++++------ conda_build/environ.py | 7 ++++--- conda_build/render.py | 4 ++-- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index a298e590c0..233cc7be82 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -419,6 +419,7 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, if post in [False, None]: print("Removing old build environment") + print("BUILD START:", m.dist()) if not dirty: if on_win: if isdir(config.short_build_prefix): @@ -429,11 +430,10 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, rm_rf(config.short_build_prefix) rm_rf(config.long_build_prefix) - # Display the name only - # Version number could be missing due to dependency on source info. - print("BUILD START:", m.dist()) - create_env(config.build_prefix, - [ms.spec for ms in m.ms_depends('build')]) + # Display the name only + # Version number could be missing due to dependency on source info. + create_env(config.build_prefix, + [ms.spec for ms in m.ms_depends('build')]) if need_source_download: # Execute any commands fetching the source (e.g., git) in the _build environment. @@ -503,7 +503,7 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, import conda_build.windows as windows windows.build(m, build_file) else: - env = environ.get_dict(m) + env = environ.get_dict(m, dirty=dirty) build_file = join(m.path, 'build.sh') if script: diff --git a/conda_build/environ.py b/conda_build/environ.py index 8e9a51e0cf..1cf1f34afb 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -143,12 +143,12 @@ def get_git_info(repo): return d -def get_dict(m=None, prefix=None): +def get_dict(m=None, prefix=None, dirty=False): if not prefix: prefix = config.build_prefix # conda-build specific vars - d = conda_build_vars(prefix) + d = conda_build_vars(prefix, dirty) # languages d.update(python_vars()) @@ -164,7 +164,7 @@ def get_dict(m=None, prefix=None): return d -def conda_build_vars(prefix): +def conda_build_vars(prefix, dirty): return { 'CONDA_BUILD': '1', 'PYTHONNOUSERSITE': '1', @@ -176,6 +176,7 @@ def conda_build_vars(prefix): 'SRC_DIR': source.get_dir(), 'HTTPS_PROXY': os.getenv('HTTPS_PROXY', ''), 'HTTP_PROXY': os.getenv('HTTP_PROXY', ''), + 'DIRTY': '1' if dirty else '', } diff --git a/conda_build/render.py b/conda_build/render.py index dcef854f8e..3b476187cc 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -99,7 +99,7 @@ def parse_or_try_download(metadata, no_download_source, verbose, return metadata, need_source_download -def render_recipe(recipe_path, no_download_source, verbose): +def render_recipe(recipe_path, no_download_source, verbose, dirty=False): with Locked(config.croot): arg = recipe_path # Don't use byte literals for paths in Python 2 @@ -129,7 +129,7 @@ def render_recipe(recipe_path, no_download_source, verbose): sys.exit(1) m = parse_or_try_download(m, no_download_source=no_download_source, - verbose=verbose) + verbose=verbose, dirty=dirty) if need_cleanup: shutil.rmtree(recipe_dir) From 590220c3db1df5fa8e14e718bb3eed6c97aedfac Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 25 May 2016 09:01:37 -0500 Subject: [PATCH 17/45] add test for dirty functionality in build scripts --- tests/test-recipes/metadata/_dirty_skip_section/bld.bat | 3 +++ tests/test-recipes/metadata/_dirty_skip_section/build.sh | 3 +++ .../test-recipes/metadata/_dirty_skip_section/meta.yaml | 6 ++++++ tests/test_build_recipes.py | 9 +++++++++ 4 files changed, 21 insertions(+) create mode 100644 tests/test-recipes/metadata/_dirty_skip_section/bld.bat create mode 100644 tests/test-recipes/metadata/_dirty_skip_section/build.sh create mode 100644 tests/test-recipes/metadata/_dirty_skip_section/meta.yaml diff --git a/tests/test-recipes/metadata/_dirty_skip_section/bld.bat b/tests/test-recipes/metadata/_dirty_skip_section/bld.bat new file mode 100644 index 0000000000..2100201c67 --- /dev/null +++ b/tests/test-recipes/metadata/_dirty_skip_section/bld.bat @@ -0,0 +1,3 @@ +:: ensure that the DIRTY environment variable is available for logic in build scripts +IF "%DIRTY%" == "1" exit 0 +exit 1 \ No newline at end of file diff --git a/tests/test-recipes/metadata/_dirty_skip_section/build.sh b/tests/test-recipes/metadata/_dirty_skip_section/build.sh new file mode 100644 index 0000000000..289cdc4b50 --- /dev/null +++ b/tests/test-recipes/metadata/_dirty_skip_section/build.sh @@ -0,0 +1,3 @@ +# ensure that the DIRTY environment variable is available for logic in build scripts +[ -n "$DIRTY" ] && exit 0 +exit 1 diff --git a/tests/test-recipes/metadata/_dirty_skip_section/meta.yaml b/tests/test-recipes/metadata/_dirty_skip_section/meta.yaml new file mode 100644 index 0000000000..82352c683b --- /dev/null +++ b/tests/test-recipes/metadata/_dirty_skip_section/meta.yaml @@ -0,0 +1,6 @@ +package: + name: conda-build-test-dirty-skip-section + version: 1.0 + +about: + summary: ensure that the DIRTY environment variable is available to bld.bat and build.sh diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index 5a9768ab68..bd19c73628 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -99,6 +99,15 @@ def test_recipe_builds(recipe): subprocess.check_call(cmd.split(), env=env) +def test_dirty_variable_available_in_build_scripts(): + cmd = 'conda build --no-anaconda-upload --dirty {}'.format(os.path.join(metadata_dir, + "_dirty_skip_section")) + subprocess.check_call(cmd.split()) + with pytest.raises(subprocess.CalledProcessError): + cmd=cmd.replace(" --dirty", "") + subprocess.check_call(cmd.split()) + + def test_checkout_tool_as_dependency(): # "hide" svn by putting a known bad one on PATH tmpdir = tempfile.mkdtemp() From 3c52a27feff978371a1ff84332d45c75ff311026 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 25 May 2016 13:11:08 -0500 Subject: [PATCH 18/45] update badge url for appveyor builds --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 92452e3998..7e3901d3b8 100644 --- a/README.rst +++ b/README.rst @@ -5,8 +5,8 @@ conda-build .. image:: https://travis-ci.org/conda/conda-build.svg?branch=master :target: https://travis-ci.org/conda/conda-build -.. image:: https://ci.appveyor.com/api/projects/status/aj0d43e7qea5amja?svg=true - :target: https://ci.appveyor.com/project/ironmancio54716/conda-build +.. image:: https://ci.appveyor.com/api/projects/status/wdis0rkcfdoeql3x?svg=true + :target: https://ci.appveyor.com/project/ContinuumAnalytics/conda-build .. image:: https://anaconda.org/conda-team/conda-build/badges/build.svg :target: https://anaconda.org/conda-team/conda-build/ From acc5af04ad916e4d0ac606655e16087f58009413 Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Wed, 25 May 2016 13:27:26 -0500 Subject: [PATCH 19/45] Drop Py3.4 from AppVeyor build matrix --- appveyor.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index e5ba399dde..f202707c25 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,10 +14,6 @@ environment: PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" - - PYTHON: "C:\\Python34_64" - PYTHON_VERSION: "3.4" - PYTHON_ARCH: "64" - - PYTHON: "C:\\Python27_32" PYTHON_VERSION: "2.7" PYTHON_ARCH: "32" @@ -26,10 +22,6 @@ environment: PYTHON_VERSION: "3.5" PYTHON_ARCH: "32" - - PYTHON: "C:\\Python34_32" - PYTHON_VERSION: "3.4" - PYTHON_ARCH: "32" - init: - ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH% %HOME% From 3cfeee81550b04ea802e2f5f8f46383269f3c71e Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Wed, 25 May 2016 15:18:34 -0500 Subject: [PATCH 20/45] add features module --- conda_build/build.py | 6 +++--- conda_build/environ.py | 8 ++++---- conda_build/features.py | 21 +++++++++++++++++++++ conda_build/metadata.py | 24 ++++-------------------- 4 files changed, 32 insertions(+), 27 deletions(-) create mode 100644 conda_build/features.py diff --git a/conda_build/build.py b/conda_build/build.py index 5355405ed5..bb6d518bb2 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -20,7 +20,7 @@ import conda.config as cc import conda.plan as plan from conda.api import get_index -from conda.compat import PY3, iteritems +from conda.compat import PY3 from conda.fetch import fetch_index from conda.install import prefix_placeholder, linked, move_to_trash from conda.lock import Locked @@ -39,7 +39,7 @@ from conda_build.create_test import (create_files, create_shell_files, create_py_files, create_pl_files) from conda_build.exceptions import indent -from conda_build.metadata import get_features +from conda_build.features import feature_list on_win = (sys.platform == 'win32') @@ -323,7 +323,7 @@ def create_env(prefix, specs, clear_cache=True): Create a conda envrionment for the given prefix and specs. ''' specs = list(specs) - for feature, value in iteritems(get_features()): + for feature, value in feature_list: if value: specs.append('%s@' % feature) diff --git a/conda_build/environ.py b/conda_build/environ.py index c01245b470..64ce7cbb5c 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -9,12 +9,12 @@ from subprocess import STDOUT, check_output, CalledProcessError, Popen, PIPE import conda.config as cc -from conda.compat import iteritems, text_type +from conda.compat import text_type from conda_build import external from conda_build import source from conda_build.config import config -from conda_build.metadata import get_features +from conda_build.features import feature_list from conda_build.scripts import prepend_bin_path @@ -163,8 +163,8 @@ def get_dict(m=None, prefix=None): d.update(system_vars(d, prefix)) # features - d.update({'FEATURE_%s' % key.upper(): str(int(value)) for key, value in - iteritems(get_features())}) + d.update({feat.upper(): str(int(value)) for feat, value in + feature_list}) return d diff --git a/conda_build/features.py b/conda_build/features.py new file mode 100644 index 0000000000..afd5bef1fa --- /dev/null +++ b/conda_build/features.py @@ -0,0 +1,21 @@ +from __future__ import absolute_import, division, print_function + +import os +import re +import sys + +from conda.compat import iteritems + + +sel_pat = re.compile('FEATURE_(\w+)') + +# list of features, where each element is a tuple(name, boolean), i.e. +# having FEATURE_A=1 and FEATURE_B=0 -> [('a', True), ('b', False)] +feature_list = [] +for key, value in iteritems(os.environ): + m = sel_pat.match(key) + if m: + if value not in ('0', '1'): + sys.exit("Error: did not expect environment variable '%s' " + "being set to '%s' (not '0' or '1')" % (key, value)) + feature_list.append((m.group(1).lower(), bool(int(value)))) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index e637da9419..e2b7985f58 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -12,6 +12,8 @@ from conda.cli.common import specs_from_url from conda_build import exceptions +from conda_build.features import feature_list + try: import yaml @@ -30,25 +32,6 @@ -@memoized -def get_features(): - """ - Returns a dict mapping feature names to booleans, corresponding to - environment variables starting with 'FEATURE_'. For example having - FEATURE_A=1 and FEATURE_B=0 will return {'a': True, 'b': False} - """ - sel_pat = re.compile('FEATURE_(\w+)') - res = {} - for key, value in iteritems(os.environ): - m = sel_pat.match(key) - if m: - if value not in ('0', '1'): - sys.exit("Error: did not expect environment variable '%s' " - "being set to '%s' (not '0' or '1')" % (key, value)) - res[m.group(1).lower()] = bool(int(value)) - return res - - def ns_cfg(): # Remember to update the docs of any of this changes plat = cc.subdir @@ -85,7 +68,8 @@ def ns_cfg(): for machine in cc.non_x86_linux_machines: d[machine] = bool(plat == 'linux-%s' % machine) - d.update(get_features()) + for feature, value in feature_list: + d[feature] = value d.update(os.environ) return d From c0bcd63572658d4e71c8a6334e6b78fcf64e647e Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Thu, 26 May 2016 13:41:49 +0100 Subject: [PATCH 21/45] MSYS2: Add build paths This mirrors what we do in conda when we activate enivornments. --- conda_build/external.py | 4 ++++ conda_build/scripts.py | 5 ++++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/conda_build/external.py b/conda_build/external.py index 1a8a427b13..76099cd8fb 100644 --- a/conda_build/external.py +++ b/conda_build/external.py @@ -14,8 +14,12 @@ def find_executable(executable): global dir_paths if sys.platform == 'win32': dir_paths = [join(config.build_prefix, 'Scripts'), + join(config.build_prefix, 'Library\\mingw-w64\\bin'), + join(config.build_prefix, 'Library\\usr\\bin'), join(config.build_prefix, 'Library\\bin'), join(cc.root_dir, 'Scripts'), + join(cc.root_dir, 'Library\\mingw-w64\\bin'), + join(cc.root_dir, 'Library\\usr\\bin'), join(cc.root_dir, 'Library\\bin'), ] else: dir_paths = [join(config.build_prefix, 'bin'), diff --git a/conda_build/scripts.py b/conda_build/scripts.py index d23d93d0af..78fd52ff5f 100644 --- a/conda_build/scripts.py +++ b/conda_build/scripts.py @@ -69,7 +69,10 @@ def prepend_bin_path(env, prefix, prepend_prefix=False): # bin_dirname takes care of bin on *nix, Scripts on win env['PATH'] = join(prefix, bin_dirname) + os.pathsep + env['PATH'] if sys.platform == "win32": - env['PATH'] = join(prefix, "Library", "bin") + os.pathsep + env['PATH'] + env['PATH'] = join(prefix, "Library", "mingw-w64", "bin") + os.pathsep + \ + join(prefix, "Library", "usr", "bin") + os.pathsep + os.pathsep + \ + join(prefix, "Library", "bin") + os.pathsep + \ + env['PATH'] prepend_prefix = True # windows has Python in the prefix. Use it. if prepend_prefix: env['PATH'] = prefix + os.pathsep + env['PATH'] From 64569b899e9295d843ef95b7eab53cbf0995d7fb Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 26 May 2016 08:59:44 -0500 Subject: [PATCH 22/45] flake8 --- conda_build/main_build.py | 3 +-- tests/test_build_recipes.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/conda_build/main_build.py b/conda_build/main_build.py index 2f9a4bf659..5cb0691199 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -324,8 +324,7 @@ def execute(args, parser): include_recipe=args.include_recipe, keep_old_work=args.keep_old_work, need_source_download=need_source_download, - dirty=args.dirty - ) + dirty=args.dirty) except (NoPackagesFound, Unsatisfiable) as e: error_str = str(e) # Typically if a conflict is with one of these diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index bd19c73628..00f2a1f60f 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -104,7 +104,7 @@ def test_dirty_variable_available_in_build_scripts(): "_dirty_skip_section")) subprocess.check_call(cmd.split()) with pytest.raises(subprocess.CalledProcessError): - cmd=cmd.replace(" --dirty", "") + cmd = cmd.replace(" --dirty", "") subprocess.check_call(cmd.split()) From 87575b763d7aa35fd6dfb88cbd121a28c5b7c745 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Thu, 26 May 2016 11:51:17 -0500 Subject: [PATCH 23/45] don't keep build env; fix duplicated comment --- conda_build/build.py | 25 ++++++++++++------------- conda_build/source.py | 2 -- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index 233cc7be82..13f714c4a2 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -420,20 +420,19 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, if post in [False, None]: print("Removing old build environment") print("BUILD START:", m.dist()) - if not dirty: - if on_win: - if isdir(config.short_build_prefix): - move_to_trash(config.short_build_prefix, '') - if isdir(config.long_build_prefix): - move_to_trash(config.long_build_prefix, '') - else: - rm_rf(config.short_build_prefix) - rm_rf(config.long_build_prefix) + if on_win: + if isdir(config.short_build_prefix): + move_to_trash(config.short_build_prefix, '') + if isdir(config.long_build_prefix): + move_to_trash(config.long_build_prefix, '') + else: + rm_rf(config.short_build_prefix) + rm_rf(config.long_build_prefix) - # Display the name only - # Version number could be missing due to dependency on source info. - create_env(config.build_prefix, - [ms.spec for ms in m.ms_depends('build')]) + # Display the name only + # Version number could be missing due to dependency on source info. + create_env(config.build_prefix, + [ms.spec for ms in m.ms_depends('build')]) if need_source_download: # Execute any commands fetching the source (e.g., git) in the _build environment. diff --git a/conda_build/source.py b/conda_build/source.py index 4ab54ebe72..d585020be7 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -367,8 +367,6 @@ def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False): elif 'git_url' in meta: git_source(meta, recipe_dir, verbose=verbose) # build to make sure we have a work directory with source in it. We want to make sure that - # build to make sure we have a work directory with source in it. We want to make sure that - # whatever version that is does not interfere with the test we run next. # whatever version that is does not interfere with the test we run next. elif 'hg_url' in meta: hg_source(meta, verbose=verbose) From 2388116edbdd5852f5d87edcd807766d4cd856cc Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Thu, 26 May 2016 13:21:54 -0500 Subject: [PATCH 24/45] only allow explicit features: debug, nomkl, opt --- conda_build/features.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/conda_build/features.py b/conda_build/features.py index afd5bef1fa..7cdce34107 100644 --- a/conda_build/features.py +++ b/conda_build/features.py @@ -1,21 +1,23 @@ from __future__ import absolute_import, division, print_function import os -import re import sys from conda.compat import iteritems -sel_pat = re.compile('FEATURE_(\w+)') +env_vars = [ + 'FEATURE_DEBUG', + 'FEATURE_NOMKL', + 'FEATURE_OPT', +] -# list of features, where each element is a tuple(name, boolean), i.e. -# having FEATURE_A=1 and FEATURE_B=0 -> [('a', True), ('b', False)] +# list of features, where each element is a tuple(name, boolean), i.e. having +# FEATURE_DEBUG=1 and FEATURE_NOMKL=0 -> [('debug', True), ('nomkl', False)] feature_list = [] for key, value in iteritems(os.environ): - m = sel_pat.match(key) - if m: + if key in env_vars: if value not in ('0', '1'): sys.exit("Error: did not expect environment variable '%s' " "being set to '%s' (not '0' or '1')" % (key, value)) - feature_list.append((m.group(1).lower(), bool(int(value)))) + feature_list.append((key[8:].lower(), bool(int(value)))) From 65556f9dfa0e2040847b682b6eb42d15e369c72c Mon Sep 17 00:00:00 2001 From: Ray Donnelly Date: Fri, 27 May 2016 12:43:04 +0100 Subject: [PATCH 25/45] Only set os.environ for non-None config vars Otherwise we end up stringifying None as 'None' and the version parsing code falls over later (if you call conda from a build script for example). --- conda_build/render.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/render.py b/conda_build/render.py index 3fa7fb5a40..88dd5e15d4 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -62,7 +62,7 @@ def set_language_env_vars(args, parser, execute=None): # Using --python, --numpy etc. is equivalent to using CONDA_PY, CONDA_NPY, etc. # Auto-set those env variables for var in conda_version.values(): - if hasattr(config, var): + if hasattr(config, var) and getattr(config, var): # Set the env variable. os.environ[var] = str(getattr(config, var)) From f11f575d03222b5a099c72a5071efc56cca99b83 Mon Sep 17 00:00:00 2001 From: Heather Kelly Date: Fri, 27 May 2016 11:43:56 -0400 Subject: [PATCH 26/45] After discussion, agreed to 255 byte (char) limit. --- conda_build/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/config.py b/conda_build/config.py index f12ab5bdea..a0f233d4ae 100644 --- a/conda_build/config.py +++ b/conda_build/config.py @@ -57,7 +57,7 @@ def get_conda_py(self): croot = abspath(expanduser('~/conda-bld')) short_build_prefix = join(cc.envs_dirs[0], '_build') - long_build_prefix = max(short_build_prefix, (short_build_prefix + 8 * '_placehold')[:128]) + long_build_prefix = max(short_build_prefix, (short_build_prefix + 25 * '_placehold')[:255]) # XXX: Make this None to be more rigorous about requiring the build_prefix # to be known before it is used. use_long_build_prefix = False From 81137d229e567c19fb07730225328bd46275d87b Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 27 May 2016 11:45:38 -0500 Subject: [PATCH 27/45] refine tests for checking GIT variables with relative paths --- .../meta.yaml | 10 ++++++++++ .../_source_git_jinja2_relative_path/meta.yaml | 10 ++++++++++ .../metadata/source_git_jinja2/meta.yaml | 2 +- tests/test_build_recipes.py | 16 ++++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 tests/test-recipes/metadata/_source_git_jinja2_relative_git_url/meta.yaml create mode 100644 tests/test-recipes/metadata/_source_git_jinja2_relative_path/meta.yaml diff --git a/tests/test-recipes/metadata/_source_git_jinja2_relative_git_url/meta.yaml b/tests/test-recipes/metadata/_source_git_jinja2_relative_git_url/meta.yaml new file mode 100644 index 0000000000..5a2834c513 --- /dev/null +++ b/tests/test-recipes/metadata/_source_git_jinja2_relative_git_url/meta.yaml @@ -0,0 +1,10 @@ +package: + name: flex + version: {{ environ.get('GIT_DESCRIBE_TAG', '0.0') }} + +build: + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} + string: {{ environ.get('GIT_DESCRIBE_NUMBER', '0') + '_JPMC' }} + +source: + git_url: ../../../../ diff --git a/tests/test-recipes/metadata/_source_git_jinja2_relative_path/meta.yaml b/tests/test-recipes/metadata/_source_git_jinja2_relative_path/meta.yaml new file mode 100644 index 0000000000..a807594584 --- /dev/null +++ b/tests/test-recipes/metadata/_source_git_jinja2_relative_path/meta.yaml @@ -0,0 +1,10 @@ +package: + name: flex + version: {{ environ.get('GIT_DESCRIBE_TAG', '0.0') }} + +build: + number: {{ environ.get('GIT_DESCRIBE_NUMBER', 0) }} + string: {{ environ.get('GIT_DESCRIBE_NUMBER', '0') + '_JPMC' }} + +source: + path: ../../../../ diff --git a/tests/test-recipes/metadata/source_git_jinja2/meta.yaml b/tests/test-recipes/metadata/source_git_jinja2/meta.yaml index e6bd0d99fe..24b27305f4 100644 --- a/tests/test-recipes/metadata/source_git_jinja2/meta.yaml +++ b/tests/test-recipes/metadata/source_git_jinja2/meta.yaml @@ -2,7 +2,7 @@ # including use cases involving expressions such as FOO[:7] or FOO.replace(...) package: name: conda-build-test-source-git-jinja2 - version: {{ GIT_DESCRIBE_TAG.replace('v', '') }} + version: {{ environ.get("GIT_DESCRIBE_TAG", "WRONG_ANSWER").replace('v', '') }} source: git_url: https://github.com/conda/conda-build diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index 00f2a1f60f..d3d8b30c23 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -62,6 +62,22 @@ def test_cached_source_not_interfere_with_versioning(): shutil.rmtree('conda_build_test_recipe') +def test_relative_path_git_versioning(): + tag = subprocess.check_output(["git", "describe", "--abbrev=0"]).rstrip() + cmd = 'conda build --output {}'.format(os.path.join(metadata_dir, + "_source_git_jinja2_relative_path")) + output = subprocess.check_output(cmd.split()) + assert tag in output + + +def test_relative_git_url_git_versioning(): + tag = subprocess.check_output(["git", "describe", "--abbrev=0"]).rstrip() + cmd = 'conda build --output {}'.format(os.path.join(metadata_dir, + "_source_git_jinja2_relative_git_url")) + output = subprocess.check_output(cmd.split()) + assert tag in output + + def test_package_test(): """Test calling conda build -t - rather than """ filename = "jinja2-2.8-py{}{}_0.tar.bz2".format(sys.version_info.major, sys.version_info.minor) From 7ac64f78fe495c1def5d740032c5242e495c9835 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 27 May 2016 15:18:23 -0500 Subject: [PATCH 28/45] don't use long prefix on windows --- conda_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 13f714c4a2..1a560fc145 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -388,7 +388,7 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, ''' if (m.get_value('build/detect_binary_files_with_prefix') or - m.binary_has_prefix_files()): + m.binary_has_prefix_files()) and not on_win: # We must use a long prefix here as the package will only be # installable into prefixes shorter than this one. config.use_long_build_prefix = True From ede9253596fabf3bccc2cd761b311c8c88956b8a Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Mon, 23 May 2016 10:51:06 -0500 Subject: [PATCH 29/45] Windows test fixes: appveyor; dirty args --- appveyor.yml | 4 +- appveyor/setup_x64.bat | 13 +++++ .../600dd186-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 614 bytes .../600dd187-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 606 bytes .../600dd188-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 592 bytes .../600dd189-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 612 bytes .../656d875f-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 610 bytes .../656d8760-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 596 bytes .../656d8763-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 612 bytes .../656d8766-2429-11d7-8bf6-00b0d03daa06.reg | Bin 0 -> 616 bytes appveyor/x64/VC_OBJECTS_PLATFORM_INFO.reg | Bin 0 -> 3460 bytes conda_build/build.py | 2 +- conda_build/metadata.py | 1 - conda_build/windows.py | 4 +- tests/test_build_recipes.py | 54 ++++++++++-------- 15 files changed, 48 insertions(+), 30 deletions(-) create mode 100755 appveyor/setup_x64.bat create mode 100755 appveyor/x64/600dd186-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/600dd187-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/600dd188-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/600dd189-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/656d875f-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/656d8760-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/656d8763-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/656d8766-2429-11d7-8bf6-00b0d03daa06.reg create mode 100755 appveyor/x64/VC_OBJECTS_PLATFORM_INFO.reg diff --git a/appveyor.yml b/appveyor.yml index f202707c25..087f7b1a13 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -60,9 +60,7 @@ install: - python setup.py install - set PATH - conda build --version - - ps: (New-Object Net.WebClient).DownloadFile("https://github.com/menpo/condaci/blob/v0.4.8/vs2008_patch.zip", "c:\vs2008_patch.zip") - - 7z x C:\vs2008_patch.zip -oc:\vs2008_patch - - call c:\vs2008_patch\setup_x64.bat + - call appveyor\setup_x64.bat - copy "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\vcvars64.bat" "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\bin\amd64\vcvarsamd64.bat" diff --git a/appveyor/setup_x64.bat b/appveyor/setup_x64.bat new file mode 100755 index 0000000000..4786b3c207 --- /dev/null +++ b/appveyor/setup_x64.bat @@ -0,0 +1,13 @@ +regedit /s x64\VC_OBJECTS_PLATFORM_INFO.reg + +regedit /s x64\600dd186-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\600dd187-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\600dd188-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\600dd189-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\656d875f-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\656d8760-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\656d8763-2429-11d7-8bf6-00b0d03daa06.reg +regedit /s x64\656d8766-2429-11d7-8bf6-00b0d03daa06.reg + +copy "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.config" "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\AMD64.VCPlatform.Express.config" +copy "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.config" "C:\Program Files (x86)\Microsoft Visual Studio 9.0\VC\vcpackages\Itanium.VCPlatform.Express.config" \ No newline at end of file diff --git a/appveyor/x64/600dd186-2429-11d7-8bf6-00b0d03daa06.reg b/appveyor/x64/600dd186-2429-11d7-8bf6-00b0d03daa06.reg new file mode 100755 index 0000000000000000000000000000000000000000..ff97081e5bf34f840451df59e100ef95512c666a GIT binary patch literal 614 zcmZXSUrPc}5XH}P(05pTs)t0=OoO1NC?)=kYJrdegJ-ncF?u zr0pxLc{}B8o4(iRvAGBTy=IfJS{u|X$Q|gFed1LNw@GC>Ucd&hL`>YK+=MsAAH%pr pUmo@;E9ZTlc_-Q5VcDiQB0Hx01SK<+2U{Eg)r{4?NXBFT#UEDIWAOk0 literal 0 HcmV?d00001 diff --git a/appveyor/x64/600dd187-2429-11d7-8bf6-00b0d03daa06.reg b/appveyor/x64/600dd187-2429-11d7-8bf6-00b0d03daa06.reg new file mode 100755 index 0000000000000000000000000000000000000000..6f218a57078f1da9198e583eea675b3e3e14beff GIT binary patch literal 606 zcmZXSUrPc}5XH}P(05pTs)t0=OoO12~mZG2VTssr9~q~jV&DypD;qEhM(iEL17K+Tk@sYWP? zP!W>zu2=9O_#CX_N+LXU*lRii`&7x*#c<_S)}i*`E@Bn_@~qggvidNWz^s=(7GnkW z9xI3EOOM`P2Ii^PO;OeMX{>xV)fpXQp^m5>sp+c6>^;3Ot16u&t~byQ>dWx%dgT8e zZPIoNOL7x(+w{FekIg;U^tzse)mowEmDquv*~ebRaGTUx#}n89mKqbcDL3KmaUQ`q oLths5gq86=PyRYdHt`{Gb|V+q6tk`tuE*X%=xlZq>wJ3;zddnc*8l(j literal 0 HcmV?d00001 diff --git a/appveyor/x64/600dd188-2429-11d7-8bf6-00b0d03daa06.reg b/appveyor/x64/600dd188-2429-11d7-8bf6-00b0d03daa06.reg new file mode 100755 index 0000000000000000000000000000000000000000..8dd2bebd55ff530673d1d773f6573b778fd28f84 GIT binary patch literal 592 zcmZXSQA+|*5QWck(0^F=sU8wdvo#2cElbi8Hwq$M7YFxqI)K znKNfbf4;k#DpR6dZ(7JxMYU)OzPWkrK;u{ic=$dG(h*({>>VkI+>6(VBDk^KAten0hGF$W-(le)Pt}#kt zRD{&LYX&|7Ux3w2Nlc^(dqZbnpQ@U=1g??-Y!B`VUgK865*>02G=RCv&CN2v6THmX z$BIP0^l0<>FlW{`M^$9oSlQ`R`#Q!$9nm{h+f+|D`+8wlbtXwoucsZ<2k`D%=KmSp zrrocwq&B5CV(uk++}^#-tm{pY7 iqptvaiupF@$=@XDCjWx^40qEwMWNNZ-LEU$Kj0V5`(O_M literal 0 HcmV?d00001 diff --git a/appveyor/x64/600dd189-2429-11d7-8bf6-00b0d03daa06.reg b/appveyor/x64/600dd189-2429-11d7-8bf6-00b0d03daa06.reg new file mode 100755 index 0000000000000000000000000000000000000000..26403d883ed73a6235e3fcf625d32416387f72d3 GIT binary patch literal 612 zcmZXSUrPc}5XH}P(05pTs)t0=HVcAeOOiB2BZ)RdT@`EF75`;KU%mR>+3+PUbN9}f zxo6JI-JkECW=a$()w`DR)X`W|u%%X7=mX@M=nOB#_w+z~L3D31& zEq5N(S3@lYs`Blt$u}TY*B#MNU1Bxes|VgOq-Pp@l~vL)St)hLWHzZaqGn3fR1=hh zs0g^T)f{{bz63inB_WXp>~&p%eabg=5nM%8$T@<$jJZ>h73q{yMMIc;t_qs~Vlerq|N}>MQUbTIT;A z-K539Jf!BS`9Zpv+V2lx{h s7wF5w9kZ@#?}S^%-P0ZZIo=8Pl>4FfRA6mu%h`i2 zbjh20b=B0h0#*5T)#4lAYv>AZq!3?Sx9Whm9nx_ee3ewx0Z|!sheWohHK1lj)l4Ip zM6d{W^Qsy65PShvaZDmSO|mys1ACVbL!V4#9dQ~dPwo;{;k3_+!Si_g_gr~&3SBqWZa3CBonV2EsU7LYu^zMcG^JN{lq8O?rybar$-C?wst3c;CJlmQ%#yUii6oj3bxf>rM!Xi$U$4G>j0obe&t>h~ zTKjx`b~IM1See$E%TrrJjlkwwXr?!iGrVJ78E;Q__-A;>yi?wX+EanGtu1E{y3i## z_v)&tYXz!ucGcnx@HKRWH&TeNu3L4$+YaeC4!%k%>VT+>zC$8g^cv7Jqid!om_)D$ z$a&QidiPK9zlLrlpU?SdqQQ zD&YChqtln8a_ahKunOHYR&F=eIh|mEj_Dog#<3o;_cUQvb(AEIucsZ@m#Mq!BL8c& zN!u?h$xX-&(Y=I^&E5Oex|xLaTES*Y>_9K{p#QM=DL)vNwrLO~Ym%fDZY0r$sB2=auK1P_{q^d(vrteBcV_3@ zmvio&{rc=`qM2f)deu^%IvQyVw$w@sy@8zLop4LJd%DBFz&qhS<9?)l6n}RTsSDkgn6P^+b-sPjvr&2}7oFx^gUB;?>9k62X0^R|YeZH)h0TyE= z_CBkK=R=RKzZ{ikjyFYB=%%r8yQ$9U6bp0$ccdGqdd%L}lvy?CByoB@?V-Lx-MowZ z-=j_1USUmcLT*UkYxLOMy`NsQNf_4#HFIKzdSM?s7Q<~)nT}_$AysNjv<`C_JRx>P o=k~T!<`;t`bYQP}=6cEg4~sUSIi11`m4egFUf literal 0 HcmV?d00001 diff --git a/appveyor/x64/656d8766-2429-11d7-8bf6-00b0d03daa06.reg b/appveyor/x64/656d8766-2429-11d7-8bf6-00b0d03daa06.reg new file mode 100755 index 0000000000000000000000000000000000000000..d945da4f4b35e5eaa2d913e081528a7e45b7f3f9 GIT binary patch literal 616 zcmZXSZA$_{6ot>{p#QM=sXiE%wrLO~Ym%fH8cDPv>Y7;VuK1P_{q^d(vk^fIGqZE< z%Q^Sn{rc=_szi}ey=o~>9gQ^sTWY0+-asy}PI;xgJ>6knV4d<#cpvLP1=g;1+w$c%|@;@od30f-{9R)e}lW zR0PCqH3uJqFTpBKNrt{WS7n(Ca+*nv*r4t3*HPdEpf(W^R@#7?iLJ=B-UyYD>z z*Jz!#DXfW&iS1JN8a>u`@3+>h6UMbc%^cs6UN|R?#c=CXrsElGM3yQYt-)LZkMUhm uxy^P)|00l>3T&F^KefbvBC2>~e?vBZ)7FM}<#Z+J_29FtHj^xm<@yE%{bZ*A literal 0 HcmV?d00001 diff --git a/appveyor/x64/VC_OBJECTS_PLATFORM_INFO.reg b/appveyor/x64/VC_OBJECTS_PLATFORM_INFO.reg new file mode 100755 index 0000000000000000000000000000000000000000..b8282bb9372a468b70d423bbcc70d78277f37093 GIT binary patch literal 3460 zcmeH}Z*LMo5XPU^P5cgoS87tzN?UD=iRIdAwbX_hi4j9M4x3YYM>tw(j9*>-%`Q;5 zqnh>|3E3>SJG1j=pPAX=&+mp}C5n{lhccB^S6AoIG7UA*HIxImF}sv~Ngv@4;Ku9| z_S<@*4aNhlx%pXd^bVO%YUx<-6=;udOEta$yd#~!ZR!l(p+2Yqz3wOt=fjS+RMAsT zrP%FqvWBe=HdCyo`idt_yaK}%HkU`BW8u)He&@qYU+XCb-$kP}}tDK3(vWvxIa<&7vuNU&`5m`N1ccv?kv)5|25`Di#TS}z9jpcl8 zcx^qT!YMt&YF=lEcWq|OZ?eG8`)RIGVf)}vymw3{^~N4qt@q|U3c%VBHT0^i4ZmoC zmJn};bCULg*-LTBT{^7)9|1czi^eIE-5IL~x?-PTr+AB^eJ?$U(p_ckl^fpMb`|Lk zb>DXRQS>$=2L^L@J-OxdEFKMRAM$=${$3FYL)C7XlB?Hx?XWSPMZ&toBD*)Swmy@O zwzpta)+f#>S}&Z|JmRxnSzqy$P_*YW|0V~rxjP#^3G*Bqk?VMO%@@nC(O9RwfXmwT z@~__!5j1RKs^>z*zG0mpVr - rather than """ filename = "jinja2-2.8-py{}{}_0.tar.bz2".format(sys.version_info.major, sys.version_info.minor) downloaded_file = os.path.join(sys.prefix, 'conda-bld', subdir, filename) - download('https://anaconda.org/conda-forge/jinja2/2.8/download/{}/{}'.format(subdir, filename), - downloaded_file) + if not os.path.isfile(downloaded_file): + download('https://anaconda.org/conda-forge/jinja2/2.8/download/{}/{}'.format(subdir, filename), # noqa + downloaded_file) subprocess.check_call(["conda", "build", "--test", downloaded_file]) From 042c392575e8eb32a8243ec8c669877c261a0252 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Sat, 28 May 2016 17:40:32 -0500 Subject: [PATCH 30/45] add new x86 (and x86_64) selectors for Intel platforms --- conda_build/metadata.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index e2b7985f58..b504f5926b 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -50,6 +50,8 @@ def ns_cfg(): win=plat.startswith('win-'), win32=bool(plat == 'win-32'), win64=bool(plat == 'win-64'), + x86=plat.endswith(('-32', '-64')), + x86_64=plat.endswith('-64'), pl=pl, py=py, lua=lua, From 168ec585c795f7d3cad63141064d60d0e471d56d Mon Sep 17 00:00:00 2001 From: whitequark Date: Sun, 29 May 2016 23:06:31 +0000 Subject: [PATCH 31/45] Add forgotten encoding argument to open() --- conda_build/build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda_build/build.py b/conda_build/build.py index 5618a605d9..b5a7a67abb 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -244,7 +244,7 @@ def create_info_files(m, files, include_recipe=True): # make sure we use '/' path separators in metadata files = [_f.replace('\\', '/') for _f in files] - with open(join(config.info_dir, 'files'), 'w') as fo: + with open(join(config.info_dir, 'files'), **mode_dict) as fo: if m.get_value('build/noarch_python'): fo.write('\n') else: From 01035c11f0fd17033fc176a99f3aaeff4f49dd3f Mon Sep 17 00:00:00 2001 From: Mike Sarahan Date: Tue, 31 May 2016 09:40:24 -0500 Subject: [PATCH 32/45] Add test strategy docstrings. --- tests/test_build.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_build.py b/tests/test_build.py index c364853b70..40cdc9140d 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -1,3 +1,7 @@ +""" +This file tests prefix finding for Windows and *nix. +""" + import os import sys @@ -17,6 +21,10 @@ def _write_prefix(filename, prefix, replacement): def test_find_prefix_files(): + """ + Write test output that has the prefix to be found, then verify that the prefix finding + identified the correct number of files. + """ # create a temporary folder prefix = os.path.join(sys.prefix, "envs", "_build") if not os.path.isdir(prefix): From 773b3713a113f3e856f6efcd8dadf47dd037f252 Mon Sep 17 00:00:00 2001 From: Benedikt Reinartz Date: Wed, 1 Jun 2016 11:40:43 +0200 Subject: [PATCH 33/45] Always use the RequestsTransport. This ensures that proxies from the environment are used even if they are not defined in the `.condarc`. --- conda_build/pypi.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/conda_build/pypi.py b/conda_build/pypi.py index e19fceecd6..d62016c04a 100644 --- a/conda_build/pypi.py +++ b/conda_build/pypi.py @@ -22,7 +22,6 @@ from conda.cli.common import spec_from_line from conda.compat import input, configparser, StringIO, string_types, PY3 -from conda.config import get_proxy_servers from conda.connection import CondaSession from conda.fetch import (download, handle_proxy_407) from conda.install import rm_rf @@ -252,13 +251,7 @@ def _build_url(self, host, handler): def get_xmlrpc_client(pypi_url): - proxies = get_proxy_servers() - - if proxies: - transport = RequestsTransport() - else: - transport = None - return ServerProxy(pypi_url, transport=transport) + return ServerProxy(pypi_url, transport=RequestsTransport()) def main(args, parser): From 927eb075d80a04a1524e4138990b44a4d78cbf77 Mon Sep 17 00:00:00 2001 From: Phil Elson Date: Mon, 26 Oct 2015 21:03:39 +0000 Subject: [PATCH 34/45] Lightened up the requirement on recipes to define numpy x.x as runtime dependencies. --- conda_build/metadata.py | 12 +++++++++--- tests/test_metadata.py | 19 ++++++++++--------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 28b8669005..0586a2257a 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -304,7 +304,7 @@ def check_bad_chrs(s, field): sys.exit("Error: bad character '%s' in %s: %s" % (c, field, s)) -def handle_config_version(ms, ver): +def handle_config_version(ms, ver, dep_type='run'): """ 'ms' is an instance of MatchSpec, and 'ver' is the version from the configuration, e.g. for ms.name == 'python', ver = 26 or None, @@ -321,7 +321,13 @@ def handle_config_version(ms, ver): else: # regular version return ms - if ver is None or (ms.strictness == 1 and ms.name == 'numpy'): + # If we don't have a configured version, or we are dealing with a simple + # numpy runtime dependency; just use "numpy"/the name of the package as + # the specification. In practice this means that a recipe which just + # defines numpy as a runtime dependency will match any version of numpy + # at install time. + if ver is None or (dep_type == 'run' and ms.strictness == 1 and + ms.name == 'numpy'): return MatchSpec(ms.name) ver = text_type(ver) @@ -467,7 +473,7 @@ def ms_depends(self, typ='run'): if ms.name == name: if self.get_value('build/noarch_python'): continue - ms = handle_config_version(ms, ver) + ms = handle_config_version(ms, ver, typ) for c in '=!@#$%^&*:;"\'\\|<>?/': if c in ms.name: diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 8be0b0ff01..42d9844aac 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -68,17 +68,18 @@ def test_python(self): MatchSpec('python x.x'), None) def test_numpy(self): - for spec, ver, res_spec in [ - ('numpy', None, 'numpy'), - ('numpy', 18, 'numpy'), - ('numpy', 110, 'numpy'), - ('numpy x.x', 17, 'numpy 1.7*'), - ('numpy x.x', 110, 'numpy 1.10*'), - ('numpy 1.9.1', 18, 'numpy 1.9.1'), - ('numpy 1.9.0 py27_2', None, 'numpy 1.9.0 py27_2'), + for spec, ver, res_spec, kwargs in [ + ('numpy', None, 'numpy', {}), + ('numpy', 18, 'numpy 1.8*', {'dep_type': 'build'}), + ('numpy', 18, 'numpy', {'dep_type': 'run'}), + ('numpy', 110, 'numpy', {}), + ('numpy x.x', 17, 'numpy 1.7*', {}), + ('numpy x.x', 110, 'numpy 1.10*', {}), + ('numpy 1.9.1', 18, 'numpy 1.9.1', {}), + ('numpy 1.9.0 py27_2', None, 'numpy 1.9.0 py27_2', {}), ]: ms = MatchSpec(spec) - self.assertEqual(handle_config_version(ms, ver), + self.assertEqual(handle_config_version(ms, ver, **kwargs), MatchSpec(res_spec)) self.assertRaises(RuntimeError, From f0dc8005617d57a573d4ef630697e42c5284b748 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Tue, 31 May 2016 14:20:52 -0500 Subject: [PATCH 35/45] search pkg subset with pypi rather than list all --- conda_build/pypi.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conda_build/pypi.py b/conda_build/pypi.py index e19fceecd6..10519e87c3 100644 --- a/conda_build/pypi.py +++ b/conda_build/pypi.py @@ -266,7 +266,8 @@ def main(args, parser): package_dicts = {} [output_dir] = args.output_dir - all_packages = client.list_packages() + # searching is faster than listing all packages + all_packages = [match["name"] for match in client.search({"name": args.packages}, "or")] all_packages_lower = [i.lower() for i in all_packages] args.created_recipes = [] From 89c2b23909f379bb737b550a1fc62024cbe81427 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Wed, 1 Jun 2016 09:08:57 -0500 Subject: [PATCH 36/45] fix missing dirty args. Cleanup. --- conda_build/environ.py | 1 + conda_build/main_build.py | 4 ++-- conda_build/metadata.py | 2 +- conda_build/render.py | 3 +-- conda_build/source.py | 20 +++++++++++--------- tests/test_build_recipes.py | 10 ++++++++++ 6 files changed, 26 insertions(+), 14 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 0a4230aa9d..18de042dfb 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -246,6 +246,7 @@ def meta_vars(meta): git_url = normpath(join(meta.path, git_url)) _x = False + if git_url: _x = verify_git_repo(git_dir, git_url, diff --git a/conda_build/main_build.py b/conda_build/main_build.py index 5cb0691199..a9d9ff880d 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -279,7 +279,7 @@ def execute(args, parser): # this fully renders any jinja templating, throwing an error if any data is missing m, need_source_download = render_recipe(recipe_dir, no_download_source=False, - verbose=False) + verbose=False, dirty=args.dirty) if m.get_value('build/noarch_python'): config.noarch = True @@ -305,7 +305,7 @@ def execute(args, parser): elif args.test: build.test(m, move_broken=False) elif args.source: - source.provide(m.path, m.get_section('source'), verbose=build.verbose) + source.provide(m.path, m.get_section('source'), verbose=build.verbose, dirty=args.dirty) print('Source tree in:', source.get_dir()) else: # This loop recursively builds dependencies if recipes exist diff --git a/conda_build/metadata.py b/conda_build/metadata.py index 28b8669005..bef2cc2696 100644 --- a/conda_build/metadata.py +++ b/conda_build/metadata.py @@ -14,7 +14,6 @@ from conda_build import exceptions from conda_build.features import feature_list - try: import yaml @@ -643,6 +642,7 @@ def _get_contents(self, permit_undefined_jinja): loader = FilteredLoader(jinja2.ChoiceLoader(loaders)) env = jinja2.Environment(loader=loader, undefined=undefined_type) + env.globals.update(ns_cfg()) env.globals.update(context_processor(self, path)) diff --git a/conda_build/render.py b/conda_build/render.py index 4d90669198..17cb3afdbd 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -76,6 +76,7 @@ def bldpkg_path(m): def parse_or_try_download(metadata, no_download_source, verbose, force_download=False, dirty=False): + if (force_download or (not no_download_source and any(var.startswith('GIT_') for var in metadata.undefined_jinja_vars))): # this try/catch is for when the tool to download source is actually in @@ -89,8 +90,6 @@ def parse_or_try_download(metadata, no_download_source, verbose, print("Warning: failed to download source. If building, will try " "again after downloading recipe dependencies.") need_source_download = True - else: - need_source_download = no_download_source else: # we have not downloaded source in the render phase. Download it in # the build phase diff --git a/conda_build/source.py b/conda_build/source.py index d585020be7..789191110b 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -75,7 +75,8 @@ def unpack(meta, verbose=False): ''' Uncompress a downloaded source. ''' src_path = download_to_cache(meta) - os.makedirs(WORK_DIR) + if not isdir(WORK_DIR): + os.makedirs(WORK_DIR) if verbose: print("Extracting download") if src_path.lower().endswith(('.tar.gz', '.tar.bz2', '.tgz', '.tar.xz', @@ -141,7 +142,7 @@ def git_source(meta, recipe_dir, verbose=False): if git_depth > 0: args += ['--depth', str(git_depth)] - check_call(args + [git_url, cache_repo_arg], cwd=recipe_dir, stdout=stdout, stderr=stderr) + check_call(args + [git_url, cache_repo_arg], stdout=stdout, stderr=stderr) assert isdir(cache_repo) # now clone into the work directory @@ -289,15 +290,15 @@ def get_repository_info(recipe_path): """This tries to get information about where a recipe came from. This is different from the source - you can have a recipe in svn that gets source via git.""" if isdir(join(recipe_path, ".git")): - origin = check_output(["git", "config", "--get", "remote.origin.url"]) - rev = check_output(["git", "rev-parse", "HEAD"]) + origin = check_output(["git", "config", "--get", "remote.origin.url"], cwd=recipe_path) + rev = check_output(["git", "rev-parse", "HEAD"], cwd=recipe_path) return "Origin {}, commit {}".format(origin, rev) elif isdir(join(recipe_path, ".hg")): - origin = check_output(["hg", "paths", "default"]) - rev = check_output(["hg", "id"]).split()[0] + origin = check_output(["hg", "paths", "default"], cwd=recipe_path) + rev = check_output(["hg", "id"], cwd=recipe_path).split()[0] return "Origin {}, commit {}".format(origin, rev) elif isdir(join(recipe_path, ".svn")): - info = check_output(["svn", "info"]) + info = check_output(["svn", "info"], cwd=recipe_path) server = re.search("Repository Root: (.*)$", info, flags=re.M).group(1) revision = re.search("Revision: (.*)$", info, flags=re.M).group(1) return "{}, Revision {}".format(server, revision) @@ -361,7 +362,7 @@ def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False): else: rm_rf(WORK_DIR) - if not os.path.exists(WORK_DIR): + if not dirty or not os.path.isdir(WORK_DIR): if any(k in meta for k in ('fn', 'url')): unpack(meta, verbose=verbose) elif 'git_url' in meta: @@ -377,7 +378,8 @@ def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False): print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) else: # no source - os.makedirs(WORK_DIR) + if not isdir(WORK_DIR): + os.makedirs(WORK_DIR) if patch: src_dir = get_dir() diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index ec459105ee..e3092b9313 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -22,6 +22,16 @@ def is_valid_dir(parent_dir, dirname): return valid +def test_CONDA_BLD_PATH(): + env = dict(os.environ) + cmd = 'conda build --no-anaconda-upload {}/source_git_jinja2'.format(metadata_dir) + with TemporaryDirectory() as tmp: + env["CONDA_BLD_PATH"] = tmp + subprocess.check_call(cmd.split(), env=env) + # trick is actually a second pass, to make sure that deletion/trash moving is working OK. + subprocess.check_call(cmd.split(), env=env) + + # TODO: this does not currently take into account post-build versioning changes with __conda_? files def test_output_build_path_git_source(): cmd = 'conda build --output {}'.format(os.path.join(metadata_dir, "source_git_jinja2")) From 94395a59b89bd47002404b55fab76e0f38723ec5 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 Jun 2016 14:35:05 -0500 Subject: [PATCH 37/45] add test for finding headers with pyyaml/yaml --- tests/test_build_recipes.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index ec459105ee..49e381053b 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -22,6 +22,29 @@ def is_valid_dir(parent_dir, dirname): return valid +@pytest.mark.skipif(sys.platform != "win32", + reason="Problem only observed on Windows with win7 sdk") +def test_header_finding(): + """ + Windows sometimes very strangely cannot find headers in %LIBRARY_INC%. This has so far + only been a problem with the recipes that use the Win 7 SDK (python 3.4 builds) + """ + cmd = 'conda build --no-anaconda-upload {}/_pyyaml_find_header'.format(metadata_dir) + output = subprocess.check_output(cmd.split()) + print(output) + assert "forcing --without-libyaml" not in output + + +def test_CONDA_BLD_PATH(): + env = dict(os.environ) + cmd = 'conda build --no-anaconda-upload {}/source_git_jinja2'.format(metadata_dir) + with TemporaryDirectory() as tmp: + env["CONDA_BLD_PATH"] = tmp + subprocess.check_call(cmd.split(), env=env) + # trick is actually a second pass, to make sure that deletion/trash moving is working OK. + subprocess.check_call(cmd.split(), env=env) + + # TODO: this does not currently take into account post-build versioning changes with __conda_? files def test_output_build_path_git_source(): cmd = 'conda build --output {}'.format(os.path.join(metadata_dir, "source_git_jinja2")) From 36f88a1be6af6fe53628b6c25b0fa51f80d439a2 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 Jun 2016 14:49:17 -0500 Subject: [PATCH 38/45] add py34 back into appveyor config --- appveyor.yml | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 087f7b1a13..3dd5d80691 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -14,6 +14,10 @@ environment: PYTHON_VERSION: "2.7" PYTHON_ARCH: "64" + - PYTHON: "C:\\Python34_64" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "64" + - PYTHON: "C:\\Python27_32" PYTHON_VERSION: "2.7" PYTHON_ARCH: "32" @@ -22,10 +26,14 @@ environment: PYTHON_VERSION: "3.5" PYTHON_ARCH: "32" + - PYTHON: "C:\\Python34_32" + PYTHON_VERSION: "3.4" + PYTHON_ARCH: "32" init: - ECHO %PYTHON% %PYTHON_VERSION% %PYTHON_ARCH% %HOME% + install: # If there is a newer build queued for the same PR, cancel this one. # The AppVeyor 'rollout builds' option is supposed to serve the same @@ -36,25 +44,24 @@ install: https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` throw "There are newer queued builds for this pull request, failing early." } - # reverse python versions for base, to avoid file permissions errors when base == test version + # these correspond to folder naming of miniconda installs on appveyor. See + # https://www.appveyor.com/docs/installed-software#python - if "%PYTHON_VERSION%" == "3.4" set "BASE_PYTHON_VERSION=3" - if "%PYTHON_VERSION%" == "3.5" set "BASE_PYTHON_VERSION=35" - if "%PYTHON_ARCH%" == "64" set "ARCH_LABEL=-x64" + # These are already installed on appveyor. Update them. - set "CONDA_ROOT=C:\Miniconda%BASE_PYTHON_VERSION%%ARCH_LABEL%" - - for /f "usebackq tokens=2,*" %%A in (`reg query HKCU\Environment /v PATH`) do set "USERPATH=%%B" - - set "USERPATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%USERPATH%" - - set "PATH=%USERPATH%;%PATH%" - - setx PATH "%USERPATH%" - - conda info + - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - conda config --set always_yes yes + - conda update -q conda + - conda info - conda update -q --all - python -c "import sys; print(sys.version)" - python -c "import sys; print(sys.executable)" - python -c "import sys; print(sys.prefix)" - - conda install -q pytest requests pycrypto git anaconda-client + - conda install -q pytest pytest-cov git anaconda-client # this is to ensure dependencies - conda install -q conda-build - - pip install auxlib pytest-cov - python --version - python -c "import struct; print(struct.calcsize('P') * 8)" - python setup.py install @@ -70,7 +77,7 @@ build: false test_script: - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - set PATH - - py.test --cov conda_build --cov-report xml tests || appveyor PushArtifact %PYTHON%/conda-bld/work/CMakeFiles/CMakeOutput.log && appveyor PushArtifact %PYTHON%/conda-bld/work/CMakeFiles/CMakeError.log + - py.test --cov conda_build --cov-report xml tests on_success: - pip install codecov From eeef794ca8af2f88f5ac50be3200ce6097e9dbcf Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Fri, 3 Jun 2016 15:47:45 -0500 Subject: [PATCH 39/45] add forgotten find header file --- .../metadata/_pyyaml_find_header/meta.yaml | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml diff --git a/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml b/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml new file mode 100644 index 0000000000..9409ef48b2 --- /dev/null +++ b/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml @@ -0,0 +1,25 @@ +# recipe from conda forge, downloaded 2016/06/03 +# problem report was https://github.com/conda/conda-build/issues/993 + +{% set version = "3.11" %} + + +package: + name: pyyaml + version: {{ version }} + +source: + fn: PyYAML-{{ version }}.tar.gz + url: https://pypi.io/packages/source/P/PyYAML/PyYAML-{{ version }}.tar.gz + md5: f50e08ef0fe55178479d3a618efe21db + +build: + number: 0 + script: + - python setup.py --with-libyaml build_ext --include-dirs="%LIBRARY_INC% --library-dirs=%LIBRARY_LIB% # [win] + - if errorlevel 1 exit 1 # [win] + +requirements: + build: + - python + - yaml From 2457c8eaa4d7743d441968404ba9441b74c7a510 Mon Sep 17 00:00:00 2001 From: Ilan Schnell Date: Fri, 3 Jun 2016 16:28:41 -0500 Subject: [PATCH 40/45] Revert "simplify file writing" This reverts commit 1f2c4d920348d499133b892b7189c56204d96624. --- conda_build/build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index edb72c6ff4..fe5b407533 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -20,6 +20,7 @@ import conda.config as cc import conda.plan as plan from conda.api import get_index +from conda.compat import PY3 from conda.fetch import fetch_index from conda.install import prefix_placeholder, linked, move_to_trash from conda.lock import Locked @@ -237,7 +238,9 @@ def create_info_files(m, files, include_recipe=True): info_index['depends'] = [' '.join(dist.rsplit('-', 2)) for dist in dists] - with open(join(config.info_dir, 'index.json'), 'w') as fo: + # Deal with Python 2 and 3's different json module type reqs + mode_dict = {'mode': 'w', 'encoding': 'utf-8'} if PY3 else {'mode': 'wb'} + with open(join(config.info_dir, 'index.json'), **mode_dict) as fo: json.dump(info_index, fo, indent=2, sort_keys=True) with open(join(config.info_dir, 'about.json'), 'w') as fo: @@ -250,7 +253,7 @@ def create_info_files(m, files, include_recipe=True): json.dump(d, fo, indent=2, sort_keys=True) if include_recipe: - with open(join(config.info_dir, 'recipe.json'), 'w') as fo: + with open(join(config.info_dir, 'recipe.json'), **mode_dict) as fo: json.dump(m.meta, fo, indent=2, sort_keys=True) if sys.platform == 'win32': From e86b89367dcdf38977f47e312668e110200cc0a6 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 4 Jun 2016 17:31:13 -0500 Subject: [PATCH 41/45] set empty build section if none exists --- conda_build/post.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/conda_build/post.py b/conda_build/post.py index 03de7d72ec..2e579e623c 100644 --- a/conda_build/post.py +++ b/conda_build/post.py @@ -375,6 +375,8 @@ def check_symlinks(files): def get_build_metadata(m): src_dir = source.get_dir() + if "build" not in m.meta: + m.meta["build"] = {} if exists(join(src_dir, '__conda_version__.txt')): print("Deprecation warning: support for __conda_version__ will be removed in Conda build 2.0." # noqa "Try Jinja templates instead: " From f5cee2602dc098d82b99bb62f94f479b34e79ea9 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sat, 4 Jun 2016 21:42:44 -0500 Subject: [PATCH 42/45] fix errant tests - add has_vcs_metadata function for when download is needed - pin libpng to make tests pass --- conda_build/render.py | 16 +++++++++++++--- .../metadata/always_include_files_glob/meta.yaml | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/conda_build/render.py b/conda_build/render.py index 17cb3afdbd..2e81a140d4 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -6,13 +6,14 @@ from __future__ import absolute_import, division, print_function +from locale import getpreferredencoding import shutil import sys import tarfile import tempfile import os from os.path import isdir, isfile, abspath -from locale import getpreferredencoding +import re import subprocess import yaml @@ -74,11 +75,20 @@ def bldpkg_path(m): return os.path.join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist()) +def has_vcs_metadata(metadata): + """returns true if recie contains metadata associated with version control systems. + If this metadata is present, a download/copy will be forced in parse_or_try_download. + """ + with open(metadata.meta_path) as f: + matches = re.findall(r"GIT_[^\.\s\'\"]+", f.read()) + # TODO: extend with other VCS systems (SVN, hg, anything else?) + return len(matches) > 0 + + def parse_or_try_download(metadata, no_download_source, verbose, force_download=False, dirty=False): - if (force_download or (not no_download_source and - any(var.startswith('GIT_') for var in metadata.undefined_jinja_vars))): + if (force_download or (not no_download_source and has_vcs_metadata(metadata))): # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. try: diff --git a/tests/test-recipes/metadata/always_include_files_glob/meta.yaml b/tests/test-recipes/metadata/always_include_files_glob/meta.yaml index df202a080e..4dd418d64d 100644 --- a/tests/test-recipes/metadata/always_include_files_glob/meta.yaml +++ b/tests/test-recipes/metadata/always_include_files_glob/meta.yaml @@ -11,4 +11,4 @@ build: requirements: build: - - libpng + - libpng 1.6.17 From 3f957c90dc3ed1b2cd3ab0fcd64e9b13292e5d85 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 5 Jun 2016 10:50:44 -0500 Subject: [PATCH 43/45] fix git path issue with cygpath --- conda_build/environ.py | 14 ++++++++- conda_build/main_build.py | 2 +- conda_build/render.py | 5 ++-- conda_build/source.py | 60 +++++++++++++++++++-------------------- tests/test_render.py | 2 +- 5 files changed, 47 insertions(+), 36 deletions(-) diff --git a/conda_build/environ.py b/conda_build/environ.py index 18de042dfb..4d57da033f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function +import logging import multiprocessing import os import sys @@ -88,6 +89,12 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): stderr=STDOUT) remote_details = remote_details.decode('utf-8') remote_url = remote_details.split('\n')[0].split()[1] + + # on windows, remote URL comes back to us as cygwin or msys format. Python doesn't + # know how to normalize it. Need to convert it to a windows path. + if sys.platform == 'win32' and remote_url.startswith('/'): + remote_url = check_output(["cygpath", '-w', remote_url]).rstrip().rstrip("\\") + if os.path.exists(remote_url): # Local filepaths are allowed, but make sure we normalize them remote_url = normpath(remote_url) @@ -95,8 +102,13 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): # If the current source directory in conda-bld/work doesn't match the user's # metadata git_url or git_rev, then we aren't looking at the right source. if remote_url.lower() != git_url.lower(): + logging.debug("\nremote does not match git_url\n") + logging.debug("Remote: " + remote_url.lower() + "\n") + logging.debug("git_url: " + git_url.lower()+ "\n") return False - except CalledProcessError: + except CalledProcessError as error: + logging.warn("Error obtaining git information. Error was: ") + logging.warn(error) return False return True diff --git a/conda_build/main_build.py b/conda_build/main_build.py index a9d9ff880d..88f2793aa5 100644 --- a/conda_build/main_build.py +++ b/conda_build/main_build.py @@ -305,7 +305,7 @@ def execute(args, parser): elif args.test: build.test(m, move_broken=False) elif args.source: - source.provide(m.path, m.get_section('source'), verbose=build.verbose, dirty=args.dirty) + source.provide(m.path, m.get_section('source'), verbose=build.verbose) print('Source tree in:', source.get_dir()) else: # This loop recursively builds dependencies if recipes exist diff --git a/conda_build/render.py b/conda_build/render.py index 2e81a140d4..4bcb05da6f 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -92,8 +92,9 @@ def parse_or_try_download(metadata, no_download_source, verbose, # this try/catch is for when the tool to download source is actually in # meta.yaml, and not previously installed in builder env. try: - source.provide(metadata.path, metadata.get_section('source'), - verbose=verbose, dirty=dirty) + if not dirty: + source.provide(metadata.path, metadata.get_section('source'), + verbose=verbose) metadata.parse_again(permit_undefined_jinja=False) need_source_download = False except subprocess.CalledProcessError: diff --git a/conda_build/source.py b/conda_build/source.py index 789191110b..0d6e2f8f5c 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -347,7 +347,7 @@ def apply_patch(src_dir, path): os.remove(patch_args[-1]) # clean up .patch_unix file -def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False): +def provide(recipe_dir, meta, verbose=False, patch=True): """ given a recipe_dir: - download (if necessary) @@ -355,36 +355,34 @@ def provide(recipe_dir, meta, verbose=False, patch=True, dirty=False): - apply patches (if any) """ - if not dirty: - if sys.platform == 'win32': - if isdir(WORK_DIR): - move_to_trash(WORK_DIR, '') - else: - rm_rf(WORK_DIR) - - if not dirty or not os.path.isdir(WORK_DIR): - if any(k in meta for k in ('fn', 'url')): - unpack(meta, verbose=verbose) - elif 'git_url' in meta: - git_source(meta, recipe_dir, verbose=verbose) - # build to make sure we have a work directory with source in it. We want to make sure that - # whatever version that is does not interfere with the test we run next. - elif 'hg_url' in meta: - hg_source(meta, verbose=verbose) - elif 'svn_url' in meta: - svn_source(meta, verbose=verbose) - elif 'path' in meta: - if verbose: - print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) - copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) - else: # no source - if not isdir(WORK_DIR): - os.makedirs(WORK_DIR) - - if patch: - src_dir = get_dir() - for patch in meta.get('patches', []): - apply_patch(src_dir, join(recipe_dir, patch)) + if sys.platform == 'win32': + if isdir(WORK_DIR): + move_to_trash(WORK_DIR, '') + else: + rm_rf(WORK_DIR) + + if any(k in meta for k in ('fn', 'url')): + unpack(meta, verbose=verbose) + elif 'git_url' in meta: + git_source(meta, recipe_dir, verbose=verbose) + # build to make sure we have a work directory with source in it. We want to make sure that + # whatever version that is does not interfere with the test we run next. + elif 'hg_url' in meta: + hg_source(meta, verbose=verbose) + elif 'svn_url' in meta: + svn_source(meta, verbose=verbose) + elif 'path' in meta: + if verbose: + print("Copying %s to %s" % (abspath(join(recipe_dir, meta.get('path'))), WORK_DIR)) + copytree(abspath(join(recipe_dir, meta.get('path'))), WORK_DIR) + else: # no source + if not isdir(WORK_DIR): + os.makedirs(WORK_DIR) + + if patch: + src_dir = get_dir() + for patch in meta.get('patches', []): + apply_patch(src_dir, join(recipe_dir, patch)) if __name__ == '__main__': diff --git a/tests/test_render.py b/tests/test_render.py index e58bba9bc0..bae26e33cd 100644 --- a/tests/test_render.py +++ b/tests/test_render.py @@ -19,4 +19,4 @@ def test_output_build_path(): sys.version_info.major, sys.version_info.minor)) if PY3: output = output.decode("UTF-8") - assert output.rstrip() == test_path + assert output.rstrip() == test_path, error From bf69e2fcdefad452637976a00d54e53a3fd23392 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Sun, 5 Jun 2016 11:19:50 -0500 Subject: [PATCH 44/45] install conda 4.0.8; fix output comparision in test_find_header --- appveyor.yml | 7 ++++++- conda_build/environ.py | 2 +- conda_build/render.py | 4 +++- conda_build/source.py | 18 +++++++----------- .../metadata/_pyyaml_find_header/meta.yaml | 2 +- .../jinja2_build_str_template_only/meta.yaml | 2 +- .../test-recipes/metadata/source_git/meta.yaml | 2 +- .../metadata/source_git_jinja2/meta.yaml | 2 +- tests/test_build_recipes.py | 10 ++++++++-- 9 files changed, 29 insertions(+), 20 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 3dd5d80691..a5ef291591 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -54,6 +54,11 @@ install: - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - conda config --set always_yes yes - conda update -q conda + - git clone https://github.com/conda/conda + - cd conda + - git checkout 4.0.8 + - python setup.py install + - cd ../ - conda info - conda update -q --all - python -c "import sys; print(sys.version)" @@ -77,7 +82,7 @@ build: false test_script: - set "PATH=%CONDA_ROOT%;%CONDA_ROOT%\Scripts;%CONDA_ROOT%\Library\bin;%PATH%" - set PATH - - py.test --cov conda_build --cov-report xml tests + - py.test --cov conda_build --cov-report xml tests on_success: - pip install codecov diff --git a/conda_build/environ.py b/conda_build/environ.py index 4d57da033f..55bbd3e41f 100644 --- a/conda_build/environ.py +++ b/conda_build/environ.py @@ -104,7 +104,7 @@ def verify_git_repo(git_dir, git_url, expected_rev='HEAD'): if remote_url.lower() != git_url.lower(): logging.debug("\nremote does not match git_url\n") logging.debug("Remote: " + remote_url.lower() + "\n") - logging.debug("git_url: " + git_url.lower()+ "\n") + logging.debug("git_url: " + git_url.lower() + "\n") return False except CalledProcessError as error: logging.warn("Error obtaining git information. Error was: ") diff --git a/conda_build/render.py b/conda_build/render.py index 4bcb05da6f..8e767e06df 100644 --- a/conda_build/render.py +++ b/conda_build/render.py @@ -97,9 +97,11 @@ def parse_or_try_download(metadata, no_download_source, verbose, verbose=verbose) metadata.parse_again(permit_undefined_jinja=False) need_source_download = False - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as error: print("Warning: failed to download source. If building, will try " "again after downloading recipe dependencies.") + print("Error was: ") + print(error) need_source_download = True else: # we have not downloaded source in the render phase. Download it in diff --git a/conda_build/source.py b/conda_build/source.py index 0d6e2f8f5c..e6c3bb49a9 100644 --- a/conda_build/source.py +++ b/conda_build/source.py @@ -94,11 +94,9 @@ def git_source(meta, recipe_dir, verbose=False): ''' Download a source from Git repo. ''' if verbose: stdout = None - stderr = None else: FNULL = open(os.devnull, 'w') stdout = FNULL - stderr = FNULL if not isdir(GIT_CACHE): os.makedirs(GIT_CACHE) @@ -125,7 +123,7 @@ def git_source(meta, recipe_dir, verbose=False): # update (or create) the cache repo if isdir(cache_repo): if meta.get('git_rev', 'HEAD') != 'HEAD': - check_call([git, 'fetch'], cwd=cache_repo, stdout=stdout, stderr=stderr) + check_call([git, 'fetch'], cwd=cache_repo, stdout=stdout) else: # Unlike 'git clone', fetch doesn't automatically update the cache's HEAD, # So here we explicitly store the remote HEAD in the cache's local refs/heads, @@ -134,15 +132,15 @@ def git_source(meta, recipe_dir, verbose=False): # but the user is working with a branch other than 'master' without # explicitly providing git_rev. check_call([git, 'fetch', 'origin', '+HEAD:_conda_cache_origin_head'], - cwd=cache_repo, stdout=stdout, stderr=stderr) + cwd=cache_repo, stdout=stdout) check_call([git, 'symbolic-ref', 'HEAD', 'refs/heads/_conda_cache_origin_head'], - cwd=cache_repo, stdout=stdout, stderr=stderr) + cwd=cache_repo, stdout=stdout) else: args = [git, 'clone', '--mirror'] if git_depth > 0: args += ['--depth', str(git_depth)] - check_call(args + [git_url, cache_repo_arg], stdout=stdout, stderr=stderr) + check_call(args + [git_url, cache_repo_arg], stdout=stdout) assert isdir(cache_repo) # now clone into the work directory @@ -151,17 +149,15 @@ def git_source(meta, recipe_dir, verbose=False): # assume the user wants the current HEAD if not checkout and git_url.startswith('.'): process = Popen(["git", "rev-parse", "HEAD"], - stdout=PIPE, stderr=PIPE, - cwd=git_url) + stdout=PIPE, cwd=git_url) output = process.communicate()[0].strip() checkout = output.decode('utf-8') if checkout and verbose: print('checkout: %r' % checkout) - check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR], - stdout=stdout, stderr=stderr) + check_call([git, 'clone', '--recursive', cache_repo_arg, WORK_DIR], stdout=stdout) if checkout: - check_call([git, 'checkout', checkout], cwd=WORK_DIR, stdout=stdout, stderr=stderr) + check_call([git, 'checkout', checkout], cwd=WORK_DIR, stdout=stdout) git_info(verbose=verbose) diff --git a/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml b/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml index 9409ef48b2..d9c04895c6 100644 --- a/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml +++ b/tests/test-recipes/metadata/_pyyaml_find_header/meta.yaml @@ -16,7 +16,7 @@ source: build: number: 0 script: - - python setup.py --with-libyaml build_ext --include-dirs="%LIBRARY_INC% --library-dirs=%LIBRARY_LIB% # [win] + - python setup.py --with-libyaml build_ext --include-dirs=%LIBRARY_INC% --library-dirs=%LIBRARY_LIB% # [win] - if errorlevel 1 exit 1 # [win] requirements: diff --git a/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml b/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml index a790a6d407..032b759b2f 100644 --- a/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml +++ b/tests/test-recipes/metadata/jinja2_build_str_template_only/meta.yaml @@ -3,7 +3,7 @@ package: version: 1.0 source: - git_url: https://github.com/conda/conda-build + git_url: ../../../../ git_tag: 1.8.1 build: diff --git a/tests/test-recipes/metadata/source_git/meta.yaml b/tests/test-recipes/metadata/source_git/meta.yaml index 7b67126930..3bd01178bb 100644 --- a/tests/test-recipes/metadata/source_git/meta.yaml +++ b/tests/test-recipes/metadata/source_git/meta.yaml @@ -3,7 +3,7 @@ package: version: 1.0 source: - git_url: https://github.com/conda/conda-build + git_url: ../../../../ git_tag: 1.8.1 requirements: diff --git a/tests/test-recipes/metadata/source_git_jinja2/meta.yaml b/tests/test-recipes/metadata/source_git_jinja2/meta.yaml index 24b27305f4..9a92b2cf29 100644 --- a/tests/test-recipes/metadata/source_git_jinja2/meta.yaml +++ b/tests/test-recipes/metadata/source_git_jinja2/meta.yaml @@ -5,7 +5,7 @@ package: version: {{ environ.get("GIT_DESCRIBE_TAG", "WRONG_ANSWER").replace('v', '') }} source: - git_url: https://github.com/conda/conda-build + git_url: ../../../../ git_tag: 1.8.1 build: diff --git a/tests/test_build_recipes.py b/tests/test_build_recipes.py index c5660593cd..e98024b435 100644 --- a/tests/test_build_recipes.py +++ b/tests/test_build_recipes.py @@ -30,8 +30,14 @@ def test_header_finding(): only been a problem with the recipes that use the Win 7 SDK (python 3.4 builds) """ cmd = 'conda build --no-anaconda-upload {}/_pyyaml_find_header'.format(metadata_dir) - output = subprocess.check_output(cmd.split()) - print(output) + try: + output = subprocess.check_output(cmd.split()) + except subprocess.CalledProcessError as error: + print(error.output) + print(os.listdir(os.path.join(sys.prefix, "envs", "_build", "Library", "include"))) + raise + if PY3: + output = output.decode("UTF-8") assert "forcing --without-libyaml" not in output From 461c389ea105cda0c8284b091791dcab58787163 Mon Sep 17 00:00:00 2001 From: Michael Sarahan Date: Mon, 6 Jun 2016 14:15:24 -0500 Subject: [PATCH 45/45] keep meta.yaml; move meta.yaml.rendered into recipe folder --- conda_build/build.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/conda_build/build.py b/conda_build/build.py index fe5b407533..590f69dd75 100644 --- a/conda_build/build.py +++ b/conda_build/build.py @@ -192,17 +192,16 @@ def create_info_files(m, files, include_recipe=True): shutil.copytree(src_path, dst_path) else: shutil.copy(src_path, dst_path) - os.rename(join(recipe_dir, "meta.yaml"), join(recipe_dir, "meta.yaml.template")) - - # store the rendered meta.yaml file, plus information about where it came from - # and what version of conda-build created it - metayaml = output_yaml(m) - with open(join(config.info_dir, "meta.yaml"), 'w') as f: - f.write("# This file created by conda-build {}\n".format(__version__)) - f.write("# meta.yaml template originally from:\n") - f.write("# " + source.get_repository_info(m.path) + "\n") - f.write("# ------------------------------------------------\n\n") - f.write(metayaml) + + # store the rendered meta.yaml file, plus information about where it came from + # and what version of conda-build created it + metayaml = output_yaml(m) + with open(join(recipe_dir, "meta.yaml.rendered"), 'w') as f: + f.write("# This file created by conda-build {}\n".format(__version__)) + f.write("# meta.yaml template originally from:\n") + f.write("# " + source.get_repository_info(m.path) + "\n") + f.write("# ------------------------------------------------\n\n") + f.write(metayaml) license_file = m.get_value('about/license_file') if license_file: