Merge pull request #950 from msarahan/output_junk_fixes

fix extraneous output when outputting build path

bdist_conda.py

@@ -16,7 +16,7 @@
 import conda.config
 from conda.cli.common import spec_from_line
 from conda_build.metadata import MetaData
-from conda_build import build, pypi
+from conda_build import build, pypi, render
 from conda_build.config import config
 from conda_build.main_build import handle_binstar_upload
 
@@ -267,13 +267,13 @@ def run(self):
  if self.binstar_upload:
  class args:
  binstar_upload = self.binstar_upload
- handle_binstar_upload(build.bldpkg_path(m), args)
+ handle_binstar_upload(render.bldpkg_path(m), args)
  else:
  no_upload_message = """\
 # If you want to upload this package to anaconda.org later, type:
 #
 # $ anaconda upload %s
-""" % build.bldpkg_path(m)
+""" % render.bldpkg_path(m)
  print(no_upload_message)
 
 

conda_build/build.py

@@ -27,9 +27,10 @@
 from conda.utils import url_path
 from conda.resolve import Resolve, MatchSpec, NoPackagesFound
 
+from conda_build import __version__
 from conda_build import environ, source, tarcheck
 from conda_build.config import config
-from conda_build.render import parse_or_try_download
+from conda_build.render import parse_or_try_download, output_yaml, bldpkg_path
 from conda_build.scripts import create_entry_points, prepend_bin_path
 from conda_build.post import (post_process, post_build,
  fix_permissions, get_build_metadata)
@@ -183,6 +184,17 @@ def create_info_files(m, files, include_recipe=True):
  shutil.copytree(src_path, dst_path)
  else:
  shutil.copy(src_path, dst_path)
+ os.rename(join(recipe_dir, "meta.yaml"), join(recipe_dir, "meta.yaml.template"))
+
+ # store the rendered meta.yaml file, plus information about where it came from
+ # and what version of conda-build created it
+ metayaml = output_yaml(m)
+ with open(join(config.info_dir, "meta.yaml"), 'w') as f:
+ f.write("# This file created by conda-build {}\n".format(__version__))
+ f.write("# meta.yaml template originally from:\n")
+ f.write("# " + source.get_repository_info(m.path) + "\n")
+ f.write("# ------------------------------------------------\n\n")
+ f.write(metayaml)
 
  license_file = m.get_value('about/license_file')
  if license_file:
@@ -229,7 +241,7 @@ def create_info_files(m, files, include_recipe=True):
 
  if sys.platform == 'win32':
  # make sure we use '/' path separators in metadata
- files = [f.replace('\\', '/') for f in files]
+ files = [_f.replace('\\', '/') for _f in files]
 
  with open(join(config.info_dir, 'files'), 'w') as fo:
  if m.get_value('build/noarch_python'):
@@ -361,14 +373,8 @@ def rm_pkgs_cache(dist):
  plan.execute_plan(rmplan)
 
 
-def bldpkg_path(m):
- '''
- Returns path to built package's tarball given its ``Metadata``.
- '''
- return join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist())
-
-
-def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_download=False):
+def build(m, post=None, include_recipe=True, keep_old_work=False,
+ need_source_download=True, verbose=True):
  '''
  Build the package with the specified metadata.
 
@@ -436,7 +442,10 @@ def build(m, post=None, include_recipe=True, keep_old_work=False, need_source_do
  try:
  os.environ['PATH'] = prepend_bin_path({'PATH': _old_path},
  config.build_prefix)['PATH']
- m, need_source_download = parse_or_try_download(m, no_download_source=False)
+ m, need_source_download = parse_or_try_download(m,
+ no_download_source=False,
+ force_download=True,
+ verbose=verbose)
  assert not need_source_download, "Source download failed. Please investigate."
  finally:
  os.environ['PATH'] = _old_path

conda_build/main_build.py

@@ -70,7 +70,9 @@ def main():
  p.add_argument(
  '-t', "--test",
  action="store_true",
- help="Test package (assumes package is already build).",
+ help="Test package (assumes package is already built). RECIPE_DIR argument can be either "
+ "recipe directory, in which case source download may be necessary to resolve package"
+ "version, or path to built package .tar.bz2 file, in which case no source is necessary.",
  )
  p.add_argument(
  '--no-test',
@@ -270,7 +272,8 @@ def execute(args, parser):
  sys.exit("Error: no such directory: %s" % recipe_dir)
 
  # this fully renders any jinja templating, throwing an error if any data is missing
- m, need_source_download = render_recipe(recipe_dir, no_download_source=False)
+ m, need_source_download = render_recipe(recipe_dir, no_download_source=False,
+ verbose=build.verbose)
  if m.get_value('build/noarch_python'):
  config.noarch = True
 

conda_build/main_render.py

@@ -8,15 +8,11 @@
 
 import sys
 
-import yaml
-
-from conda.compat import PY3
 from conda.cli.common import add_parser_channels
 from conda.cli.conda_argparse import ArgumentParser
 
 from conda_build import __version__
-from conda_build.build import bldpkg_path
-from conda_build.render import render_recipe, set_language_env_vars
+from conda_build.render import render_recipe, set_language_env_vars, bldpkg_path, output_yaml
 from conda_build.utils import find_recipe
 from conda_build.completers import (RecipeCompleter, PythonVersionCompleter, RVersionsCompleter,
  LuaVersionsCompleter, NumPyVersionCompleter)
@@ -41,7 +37,7 @@ def get_render_parser():
  version='conda-build %s' % __version__,
  )
  p.add_argument(
- '-n', "--no_source",
+ '-n', "--no-source",
  action="store_true",
  help="When templating can't be completed, do not obtain the \
 source to try fill in related template variables.",
@@ -100,37 +96,6 @@ def get_render_parser():
  return p
 
 
-# Next bit of stuff is to support YAML output in the order we expect.
-# http://stackoverflow.com/a/17310199/1170370
-class MetaYaml(dict):
- fields = ["package", "source", "build", "requirements", "test", "about", "extra"]
-
- def to_omap(self):
- return [(field, self[field]) for field in MetaYaml.fields if field in self]
-
-
-def represent_omap(dumper, data):
- return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.to_omap())
-
-
-def unicode_representer(dumper, uni):
- node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni)
- return node
-
-
-class IndentDumper(yaml.Dumper):
- def increase_indent(self, flow=False, indentless=False):
- return super(IndentDumper, self).increase_indent(flow, False)
-
-
-yaml.add_representer(MetaYaml, represent_omap)
-if PY3:
- yaml.add_representer(str, unicode_representer)
- unicode = None # silence pyflakes about unicode not existing in py3
-else:
- yaml.add_representer(unicode, unicode_representer)
-
-
 def main():
  p = get_render_parser()
  p.add_argument(
@@ -147,22 +112,21 @@ def main():
  choices=RecipeCompleter(),
  help="Path to recipe directory.",
  )
-
+ # this is here because we have a different default than build
+ p.add_argument(
+ '--verbose',
+ action='store_true',
+ help='Enable verbose output from download tools and progress updates',
+ )
  args = p.parse_args()
  set_language_env_vars(args, p)
 
- metadata = render_recipe(find_recipe(args.recipe), no_download_source=args.no_source)
+ metadata, _ = render_recipe(find_recipe(args.recipe), no_download_source=args.no_source,
+ verbose=args.verbose)
  if args.output:
  print(bldpkg_path(metadata))
  else:
- output = yaml.dump(MetaYaml(metadata.meta), Dumper=IndentDumper,
- default_flow_style=False, indent=4)
- if args.file:
- with open(args.file, "w") as f:
- f.write(output)
- else:
- print(output)
-
+ print(output_yaml(metadata, args.file))
 
 if __name__ == '__main__':
  main()

conda_build/post.py

@@ -376,26 +376,26 @@ def check_symlinks(files):
 def get_build_metadata(m):
  src_dir = source.get_dir()
  if exists(join(src_dir, '__conda_version__.txt')):
- print("Deprecation warning: support for __conda_version__ will be removed in Conda build 2.0."
+ print("Deprecation warning: support for __conda_version__ will be removed in Conda build 2.0." # noqa
  "Try Jinja templates instead: "
- "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables")
+ "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa
  with open(join(src_dir, '__conda_version__.txt')) as f:
  version = f.read().strip()
  print("Setting version from __conda_version__.txt: %s" % version)
  m.meta['package']['version'] = version
  if exists(join(src_dir, '__conda_buildnum__.txt')):
- print("Deprecation warning: support for __conda_buildnum__ will be removed in Conda build 2.0."
+ print("Deprecation warning: support for __conda_buildnum__ will be removed in Conda build 2.0." # noqa
  "Try Jinja templates instead: "
- "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables")
+ "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa
  with open(join(src_dir, '__conda_buildnum__.txt')) as f:
  build_number = f.read().strip()
  print("Setting build number from __conda_buildnum__.txt: %s" %
  build_number)
  m.meta['build']['number'] = build_number
  if exists(join(src_dir, '__conda_buildstr__.txt')):
- print("Deprecation warning: support for __conda_buildstr__ will be removed in Conda build 2.0."
+ print("Deprecation warning: support for __conda_buildstr__ will be removed in Conda build 2.0." # noqa
  "Try Jinja templates instead: "
- "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables")
+ "http://conda.pydata.org/docs/building/environment-vars.html#git-environment-variables") # noqa
  with open(join(src_dir, '__conda_buildstr__.txt')) as f:
  buildstr = f.read().strip()
  print("Setting version from __conda_buildstr__.txt: %s" % buildstr)

conda_build/render.py

@@ -15,6 +15,8 @@
 from locale import getpreferredencoding
 import subprocess
 
+import yaml
+
 from conda.compat import PY3
 from conda.lock import Locked
 
@@ -65,25 +67,38 @@ def set_language_env_vars(args, parser, execute=None):
  os.environ[var] = str(getattr(config, var))
 
 
-def parse_or_try_download(metadata, no_download_source):
- if not no_download_source:
+def bldpkg_path(m):
+ '''
+ Returns path to built package's tarball given its ``Metadata``.
+ '''
+ return os.path.join(config.bldpkgs_dir, '%s.tar.bz2' % m.dist())
+
+
+def parse_or_try_download(metadata, no_download_source, verbose, force_download=False):
+ if (("version" not in metadata.meta["package"] or
+ not metadata.meta["package"]["version"]) and
+ not no_download_source) or force_download:
  # this try/catch is for when the tool to download source is actually in
  # meta.yaml, and not previously installed in builder env.
  try:
- source.provide(metadata.path, metadata.get_section('source'))
+ source.provide(metadata.path, metadata.get_section('source'), verbose=verbose)
  metadata.parse_again(permit_undefined_jinja=False)
  need_source_download = False
  except subprocess.CalledProcessError:
  print("Warning: failed to download source. If building, will try "
-  "again after downloading recipe dependencies.")
+ "again after downloading recipe dependencies.")
  need_source_download = True
+ else:
+ need_source_download = no_download_source
  else:
- metadata.parse_again(permit_undefined_jinja=False)
- need_source_download = no_download_source
+ # we have not downloaded source in the render phase. Download it in
+ # the build phase
+ need_source_download = True
+ metadata.parse_again(permit_undefined_jinja=False)
  return metadata, need_source_download
 
 
-def render_recipe(recipe_path, no_download_source):
+def render_recipe(recipe_path, no_download_source, verbose):
  with Locked(config.croot):
  arg = recipe_path
  # Don't use byte literals for paths in Python 2
@@ -112,9 +127,51 @@ def render_recipe(recipe_path, no_download_source):
  sys.stderr.write(e.error_msg())
  sys.exit(1)
 
- m = parse_or_try_download(m, no_download_source=no_download_source)
+ m = parse_or_try_download(m, no_download_source=no_download_source,
+ verbose=verbose)
 
  if need_cleanup:
  shutil.rmtree(recipe_dir)
 
  return m
+
+
+# Next bit of stuff is to support YAML output in the order we expect.
+# http://stackoverflow.com/a/17310199/1170370
+class _MetaYaml(dict):
+ fields = ["package", "source", "build", "requirements", "test", "about", "extra"]
+
+ def to_omap(self):
+ return [(field, self[field]) for field in _MetaYaml.fields if field in self]
+
+
+def _represent_omap(dumper, data):
+ return dumper.represent_mapping(u'tag:yaml.org,2002:map', data.to_omap())
+
+
+def _unicode_representer(dumper, uni):
+ node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni)
+ return node
+
+
+class _IndentDumper(yaml.Dumper):
+ def increase_indent(self, flow=False, indentless=False):
+ return super(_IndentDumper, self).increase_indent(flow, False)
+
+yaml.add_representer(_MetaYaml, _represent_omap)
+if PY3:
+ yaml.add_representer(str, _unicode_representer)
+ unicode = None # silence pyflakes about unicode not existing in py3
+else:
+ yaml.add_representer(unicode, _unicode_representer)
+
+
+def output_yaml(metadata, filename=None):
+ output = yaml.dump(_MetaYaml(metadata.meta), Dumper=_IndentDumper,
+ default_flow_style=False, indent=4)
+ if filename:
+ with open(filename, "w") as f:
+ f.write(output)
+ return("Wrote yaml to %s" % filename)
+ else:
+ return(output)