Merge pull request #359 from broadinstitute/dp-build-optim

build optimizations

.gitignore

@@ -65,3 +65,7 @@ easy-deploy/data/config.yaml
 easy-deploy/data/viral-ngs/
 
 easy-deploy/.vagrant/
+
+tools/build/
+tools/conda-cache/
+tools/conda-tools/

.travis.yml

@@ -3,13 +3,15 @@ sudo: false
 
 cache:
  directories:
+  - $HOME/misc_cache
   - $HOME/virtualenv
   - $HOME/miniconda
 
 env:
   global:
-  - CACHE_DIR="$HOME/virtualenv"
+  - CACHE_DIR="$HOME/misc_cache"
   - MINICONDA_DIR="$HOME/miniconda"
+  - PIP_DIR="$HOME/virtualenv"
   - GATK_PATH="$CACHE_DIR/bin_bundles/GenomeAnalysisTK-3.3-0-g37228af"
   - NOVOALIGN_PATH="$CACHE_DIR/bin_bundles/novocraft_v3"
   - PYTHONIOENCODING=UTF8
@@ -20,20 +22,6 @@ python:
   - 3.4
   - 3.5
 
-# Container pkgs for building diamond.
-# These versioned and sources hacks should be removed once travis upgrades to
-# Ubuntu 14.04.
-addons:
-  apt:
-    sources:
-    - boost-latest # For newer boost
-    - ubuntu-toolchain-r-test # For newer gcc
-    packages:
-    - cmake
-    - gcc-4.9
-    - g++-4.9
-    - libboost1.55-all-dev
-
 git:
   depth: 3
 
@@ -48,15 +36,11 @@ install:
   - travis/install-tools.sh
 
 before_script:
-  - flake8 --exit-zero .
+  - travis/flake8.sh
 
 script:
   - travis/tests-unit.sh
   - travis/tests-long.sh
 
 after_success:
   - coveralls
-
-notifications:
-  slack:
-    secure: dVmGF86ZaVkL7LhqmcjqTgmmc8ONZfLH2H2GO5AUHce3q61dA4tuZrfW0T+6puQWtIvvMC+5ioD0vRce+3iZELs83KR4Xm1vqNu6RZdAHx/ec3QQU4/7ulcEs50wWQl+WmyRqciGaBThUv+Un1K6AA2HYTcb8gQaFkpPmI3huZo=

install_tools.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+''' This script installs every Tool needed by viral-ngs
+'''
+
+from __future__ import print_function
+import os.path
+import sys
+import timeit
+import tools
+from tools import *
+import util.file
+
+__author__ = "[email protected]"
+
+def install_all_tools():
+    sumtime = 0.0
+    n_tools = 0
+    n_success = 0
+    for tool_class in tools.all_tool_classes():
+        t = tool_class()
+        print("installing %s .. " % tool_class.__name__, end="")
+        sys.stdout.flush()
+        runtime = timeit.timeit(t.install)
+        sumtime += runtime
+        success = t.is_installed()
+        print("SUCCESS" if success else "FAILED", end="")
+        print(" (%0.1f seconds)" % runtime)
+        sys.stdout.flush()
+        if success:
+            n_success += 1
+        n_tools += 1
+    print("Total %d tools attempted, %d succeeded, %d failed, cumulative install time %0.1f seconds" % (
+        n_tools, n_success, n_tools - n_success, sumtime))
+    return (n_tools == n_success)
+
+
+if __name__ == '__main__':
+    print("this install script is %s" % (
+        os.path.abspath(os.path.expanduser(__file__))))
+    print("installing tools into: %s (build, conda-tools, conda-cache)" % (
+        os.path.join(util.file.get_project_path(), 'tools')))
+    sys.exit(0 if install_all_tools() else 1)

requirements-conda.txt

@@ -0,0 +1,17 @@
+blast=2.2.31
+bmtagger=3.101
+diamond=0.7.10=boost1.60_1
+kraken-all=0.10.6_eaf8fb68
+last=719
+mafft=7.221
+mummer=3.23
+muscle=3.8.1551
+mvicuna=1.0
+novoalign=3.03.02
+picard=1.126
+prinseq=0.20.4
+samtools=1.2
+snpeff=4.1l
+trimmomatic=0.35
+trinity=date.2011_11_26
+vphaser2=2.0

requirements-tests.txt

@@ -1,5 +1,4 @@
 coveralls==1.1
-flake8<=3
 pycodestyle
 mock==2.0.0
 six<2

test/unit/test_tools.py

@@ -1,50 +1,17 @@
 # Unit tests for tools/__init__.py
 
-__author__ = "dpark@broadinstitute.org"
+__author__ = "yesimon@broadinstitute.org"
 
 import tools
 from tools import *
-import unittest
-import tempfile
-import shutil
-import os
-import logging
-import util.cmd
-import util.file
-from test import TestCaseWithTmp
-import operator
 import pytest
 
-log = logging.getLogger(__name__)
-util.cmd.setup_logger('INFO')
-
-
-def iter_leaf_subclasses(aClass):
-    "Iterate over subclasses at all levels that don't themselves have a subclass"
-    isLeaf = True
-    for subclass in sorted(aClass.__subclasses__(), key=operator.attrgetter("__name__")):
-        isLeaf = False
-        for leafClass in iter_leaf_subclasses(subclass):
-            if not getattr(leafClass, '_skiptest', False):
-                yield leafClass
-    if isLeaf:
-        yield aClass
-
-
-def all_tool_tests():
-    for tool_class in iter_leaf_subclasses(tools.Tool):
-        yield tool_class
-
-
-@pytest.fixture(params=all_tool_tests())
+@pytest.fixture(params=tools.all_tool_classes())
 def tool_class(request):
     print(request.param)
     return request.param
 
-
 def test_tool_install(tool_class):
     t = tool_class()
     t.install()
-    assert t.is_installed(), "installation of tool %s failed" % tool_class.__name__
-    log.info(".. installation of %s succeeded with installer %s" %
-             (tool_class.__name__, t.installed_method.__class__.__name__))
+    assert t.is_installed()

tools/__init__.py

@@ -3,6 +3,8 @@
 __author__ = "[email protected],[email protected]"
 
 import collections
+import json
+import operator
 import os
 import re
 import logging
@@ -11,7 +13,6 @@
 import subprocess
 import util.file
 import util.misc
-import json
 
 try:
     # Python 3.x
@@ -39,6 +40,21 @@
 _log = logging.getLogger(__name__)
 
 
+def iter_leaf_subclasses(aClass):
+    "Iterate over subclasses at all levels that don't themselves have a subclass"
+    isLeaf = True
+    for subclass in sorted(aClass.__subclasses__(), key=operator.attrgetter("__name__")):
+        isLeaf = False
+        for leafClass in iter_leaf_subclasses(subclass):
+            if not getattr(leafClass, '_skiptest', False):
+                yield leafClass
+    if isLeaf:
+        yield aClass
+
+def all_tool_classes():
+    return iter_leaf_subclasses(Tool)
+
+
 def get_tool_by_name(name):
     if name not in installed_tools:
         raise NotImplementedError
@@ -271,14 +287,14 @@ def __init__(
 
         # if the env is being overridden, or if we could not find an active conda env
         if env_root_path or env or not self.env_path:
-            env_root_path = env_root_path or os.path.join(util.file.get_build_path(), 'conda-tools')
+            env_root_path = env_root_path or os.path.join(util.file.get_project_path(), 'tools', 'conda-tools')
             env = env or 'default'
             self.env_path = os.path.realpath(os.path.expanduser(
                 os.path.join(env_root_path, env)))
 
         # set an env variable to the conda cache path. this env gets passed to the
         # the subprocess, and the variable instructs conda where to place its cache files
-        conda_cache_path = conda_cache_path or os.path.join(util.file.get_build_path(), 'conda-cache')
+        conda_cache_path = conda_cache_path or os.path.join(util.file.get_project_path(), 'tools', 'conda-cache')
         self.conda_cache_path = os.path.realpath(os.path.expanduser(conda_cache_path))
         self.conda_env = os.environ
         old_envs_path = os.environ.get('CONDA_DEFAULT_ENV')

tools/blast.py

@@ -3,28 +3,10 @@
 import os
 import util.misc
 
-URL_PREFIX = 'ftp://ftp.ncbi.nlm.nih.gov/blast/executables' \
-            '/blast+/2.2.29/ncbi-blast-2.2.29+-'
-
 TOOL_NAME = "blast"
 TOOL_VERSION = "2.2.31"
 
 
-def get_url():
-    """ creates the download url for this tool """
-    uname = os.uname()
-    if uname[0] == 'Darwin':
-        os_str = 'universal-macosx'
-    elif uname[0] == 'Linux':
-        if uname[4].endswith('64'):
-            os_str = 'x64-linux'
-        else:
-            os_str = 'ia32-linux'
-    else:
-        raise NotImplementedError('OS {} not implemented'.format(uname[0]))
-    return URL_PREFIX + os_str + '.tar.gz'
-
-
 class BlastTools(tools.Tool):
     """'Abstract' base class for tools in the blast+ suite.
        Subclasses must define class member subtool_name."""
@@ -37,22 +19,8 @@ def __init__(self, install_methods=None):
         ]
         self.subtool_name = self.subtool_name if hasattr(self, "subtool_name") else "blastn"
         if install_methods is None:
-            target_rel_path = 'ncbi-blast-2.2.29+/bin/' + self.subtool_name
             install_methods = []
             install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION))
-            install_methods.append(
-                tools.DownloadPackage(
-                    get_url(),
-                    target_rel_path,
-                    post_download_command=' '.join(
-                        ['rm'] + [
-                            'ncbi-blast-2.2.29+/bin/' + f for f in unwanted
-                        ]
-                    ),
-                    post_download_ret=None
-                )
-            )
-        #tools.Tool.__init__(self, install_methods=install_methods)
         super(BlastTools, self).__init__(install_methods=install_methods)
 
     def execute(self, *args):

tools/bmtagger.py

@@ -32,7 +32,6 @@ def __init__(self, install_methods=None):
         if install_methods is None:
             install_methods = []
             install_methods.append(tools.CondaPackage(TOOL_NAME, executable=self.subtool_name, version=TOOL_VERSION))
-            install_methods.append(DownloadBmtagger(self.subtool_name))
         tools.Tool.__init__(self, install_methods=install_methods)
 
     def execute(self, *args):
@@ -125,47 +124,3 @@ def build_database(self, fasta_files, database_prefix_path):
         self.execute(*args)
 
         return database_prefix_path
-
-
-class DownloadBmtagger(tools.InstallMethod):
-    """ InstallMethod class for downloading platform-specific bmtagger """
-    executables = ['bmtagger.sh', 'bmfilter', 'extract_fullseq', 'srprism']
-
-    def __init__(self, subtool_name):
-        self.installed = False
-        self.target_dir = os.path.join(util.file.get_build_path(), 'bmtagger')
-        self.target_path = os.path.join(self.target_dir, subtool_name)
-        tools.InstallMethod.__init__(self)
-
-    def is_installed(self):
-        return self.installed
-
-    def executable_path(self):
-        return self.installed and self.target_path or None
-
-    def verify_install(self):
-        """ confirms that the tools are present and executable """
-        self.installed = all(
-            os.access(
-                os.path.join(self.target_dir, executable), (os.X_OK | os.R_OK)) for executable in self.executables
-        )
-        return self.installed
-
-    def _attempt_install(self):
-        if self.verify_install():
-            return
-        util.file.mkdir_p(self.target_dir)
-        url_base = 'ftp://ftp.ncbi.nlm.nih.gov/pub/agarwala/bmtagger/'
-        uname = os.uname()
-        if uname[0] == 'Darwin':
-            url_base += 'mac-os/'
-        elif uname[0] != 'Linux' or not uname[4].endswith('64'):
-            _log.debug('OS %s not implemented', uname[0])
-            return
-        for executable in self.executables:
-            path = os.path.join(self.target_dir, executable)
-            url = url_base + executable
-            _log.info('Downloading from %s ...', url)
-            urlretrieve(url, path)
-            os.system('chmod +x ' + path)
-        self.verify_install()

tools/diamond.py

@@ -18,6 +18,7 @@
 log = logging.getLogger(__name__)
 
 
+@tools.skip_install_test(condition=tools.is_osx())
 class Diamond(tools.Tool):
 
     SUBCOMMANDS = ['makedb', 'blastx', 'blastp', 'view']