Merge pull request #76 from broadinstitute/dpark-dev

more docs, add align_and_fix python command

.gitignore

@@ -1,5 +1,7 @@
 VERSION
 test/output/
+
+# Sphinx documentation
 docs/_build/
 
 # Mac OSX

assembly.py

@@ -17,8 +17,9 @@
 log = logging.getLogger(__name__)
 
 
-def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000, outFastqs=None):
+def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000):
     ''' Take reads through Trimmomatic, Prinseq, and subsampling.
+        This should probably move over to read_utils or taxon_filter.
     '''
 
     # BAM -> fastq
@@ -57,6 +58,9 @@ def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000, outFastqs=No
     os.unlink(purgefq[1])
 
     # Fastq -> BAM
+    # Note: this destroys RG IDs! We should instead just pull the list
+    # of IDs out of purge_unmated, random.sample them ourselves, and
+    # use FilterSamReadsTool to go straight from inBam -> outBam
     tmp_bam = util.file.mkstempfname('.subsamp.bam')
     tmp_header = util.file.mkstempfname('.header.sam')
     tools.picard.FastqToSamTool().execute(
@@ -65,11 +69,6 @@ def trim_rmdup_subsamp_reads(inBam, clipDb, outBam, n_reads=100000, outFastqs=No
     tools.samtools.SamtoolsTool().reheader(tmp_bam, tmp_header, outBam)
     os.unlink(tmp_bam)
     os.unlink(tmp_header)
-
-    # Save fastqs if requested
-    if outFastqs:
-        shutil.copyfile(subsampfq[0], outFastqs[0])
-        shutil.copyfile(subsampfq[1], outFastqs[1])
     os.unlink(subsampfq[0])
     os.unlink(subsampfq[1])
 
@@ -83,14 +82,16 @@ def assemble_trinity(inBam, outFasta, clipDb, n_reads=100000, outReads=None):
         subsamp_bam = outReads
     else:
         subsamp_bam = util.file.mkstempfname('.subsamp.bam')
+
+    trim_rmdup_subsamp_reads(inBam, clipDb, subsamp_bam, n_reads=n_reads)
     subsampfq = list(map(util.file.mkstempfname, ['.subsamp.1.fastq', '.subsamp.2.fastq']))
-    trim_rmdup_subsamp_reads(inBam, clipDb, subsamp_bam,
-        n_reads=n_reads, outFastqs=subsampfq)
+    tools.picard.SamToFastqTool().execute(subsamp_bam, subsampfq[0], subsampfq[1])
     tools.trinity.TrinityTool().execute(subsampfq[0], subsampfq[1], outFasta)
-    if not outReads:
-        os.unlink(subsamp_bam)
     os.unlink(subsampfq[0])
     os.unlink(subsampfq[1])
+
+    if not outReads:
+        os.unlink(subsamp_bam)
 
 def parser_assemble_trinity(parser=argparse.ArgumentParser()):
     parser.add_argument('inBam',

docs/broad_utils.rst

@@ -1,5 +1,5 @@
 broad_utils.py - for data generated at the Broad Institute
-=====================================
+==========================================================
 
 .. argparse::
     :module:  broad_utils

docs/conf.py

@@ -21,14 +21,22 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.dirname(os.path.abspath('.')))
 
-import util.version
-
 # -- Mock out the heavyweight pip packages, esp those that require C ----
 import mock
 MOCK_MODULES = ['numpy', 'scipy', 'matplotlib', 'pysam',
   'Bio', 'Bio.AlignIO', 'Bio.SeqIO', 'Bio.Data.IUPACData']
 for mod_name in MOCK_MODULES:
    sys.modules[mod_name] = mock.Mock()
+
+# -- Obtain GIT version --
+import subprocess
+def _git_version():
+    cmd = ['git', 'describe', '--tags', '--always']  # omit "--dirty" from doc build
+    out = subprocess.check_output(cmd)
+    if type(out) != str:
+        out = out.decode('utf-8')
+    return out.strip()
+__version__ = _git_version()
 
 # -- General configuration ------------------------------------------------
 
@@ -65,7 +73,7 @@
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-release = util.version.get_version()
+release = __version__
 version = '.'.join(release.split('.')[:2])
 
 # The language for content autogenerated by Sphinx. Refer to documentation
@@ -76,7 +84,7 @@
 # non-false value, then it is used:
 #today = ''
 # Else, today_fmt is used as the format for a strftime call.
-#today_fmt = '%B %d, %Y'
+today_fmt = '%Y-%m-%d'
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -149,7 +157,7 @@
 
 # If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
 # using the given strftime format.
-#html_last_updated_fmt = '%b %d, %Y'
+html_last_updated_fmt = '%Y-%m-%d. {}'.format(release)
 
 # If true, SmartyPants will be used to convert quotes and dashes to
 # typographically correct entities.

docs/description.rst

@@ -0,0 +1,33 @@
+Description of the methods
+==========================
+
+Much more documentation to come...
+
+TO DO: here we will put a high level description of the various tools that
+exist here, perhaps with some pictures and such. We will describe why we
+used certain tools and approaches / how other approaches fell short / what
+kinds of problems certain steps are trying to solve.  Perhaps some links to
+papers and such.  Kind of a mini-methods paper here.
+
+
+Viral genome analysis
+---------------------
+
+*De novo* assembly, reference assisted assembly improvements,
+gene annotaion, species-level variation, within-host variation, etc.
+
+
+Taxonomic read filtration
+-------------------------
+
+Especially human read depletion (prior to submission to NCBI SRA).
+But also the part where we restrict to a particular taxa of interest
+(the species you're studying).
+
+
+Taxonomic read identification
+-----------------------------
+
+Nothing much here at the moment. That comes later, but we will later
+integrate it when it's ready.
+

docs/index.rst

@@ -14,6 +14,7 @@ Contents
    :maxdepth: 2
    :numbered:
 
+   description
    install
    cmdline
    pipeuse

docs/install.rst

@@ -70,3 +70,7 @@ Alternatively, if you are using the Snakemake pipelines, you can create
 a dictionary called "env_vars" in the config.json file for Snakemake,
 and the pipelines will automatically set all environment variables prior
 to running any scripts.
+
+The version of MOSAIK we use seems to fail compile on GCC-4.9 but compiles
+fine on GCC-4.4. We have not tried intermediate versions of GCC, nor the
+latest versions of MOSAIK.

docs/interhost.rst

@@ -1,5 +1,5 @@
 interhost.py - species and population-level genetic variation
-=====================================
+=============================================================
 
 .. argparse::
     :module:  interhost

docs/read_utils.rst

@@ -1,5 +1,5 @@
 read_utils.py - utilities that manipulate bam and fastq files
-=====================================
+=============================================================
 
 .. argparse::
     :module:  read_utils

docs/reports.rst

@@ -1,5 +1,5 @@
 reports.py - produce various metrics and reports
-=====================================
+================================================
 
 .. argparse::
     :module:  reports

docs/taxon_filter.rst

@@ -1,5 +1,5 @@
 taxon_filter.py - tools for taxonomic removal or filtration of reads
-=====================================
+====================================================================
 
 .. argparse::
     :module:  taxon_filter