Merge pull request #19 from arteria-project/devel

Merge devel branch for new release

bcl2fastq/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.0"
+__version__ = "1.1.1"

bcl2fastq/app.py

@@ -15,7 +15,8 @@ def routes(**kwargs):
  url(r"/api/1.0/versions", VersionsHandler, name="versions", kwargs=kwargs),
  url(r"/api/1.0/start/([\w_-]+)", StartHandler, name="start", kwargs=kwargs),
  url(r"/api/1.0/status/(\d*)", StatusHandler, name="status", kwargs=kwargs),
- url(r"/api/1.0/stop/([\d|all]*)", StopHandler, name="stop", kwargs=kwargs)
+ url(r"/api/1.0/stop/([\d|all]*)", StopHandler, name="stop", kwargs=kwargs),
+ url(r"/api/1.0/logs/([\w_-]+)", Bcl2FastqLogHandler, name="logs", kwargs=kwargs)
  ]
 
 def start():

bcl2fastq/handlers/bcl2fastq_handlers.py

@@ -6,9 +6,12 @@
 from bcl2fastq.lib.jobrunner import LocalQAdapter
 from bcl2fastq.lib.bcl2fastq_utils import BCL2FastqRunnerFactory, Bcl2FastqConfig
 from bcl2fastq import __version__ as version
+from bcl2fastq.lib.bcl2fastq_logs import Bcl2FastqLogFileProvider
+from arteria.exceptions import ArteriaUsageException
 from arteria.web.state import State
 from arteria.web.handlers import BaseRestHandler
 
+
 log = logging.getLogger(__name__)
 
 class Bcl2FastqServiceMixin:
@@ -59,6 +62,7 @@ def initialize(self, config):
  to subclasses.
  """
  self.config = config
+ self.bcl2fastq_log_file_provider = Bcl2FastqLogFileProvider(self.config)
 
 
 class VersionsHandler(BaseBcl2FastqHandler):
@@ -109,7 +113,7 @@ def create_config_from_request(self, runfolder, request_body):
  runfolder_input = "{0}/{1}".format(runfolder_base_path, runfolder)
 
  if not os.path.isdir(runfolder_input):
- raise RuntimeError("No such file: {0}".format(runfolder_input))
+ raise ArteriaUsageException("No such file: {0}".format(runfolder_input))
 
  if "bcl2fastq_version" in request_data:
  bcl2fastq_version = request_data["bcl2fastq_version"]
@@ -172,8 +176,7 @@ def post(self, runfolder):
  cmd = job_runner.construct_command()
  job_runner.symlink_output_to_unaligned()
 
- log_base_path = self.config["bcl2fastq_logs_path"]
- log_file = "{0}/{1}.log".format(log_base_path, runfolder)
+ log_file = self.bcl2fastq_log_file_provider.log_file_path(runfolder)
 
  job_id = self.runner_service().start(
  cmd,
@@ -202,11 +205,12 @@ def post(self, runfolder):
 
  self.set_status(202, reason="started processing")
  self.write_json(response_data)
- except RuntimeError as e:
+ except ArteriaUsageException as e:
  log.warning("Failed starting {0}. Message: {1}".format(runfolder, e.message))
  self.send_error(status_code=500, reason=e.message)
 
 
+
 class StatusHandler(BaseBcl2FastqHandler, Bcl2FastqServiceMixin):
  """
  Get the status of one or all jobs.
@@ -252,10 +256,28 @@ def post(self, job_id):
  self.runner_service().stop(job_id)
  self.set_status(200)
  else:
- ValueError("Unknown job to stop")
- except ValueError as e:
+ ArteriaUsageException("Unknown job to stop")
+ except ArteriaUsageException as e:
  log.warning("Failed stopping job: {}. Message: ".format(job_id, e.message))
  self.send_error(500, reason=e.message)
 
 
+class Bcl2FastqLogHandler(BaseBcl2FastqHandler):
+ """
+ Gets the content of the log for a particular runfolder
+ """
 
+ def get(self, runfolder):
+ """
+ Get the content of the log for a particular runfolder
+ :param runfolder:
+ :return:
+ """
+ try:
+ log_content = self.bcl2fastq_log_file_provider.get_log_for_runfolder(runfolder)
+ response_data = {"runfolder": runfolder, "log": log_content}
+ self.set_status(200)
+ self.write_json(response_data)
+ except IOError as e:
+ log.warning("Problem with accessing {}, message: {}".format(runfolder, e.message))
+ self.send_error(500, reason=e.message)

bcl2fastq/lib/bcl2fastq_logs.py

@@ -0,0 +1,18 @@
+
+
+class Bcl2FastqLogFileProvider:
+
+ def __init__(self, config):
+ self.config = config
+
+ def log_file_path(self, runfolder):
+ log_base_path = self.config["bcl2fastq_logs_path"]
+ log_file = "{0}/{1}.log".format(log_base_path, runfolder)
+ return log_file
+
+ def get_log_for_runfolder(self, runfolder):
+ log_path = self.log_file_path(runfolder)
+ with open(log_path) as f:
+ file_content = f.read()
+ return file_content
+

bcl2fastq/lib/bcl2fastq_utils.py

@@ -7,6 +7,8 @@
 
 from illuminate.metadata import InteropMetadata
 
+
+from arteria.exceptions import ArteriaUsageException
 from bcl2fastq.lib.illumina import Samplesheet
 
 log = logging.getLogger(__name__)
@@ -169,7 +171,8 @@ def build_index_string(length_tuple):
  difference = length_of_index_read - length_of_index_in_samplesheet
 
  if not difference >= 0:
- raise ValueError("Sample sheet indicates that index is longer than what was read by the sequencer!")
+ raise ArteriaUsageException("Sample sheet indicates that index is "
+ "longer than what was read by the sequencer!")
 
  if length_of_index_in_samplesheet == 0:
  # If there is no index in the samplesheet, ignore it in the base-mask
@@ -427,8 +430,9 @@ def construct_command(self):
  is_single_read_run)
  base_masks_as_set = set(lanes_and_base_mask.values())
 
- assert len(base_masks_as_set) is 1, "For bcl2fastq 1.8.4 there is no support for " \
- "mixing different bases masks for different lanes"
+ if len(base_masks_as_set) is 1:
+ raise ArteriaUsageException("For bcl2fastq 1.8.4 there is no support for "
+ "mixing different bases masks for different lanes")
 
  # Here we are forced to use the same bases mask was always used for all lanes.
  commandline_collection.append("--use_bases_mask " + lanes_and_base_mask.values()[0])

requirements/prod

@@ -1,7 +1,7 @@
 jsonpickle==0.9.2
 tornado==4.2.1
 git+https://github.com/johandahlberg/localq.git@with_shell_true # Get from pip in future - localq
-git+https://github.com/arteria-project/[email protected].0#egg=arteria-core
+git+https://github.com/arteria-project/[email protected].1#egg=arteria-core
 illuminate==0.6.2
 pandas==0.14.1
 

tests/test_bcl2fastq_handlers.py

@@ -7,6 +7,7 @@
 
 from bcl2fastq.handlers.bcl2fastq_handlers import *
 from bcl2fastq.lib.bcl2fastq_utils import BCL2Fastq2xRunner, BCL2FastqRunner
+from bcl2fastq.lib.bcl2fastq_logs import Bcl2FastqLogFileProvider
 from bcl2fastq.app import routes
 from tornado.web import Application
 from test_utils import FakeRunner
@@ -129,3 +130,13 @@ def test_stop_handler(self):
  def test_exception_stop_handler(self):
  response = self.fetch(self.API_BASE + "/stop/lll", method="POST", body = "")
  self.assertEqual(response.code, 500)
+
+ def test_get_logs(self):
+ with mock.patch.object(Bcl2FastqLogFileProvider, "get_log_for_runfolder", return_value="This is a string"):
+ response = self.fetch(self.API_BASE + "/logs/coolest_runfolder", method="GET")
+ self.assertEqual(response.code, 200)
+ self.assertEqual(json.loads(response.body)["log"], "This is a string")
+
+ def test_get_logs_trying_to_reach_other_files(self):
+ response = self.fetch(self.API_BASE + "/logs/../../../etc/shadow", method="GET")
+ self.assertEqual(response.code, 404)

tests/test_bcl2fastq_logs.py

@@ -0,0 +1,35 @@
+
+import unittest
+from mock import MagicMock, patch, mock_open
+
+from bcl2fastq.lib.bcl2fastq_logs import Bcl2FastqLogFileProvider
+
+class TestBcl2FastqLogFileProvider(unittest.TestCase):
+
+ fake_file_content = """
+ This is a nice multi-line
+ string for
+ you
+ my friend
+ """
+ fake_path = "/fake/path"
+ mock_config = MagicMock()
+ mock_config.__getitem__.return_value = fake_path
+ runfolder = "160218_ST-E00215_0070_BHKGLFCCXX"
+
+ log_filer_provider = Bcl2FastqLogFileProvider(mock_config)
+
+ def test_log_file_path(self):
+ log_file = self.log_filer_provider.log_file_path(self.runfolder)
+ self.assertEqual(log_file, "{}/{}.log".format(self.fake_path, self.runfolder))
+
+ def test_get_log_for_runfolder(self):
+ with patch("__builtin__.open", mock_open(read_data=self.fake_file_content), create=True):
+ file_content = self.log_filer_provider.get_log_for_runfolder(self.runfolder)
+ self.assertEqual(file_content, self.fake_file_content)
+
+ def test_get_log_for_runfolder_does_not_exist(self):
+ with self.assertRaises(IOError):
+ self.log_filer_provider.get_log_for_runfolder(self.runfolder)
+
+

tests/tests_bcl2fastq_utils.py

@@ -93,7 +93,7 @@ def test_get_bases_mask_per_lane_from_samplesheet_invalid_length_combo(self):
  mock_read_index_lengths = {2: 4, 3: 4}
  samplesheet = Samplesheet(TestBcl2FastqConfig.samplesheet_file)
 
- with self.assertRaises(ValueError):
+ with self.assertRaises(ArteriaUsageException):
  Bcl2FastqConfig. \
  get_bases_mask_per_lane_from_samplesheet(samplesheet, mock_read_index_lengths, False)