Chemshell integration issue #120

Added rest of code to make the basic functionality work. Longbow now has overrides in the submission portion of its core such that the chemshell wrapper can be used natively without going via qsub scripts. The unit tests for the existing longbow core code required fixing, however the chemshell plugin current does not have tests.
HECBioSim · Feb 26, 2019 · 2a1f14f · 2a1f14f
1 parent d782b14
commit 2a1f14f
Show file tree

Hide file tree

Showing 6 changed files with 81 additions and 48 deletions.
diff --git a/longbow/apps/chemshell.py b/longbow/apps/chemshell.py
@@ -44,9 +44,9 @@
 
 
 EXECDATA = {
-    "chemsh.x": {
+    "chemsh": {
         "subexecutables": [],
-        "requiredfiles": ["<"],
+        "requiredfiles": ["<"]
     }
 }
 
@@ -82,15 +82,25 @@ def submithook(job):
     LOG.info("Chemshell plugin overrides the native submit function.")
 
     # Change into the working directory and submit the job.
-    cmd = ["cd " + job["destdir"] + "\n", "qsub " + job["subfile"]]
+    cmd = ["cd " + job["destdir"] + "\n", job["executableargs"]]
 
     shellout = shellwrappers.sendtossh(job, cmd)
 
     # Find job id.
     try:
 
-        # Do the regex in Longbow rather than in the subprocess.
-        jobid = re.search(r'\d+', shellout[0]).group()
+        # Split the shell output down to a list of lines.
+        lines = shellout[0].split('\n')
+
+        # Find line with keyword "submitted".
+        for line in lines:
+
+            if "submitted" in line:
+
+                jobidline = line
+                break
+
+        jobid = re.search(r'\d+', jobidline).group()
 
     except AttributeError:
 

diff --git a/longbow/entrypoints.py b/longbow/entrypoints.py
@@ -70,7 +70,7 @@
 import longbow.staging as staging
 
 PYTHONVERSION = "{0}.{1}".format(sys.version_info[0], sys.version_info[1])
-LONGBOWVERSION = "1.5.2"
+LONGBOWVERSION = "1.5.3-dev"
 
 LOG = logging.getLogger("longbow")
 

diff --git a/longbow/scheduling.py b/longbow/scheduling.py
@@ -403,16 +403,16 @@ def submit(jobs):
 
                 getattr(schedulers, scheduler.lower()).submit(job)
 
-                LOG.info("Job '%s' submitted with id '%s'", item, job["jobid"])
+            LOG.info("Job '%s' submitted with id '%s'", item, job["jobid"])
 
-                job["laststatus"] = "Queued"
+            job["laststatus"] = "Queued"
 
-                # Increment the queue counter by one (used to count the slots).
-                jobs["lbowconf"][job["resource"] + "-" + "queue-slots"] = \
-                    str(int(jobs["lbowconf"][job["resource"] +
-                            "-" + "queue-slots"]) + 1)
+            # Increment the queue counter by one (used to count the slots).
+            jobs["lbowconf"][job["resource"] + "-" + "queue-slots"] = \
+                str(int(jobs["lbowconf"][job["resource"] +
+                        "-" + "queue-slots"]) + 1)
 
-                submitted += 1
+            submitted += 1
 
         # Submit method can't be found.
         except AttributeError:

diff --git a/setup.py b/setup.py
@@ -56,7 +56,7 @@
 
 # Setup
 setup(name='Longbow',
-      version='1.5.2',
+      version='1.5.3-dev',
       description='Biomolecular simulation remote job submission tool.',
       long_description=open('README.rst').read(),
       author='James T Gebbie-Rayet, Gareth B Shannon',

diff --git a/tests/unit/scheduling/test_prepare.py b/tests/unit/scheduling/test_prepare.py
@@ -58,9 +58,10 @@ def test_prepare_single(mock_prepare):
 
     jobs = {
         "job-one": {
+            "executable": "pmemd.MPI",
+            "jobid": "test456",
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456",
             "subfile": ""
         }
     }
@@ -81,21 +82,24 @@ def test_prepare_multiple(mock_prepare):
 
     jobs = {
         "job-one": {
+            "executable": "pmemd.MPI",
+            "jobid": "test123",
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test123",
             "subfile": ""
         },
         "job-two": {
+            "executable": "pmemd.MPI",
+            "jobid": "test456",
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456",
             "subfile": ""
         },
         "job-three": {
+            "executable": "pmemd.MPI",
+            "jobid": "test789",
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test789",
             "subfile": ""
         }
     }
@@ -115,9 +119,10 @@ def test_prepare_attrexcept(mock_prepare):
 
     jobs = {
         "job-one": {
+            "executable": "pmemd.MPI",
+            "jobid": "test456",
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456",
             "subfile": ""
         }
     }
@@ -138,9 +143,10 @@ def test_prepare_ownscript(mock_prepare):
 
     jobs = {
         "job-one": {
+            "executable": "pmemd.MPI",
+            "jobid": "test456",
             "resource": "test-machine",
             "scheduler": "LSF",
-            "jobid": "test456",
             "subfile": "test.lsf",
             "upload-include": "file1, file2, file3"
         }

diff --git a/tests/unit/scheduling/test_submit.py b/tests/unit/scheduling/test_submit.py
@@ -60,9 +60,10 @@ def test_submit_single(mock_isdir, mock_submit):
     jobs = {
         "lbowconf": {},
         "job-one": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test456"
+            "scheduler": "LSF"
         }
     }
 
@@ -86,19 +87,22 @@ def test_submit_multiplesame(mock_isdir, mock_lsf):
     jobs = {
         "lbowconf": {},
         "job-one": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test123"
+            "scheduler": "LSF"
         },
         "job-two": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test456"
+            "scheduler": "LSF"
         },
         "job-three": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test789"
+            "scheduler": "LSF"
         }
     }
 
@@ -123,16 +127,19 @@ def test_submit_multiplediff(mock_isdir, mock_lsf, mock_pbs, mock_slurm):
     jobs = {
         "lbowconf": {},
         "job-one": {
+            "executable": "pmemd.MPI",
             "resource": "lsf-machine",
             "scheduler": "LSF",
             "jobid": "test123"
         },
         "job-two": {
+            "executable": "pmemd.MPI",
             "resource": "pbs-machine",
             "scheduler": "pbs",
             "jobid": "test456"
         },
         "job-three": {
+            "executable": "pmemd.MPI",
             "resource": "slurm-machine",
             "scheduler": "Slurm",
             "jobid": "test789"
@@ -165,9 +172,10 @@ def test_submit_filewrite(mock_isdir, mock_submit, mock_savini):
             "recoveryfile": "recovery-YYMMDD-HHMMSS"
         },
         "job-one": {
-            "resource": "test-machine",
-            "scheduler": "LSF",
             "jobid": "test456",
+            "executable": "pmemd.MPI",
+            "resource": "test-machine",
+            "scheduler": "LSF"
         }
     }
 
@@ -193,9 +201,10 @@ def test_submit_fileuninit(mock_isdir, mock_submit, mock_savini):
             "recoveryfile": ""
         },
         "job-one": {
-            "resource": "test-machine",
-            "scheduler": "LSF",
             "jobid": "test456",
+            "executable": "pmemd.MPI",
+            "resource": "test-machine",
+            "scheduler": "LSF"
         }
     }
 
@@ -221,9 +230,10 @@ def test_submit_fileexcept1(mock_isdir, mock_submit, mock_savini):
             "recoveryfile": "recovery-YYMMDD-HHMMSS"
         },
         "job-one": {
-            "resource": "test-machine",
-            "scheduler": "LSF",
             "jobid": "test456",
+            "executable": "pmemd.MPI",
+            "resource": "test-machine",
+            "scheduler": "LSF"
         }
     }
 
@@ -248,9 +258,10 @@ def test_submit_fileexcept2(mock_isdir, mock_submit, mock_savini):
             "recoveryfile": "recovery-YYMMDD-HHMMSS"
         },
         "job-one": {
-            "resource": "test-machine",
-            "scheduler": "LSF",
             "jobid": "test456",
+            "executable": "pmemd.MPI",
+            "resource": "test-machine",
+            "scheduler": "LSF"
         }
     }
 
@@ -273,9 +284,10 @@ def test_submit_attrexcept(mock_isdir, mock_submit, mock_savini):
     jobs = {
         "lbowconf": {},
         "job-one": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test456"
+            "scheduler": "LSF"
         }
     }
 
@@ -300,9 +312,10 @@ def test_submit_submitexcept(mock_isdir, mock_submit, mock_savini):
     jobs = {
         "lbowconf": {},
         "job-one": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test456"
+            "scheduler": "LSF"
         }
     }
 
@@ -328,9 +341,10 @@ def test_submit_queueexcept(mock_isdir, mock_submit, mock_savini):
     jobs = {
         "lbowconf": {},
         "job-one": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test456"
+            "scheduler": "LSF"
         }
     }
 
@@ -358,19 +372,22 @@ def test_submit_queueinfo(mock_isdir, mock_submit, mock_savini):
             "test-machine-queue-max": 0
         },
         "job-one": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test123"
+            "scheduler": "LSF"
         },
         "job-two": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test456"
+            "scheduler": "LSF"
         },
         "job-three": {
+            "jobid": "test456",
+            "executable": "pmemd.MPI",
             "resource": "test-machine",
-            "scheduler": "LSF",
-            "jobid": "test789"
+            "scheduler": "LSF"
         }
     }