Adding pipeline hooks to get SLURM job information

workflow/Snakefile

@@ -41,6 +41,7 @@ with open(join('config', 'cluster.json')) as fh:
 
 # Imported rules
 include: join("rules", "common.smk")
+include: join("rules", "hooks.smk")
 include: join("rules", "paired-end.smk")
 
 # Targets of the pipeline to build the DAG,
@@ -132,4 +133,4 @@ rule all:
  provided(
  [join(workpath,"Project","Project.contig.classification.html")],
  do_aggregate
- )
+ )

workflow/rules/hooks.smk

@@ -0,0 +1,79 @@
+# Adding handlers for displaying status of the 
+# pipeline and for getting job information for 
+# previously submitted jobs using `jobby`:
+# https://github.com/OpenOmics/scribble/blob/main/scripts/jobby/jobby
+if config['options']['mode'] == 'slurm':
+ onstart:
+ shell(
+ """
+ # Move any job information for a previous
+ # instance of the pipeline to logfiles
+ sleep 5; rm -f COMPLETED FAILED RUNNING;
+ touch RUNNING
+ for f in job_information_*.tsv; do
+ # Skip over non-existant files
+ [ -e "${{f}}" ] || continue
+ mv ${{f}} logfiles/;
+ done
+
+ for f in failed_jobs_*.tsv; do
+ # Skip over non-existant files
+ [ -e "${{f}}" ] || continue
+ mv ${{f}} logfiles/;
+ done
+ """
+ )
+
+ onsuccess:
+ shell(
+ """
+ # Get job information on all
+ # previously submitted jobs 
+ sleep 15; rm -f COMPLETED FAILED RUNNING;
+ timestamp=$(date +"%Y-%m-%d_%H-%M-%S");
+ ./workflow/scripts/jobby \\
+ $(grep --color=never "^Submitted .* external jobid" logfiles/snakemake.log \\
+ | awk '{{print $NF}}' \\
+ | sed "s/['.]//g" \\
+ | sort \\
+ | uniq \\
+ | tr "\\n" " "
+ ) \\
+ > job_information_${{timestamp}}.tsv
+
+ # Get information on any child 
+ # job(s) that may have failed 
+ grep --color=never \\
+ '^jobid\\|FAILED' \\
+ job_information_${{timestamp}}.tsv \\
+ > failed_jobs_${{timestamp}}.tsv
+ touch COMPLETED 
+ """
+ )
+
+ onerror:
+ shell(
+ """
+ # Get job information on all
+ # previously submitted jobs 
+ sleep 15; rm -f COMPLETED FAILED RUNNING;
+ timestamp=$(date +"%Y-%m-%d_%H-%M-%S");
+ ./workflow/scripts/jobby \\
+ $(grep --color=never "^Submitted .* external jobid" logfiles/snakemake.log \\
+ | awk '{{print $NF}}' \\
+ | sed "s/['.]//g" \\
+ | sort \\
+ | uniq \\
+ | tr "\\n" " "
+ ) \\
+ > job_information_${{timestamp}}.tsv
+
+ # Get information on any child 
+ # job(s) that may have failed 
+ grep --color=never \\
+ '^jobid\\|FAILED' \\
+ job_information_${{timestamp}}.tsv \\
+ > failed_jobs_${{timestamp}}.tsv
+ touch FAILED
+ """
+ )