instead of submission scripts, save slurm flags as an env var

slurm/README.md

@@ -8,10 +8,9 @@ At the moment, our slurm utilities are written specifically for Stanford's Sherl
 
 # Setup
 1. You *must* clone dynadojo from github into a location that is shared to all nodes, usually $HOME. The singularity container does NOT contain a copy of the code, only the dependencies. The git repo gets mounted onto the container by specifying the $DD_REPO_DIR env var (see `dynadojo/slurm/<cluster>/env.sh`). We do this so you can live-edit your params in `dynadojo/experiments/params.py`.
-2. Edit the environment variables in `dynadojo/slurm/<cluster>/env.sh`
+2. Edit the submission flags and environment variables in `dynadojo/slurm/<cluster>/env.sh`
     1. Note that `$DD_SCRATCH_DIR/$DD_OUTPUT_DIR` gets mounted at experiments/outputs which is the default output path!
-3. Edit the sbatch submission flags in `dynadojo/slurm/<cluster>/submit.sh`
-4. Create whatever folders you need to create for logs, output, etc...
+3. Create whatever folders you need to create for logs, output, etc...
 
 
 # Make params file 

slurm/quest/env.sh

@@ -1,12 +1,18 @@
 #!/bin/bash
 
-# QUEST CLUSTER SETTINGS
+# QUEST SLURM SETTINGS
+ACCOUNT='-A p32141'
+PARTITION='--partition short'
+OUTPUT='-o /home/ctb3982/logs/out/%A_%a.out'
+ERROR='/home/ctb3982/logs/err/%A_%a.err'
 
+# ENVIRONMENT VARS
 export DD_SINGULARITY_IMAGE_LOCATION=$HOME/simg                 #where you want to store the singularity image
 export DD_REPO_DIR=$HOME                                        #parent directory of where you cloned dynadojo
 export DD_SCRATCH_DIR=$HOME                                     #your scratch directory  
 export DD_OUTPUT_DIR=questput                                   #name of folder in scratch to put output 
 export DD_IMAGE_REPO=docker://carynbear/dynadojo:sherlock       #docker image to pull
 export DD_CLUSTER=quest                                         #cluster name
+export DD_SLURM_ARGS="$ACCOUNT $PARTITION $OUTPUT $ERROR"
 
 echo "Setting up environment for $DD_CLUSTER"

slurm/scripts/srun_setup.sh → slurm/scripts/_setup.sh

@@ -23,6 +23,7 @@ if [ ! -f "$ENV_SCRIPT" ]; then
   echo -e "${RED} Environment configuration script $ENV_SCRIPT does not exist ${NC}";
   return 0 2> /dev/null || exit 0
 else
+    echo "Running $ENV_SCRIPT"
     chmod +x $ENV_SCRIPT #make the script executable
     source $ENV_SCRIPT
 fi

slurm/scripts/sbatch_run.sh

@@ -2,7 +2,7 @@
 
 #Run the setup script to check the directory and set the environment variables
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source ${__dir}/srun_setup.sh
+source ${__dir}/_setup.sh
 
 
 #ask for challenge, system, and algorithm
@@ -42,12 +42,5 @@ params_file=$opt
 echo ""
 read -p "List of jobs to run (comma separated, no spaces): " jobs
 
-SUBMIT_SCRIPT="slurm/$DD_CLUSTER/submit.sh"
-#check if the submit script exists
-if [ ! -f "$SUBMIT_SCRIPT" ]; then
-  echo -e "${RED} Submit script $SUBMIT_SCRIPT does not exist ${NC}";
-  return 0 2> /dev/null || exit 0
-else
-    chmod +x $SUBMIT_SCRIPT #make the script executable
-    ./$SUBMIT_SCRIPT ${__dir}/../jobscripts/sbatch/run.sbatch $params_file $jobs
-fi
+
+sbatch $DD_SLURM_ARGS --export=all ${__dir}/../jobscripts/sbatch/run.sbatch $params_file $jobs

slurm/scripts/sbatch_submit.sh

@@ -2,14 +2,7 @@
 
 #Run the setup script to check the directory and set the environment variables
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source ${__dir}/srun_setup.sh
+source ${__dir}/_setup.sh
 
-SUBMIT_SCRIPT="slurm/$DD_CLUSTER/submit.sh"
-#check if the submit script exists
-if [ ! -f "$SUBMIT_SCRIPT" ]; then
-  echo -e "${RED} Submit script $SUBMIT_SCRIPT does not exist ${NC}";
-  return 0 2> /dev/null || exit 0
-else
-    chmod +x $SUBMIT_SCRIPT #make the script executable
-    ./$SUBMIT_SCRIPT "$@" #run the submit script passing all arguments
-fi
+
+sbatch $DD_SLURM_ARGS --export=all "$@"

slurm/scripts/srun_check.sh

@@ -3,7 +3,7 @@
 
 #Run the setup script to check the directory and set the environment variables
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source ${__dir}/srun_setup.sh
+source ${__dir}/_setup.sh
 
 # Warn user that the directory must be relative to the singularity container
 echo "Warning: The data directory must be relative to the singularity container."
@@ -41,4 +41,4 @@ data_dir=$opt
 
 
 chmod +x slurm/jobscripts/check.sh #make the script executable
-srun --export=all -c 1 slurm/jobscripts/check.sh $data_dir
+srun $DD_SLURM_ARGS --export=all -c 1 slurm/jobscripts/check.sh $data_dir

slurm/scripts/srun_interactive.sh

@@ -2,9 +2,9 @@
 
 #Run the setup script to check the directory and set the environment variables
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source ${__dir}/srun_setup.sh
+source ${__dir}/_setup.sh
 
 echo "Running interactive singularity shell."
 chmod +x slurm/jobscripts/interactive.sh
 unset PROMPT_COMMAND
-srun --export=all  -c 1 --pty slurm/jobscripts/interactive.sh
+srun $DD_SLURM_ARGS --export=all  -c 1 --pty slurm/jobscripts/interactive.sh

slurm/scripts/srun_make.sh

@@ -15,7 +15,7 @@ n() {
 
 #Run the setup script to check the directory and set the environment variables
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source ${__dir}/srun_setup.sh
+source ${__dir}/_setup.sh
 
 
 #ask for challenge, system, and algorithm
@@ -32,10 +32,10 @@ select yn in "Y" "N"; do
            read -p "Challenge "fc","fts","fe"]: " challenge
            read -p "System: " system
            read -p "Algorithm: " algo
-           echo "Run $challenge $system $algo?";;
+           echo "Make $challenge $system $algo?";;
       *) echo "Invalid option. Try again.";continue;;
     esac
 done
 
 chmod +x slurm/jobscripts/make.sh #make the script executable
-srun --export=all -c 1 slurm/jobscripts/make.sh $challenge $system $algo
+srun $DD_SLURM_ARGS --export=all -c 1 slurm/jobscripts/make.sh $challenge $system $algo

slurm/scripts/srun_submit.sh

@@ -2,6 +2,7 @@
 
 #Run the setup script to check the directory and set the environment variables
 __dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-source ${__dir}/srun_setup.sh
+source ${__dir}/_setup.sh
 
-srun --export=all  -c 1 "$@"
+if 
+srun $DD_SLURM_ARGS --export=all  -c 1 "$@"

slurm/sherlock/env.sh

@@ -1,11 +1,17 @@
 #!/bin/bash
 
+# SHERLOCK SLURM ARGS
+PARTITION='-p normal'
+OUTPUT='-o ${SCRATCH}/logs/out/%A_%a.out'
+ERROR='-e ${SCRATCH}/logs/err/%A_%a.err'
+
 # SHERLOCK CLUSTER SETTINGS
 export DD_SINGULARITY_IMAGE_LOCATION=$GROUP_HOME/$USER/simg     #where you want to store the singularity image
 export DD_REPO_DIR=$HOME                                        #parent directory of where you cloned dynadojo
 export DD_SCRATCH_DIR=$SCRATCH                                  #your scratch directory  
 export DD_OUTPUT_DIR=sherput                                    #name of folder in scratch to put output
 export DD_IMAGE_REPO=docker://carynbear/dynadojo:sherlock       #docker image to pull
 export DD_CLUSTER=sherlock                                      #cluster name
+export DD_SLURM_ARGS="$PARTITION $OUTPUT $ERROR"
 
 echo "Setting up environment for $DD_CLUSTER"