diff --git a/README.md b/README.md index edafe3b..7690db9 100644 --- a/README.md +++ b/README.md @@ -31,14 +31,23 @@ Compute Canada does not allow any GUI (X) applications to be run, even in intera ## BIDS Apps: -Running bidsBatch lists the usage and displays bids-apps that are deployed on the system (contents of bids-apps.tsv). For default app options, please also see the bids-apps.tsv file. Note: when you run bidsBatch to process a dataset, a ```./jobs/``` folder is created in the current working directory. +The bidsBatch wrapper uses SLURM to parallelize over *subjects*, and by default will run a job for every subject in your participants.tsv file. +bidsBatch uses SLURM Arrays, so it groups all the different jobs under a single job ID, with each array job indicated as _. + +Running bidsBatch lists the usage and displays bids-apps that are deployed on the system (contents of bids-apps.tsv). For default app options, please also see the bids-apps.tsv file. Note: when you run bidsBatch to process a dataset, a ```jobs/``` folder is created in the output directory you specify. + ### Example: Running fmriprep on your bids dataset ``` -mkdir fmriprep_work -cd fmriprep_work -bidsBatch fmriprep 1.0.4 ~/my-bids-dataset ~/my-bids-dataset/derivatives/fmriprep-v1.0.4 participant +bidsBatch fmriprep_1.0.4 ~/my-bids-dataset ~/my-bids-dataset/derivatives/fmriprep-v1.0.4 participant +``` +To get usage/options for a particular bids app, leave out the bids-app required arguments, e.g.: + ``` +bidsBatch fmriprep_1.0.4 +``` + + ## Wrappers: diff --git a/bids-apps.tsv b/bids-apps.tsv index 31a0ccd..df2c4ef 100644 --- a/bids-apps.tsv +++ b/bids-apps.tsv @@ -1,15 +1,10 @@ -app_name tag repo base_options job_template -fmriprep 1.0.4 poldracklab --nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor -fmriprep 1.0.0 poldracklab --nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor -fmriprep 1.0.0-rc8 poldracklab --nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} -mriqc 0.10.0 poldracklab --no-sub --n_procs 8 --mem_gb 32000 -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} -niak latest bids --n_thread 8 -brainiak-srm latest bids -tracula v6.0.0-4 bids -prepdwi 0.0.3b khanlab 2core8gb16h -magetbrain latest bids -spm latest bids -ndmg v0.0.50 bids -cpac v1.0.2_disable_log_2 bids -mrtrix3_connectome 0.2.2 bids -qsm-sstv v0.0.1 khanlab --SS_TV_lagrange_parameter 0.4 --mag_match_pattern *part-mag_echo* --phase_match_pattern *part-phase_echo* --keep_unnecessary_outputs +name url base_options job_template +fmriprep_1.0.6 docker://poldracklab/fmriprep:1.0.6 --nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor +fmriprep_1.0.4 docker://poldracklab/fmriprep:1.0.4 --nthreads 8 --n_cpus 8 --omp-nthreads 8 --mem_mb 32000 --debug -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} --fs-license-file /project/6007967/akhanf/opt/freesurfer/.license --resource-monitor +mriqc_0.10.1 docker://poldracklab/mriqc:0.10.1 --no-sub --n_procs 8 --mem_gb 32000 -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} +mriqc_0.10.0 docker://poldracklab/mriqc:0.10.0 --no-sub --n_procs 8 --mem_gb 32000 -w /scratch/${USER}/${SLURM_JOB_ID}_${SLURM_ARRAY_TASK_ID} +prepdwi_0.0.4b shub://khanlab/prepdwi:0.0.4b --no-bedpost 2core8gb16h +prepdwi_0.0.4b_bedpost shub://khanlab/prepdwi:0.0.4b --n_cpus 32 Fat +prepdwi_0.0.3e file:///project/6007967/akhanf/singularity/bids-apps/khanlab_prepdwi_0.0.3e.img --no-bedpost 2core8gb16h +qsm_sstv_v0.0.1 shub://khanlab/qsm_sstv:v0.0.1 --SS_TV_lagrange_parameter 0.4 --mag_match_pattern *part-mag_echo* --phase_match_pattern *part-phase_echo* --keep_unnecessary_outputs +mp2rage_correction_v0.0.1 shub://khanlab/mp2rage_correction:v0.0.1 ShortSkinny diff --git a/bidsBatch b/bidsBatch index ff7d91f..a47678e 100755 --- a/bidsBatch +++ b/bidsBatch @@ -8,10 +8,10 @@ function usage { echo " Loops through an input subject_list_txt (txt with subj id on each line)" echo " Can be used to run scripts that generally take command-line parameters as:" echo "" - echo "Usage: $cmdname " + echo "Usage: $cmdname " echo "" echo "Available apps:" - cat $app_list | awk -F '\t' '{print $1 "\t" $2}' + cat $app_list | awk -F '\t' '{print $1 "\t" $2}' | column -t echo "" echo "optional flags:" echo "" @@ -33,6 +33,87 @@ function usage { } +function getContainer { + +app_name=$1 +bids_app_dir=$SINGULARITY_DIR/bids-apps + +#get matching line for app_name +linenum=`awk -F '\t' '{print $1}' $app_list | grep -n $app_name` +linenum=${linenum%%:*} + + +if [ ! -n "$linenum" ] +then + echo "${app_name} does not exist in bids-app list!" >&2 + exit 1 +fi + +url=`cat $app_list | head -n $linenum | tail -n 1 | awk -F '\t' '{print $2}'` +base_opts=`cat $app_list | head -n $linenum | tail -n 1 | awk -F '\t' '{print $3}'` +app_job_template=`cat $app_list | head -n $linenum | tail -n 1 | awk -F '\t' '{print $4}'` + +hub=${url%%:*} +if [ "$hub" = "docker" ] +then + #location for deployed docker images + repo=${url#*://} + repo=${repo%%/*} + + app=${url##*/} + app=${app%:*} + + tag=${url##*:} + + singularity_image=$bids_app_dir/${repo}_${app}_${tag}.img +if [ ! -e $singularity_image ] + then + echo "Singularity image: $singularity_image not found," >&2 + echo " run deploy_bids-apps first on your local machine" >&2 + exit 1 + fi + + +fi + +if [ "$hub" = "shub" ] +then + #use shub-cache to download if not exist, and return path to image + singularity_image=`shub-cache $url` + if [ ! "$?" = 0 -o ! -n "$singularity_image" -o ! -e "$singularity_image" ] + then + echo "Failed to pull $url, exiting" >&2 + exit 1 + fi + + +fi + +if [ "$hub" = "file" ] +then + singularity_image=${url##file://} + + if [ ! -e "$singularity_image" ] + then + echo "File $singularity_image does not exist, exiting" >&2 + exit 1 + fi + + +fi + + +if [ "$override_template" = "0" -a -n "$app_job_template" ] +then + job_template=$app_job_template + echo "Using job-template specified in bids-apps.tsv, $app_job_template" >&2 +fi + + + + +} + execpath=`dirname $0` execpath=`realpath $execpath` @@ -83,57 +164,60 @@ done shift $((OPTIND-1)) -if [ "$#" -lt 5 ] +if [ "$#" -lt 1 ] then usage exit 1 fi -app_name=$1 -tag=$2 -bids_dir=`realpath $3` -out_dir=$4 -level=$5 +app_name=$1 +shift 1 -mkdir -p $out_dir -out_dir=`realpath $out_dir` - -shift 5 -options=$@ +getContainer $app_name -bids_app_dir=$SINGULARITY_DIR/bids-apps -repo=`grep "${app_name} ${tag} " $app_list | awk -F '\t' '{print $3}'` - -if [ ! -n "$repo" ] +if [ "$#" -lt 3 ] then - echo "${app_name} ${tag} does not exist in bids-app list!" >&2 + usage + echo "Usage for bids app:" >&2 + if [ -e $singularity_image ] + then + singularity run $singularity_image + fi + exit 1 fi -base_opts=`grep "${app_name} ${tag} " $app_list | awk -F '\t' '{print $4}'` -app_job_template=`grep "${app_name} ${tag} " $app_list | awk -F '\t' '{print $5}'` +bids_dir=`realpath $1` +out_dir=$2 +level=$3 -if [ "$override_template" = "0" -a -n "$app_job_template" ] +#check validity of participant +level_part=${level:0:5} +if [ "$level_part" != "group" -a "$level_part" != "parti" ] then - job_template=$app_job_template - echo "Using job-template specified in bids-apps.tsv, $app_job_template" >&2 + echo "Required argument 3 to bids apps must be analysis level (participant/group)" >&2 + exit 1 fi +mkdir -p $out_dir +out_dir=`realpath $out_dir` +shift 3 +options=$@ -singularity_image=$bids_app_dir/${repo}_${app_name}_${tag}.img +#check if participant_label is used as an option +#and quit if it is found -if [ ! -e $singularity_image ] - then - echo "Singularity image: $singularity_image not found," >&2 - echo " run deploy_bids-apps first on your local machine" >&2 - exit 1 - fi +if [[ $options = *--participant_label* ]] +then + echo "ERROR: --participant_label should not be used with bidsBatch, use: bidsBatch -s instead" >&2 + exit 1 +fi @@ -205,7 +289,7 @@ cp $execpath/job-templates/${job_template}.job $job echo "#SBATCH --job-name=$app_name" >> $job - +#FIX THIS LATER TO ALLOW FOR participant1 etc.., group1, group2 ... if [ "$level" == "participant" ] then #submit with array, using subj and subjlist, indexed by line number (1-N) @@ -216,12 +300,12 @@ then #strip off sub- from sub-LABEL echo "subj=\${subj##sub-}" >> $job echo "echo SINGULARITY_IMAGE: $singularity_image" >> $job - echo "echo CMD: --participant_label \$subj $base_opts $options $bids_dir $out_dir $level" >>$job + echo "echo CMD: $bids_dir $out_dir $level --participant_label \$subj $base_opts $options" >>$job echo "echo START_TIME: \`date\`" >>$job echo "SCRATCH_DIR=/scratch/${USER}/\${SLURM_ARRAY_JOB_ID}_\${SLURM_ARRAY_TASK_ID}" >> $job echo "mkdir -p \$SCRATCH_DIR" >> $job echo "cd `pwd`" >> $job - echo singularity run $SINGULARITY_OPTS $singularity_image --participant_label \$subj $base_opts $options $bids_dir $out_dir $level >> $job + echo singularity run $SINGULARITY_OPTS $singularity_image $bids_dir $out_dir $level --participant_label \$subj $base_opts $options >> $job echo "RETURNVAL=\$?" >> $job echo "rm -rf \$SCRATCH_DIR" >> $job echo "echo RETURNVAL=\$RETURNVAL" >>$job @@ -233,12 +317,12 @@ else #submit without arrays, for group level echo "#SBATCH --output=$job_dir/${app_name}.$level.%A.out" >> $job echo "echo SINGULARITY_IMAGE: $singularity_image" >> $job - echo "echo CMD: $base_opts $options $bids_dir $out_dir $level" >>$job + echo "echo CMD: $bids_dir $out_dir $level $base_opts $options" >>$job echo "echo START_TIME: \`date\`" >>$job echo "export SCRATCH_DIR=/scratch/${USER}/\${SLURM_JOB_ID}" >> $job echo "mkdir -p \$SCRATCH_DIR" >> $job echo "cd `pwd`" >> $job - echo singularity run $SINGULARITY_OPTS $singularity_image $base_opts $options $bids_dir $out_dir $level >> $job + echo singularity run $SINGULARITY_OPTS $singularity_image $bids_dir $out_dir $level $base_opts $options >> $job echo "RETURNVAL=\$?" >> $job echo "rm -rf \$SCRATCH_DIR" >> $job echo "echo RETURNVAL=\$RETURNVAL" >>$job diff --git a/deploy_bids-apps b/deploy_bids-apps index dbe41e7..dae7629 100755 --- a/deploy_bids-apps +++ b/deploy_bids-apps @@ -11,13 +11,24 @@ Nlines=`cat $app_list | wc -l` for i in `seq 2 $Nlines` do - app=`cat $app_list | head -n $i | tail -n 1 | awk -F '\t' '{print $1}'` - tag=`cat $app_list | head -n $i | tail -n 1 | awk -F '\t' '{print $2}'` - repo=`cat $app_list | head -n $i | tail -n 1 | awk -F '\t' '{print $3}'` - base_opts=`cat $app_list | head -n $i | tail -n 1 | awk -F '\t' '{print $4}'` - docker_name=${repo}/${app}:${tag} - out_name=${repo}_${app}_${tag}.img - out_img=$remote_dir/$out_name + appname=`cat $app_list | head -n $i | tail -n 1 | awk -F '\t' '{print $1}'` + url=`cat $app_list | head -n $i | tail -n 1 | awk -F '\t' '{print $2}'` + + + hub=${url%%:*} + if [ "$hub" = "docker" ] + then + docker_name=${url#docker://} + + repo=${url#*://} + repo=${repo%%/*} + + app=${url##*/} + app=${app%:*} + + tag=${url##*:} + + out_img=$remote_dir/${repo}_${app}_${tag}.img if [ ! -e $out_img ] then @@ -37,4 +48,5 @@ do echo Skipping $out_img, already exists ... fi + fi done