From f0bfd544419ce982a591add10ec194e8d265d83b Mon Sep 17 00:00:00 2001 From: z0on Date: Fri, 18 Feb 2022 13:55:50 -0600 Subject: [PATCH] ls6_launcher added --- automatic_RAD_pipeline.sh | 32 +-- ls6_launcher_creator.py | 461 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 477 insertions(+), 16 deletions(-) create mode 100755 ls6_launcher_creator.py diff --git a/automatic_RAD_pipeline.sh b/automatic_RAD_pipeline.sh index ee7d316..c70f943 100644 --- a/automatic_RAD_pipeline.sh +++ b/automatic_RAD_pipeline.sh @@ -92,7 +92,7 @@ $HOME/edirect/esearch -db sra -query $BioProject | efetch --format runinfo |cut for A in `cat $BioProject.SRR`;do echo "fastq-dump-orig.2.10.5 $A">>gets; done -ls5_launcher_creator.py -j gets -n gets -a tagmap -e matz@utexas.edu -t 12:00:00 -w 24 -q normal +s2_launcher_creator.py -j gets -n gets -a tagmap -e matz@utexas.edu -t 12:00:00 -w 24 -q normal getsjob=$(sbatch gets.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') Q= -log10(Perror)*10 @@ -120,25 +120,25 @@ export GENOME_REF=all_cc.fasta for file in *.fastq; do echo "cutadapt --format fastq -q 15,15 -a AGATCGGA -m $TagLen -l $TagLen -o ${file/.fastq/}.trim0 $file > ${file}_trimlog.txt && head -12000000 ${file/.fastq/}.trim0 > ${file/.fastq/}.trim && rm ${file/.fastq/}.trim0" >> trim; done -ls5_launcher_creator.py -j trim -n trim -a tagmap -e matz@utexas.edu -t 1:00:00 -w 48 -q normal +s2_launcher_creator.py -j trim -n trim -a tagmap -e matz@utexas.edu -t 1:00:00 -w 48 -q normal trimjob=$(sbatch trim.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # converting fastq to fasta (for kmer analysis) >f2f for F in `ls *fastq`; do echo "paste - - - - < ${F/.fastq/}.trim | cut -f 1,2 | sed 's/^@/>/' | tr \"\t\" \"\n\" > ${F/.fastq/}.fasta" >>f2f ;done -ls5_launcher_creator.py -j f2f -n f2f -a tagmap -e matz@utexas.edu -t 0:05:00 -w 48 -q normal +s2_launcher_creator.py -j f2f -n f2f -a tagmap -e matz@utexas.edu -t 0:05:00 -w 48 -q normal f2fjob=$(sbatch --dependency=afterok:$trimjob f2f.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # analyzing kmers, removing singletons from each file (kmerer.pl) >jelly for F in `ls *fastq`; do echo "jellyfish count -m $TagLen -s 100M -t 1 -C ${F/.fastq/}.fasta -o ${F/.fastq/}.jf && jellyfish dump ${F/.fastq/}.jf | kmerer.pl - 2 > ${F/.fastq/}.kmers.fa">>jelly;done -ls5_launcher_creator.py -j jelly -n jelly -a tagmap -e matz@utexas.edu -t 0:10:00 -w 24 -q normal +s2_launcher_creator.py -j jelly -n jelly -a tagmap -e matz@utexas.edu -t 0:10:00 -w 24 -q normal jellyjob=$(sbatch --dependency=afterok:$f2fjob jelly.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # merging kmer lists and filtering kmers aiming to get major alleles # clustering, concatenating into fake genome (10 chromosomes), and indexing it echo "ls *kmers.fa > all.kf && mergeKmers.pl all.kf minDP=10 minInd=5 > kmers.tab && cat kmers.tab | shuf | awk '{print \">\"\$1\"\n\"\$2}' > all.fasta && cd-hit-est -i all.fasta -o all.clust -aL 1 -aS 1 -g 1 -c $MatchFrac -M 0 -T 0 && concatFasta.pl fasta=all.clust num=10 && bowtie2-build all_cc.fasta all_cc.fasta && samtools faidx all_cc.fasta" >mk -ls5_launcher_creator.py -j mk -n mk -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j mk -n mk -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal mergejob=$(sbatch --dependency=afterok:$jellyjob mk.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') #mergejob=$(sbatch mk.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') @@ -148,7 +148,7 @@ for F in `ls *.fastq`; do REF=all_cc.fasta echo "bowtie2 --no-unal -x $REF -U ${F/.fastq/}.trim -S ${F/.fastq/}.sam && samtools sort -O bam -o ${F/.fastq/}.bam ${F/.fastq/}.sam && samtools index ${F/.fastq/}.bam">>maps done -ls5_launcher_creator.py -j maps -n maps -a tagmap -e matz@utexas.edu -t 2:00:00 -w 24 -q normal +s2_launcher_creator.py -j maps -n maps -a tagmap -e matz@utexas.edu -t 2:00:00 -w 24 -q normal mapsjob=$(sbatch --dependency=afterok:$mergejob maps.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # quality assessment, removing bams with log(coverage)<3SD @@ -158,7 +158,7 @@ FILTERSQ='-uniqueOnly 1 -remove_bads 1 -minMapQ 20 -minInd $MI' TODOQ="-doQsDist 1 -doDepth 1 -doCounts 1 -dumpCounts 2" echo 'export NIND=`cat bams | wc -l`; export MI=`echo "($NIND*$MinIndPerc+0.5)/1" | bc`' >calc1 echo "ls *.bam > bams && source calc1 && angsd -b bams -r chr1:1-1000000 -GL 1 $FILTERSQ $TODOQ -P 12 -out dd && Rscript ~/bin/plotQC.R prefix=dd">a0 -ls5_launcher_creator.py -j a0 -n a0 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j a0 -n a0 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal qjob=$(sbatch --dependency=afterok:$mapsjob a0.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') #------------ examine dd.pdf, decide on MinIndPerc and whether bams.qc is reasonable @@ -173,7 +173,7 @@ FILTERS0='-minInd $MI -uniqueOnly 1 -remove_bads 1 -minMapQ 20 -minQ 20 -snp_pva TODO0='-doMajorMinor 1 -doMaf 1 -doCounts 1 -makeMatrix 1 -doIBS 1 -doCov 1 -doPost 1 -doGlf 2' echo 'export NIND=`cat bams.qc | wc -l`; export MI=`echo "($NIND*$MinIndPerc+0.5)/1" | bc`' >calc1 echo "source calc1 && angsd -b bams.qc -GL 1 $FILTERS0 $TODO0 -P 12 -out myresult && Rscript ~/bin/detect_clones.R bams.qc myresult.ibsMat 0.15">a1 -ls5_launcher_creator.py -j a1 -n a1 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j a1 -n a1 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal # a1job=$(sbatch --dependency=afterok:$qjob a1.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') a1job=$(sbatch a1.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') @@ -187,7 +187,7 @@ FILTERS1='-minInd $MI2 -uniqueOnly 1 -remove_bads 1 -minMapQ 20 -minQ 20 -snp_pv TODO1='-doMajorMinor 1 -doMaf 1 -doCounts 1 -makeMatrix 1 -doIBS 1 -doCov 1 -doPost 1 -doGlf 2' echo 'cat bams.nr | sort > bams.NR && mv bams.NR bams.nr && export NIND2=`cat bams.nr | wc -l`; export MI2=`echo "($NIND2*$MinIndPerc+0.5)/1" | bc`; export MAF=`echo "3/(2*$NIND2)" | bc -l`' >calc2 echo "source calc2 && angsd -b bams.nr -GL 1 $FILTERS1 $TODO1 -P 12 -out myresult2 && Rscript ~/bin/pcaStructure.R myresult2.ibsMat">a2 -ls5_launcher_creator.py -j a2 -n a2 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 +s2_launcher_creator.py -j a2 -n a2 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal a2job=$(sbatch --dependency=afterok:$a1job a2.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') #a2job=$(sbatch a2.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') @@ -196,7 +196,7 @@ a2job=$(sbatch --dependency=afterok:$a1job a2.slurm | grep "Submitted batch job" module load python2 echo 'python ~/pcangsd/pcangsd.py -beagle myresult2.beagle.gz -admix -o pcangsd -inbreed 2 -kinship -selection -threads 12' >adm -ls5_launcher_creator.py -j adm -n adm -a tagmap -e matz@utexas.edu -t 0:10:00 -w 1 -q normal +s2_launcher_creator.py -j adm -n adm -a tagmap -e matz@utexas.edu -t 0:10:00 -w 1 -q normal admjob=$(sbatch --dependency=afterok:$a2job adm.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # ------- genetic diversity stats based on all well-genotyped sites (including invariants) @@ -209,21 +209,21 @@ FILTERS='-minInd $MI2 -uniqueOnly 1 -skipTriallelic 1 -minMapQ 20 -minQ 20 -doHW TODO="-doSaf 1 -anc $REF -ref $GENOME_REF -doMajorMinor 1 -doMaf 1 -dosnpstat 1 -doPost 1 -doGlf 2" echo 'export NIND2=`cat bams.nr | wc -l`; export MI2=`echo "($NIND2*$MinIndPerc+0.5)/1" | bc`' >calc2 echo "source calc2 && angsd -b bams.nr -r chr5 -GL 1 -P 12 $FILTERS $TODO -out chr5 && realSFS chr5.saf.idx -P 12 -fold 1 > chr5.sfs && realSFS saf2theta chr5.saf.idx -outname chr5 -sfs chr5.sfs -fold 1 && thetaStat do_stat chr5.thetas.idx -outnames chr5 && grep \"chr\" chr5.pestPG | awk '{ print \$4/\$14}' >piPerChrom">sfsj -ls5_launcher_creator.py -j sfsj -n sfsj -t 2:00:00 -e matz@utexas.edu -w 1 -a tagmap -q normal +s2_launcher_creator.py -j sfsj -n sfsj -t 2:00:00 -e matz@utexas.edu -w 1 -a tagmap -q normal sfsjob=$(sbatch --dependency=afterok:$a1job sfsj.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') #sfsjob=$(sbatch sfsj.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # individual heterozygosities (proportion of heterozygotes across SNPs that pass sfsjob filters) echo "Rscript ~/bin/heterozygosity_beagle.R chr5.beagle.gz >indHets" >bg -ls5_launcher_creator.py -j bg -n bg -a tagmap -e matz@utexas.edu -t 12:00:00 -w 1 -q normal +s2_launcher_creator.py -j bg -n bg -a tagmap -e matz@utexas.edu -t 12:00:00 -w 1 -q normal sbatch --dependency=afterok:$sfsjob bg.slurm # heterozygosity_beagle.R script (by Nathaniel Pope) outputs *_zygosity.RData R data bundle containing AFS (rows) for each individual (columns). The proportion of heterozygotes is the second row. Individual heterozygosities are also printed out to STDOUT (in the case above, saved to text file indHets) # relatedness with NgsRelate echo 'export NIND2=`cat bams.nr | wc -l`; export NS=``zcat g3.mafs.gz | wc -l`' >calc3 echo 'source calc3 && zcat g3.mafs.gz | cut -f5 |sed 1d >freq && ngsRelate -g g3.glf.gz -n $NIND -f freq >g3.relatedness' >rel -ls5_launcher_creator.py -j rel -n rel -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j rel -n rel -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal reljob=$(sbatch rel.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') @@ -243,13 +243,13 @@ FILTERS1='-minInd $MI2 -uniqueOnly 1 -remove_bads 1 -minMapQ 20 -minQ 20 -snp_pv TODO1='-doMajorMinor 1 -doMaf 1 -doCounts 1 -doPost 1 -doGlf 3' echo 'cat bams.nr | sort > bams.NR && mv bams.NR bams.nr && export NIND2=`cat bams.nr | wc -l`; export MI2=`echo "($NIND2*$MinIndPerc+0.5)/1" | bc`' >calc2 echo "source calc2 && angsd -b bams.nr -GL 1 $FILTERS1 $TODO1 -P 12 -out g3">g3 -ls5_launcher_creator.py -j g3 -n g3 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j g3 -n g3 -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal g3job=$(sbatch g3.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') # individual inbreeding coeffs echo 'zcat g3.glf.gz | ngsF --glf - --n_ind 44 --n_sites 909052 --out inbr' >nf -ls5_launcher_creator.py -j nf -n nf -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j nf -n nf -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal ngsFjob=$(sbatch --dependency=afterok:$g3job nf.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') #ngsFjob=$(sbatch nf.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') @@ -258,7 +258,7 @@ zcat g3.mafs.gz | cut -f5 |sed 1d >freq # relatedness with NgsRelate echo 'export NIND2=`cat bams.nr | wc -l`; export NS=``zcat g3.mafs.gz | wc -l`' >calc3 echo 'source calc3 && zcat g3.mafs.gz | cut -f5 |sed 1d >freq && ngsRelate -g g3.glf.gz -n $NIND -f freq >g3.relatedness' >rel -ls5_launcher_creator.py -j rel -n rel -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal +s2_launcher_creator.py -j rel -n rel -a tagmap -e matz@utexas.edu -t 2:00:00 -w 1 -q normal reljob=$(sbatch rel.slurm | grep "Submitted batch job" | perl -pe 's/\D//g') diff --git a/ls6_launcher_creator.py b/ls6_launcher_creator.py new file mode 100755 index 0000000..88a0d5b --- /dev/null +++ b/ls6_launcher_creator.py @@ -0,0 +1,461 @@ +#!/usr/bin/env python3 + +# Create TACC launchers for Stampede and Lonestar, and just run commands otherwise. +# Depending on the host, launcher_creator.py generated a SLURM or SGE launcher as appropriate. +# +# Copyright 2013, Regents Of The University Of Texas At Austin +# Originally written by Scott Hunicke-Smith +# New version with SLURM support by Benjamin Goetz + +try: + import argparse + import subprocess +except: + print ('Try typing "module load python" and then running this again.') + import sys + sys.exit(1) + +import os +import sys +import re + +lonestar_parameters='''#!/bin/bash +# +# Simple SGE script for submitting multiple serial +# jobs (e.g. parametric studies) using a script wrapper +# to launch the jobs. +# +# To use, build the launcher executable and your +# serial application(s) and place them in your WORKDIR +# directory. Then, edit the CONTROL_FILE to specify +# each executable per process. +#------------------------------------------------------- +#------------------------------------------------------- +# +# <------ Setup Parameters ------> +# +#$ -N {name} +#$ -pe {wayness}way {total_threads} +#$ -q {queue} +#$ -o {name}.o$JOB_ID +#$ -l h_rt={time} +#$ -V +{email_line}{hold_line}#$ -cwd +{allocation_line} +''' + +stampede_parameters='''#!/bin/bash +#SBATCH -J {name} +#SBATCH -n {num_jobs} +{num_nodes_line}#SBATCH -p {queue} +#SBATCH -o {name}.o%j +#SBATCH -e {name}.e%j +#SBATCH -t {time} +{allocation_line} +{email_line}''' + +bash_and_modules=''' +module load launcher +{modules} + +{bash_commands} +''' + +parametric_job_submission=''' +export EXECUTABLE=$LAUNCHER_DIR/init_launcher +export CONTROL_FILE={control_file} +$LAUNCHER_DIR/paramrun $EXECUTABLE $CONTROL_FILE + +''' + +parametric_job_submission_ls6=''' + +export LAUNCHER_PLUGIN_DIR=$LAUNCHER_DIR/plugins +export LAUNCHER_RMI=SLURM +export LAUNCHER_JOB_FILE={jobfile} +export OMP_NUM_THREADS=64 + +$LAUNCHER_DIR/paramrun +''' + +def file_len(fname): + f = open(fname) + contents = f.readlines() + # Only count non-empty lines and count the + # last line even if it has no carriage return + i = 0 + for line in contents: + stripped_line = line.strip() + #comment lines + if re.match('^\s*#', stripped_line): + continue + if not stripped_line: + continue + else: + i+=1 + f.close() + return i + + +def generate_lonestar_launcher(results, launcher_file): + """ + Write a Lonestar launcher SGE script. + """ + + # Must account (on Lonestar6) for normal having 128 threads per node; + # if results.queue == 'largemem': + # threads_per_node = 24 + # else: + threads_per_node = 128 + + if not results.wayness: + results.wayness = 128 + + if results.allocation: + allocation_line = '#$ -A {}'.format(results.allocation) + else: + allocation_line = '' + + if results.email: + email_line = '#$ -M {}\n#$ -m be\n'.format(results.email) + else: + email_line = '' + + if results.hold and not results.submit_cmd: + hold_line='#$ -hold_jid {}\n'.format(results.hold) + else: + hold_line='' + + if not results.num_nodes: + if results.job: + num_nodes = file_len(results.job) + else: + num_nodes = 1 + + if num_nodes == 0: + print ('Your job file appears to be empty. Please fix this error.') + return + + while num_nodes % results.wayness != 0: + num_nodes += 1 + total_threads = num_nodes * threads_per_node / results.wayness + else: + num_nodes = results.num_nodes + total_threads = results.num_nodes * threads_per_node + + if not results.submit_cmd: + print ('Job file has %i lines.' % num_nodes) + + if not results.submit_cmd: + print ('Using {} nodes.'.format(num_nodes)) + print ('Writing to {}.'.format(results.launcher)) + + launcher_file.write(lonestar_parameters.format( + name=results.name, + wayness=results.wayness, + total_threads=total_threads, + walrus_total_threads=total_threads, + queue=results.queue, + time=results.time, + email_line=email_line, + hold_line=hold_line, + allocation_line=allocation_line)) + + launcher_file.write(bash_and_modules.format(modules=results.modules, + bash_commands=results.bash_commands)) + + if results.job: + launcher_file.write(parametric_job_submission.format(control_file=results.job)) + + +def generate_stampede_launcher(results, launcher_file): + """ + Write a Stampede launcher SLURM script. + """ + + # Must account (on Stampede) for normal having 16 threads per node + if results.queue == 'largemem': + threads_per_node = 32 + else: + threads_per_node = 16 + + num_nodes_line = '' + # if not results.wayness: + # results.wayness = 16 + # num_cores_line = '' + + if results.allocation: + allocation_line = '#SBATCH -A {}'.format(results.allocation) + else: + allocation_line = '' + + if results.email: + email_line = '#SBATCH --mail-type=ALL\n#SBATCH --mail-user={}\n'.format(results.email) + else: + email_line = '' + + if not results.num_nodes: + # If a number of nodes is not explicitly specified, calculate based on number of commands in jobs file, and optional wayness. + if results.job: + num_jobs = file_len(results.job) + if num_jobs == 0: + print ('Your job file appears to be empty. Please fix this error.') + return + else: + num_jobs = 1 + + if results.wayness: + num_nodes, remainder = divmod(num_jobs, results.wayness) + if remainder != 0: + num_nodes += 1 + num_nodes_line = '#SBATCH -N {}\n'.format(num_nodes) + else: + num_nodes, remainder = divmod(num_jobs, 16) + if remainder != 0: + num_nodes += 1 + else: + # A number of nodes has been specified. Default wayness is 16 on Stampede. Use specified wayness to calculate total tasks. + num_nodes = results.num_nodes + num_nodes_line = '#SBATCH -N {}\n'.format(num_nodes) + + if results.wayness: + num_jobs = num_nodes * results.wayness + else: + num_jobs = num_nodes * 16 + + if not results.submit_cmd: + print ('Using {} nodes.'.format(num_nodes)) + print ('Writing to {}.'.format(results.launcher)) + + launcher_file.write(stampede_parameters.format( + name=results.name, + num_jobs=num_jobs, + num_nodes_line=num_nodes_line, + queue=results.queue, + time=results.time, + email_line=email_line, + allocation_line=allocation_line)) + + launcher_file.write(bash_and_modules.format(modules=results.modules, + bash_commands=results.bash_commands)) + + if results.job: + launcher_file.write(parametric_job_submission.format(control_file=results.job)) + + +def generate_ls6_launcher(results, launcher_file): + """ + Write a Lonestar 6 launcher SLURM script. + """ + + # Must account (on Lonestar 6) for normal having 128 threads per node +# if results.queue == 'largemem': +# threads_per_node = 32 +# elif results.queue == 'hugemem': +# threads_per_node = 20 +# else: + threads_per_node = 128 + + num_nodes_line = '' + if not results.wayness: + results.wayness = 128 + num_cores_line = '' + + if results.allocation: + allocation_line = '#SBATCH -A {}'.format(results.allocation) + else: + allocation_line = '' + + if results.email: + email_line = '#SBATCH --mail-type=ALL\n#SBATCH --mail-user={}\n'.format(results.email) + else: + email_line = '' + + if not results.num_nodes: + # If a number of nodes is not explicitly specified, calculate based on number of commands in jobs file, and optional wayness. + if results.job: + num_jobs = file_len(results.job) + if num_jobs == 0: + print ('Your job file appears to be empty. Please fix this error.') + return + else: + num_jobs = 1 + + if results.wayness: + num_nodes, remainder = divmod(num_jobs, results.wayness) + if remainder != 0: + num_nodes += 1 + num_nodes_line = '#SBATCH -N {}\n'.format(num_nodes) + else: + num_nodes, remainder = divmod(num_jobs, 24) + if remainder != 0: + num_nodes += 1 + else: + # A number of nodes has been specified. Default wayness is 16 on Stampede. Use specified wayness to calculate total tasks. + num_nodes = results.num_nodes + num_nodes_line = '#SBATCH -N {}\n'.format(num_nodes) + + if results.wayness: + num_jobs = num_nodes * results.wayness + else: + num_jobs = num_nodes * 24 + + if not results.submit_cmd: + print ('Using {} nodes.'.format(num_nodes)) + print ('Writing to {}.'.format(results.launcher)) + + launcher_file.write(stampede_parameters.format( + name=results.name, + num_jobs=num_jobs, + num_nodes_line=num_nodes_line, + queue=results.queue, + time=results.time, + email_line=email_line, + allocation_line=allocation_line)) + + launcher_file.write(bash_and_modules.format(modules=results.modules, + bash_commands=results.bash_commands)) + + if results.job: + launcher_file.write(parametric_job_submission_ls6.format(jobfile=results.job)) + + +def check_time_and_queue(results): + host = subprocess.check_output(["hostname", "-f"]).split(b'.')[1] + queues = {'ls4':['normal','development','largemem','gpu','vis','serial'], + 'stampede':['normal','development','largemem','serial','large','request','normal-mic','gpu','gpudev','vis']} + + if host in queues and results.queue not in queues[host]: + sys.exit( 'Your queue selection "' + results.queue + '" is not available.' + '\n' + + 'Available queues: ' + ', '.join(queues[host])) + + if results.time == None: + parser.print_help() + sys.exit('You did not give a job time (-t hh:mm:ss).') + + time_match = re.match(r'(\d?\d):(\d\d):(\d\d)', results.time) + if not time_match: + sys.exit( 'Your time argument -t ' + results.time + ' is not in the right format.' + '\n' + + 'Please change it to match (-t hh:mm:ss).' ) + + hh = int(time_match.group(1)) + mm = int(time_match.group(2)) + ss = int(time_match.group(3)) + + if hh == 0 and mm == 0 and ss == 0: sys.exit( 'Please enter a non-zero time value.' ) + if mm > 59 or ss > 59: sys.exit( results.time + ' is not a valid time.' ) + + if host == "ls4" and results.queue == 'development': time_limit = 1 + elif host == "ls4" and results.queue == 'serial': time_limit = 12 + elif host == "stampede" and results.queue == 'development': time_limit = 2 + elif host == "stampede" and \ + results.queue in ['gpudev','visdev']: time_limit = 4 + elif host == "stampede" and results.queue == 'vis': time_limit = 8 + elif host == "stampede" and results.queue == 'serial': time_limit = 12 + elif host == "stampede" and \ + results.queue in ['normal','largemem','normal-mic']: time_limit = 48 + elif host == "ls5" and results.queue == 'development': time_limit = 2 + elif host == "ls5" and results.queue == 'gpu': time_limit = 24 + elif host == "ls5" and results.queue == 'vis': time_limit = 8 + elif host == "ls5" and \ + results.queue in ['normal','largemem','hugemem']: time_limit = 48 + else: time_limit = 24 + + if (hh > time_limit) or (hh == time_limit and (mm > 0 or ss > 0)): + sys.exit( 'Your time argument -t ' + results.time + ' exceeds the ' + + str(time_limit) + ' hour limit for the ' + results.queue + ' queue' + '\n' + + 'on this machine. Please change queue or reduce requested time.' ) + + +class ArgumentParserDefaultHelp(argparse.ArgumentParser): + '''Simple variation on ArgumentParser. When an error is thrown, this inherited class prints the entire help message.''' + + def error(self, message): + self.print_help() + sys.stderr.write('\nerror: {}\n\n'.format(message)) + sys.exit(2) + + +def main(): + # Get environment variables, os.environ.get returns 'none' if the environment variable isn't defined + env_allocation = os.environ.get('ALLOCATION') + env_email_address = os.environ.get('EMAIL_ADDRESS') + + parser = ArgumentParserDefaultHelp(description='''Create TACC launchers for Stampede and Lonestar, and just run commands otherwise. +Depending on the host, launcher_creator.py generates a SLURM or SGE launcher as appropriate. +Report problems to rt-other@ccbb.utexas.edu''') + required = parser.add_argument_group('Required') + required.add_argument('-n', action='store', dest='name', required=True, help='The name of your job.') + required.add_argument('-t', action='store', dest='time', required=True, help='The time you want to give to your job. Format: hh:mm:ss') + commands = parser.add_argument_group('Commands', 'You must use at least one of these options to submit your commands for TACC.') + commands.add_argument('-j', action='store', dest='job', help='The name of the job file containing your commands.') + commands.add_argument('-b', action='store', dest='bash_commands', default='', help='A string of Bash commands that are executed before the parametric jobs are launched.') + optional = parser.add_argument_group('Optional') + optional.add_argument('-q', action='store', dest='queue', default='development', help='The TACC allocation for job submission. Default="development"') + optional.add_argument('-a', '-A', action='store', dest='allocation', nargs='?', default=env_allocation, help='The TACC allocation for job submission. You can set a default ALLOCATION environment variable.') + optional.add_argument('-m', action='store', dest='modules', default='', help='A list of module commands. The "launcher" module is always automatically included. Example: -m "module swap intel gcc; module load bedtools"') + optional.add_argument('-w', action='store', dest='wayness', type=int, help='Wayness: the number of commands you want to give each node. Default= 12 on Lonestar, 16 on Stampede.') + optional.add_argument('-N', action='store', dest='num_nodes', type=int, help='Number of nodes to request. You probably don\'t need this option. You ONLY need it if you want to run a job list that isn\'t defined at the time you submit the launcher.') + optional.add_argument('-e', action='store', dest='email', nargs='?', const=env_email_address, help='Your email address if you want to receive an email from Lonestar when your job starts and ends. Without an argument, it will use a default EMAIL_ADDRESS environment variable.') + optional.add_argument('-l', action='store', dest='launcher', help='The name of the *.sge launcher script that will be created. Default=".sge"') + optional.add_argument('-s', dest='submit_cmd', action='store_true', help='Echoes the launcher filename to stdout.') + deprecated = parser.add_argument_group('Deprecated', 'Rewrite any scripts that use this.') + deprecated.add_argument('-H', action='store', dest='hold', help='Hold job in queue until the job specified in this option finishes.') + + results = parser.parse_args() + + # Handle allocation defaults + # if not results.allocation: + # results.allocation = os.environ.get('ALLOCATION') + + # Check for required parameters + # if results.name == None: + # print 'You did not give a job name (-n ).' + # parser.print_help() + # return + + # Check for valid time and queue entries + check_time_and_queue(results) + + # Check for valid email address, if email is given + if results.email: + if not re.match(r'^.+@.+\..{2,4}$', results.email): + print ('Not a valid email address, please check it.') + return + + if not (results.job or results.bash_commands): + if os.path.isfile('commands'): + results.job = 'commands' + else: + print ('-== Neither a job file nor bash commands given. You need at least one to do anything! ==-') + return + + if not results.submit_cmd: + print ('Project {}.'.format(results.name)) + print ('Using job file {}.'.format(results.job)) + print ('Using {} queue.'.format(results.queue)) + print ('For {} time.'.format(results.time)) + print ('Using {} allocation.'.format(results.allocation)) + + # if (results.hold != None) and not results.submit_cmd: + # print 'Holding until {} finishes.'.format(results.hold) + + if not results.submit_cmd: + if results.email != None: + print ('Sending start/stop email to {}.'.format(results.email)) + else: + print ('Not sending start/stop email.') + + + host = subprocess.check_output(["hostname", "-f"]).split(b'.')[1] + + if not results.launcher: + results.launcher = '{}.slurm'.format(results.name) + + + launcher_file = open(results.launcher, 'w') + generate_ls6_launcher(results, launcher_file) + launcher_file.close() + +if __name__ == '__main__': + main()