Skip to content

Commit

Permalink
Merge pull request #1635 from emkemp/fix/lisf76_scripts
Browse files Browse the repository at this point in the history
Added FOC files and script updates
  • Loading branch information
jvgeiger authored Nov 8, 2024
2 parents 5bc25f8 + 131b787 commit 966f63b
Show file tree
Hide file tree
Showing 356 changed files with 91,683 additions and 800 deletions.
Original file line number Diff line number Diff line change
@@ -1,35 +1,33 @@
#!/bin/sh
#SBATCH --job-name=smapeopl
#SBATCH --time=0:20:00
#SBATCH --account s1189
#SBATCH --time=0:30:00
#SBATCH --account=NWP601
#SBATCH --output smapeopl.slurm.out
#SBATCH --ntasks=13 --ntasks-per-node=1
#SBATCH --mail-type=ALL
#SBATCH --qos=debug
#--------------------------------------------------------------------------
#SBATCH --ntasks=13 --ntasks-per-node=13
#SBATCH --cluster-constraint=blue
#SBATCH --exclusive
#SBATCH --mem=0
#------------------------------------------------------------------------------
#
# SCRIPT: run_smap_e_opl_discover.sh
# SCRIPT: run_ldt_noah39_smapeopl_hpc11.sh
#
# Batch script for running LDT to generate SMAP_E_OPL retrievals with
# LDT. Customized for NASA Discover supercomputer running SLURM batch
# queueing system.
# Batch script for running LDT to generate Noah39-based SMAP_E_OPL retrievals
# with LDT. Customized for USAF HPC11 supercomputer running SLURM batch
# queueing system. NOTE: Internal paths must be customized before use.
#
# USAGE: run_smap_e_opl_discover.sh $startdate $starthour $enddate \
# $endhour $lsm
# USAGE: run_ldt_noah39_smapeopl_hpc11.sh $startdate $starthour $enddate \
# $endhour
# where $startdate and $enddate specify the inclusive UTC
# date range to run SMAP_E_OPL retrievals (formatted YYYY-MM-DD);
# $starthour and $endhour specify the hours of the respective
# dates (formatted HH); and $lsm is the LSM name, used to select
# the ldt.config template file.
# date range to run SMAP_E_OPL retrievals (formatted YYYY-MM-DD
# or YYYYMMDD); $starthour and $endhour specify the hours of the
# respective dates (formatted HH)
#
# Based on Korn shell script provided by Pang-Wei Liu.
#
# REVISION HISTORY:
# 25 Jan 2024: Eric Kemp. Initial specification.
# 26 Jan 2024: Eric Kemp. Added in-script parallelization following
# autotuning scripts.
# 05 Nov 2024: Eric Kemp. Initial specification.
#
#--------------------------------------------------------------------------
#------------------------------------------------------------------------------

ulimit -s unlimited

Expand All @@ -40,15 +38,14 @@ if [ ! -z $SLURM_SUBMIT_DIR ] ; then
fi

# Environment
module purge
unset LD_LIBRARY_PATH
module use --append /home/emkemp/privatemodules
module load lisf_7.5_intel_2021.4.0_s2s
module use --append /ccs/home/emkemp/hpc11/privatemodules
module load lisf_7.6_prgenv_cray_8.5.0_cpe_23.12
module load afw-python/3.11-202406

# Paths on local system
SCRIPTDIR=/discover/nobackup/projects/usaf_lis/emkemp/AFWA/lis76_smap_e_opl_scripting/scripts
BINDIR=/discover/nobackup/projects/usaf_lis/emkemp/AFWA/lis76_smap_e_opl_scripting/bin
TMPLDIR=/discover/nobackup/projects/usaf_lis/emkemp/AFWA/lis76_smap_e_opl_scripting/tmpl
# Paths on local system. Must be customized before running script.
SCRIPTDIR=/lustre/storm/nwp601/proj-shared/emkemp/LISFV7.6/ldt/noah39_smap_e_opl/scripts
BINDIR=/lustre/storm/nwp601/proj-shared/emkemp/LISFV7.6/ldt/noah39_smap_e_opl/bin
TMPLDIR=/lustre/storm/nwp601/proj-shared/emkemp/LISFV7.6/ldt/noah39_smap_e_opl/tmpl

# Get the command line arguments to specify the training period
if [ -z "$1" ] ; then
Expand All @@ -67,9 +64,6 @@ if [ -z "$4" ] ; then
echo "ERROR, Missing end hour for SMAP_E_OPL retrievals!"
exit 1
fi
if [ -z "$5" ] ; then
echo "ERROR, missing LSM option!" && exit 1
fi

# Use the command line arguments to set start and end datetime limits.
# Sanity check and report errors if found.
Expand All @@ -90,17 +84,8 @@ end_yyyymmddhh=$(date -d "$end_dt" +%Y%m%d%H)

# Use the LSM command line argument to select the appropriate ldt.config
# template.
lsm="$5"
if [ "$lsm" = "noah" ] ; then
tmplfile="$TMPLDIR/ldt.config.smapeopl.noah.tmpl"
elif [ "$lsm" = "noahmp" ] ; then
tmplfile="$TMPLDIR/ldt.config.smapeopl.noahmp.tmpl"
elif [ "$lsm" = "jules" ] ; then
tmplfile="$TMPLDIR/ldt.config.smapeopl.jules.tmpl"
else
echo "ERR, invalid LSM option, must be noah, noahmp, or jules"
exit 1
fi
lsm="noah"
tmplfile="$TMPLDIR/ldt.config.smapeopl.noah.tmpl"
if [ ! -e $tmplfile ] ; then
echo "ERR, $tmplfile not found!" && exit 1
fi
Expand All @@ -127,8 +112,9 @@ while [ "$cur_yyyymmddhh" -le "$end_yyyymmddhh" ] ; do
"$tmplfile" $i || exit 1

# Run LDT
srun --ntasks=1 --nodes=1 --exclusive \
srun --ntasks=1 --cpus-per-task=1 \
$BINDIR/LDT ldt.config.smapeopl."$cur_yyyymmddhh" &

PIDS+=($!)
actives+=(1)
yyyymmddhh+=("$cur_yyyymmddhh")
Expand All @@ -151,6 +137,8 @@ while true; do
fi
pid="${PIDS[$i]}"
# See if pid is still running
#echo `ps --pid "$pid"`

ps --pid "$pid" > /dev/null
if [ "$?" -ne 0 ] ; then # ps doesn't see it, so it terminated
wait "$pid"
Expand Down Expand Up @@ -183,4 +171,6 @@ unset actives
unset yyyymmddhh

# The end
echo "INFO, All NoahMP401 SMAP_E_OPL tasks completed at `date`"
touch noah39.smapeopl.job.done
exit 0
Original file line number Diff line number Diff line change
Expand Up @@ -9,24 +9,23 @@
#SBATCH --mem=0
#--------------------------------------------------------------------------
#
# SCRIPT: run_smap_e_opl_hpc11.sh
# SCRIPT: run_ldt_noah39_smapeopl_hpc11.sh
#
# Batch script for running LDT to generate SMAP_E_OPL retrievals with
# LDT. Customized for USAF HPC11 supercomputer running SLURM batch
# queueing system.
# Batch script for running LDT to generate NoahMP401-based SMAP_E_OPL
# retrievals with LDT. Customized for USAF HPC11 supercomputer running SLURM
# batch queueing system. NOTE: Internal paths must be customized before use.
#
# USAGE: run_smap_e_opl_hpc11.sh $startdate $starthour $enddate \
# $endhour $lsm
# USAGE: run_ldt_noahmp401_smapeopl_hpc11.sh $startdate $starthour $enddate \
# $endhour
# where $startdate and $enddate specify the inclusive UTC
# date range to run SMAP_E_OPL retrievals (formatted YYYY-MM-DD
# or YYYYMMDD); $starthour and $endhour specify the hours of the
# respective dates (formatted HH); and $lsm is the LSM name, used
# to select the ldt.config template file.
# respective dates (formatted HH).
#
# Based on Korn shell script provided by Pang-Wei Liu.
#
# REVISION HISTORY:
# 29 Mar 2024: Eric Kemp. Initial specification, based on Discover script.
# 05 Nov 2024: Eric Kemp. Initial specification.
#
#--------------------------------------------------------------------------

Expand All @@ -40,13 +39,13 @@ fi

# Environment
module use --append /ccs/home/emkemp/hpc11/privatemodules
module load lisf_7.6_prgenv_cray_8.5.0_cpe_23.12_draft
module load afw-python/3.10-202312
module load lisf_7.6_prgenv_cray_8.5.0_cpe_23.12
module load afw-python/3.11-202406

# Paths on local system
SCRIPTDIR=/lustre/storm/nwp601/proj-shared/emkemp/lis76_smap_e_opl/scripts
BINDIR=/lustre/storm/nwp601/proj-shared/emkemp/lis76_smap_e_opl/bin
TMPLDIR=/lustre/storm/nwp601/proj-shared/emkemp/lis76_smap_e_opl/tmpl
# Paths on local system. Must be customized before running script.
SCRIPTDIR=/lustre/storm/nwp601/proj-shared/emkemp/LISFV7.6/ldt/noahmp401_smap_e_opl/scripts
BINDIR=/lustre/storm/nwp601/proj-shared/emkemp/LISFV7.6/ldt/noahmp401_smap_e_opl/bin
TMPLDIR=/lustre/storm/nwp601/proj-shared/emkemp/LISFV7.6/ldt/noahmp401_smap_e_opl/tmpl

# Get the command line arguments to specify the training period
if [ -z "$1" ] ; then
Expand Down Expand Up @@ -88,17 +87,9 @@ end_yyyymmddhh=$(date -d "$end_dt" +%Y%m%d%H)

# Use the LSM command line argument to select the appropriate ldt.config
# template.
lsm="$5"
if [ "$lsm" = "noah" ] ; then
tmplfile="$TMPLDIR/ldt.config.smapeopl.noah.tmpl"
elif [ "$lsm" = "noahmp" ] ; then
tmplfile="$TMPLDIR/ldt.config.smapeopl.noahmp.tmpl"
elif [ "$lsm" = "jules" ] ; then
tmplfile="$TMPLDIR/ldt.config.smapeopl.jules.tmpl"
else
echo "ERR, invalid LSM option, expected noah, noahmp, or jules; got $lsm"
exit 1
fi
lsm="noahmp"
tmplfile="$TMPLDIR/ldt.config.smapeopl.noahmp.tmpl"

if [ ! -e $tmplfile ] ; then
echo "ERR, $tmplfile not found!" && exit 1
fi
Expand Down Expand Up @@ -150,7 +141,7 @@ while true; do
fi
pid="${PIDS[$i]}"
# See if pid is still running
echo `ps --pid "$pid"`
#echo `ps --pid "$pid"`

ps --pid "$pid" > /dev/null
if [ "$?" -ne 0 ] ; then # ps doesn't see it, so it terminated
Expand Down Expand Up @@ -184,5 +175,6 @@ unset actives
unset yyyymmddhh

# The end
touch smapeopl.job.done
echo "INFO, All NoahMP401 SMAP_E_OPL tasks completed at `date`"
touch noahmp401.smapeopl.job.done
exit 0
48 changes: 48 additions & 0 deletions ldt/utils/usaf/USAFSI/run_ldt_usafsi_hpc11.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
#!/bin/sh
#SBATCH --job-name=ldt
#SBATCH --time=1:00:00
#SBATCH --account=NWP601
#SBATCH --output ldt.slurm.out
#SBATCH --ntasks=1
#SBATCH --cluster-constraint=blue
#SBATCH --exclusive
#SBATCH --mem=0
#------------------------------------------------------------------------------
#
# SCRIPT: run_ldt_usafsi_hpc11.sh
#
# Batch script for running LDT to generate USAFSI snow and ice analysis.
#
# REVISION HISTORY:
# 05 Nov 2024: Eric Kemp, SSAI. Initial specification.
#------------------------------------------------------------------------------

ulimit -s unlimited

# When a batch script is started, it starts in the user's home directory.
# Change to the directory where job was submitted.
if [ ! -z $SLURM_SUBMIT_DIR ] ; then
cd $SLURM_SUBMIT_DIR || exit 1
fi

# Environment
module use --append /ccs/home/emkemp/hpc11/privatemodules
module load lisf_7.6_prgenv_cray_8.5.0_cpe_23.12
module load afw-python/3.11-202406

ldtconfig=ldt.config.foc.nrt_sf.usafsi.76

# Sanity checks
if [ ! -e ./$ldtconfig ] ; then
echo "ERROR, ./$ldtconfig not found!" && exit 1
fi
if [ ! -e ./LDT ] ; then
echo "ERROR, ./LDT does not exist!" && exit 1
fi

echo `date`
srun ./LDT $ldtconfig || exit 1
echo `date`

# The end
exit 0
51 changes: 51 additions & 0 deletions ldt/utils/usaf/bridge/run_ldt_bridge_noah39_hpc11.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/sh
#SBATCH --job-name=ldt
#SBATCH --time=1:00:00
#SBATCH --account=NWP601
#SBATCH --output ldt.slurm.out
#SBATCH --ntasks=1
#SBATCH --cluster-constraint=blue
#SBATCH --exclusive
#SBATCH --mem=0
#------------------------------------------------------------------------------
#
# SCRIPT: run_ldt_bridge_noah39_hpc11.sh
#
# Batch script for running NRT-Streamflow to MR "bridge" step, i.e., converting
# LIS LSM restart file from NRT-Streamflow run into version usable by LIS MR
# run. Customized for Noah39 runs (only LSM restart file is processed, not
# routing). Also customized for HPC11.
#
# REVISION HISTORY:
# 05 Nov 2024: Eric Kemp, SSAI. Initial specification.
#------------------------------------------------------------------------------

ulimit -s unlimited

# When a batch script is started, it starts in the user's home directory.
# Change to the directory where job was submitted.
if [ ! -z $SLURM_SUBMIT_DIR ] ; then
cd $SLURM_SUBMIT_DIR || exit 1
fi

# Environment
module use --append /ccs/home/emkemp/hpc11/privatemodules
module load lisf_7.6_prgenv_cray_8.5.0_cpe_23.12
module load afw-python/3.11-202406

ldtconfig=ldt.config.foc.bridge.noah39.76

# Sanity checks
if [ ! -e ./$ldtconfig ] ; then
echo "ERROR, ./$ldtconfig not found!" && exit 1
fi
if [ ! -e ./LDT ] ; then
echo "ERROR, ./LDT does not exist!" && exit 1
fi

echo `date`
srun ./LDT $ldtconfig || exit 1
echo `date`

# The end
exit 0
51 changes: 51 additions & 0 deletions ldt/utils/usaf/bridge/run_ldt_bridge_noahmp401_hpc11.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
#!/bin/sh
#SBATCH --job-name=ldt
#SBATCH --time=1:00:00
#SBATCH --account=NWP601
#SBATCH --output ldt.slurm.out
#SBATCH --ntasks=1
#SBATCH --cluster-constraint=blue
#SBATCH --exclusive
#SBATCH --mem=0
#------------------------------------------------------------------------------
#
# SCRIPT: run_ldt_bridge_noahmp401_hpc11.sh
#
# Batch script for running NRT-Streamflow to MR "bridge" step, i.e., converting
# LIS LSM restart file from NRT-Streamflow run into version usable by LIS MR
# run. Customized for NoahMP401 runs (only LSM restart file is processed, not
# routing). Also customized for HPC11.
#
# REVISION HISTORY:
# 05 Nov 2024: Eric Kemp, SSAI. Initial specification.
#------------------------------------------------------------------------------

ulimit -s unlimited

# When a batch script is started, it starts in the user's home directory.
# Change to the directory where job was submitted.
if [ ! -z $SLURM_SUBMIT_DIR ] ; then
cd $SLURM_SUBMIT_DIR || exit 1
fi

# Environment
module use --append /ccs/home/emkemp/hpc11/privatemodules
module load lisf_7.6_prgenv_cray_8.5.0_cpe_23.12
module load afw-python/3.11-202406

ldtconfig=ldt.config.foc.bridge.noahmp401.76

# Sanity checks
if [ ! -e ./$ldtconfig ] ; then
echo "ERROR, ./$ldtconfig not found!" && exit 1
fi
if [ ! -e ./LDT ] ; then
echo "ERROR, ./LDT does not exist!" && exit 1
fi

echo `date`
srun ./LDT $ldtconfig || exit 1
echo `date`

# The end
exit 0
Loading

0 comments on commit 966f63b

Please sign in to comment.