Merge branch 'NOAA-EMC:develop' into develop

.github/workflows/docs.yaml

@@ -17,7 +17,7 @@ on:
 jobs:
 
   documentation:
-    
+
     permissions:
       pull-requests: 'write'
 
@@ -56,7 +56,8 @@ jobs:
         path: artifact/doc_warnings.log
         if-no-files-found: ignore
 
-    - name: Comment ReadDocs
+    - name: Comment ReadDocs Link in PR
+      if: github.event_name == 'pull_request'
       uses: actions/github-script@v6
       with:
         script: |
@@ -68,5 +69,5 @@ jobs:
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
-           body: message 
-          })     
+           body: message
+          })

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -16,6 +16,3 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
-
-skip_ci_on_hosts:
-  - hercules

ci/cases/pr/C96_atm3DVar.yaml

@@ -15,6 +15,3 @@ arguments:
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
-
-skip_ci_on_hosts:
-  - hercules

ci/scripts/check_ci.sh

@@ -22,7 +22,7 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
    ;;

ci/scripts/driver.sh

@@ -34,7 +34,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
@@ -173,7 +173,7 @@ for pr in ${pr_list}; do
   # we need to exit this instance of the driver script
   #################################################################
   if [[ ${ci_status} -ne 0 ]]; then
-     build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "{pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
+     build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
      if [[ "${build_PID_check}" -ne "$$" ]]; then
         echo "Driver build PID: ${build_PID_check} no longer running this build ... exiting"
         exit 0

ci/scripts/driver_weekly.sh

@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

ci/scripts/run_ci.sh

@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
    ;;

env/HERCULES.env

@@ -12,7 +12,7 @@ fi
 
 step=$1
 
-export npe_node_max=40
+export npe_node_max=80
 export launcher="srun -l --export=ALL"
 export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
@@ -26,19 +26,164 @@ export KMP_AFFINITY=scatter
 export OMP_STACKSIZE=2048000
 export NTHSTACK=1024000000
 #export LD_BIND_NOW=1
+export I_MPI_EXTRA_FILESYSTEM=1
+export I_MPI_EXTRA_FILESYSTEM_LIST=lustre
 
 ulimit -s unlimited
 ulimit -a
 
-if [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \
-    [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then
+case ${step} in
+ "prep" | "prepbufr")
+
+    nth_max=$((npe_node_max / npe_node_prep))
+
+    export POE="NO"
+    export BACK=${BACK:-"YES"}
+    export sys_tp="HERCULES"
+    export launcher_PREP="srun"
+ ;;
+ "preplandobs")
+
+    export APRUN_CALCFIMS="${launcher} -n 1"
+ ;;
+ "waveinit" | "waveprep" | "wavepostsbs" | "wavepostbndpnt" | "wavepostpnt" | "wavepostbndpntbll")
 
     export CFP_MP="YES"
-    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    [[ "${step}" = "waveprep" ]] && export MP_PULSE=0
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
-elif [[ "${step}" = "fcst" ]]; then
+ ;;
+ "atmanlrun")
+
+    nth_max=$((npe_node_max / npe_node_atmanlrun))
+
+    export NTHREADS_ATMANL=${nth_atmanlrun:-${nth_max}}
+    [[ ${NTHREADS_ATMANL} -gt ${nth_max} ]] && export NTHREADS_ATMANL=${nth_max}
+    export APRUN_ATMANL="${launcher} -n ${npe_atmanlrun} --cpus-per-task=${NTHREADS_ATMANL}"
+ ;;
+ "atmensanlrun")
+
+    nth_max=$((npe_node_max / npe_node_atmensanlrun))
+
+    export NTHREADS_ATMENSANL=${nth_atmensanlrun:-${nth_max}}
+    [[ ${NTHREADS_ATMENSANL} -gt ${nth_max} ]] && export NTHREADS_ATMENSANL=${nth_max}
+    export APRUN_ATMENSANL="${launcher} -n ${npe_atmensanlrun} --cpus-per-task=${NTHREADS_ATMENSANL}"
+ ;;
+ "aeroanlrun")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_aeroanlrun))
+
+    export NTHREADS_AEROANL=${nth_aeroanlrun:-${nth_max}}
+    [[ ${NTHREADS_AEROANL} -gt ${nth_max} ]] && export NTHREADS_AEROANL=${nth_max}
+    export APRUN_AEROANL="${launcher} -n ${npe_aeroanlrun} --cpus-per-task=${NTHREADS_AEROANL}"
+ ;;
+ "landanl")
+
+    nth_max=$((npe_node_max / npe_node_landanl))
+
+    export NTHREADS_LANDANL=${nth_landanl:-${nth_max}}
+    [[ ${NTHREADS_LANDANL} -gt ${nth_max} ]] && export NTHREADS_LANDANL=${nth_max}
+    export APRUN_LANDANL="${launcher} -n ${npe_landanl} --cpus-per-task=${NTHREADS_LANDANL}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+ ;;
+ "ocnanalbmat")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalbmat))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalbmat:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalbmat} --cpus-per-task=${NTHREADS_OCNANAL}"
+ ;;
+ "ocnanalrun")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalrun))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalrun:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalrun} --cpus-per-task=${NTHREADS_OCNANAL}"
+ ;;
+ "ocnanalchkpt")
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_ocnanalchkpt))
+
+    export NTHREADS_OCNANAL=${nth_ocnanalchkpt:-${nth_max}}
+    [[ ${NTHREADS_OCNANAL} -gt ${nth_max} ]] && export NTHREADS_OCNANAL=${nth_max}
+    export APRUN_OCNANAL="${launcher} -n ${npe_ocnanalchkpt} --cpus-per-task=${NTHREADS_OCNANAL}"
+ ;;
+ "anal" | "analcalc")
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_anal))
+
+    export NTHREADS_GSI=${nth_anal:-${nth_max}}
+    [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${nth_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    npe_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${npe_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    npe_gausfcanl=${npe_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+ ;;
+ "sfcanl")
+    nth_max=$((npe_node_max / npe_node_sfcanl))
+
+    export NTHREADS_CYCLE=${nth_sfcanl:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    npe_sfcanl=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${npe_sfcanl} --cpus-per-task=${NTHREADS_CYCLE}"
+ ;;
+ "eobs")
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_eobs))
+
+    export NTHREADS_GSI=${nth_eobs:-${nth_max}}
+    [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_eobs}} --cpus-per-task=${NTHREADS_GSI}"
+ ;;
+ "eupd")
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    nth_max=$((npe_node_max / npe_node_eupd))
+
+    export NTHREADS_ENKF=${nth_eupd:-${nth_max}}
+    [[ ${NTHREADS_ENKF} -gt ${nth_max} ]] && export NTHREADS_ENKF=${nth_max}
+    export APRUN_ENKF="${launcher} -n ${npe_enkf:-${npe_eupd}} --cpus-per-task=${NTHREADS_ENKF}"
+ ;;
+ "fcst" | "efcs")
 
     export OMP_STACKSIZE=512M
     if [[ "${CDUMP}" =~ "gfs" ]]; then
@@ -53,17 +198,110 @@ elif [[ "${step}" = "fcst" ]]; then
     # With ESMF threading, the model wants to use the full node
     export APRUN_UFS="${launcher} -n ${ntasks}"
     unset nprocs ppn nnodes ntasks
+ ;;
 
-elif [[ "${step}" = "upp" ]]; then
+ "upp")
 
     nth_max=$((npe_node_max / npe_node_upp))
 
     export NTHREADS_UPP=${nth_upp:-1}
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
     export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}"
-
-elif [[ "${step}" = "atmos_products" ]]; then
+ ;;
+ "atmos_products")
 
     export USE_CFP="YES"  # Use MPMD for downstream product generation
+ ;;
+ "ecen")
 
-fi
+    nth_max=$((npe_node_max / npe_node_ecen))
+
+    export NTHREADS_ECEN=${nth_ecen:-${nth_max}}
+    [[ ${NTHREADS_ECEN} -gt ${nth_max} ]] && export NTHREADS_ECEN=${nth_max}
+    export APRUN_ECEN="${launcher} -n ${npe_ecen} --cpus-per-task=${NTHREADS_ECEN}"
+
+    export NTHREADS_CHGRES=${nth_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${npe_node_max} ]] && export NTHREADS_CHGRES=${npe_node_max}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${nth_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
+    export APRUN_CALCINC="${launcher} -n ${npe_ecen} --cpus-per-task=${NTHREADS_CALCINC}"
+
+ ;;
+ "esfc")
+
+    nth_max=$((npe_node_max / npe_node_esfc))
+
+    export NTHREADS_ESFC=${nth_esfc:-${nth_max}}
+    [[ ${NTHREADS_ESFC} -gt ${nth_max} ]] && export NTHREADS_ESFC=${nth_max}
+    export APRUN_ESFC="${launcher} -n ${npe_esfc} --cpus-per-task=${NTHREADS_ESFC}"
+
+    export NTHREADS_CYCLE=${nth_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
+    export APRUN_CYCLE="${launcher} -n ${npe_esfc} --cpus-per-task=${NTHREADS_CYCLE}"
+
+ ;;
+ "epos")
+
+    nth_max=$((npe_node_max / npe_node_epos))
+
+    export NTHREADS_EPOS=${nth_epos:-${nth_max}}
+    [[ ${NTHREADS_EPOS} -gt ${nth_max} ]] && export NTHREADS_EPOS=${nth_max}
+    export APRUN_EPOS="${launcher} -n ${npe_epos} --cpus-per-task=${NTHREADS_EPOS}"
+
+ ;;
+ "postsnd")
+
+    export CFP_MP="YES"
+
+    nth_max=$((npe_node_max / npe_node_postsnd))
+
+    export NTHREADS_POSTSND=${nth_postsnd:-1}
+    [[ ${NTHREADS_POSTSND} -gt ${nth_max} ]] && export NTHREADS_POSTSND=${nth_max}
+    export APRUN_POSTSND="${launcher} -n ${npe_postsnd} --cpus-per-task=${NTHREADS_POSTSND}"
+
+    export NTHREADS_POSTSNDCFP=${nth_postsndcfp:-1}
+    [[ ${NTHREADS_POSTSNDCFP} -gt ${nth_max} ]] && export NTHREADS_POSTSNDCFP=${nth_max}
+    export APRUN_POSTSNDCFP="${launcher} -n ${npe_postsndcfp} ${mpmd_opt}"
+
+ ;;
+ "awips")
+
+    nth_max=$((npe_node_max / npe_node_awips))
+
+    export NTHREADS_AWIPS=${nth_awips:-2}
+    [[ ${NTHREADS_AWIPS} -gt ${nth_max} ]] && export NTHREADS_AWIPS=${nth_max}
+    export APRUN_AWIPSCFP="${launcher} -n ${npe_awips} ${mpmd_opt}"
+
+ ;;
+ "gempak")
+
+    export CFP_MP="YES"
+
+    if [[ ${CDUMP} == "gfs" ]]; then
+        npe_gempak=${npe_gempak_gfs}
+        npe_node_gempak=${npe_node_gempak_gfs}
+    fi
+
+    nth_max=$((npe_node_max / npe_node_gempak))
+
+    export NTHREADS_GEMPAK=${nth_gempak:-1}
+    [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
+    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+
+ ;;
+ "fit2obs")
+
+    nth_max=$((npe_node_max / npe_node_fit2obs))
+
+    export NTHREADS_FIT2OBS=${nth_fit2obs:-1}
+    [[ ${NTHREADS_FIT2OBS} -gt ${nth_max} ]] && export NTHREADS_FIT2OBS=${nth_max}
+    export MPIRUN="${launcher} -n ${npe_fit2obs} --cpus-per-task=${NTHREADS_FIT2OBS}"
+
+ ;;
+ *)
+    # Some other job not yet defined here
+    echo "WARNING: The job step ${step} does not specify Hercules-specific resources"
+ ;;
+esac

jobs/JGFS_ATMOS_GEMPAK_META

@@ -6,7 +6,7 @@
 # GFS GEMPAK META PRODUCT GENERATION
 ############################################
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -e "base"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "gempak_meta" -c "base"
 
 
 ###############################################

modulefiles/module_base.hercules.lua

@@ -8,7 +8,9 @@ prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack
 
 load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
 load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("intel-oneapi-mkl", os.getenv("intel_mkl_ver")))
 load(pathJoin("python", os.getenv("python_ver")))
+load(pathJoin("perl", os.getenv("perl_ver")))
 
 -- TODO load NCL once the SAs remove the 'depends_on' statements within it
 --      NCL is a static installation and does not depend on any libraries