From 7085f9ff8ab0581af25a79953294c84097f8d19d Mon Sep 17 00:00:00 2001 From: kayeekayee Date: Fri, 17 May 2024 08:49:02 -0600 Subject: [PATCH] Temporary fix for Rocky8 (rt_dev1) (#58) * Temporary fix for Rocky 8 * replace gfs_utils.fd pointer to KaYee * Rocky8 pointer update * Increase more wallclock time to run 120hrs --- .gitmodules | 2 +- INFO | 4 ++++ jobs/rocoto/init.sh | 4 ++-- jobs/rocoto/remapgrib.ksh | 5 +++-- modulefiles/module_base.hera.lua | 7 +++---- modulefiles/module_gwci.hera.lua | 4 ++-- modulefiles/module_gwsetup.hera.lua | 4 ++-- sorc/gfs_utils.fd | 2 +- sorc/ufs_model.fd | 2 +- sorc/ufs_utils.fd | 2 +- ush/load_fv3gfs_modules.sh | 3 ++- versions/build.hera.ver | 2 +- versions/run.hera.ver | 2 +- versions/run.spack.ver | 2 +- workflow/gsl_template_hera.xml | 2 +- 15 files changed, 26 insertions(+), 21 deletions(-) diff --git a/.gitmodules b/.gitmodules index ed2f92d45c..bfda9ae9aa 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,7 +7,7 @@ url = https://github.com/NOAA-EMC/wxflow [submodule "sorc/gfs_utils.fd"] path = sorc/gfs_utils.fd - url = https://github.com/NOAA-EMC/gfs-utils + url = https://github.com/kayeekayee/gfs-utils [submodule "sorc/ufs_utils.fd"] path = sorc/ufs_utils.fd url = https://github.com/NOAA-GSL/UFS_UTILS.git diff --git a/INFO b/INFO index a667b9aa10..fde79d6fcd 100644 --- a/INFO +++ b/INFO @@ -1,3 +1,7 @@ +04-10-24 +======== + temporary fix for Rocky 8 + 03-28-24 ======== change dt_inner to 50 for aerosol aware Thompson diff --git a/jobs/rocoto/init.sh b/jobs/rocoto/init.sh index 48fce24822..94ceafda7f 100755 --- a/jobs/rocoto/init.sh +++ b/jobs/rocoto/init.sh @@ -21,8 +21,8 @@ status=$? [[ $status -ne 0 ]] && exit $status #JKH load ufs-utils modules -module use /scratch1/BMC/gsd-fv3-dev/Judy.K.Henderson/test/gsl_ufs_dev/sorc/ufs_utils.fd/modulefiles -module load build.hera.intel +module use /scratch1/BMC/gsd-fv3/rtruns/modulefiles +module load build.hera.gfsinit module list ############################################################### diff --git a/jobs/rocoto/remapgrib.ksh b/jobs/rocoto/remapgrib.ksh index b210f93021..d18a979681 100755 --- a/jobs/rocoto/remapgrib.ksh +++ b/jobs/rocoto/remapgrib.ksh @@ -4,8 +4,8 @@ # 236 201 244 130 224 242 # initialize -module load intel/2022.1.2 -module load wgrib2/2.0.8 +module load gnu/13.2.0 intel/2023.2.0 netcdf/4.7.0 wgrib2/3.1.2_ncep +module list ECHO=echo MKDIR=mkdir LN=ln @@ -20,6 +20,7 @@ echo "fcst = ${fcst}" echo "GRID_NAMES = ${GRID_NAMES}" echo +echo `which wgrib2` ######################### # Grid Definitions ######################### diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 311fb0a1cf..bf970bbaee 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -2,9 +2,8 @@ help([[ Load environment to run GFS on Hera ]]) -spack_stack_ver=(os.getenv("spack_stack_ver") or "None") -spack_env=(os.getenv("spack_env") or "None") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") +local spack_mod_path=(os.getenv("spack_mod_path") or "None") +prepend_path("MODULEPATH", spack_mod_path) load(pathJoin("stack-intel", (os.getenv("stack_intel_ver") or "None"))) load(pathJoin("stack-intel-oneapi-mpi", (os.getenv("stack_impi_ver") or "None"))) @@ -42,7 +41,7 @@ setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/dev-gfsv17/modulefiles")) load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None"))) prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua index 1aecddf549..75b972b69b 100644 --- a/modulefiles/module_gwci.hera.lua +++ b/modulefiles/module_gwci.hera.lua @@ -2,13 +2,13 @@ help([[ Load environment to run GFS workflow setup scripts on Hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("2021.5.0"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) load(pathJoin("netcdf-c", os.getenv("4.9.2"))) -load(pathJoin("netcdf-fortran", os.getenv("4.6.0"))) +load(pathJoin("netcdf-fortran", os.getenv("4.6.1"))) load(pathJoin("nccmp","1.9.0.1")) load(pathJoin("wgrib2", "2.0.8")) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index c86cac7b02..06cf566850 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -4,10 +4,10 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/gsi-addon-dev-rocky8/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local python_ver=os.getenv("python_ver") or "3.10.8" +local python_ver=os.getenv("python_ver") or "3.11.6" load(pathJoin("stack-intel", stack_intel_ver)) load(pathJoin("python", python_ver)) diff --git a/sorc/gfs_utils.fd b/sorc/gfs_utils.fd index 7d3b08e87c..5198f6376b 160000 --- a/sorc/gfs_utils.fd +++ b/sorc/gfs_utils.fd @@ -1 +1 @@ -Subproject commit 7d3b08e87c07cfa54079442d245ac7e9ab1cd9f4 +Subproject commit 5198f6376b3d769f9ce88c675f066e1c17a46f98 diff --git a/sorc/ufs_model.fd b/sorc/ufs_model.fd index 04733d431d..7088634d67 160000 --- a/sorc/ufs_model.fd +++ b/sorc/ufs_model.fd @@ -1 +1 @@ -Subproject commit 04733d431d034ec2f3c03648b646d2c87c79ba0c +Subproject commit 7088634d67da1650de3f3f3789a1c8db250d49be diff --git a/sorc/ufs_utils.fd b/sorc/ufs_utils.fd index d805336652..7dcadcaf8d 160000 --- a/sorc/ufs_utils.fd +++ b/sorc/ufs_utils.fd @@ -1 +1 @@ -Subproject commit d80533665273705b2041c494ed3c1bd50f263265 +Subproject commit 7dcadcaf8db44ecd4ef46a1802c666c526a5e8c8 diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index b4f23fa331..57a188b93e 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -16,7 +16,8 @@ source "${HOMEgfs}/modulefiles/module-setup.sh.inc" source "${HOMEgfs}/versions/run.ver" # Load our modules: -module use "${HOMEgfs}/modulefiles" +#KYWmodule use "${HOMEgfs}/modulefiles" +module use "/scratch1/BMC/gsd-fv3/rtruns/modulefiles/" if [[ -d /lfs/f1 ]]; then # We are on WCOSS2 (Cactus or Dogwood) diff --git a/versions/build.hera.ver b/versions/build.hera.ver index 263e967a97..337d5c32da 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,5 +1,5 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 source "${HOMEgfs:-}/versions/build.spack.ver" export spack_mod_path="/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-${spack_stack_ver}/envs/${spack_env}/install/modulefiles/Core" diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 4529d34821..b358f9d495 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -1,6 +1,6 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export spack_env=gsi-addon-dev +export spack_env=gsi-addon-dev-rocky8 export hpss_ver=hpss export ncl_ver=6.6.2 diff --git a/versions/run.spack.ver b/versions/run.spack.ver index c1c13f58df..80fa6acd1a 100644 --- a/versions/run.spack.ver +++ b/versions/run.spack.ver @@ -1,5 +1,5 @@ export spack_stack_ver=1.5.1 -export spack_env=gsi-addon +export spack_env=gsi-addon-dev-rocky8 export python_ver=3.10.8 export jasper_ver=2.0.32 diff --git a/workflow/gsl_template_hera.xml b/workflow/gsl_template_hera.xml index 8d7caf4b33..b796f4ae82 100644 --- a/workflow/gsl_template_hera.xml +++ b/workflow/gsl_template_hera.xml @@ -80,7 +80,7 @@ gsd-fv3 batch hera - 04:10:00 + 06:00:00 56:ppn=40:tpp=1 &NATIVE_STR;