From 9da77029e07b1ad5b858923d7ea377ec9ab0b7ec Mon Sep 17 00:00:00 2001 From: William <14932329+WillyChap@users.noreply.github.com> Date: Fri, 12 Jan 2024 11:29:49 -0700 Subject: [PATCH 1/9] Update config_compilers.xml --- config/cesm/machines/config_compilers.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index 73d44b42306..3f80ebb6fbc 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -242,6 +242,7 @@ using a fortran linker. -qopenmp + -L/glade/u/apps/opt/conda/envs/npl-2023b/lib/python3.10/config-3.10-x86_64-linux-gnu -L/glade/u/apps/opt/conda/envs/npl-2023b/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lrt -lm -lm mpicc mpicxx From 72e7c51980b7807d261f07dd80408586b0f9b72d Mon Sep 17 00:00:00 2001 From: willychap Date: Fri, 12 Jan 2024 12:54:26 -0700 Subject: [PATCH 2/9] update config_compilers. --- config/cesm/machines/config_compilers.xml | 24 ----------------------- 1 file changed, 24 deletions(-) diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index 3f80ebb6fbc..e79e6561fa3 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -757,30 +757,6 @@ using a fortran linker. - - - -march=core-avx2 -no-fma - - - -march=core-avx2 -no-fma - - - - - - -march=core-avx2 -no-fma - - - -march=core-avx2 -no-fma -qno-opt-dynamic-align -fp-model precise -std=gnu99 - -std=gnu89 - - - -L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf - -L$(PNETCDF_PATH)/lib -lpnetcdf - - - - From 966a9bce9882e57747052d09feeb1925e161226e Mon Sep 17 00:00:00 2001 From: James Edwards Date: Fri, 12 Jan 2024 15:08:09 -0700 Subject: [PATCH 3/9] add forpy_mod.F90 to share code and link python in ldflags --- config/cesm/machines/config_compilers.xml | 32 ++++++++++++++++++++--- src/build_scripts/buildlib.csm_share | 2 ++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index e79e6561fa3..185a07eba94 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -746,18 +746,44 @@ using a fortran linker. + $ENV{NETCDF} + $ENV{PNETCDF} + + -lnetcdff -lnetcdf + + + + -qno-opt-dynamic-align -fp-model precise -std=gnu99 -std=gnu89 + -march=core-avx2 -no-fma + + + -march=core-avx2 -no-fma + + + -L/glade/u/apps/derecho/23.09/spack/opt/spack/python/3.10.12/gcc/7.5.0/lmsy/lib -lpython3.10 + + + + + + -march=core-avx2 -no-fma + + + -march=core-avx2 -no-fma -qno-opt-dynamic-align -fp-model precise -std=gnu99 + -std=gnu89 - $ENV{NETCDF} - $ENV{PNETCDF} - -lnetcdff -lnetcdf + -L$(NETCDF_PATH)/lib -lnetcdff -lnetcdf + -L$(PNETCDF_PATH)/lib -lpnetcdf + + -O2 diff --git a/src/build_scripts/buildlib.csm_share b/src/build_scripts/buildlib.csm_share index a16975c7a0e..3598a5ed18c 100755 --- a/src/build_scripts/buildlib.csm_share +++ b/src/build_scripts/buildlib.csm_share @@ -45,7 +45,9 @@ def buildlib(bldroot, installpath, caseroot): ############################################################################### with Case(caseroot, read_only=False) as case: cimeroot = case.get_value("CIMEROOT") + srcroot = case.get_value("SRCROOT") filepath = [os.path.join(caseroot,"SourceMods","src.share"), + os.path.join(srcroot, "forpy"), os.path.join(cimeroot,"src","drivers","mct","shr"), os.path.join(cimeroot,"src","share","streams"), os.path.join(cimeroot,"src","share","util"), From d4435be314c49ee331f733702df5ca7a73183bcc Mon Sep 17 00:00:00 2001 From: William <14932329+WillyChap@users.noreply.github.com> Date: Mon, 15 Jan 2024 12:25:25 -0700 Subject: [PATCH 4/9] Update config_compilers.xml --- config/cesm/machines/config_compilers.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index 185a07eba94..f11cc5bd6b9 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -242,7 +242,6 @@ using a fortran linker. -qopenmp - -L/glade/u/apps/opt/conda/envs/npl-2023b/lib/python3.10/config-3.10-x86_64-linux-gnu -L/glade/u/apps/opt/conda/envs/npl-2023b/lib -lpython3.10 -lcrypt -lpthread -ldl -lutil -lrt -lm -lm mpicc mpicxx From 4240b59d2da05833b19ad2eafe187744b9ecc8cb Mon Sep 17 00:00:00 2001 From: willychap Date: Thu, 18 Jan 2024 10:25:10 -0700 Subject: [PATCH 5/9] changing compiler flags - forpy --- .DS_Store | Bin 0 -> 6148 bytes config/.DS_Store | Bin 0 -> 6148 bytes config/cesm/.DS_Store | Bin 0 -> 6148 bytes config/cesm/machines/config_compilers.xml | 4 +--- 4 files changed, 1 insertion(+), 3 deletions(-) create mode 100644 .DS_Store create mode 100644 config/.DS_Store create mode 100644 config/cesm/.DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..b89f7124a55a2c7affb56a18d7b0e41a3bdaecca GIT binary patch literal 6148 zcmeHKO-jQ+6rO2on^J@<6vS1)tzcWL72JeaFJMF$Dm5{+24ki)sacdl4xktEAfCtf zW+sbZz@3PF4`#miJCpe!^TQb9&LZeBW-`VMXowt@3PE>esHT$w#*Br-Sk(naEU^Bc4-g^2t@ z_x!7Hl8w!+GZCj=7*FQ1APgpua(f-dfyf-Oh=WAdx+-9_jrQ2wT`muMC;feEFdVM- zt!1x&GyvncyIQr4?Y)E3i}7>x5{oxQB?s1}Y-%jv9hB0lUfp>diFgYBqNd0tBnF59 zVt^P}HwN@cV06}P;j~_2fEf6Z0o)%fXo$APT%p`Lpu_7k#v6zzpyOKt(b8yZ%oRce zgsW0ORm$}fgR64zTRP9ym@8D}jLVr}96fXKc;Rw(@LMvSaa$qv!~ii+W}vQ26VLy1 z_+=^|`Q;L_hyh~YpE1B2BX{J$qV(DNZFzXsTF~yHpO8^+SkL<{xid(2d Zo~4a_Xp!7&hmEqov+o5^^Lk&H=E|I_b3W4_w#Ay`jcz4E`&&eO80}y zI4XwL)~QHyKTe~GEQrGhQf{u|G!%s^rfHbTT2BSchS?ZeyR+Fr=eXOld;R{rWzRa@ z!yXt%?fJZ6Ztv}%oDZLpQ7T>(r5qTSvZ=9xS5Q7z^z2R2M5GVktMaQnLSldzAO?tm zRb@b*2xfCt37~Zo1H{0O4B-A?Lql{lmI~$80UchSF>WBDfR1koL|dbyu~Y~V5UxrA zRVmj`46e$-Z|giqW2sP;GcIR_arDg9sDOaJa0vhd_mLeLRB;=1$a6H7 V3UL(l%W^=v2uMPxBL;qffe#>FO1=O9 literal 0 HcmV?d00001 diff --git a/config/cesm/.DS_Store b/config/cesm/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..c109f5a4631b527f35266ac527e08713022da243 GIT binary patch literal 6148 zcmeHK%}N6?5T3NvrWBzE1-%8l7HqAG#miFb3%H^OmAY$-F0Px>ZSA2H_O37Fi}*aw zB&k@c;7P>Jz~r0E&xU-vWHtaGTBB|qpaK9Al`vPr;R~TS>4M~(hftVjM36ua(r}#i z7qZ#$7a5>;C*ctSxP^Q8^!~zFhZuVe`*D;GYqd9#%NJHwi?S%o^4h=GV}BS7M`o}a*L3keZ#+}Ogu}+3Tl=KFsAnNuo03@`&r#DF^jWqFDA%v)s!n1P=$K<9%(CG;F-7R}Lt1N}Zyyg*2THr*u% zl|#>AW)UMO!j>Z1QsJ%`!j_|7xjfHdX3>^|Fe~G8+{(h;P=r|>{mP_+@GSDk3@`)p z4CL*!LjC{b`}_ZV5zm+bX5e2jAPQ~2-NG%o-ny_k>a_~>29<>JGK(K2XqZ-vxzvg` bQH`Kqk%8zr%q*e@g?|JL4LmRdzskTnOUq3t literal 0 HcmV?d00001 diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index f11cc5bd6b9..f4263ce8f92 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -749,6 +749,7 @@ using a fortran linker. $ENV{PNETCDF} -lnetcdff -lnetcdf + -L/glade/work/wchapman/miniconda3.1/envs/cesmTORCH/lib -lpython3.11 -Wl,-rpath /glade/work/wchapman/miniconda3.1/envs/cesmTORCH/lib/ @@ -761,9 +762,6 @@ using a fortran linker. -march=core-avx2 -no-fma - - -L/glade/u/apps/derecho/23.09/spack/opt/spack/python/3.10.12/gcc/7.5.0/lmsy/lib -lpython3.10 - From 82b3fbd45b73f7e627781c266056fe10242d0d1e Mon Sep 17 00:00:00 2001 From: willychap Date: Fri, 19 Jan 2024 11:42:35 -0700 Subject: [PATCH 6/9] update config compile with new miniconda --- .DS_Store | Bin 6148 -> 6148 bytes config/.DS_Store | Bin 6148 -> 6148 bytes config/cesm/.DS_Store | Bin 6148 -> 6148 bytes config/cesm/machines/config_compilers.xml | 2 +- 4 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.DS_Store b/.DS_Store index b89f7124a55a2c7affb56a18d7b0e41a3bdaecca..8546179beaf523c3977fc0cb74ca0760f3434c48 100644 GIT binary patch delta 22 dcmZoMXffC@hmq6B(A+{t!PMMn^HRomF#t^621x(_ delta 22 dcmZoMXffC@hmq6R(85SZ!NlBr^HRomF#t@<21x(_ diff --git a/config/.DS_Store b/config/.DS_Store index 90f221c2a972f89c2a91088685ce9a17dba3a602..0f229af3af0d8cc0a08ddcb32f2699cf13cfd167 100644 GIT binary patch delta 22 dcmZoMXffC@gOSt7(A+{t!PMMn^FqdCF#t@F21Nh> delta 22 dcmZoMXffC@gOStN(85SZ!NlBr^FqdCF#t?|21Nh> diff --git a/config/cesm/.DS_Store b/config/cesm/.DS_Store index c109f5a4631b527f35266ac527e08713022da243..7a6cfeca5125765e4a49e382e6ba29749fc00c59 100644 GIT binary patch delta 71 zcmZoMXffEJ&BVBCvJO+Brc`yck)gSTj)JMVQLT$ENV{PNETCDF} -lnetcdff -lnetcdf - -L/glade/work/wchapman/miniconda3.1/envs/cesmTORCH/lib -lpython3.11 -Wl,-rpath /glade/work/wchapman/miniconda3.1/envs/cesmTORCH/lib/ + -L/glade/work/wchapman/miniconda3.2/lib -lpython3.11 -Wl,-rpath /glade/work/wchapman/miniconda3.2/lib/ From ed4799f5787a7df6cf0b231d591d87c3977ec172 Mon Sep 17 00:00:00 2001 From: willychap Date: Mon, 22 Jan 2024 14:12:02 -0700 Subject: [PATCH 7/9] update DS store --- .DS_Store | Bin 6148 -> 6148 bytes config/.DS_Store | Bin 6148 -> 6148 bytes config/cesm/.DS_Store | Bin 6148 -> 6148 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/.DS_Store b/.DS_Store index 8546179beaf523c3977fc0cb74ca0760f3434c48..28f605bb9b468a658acfade3c03b7b4488bd0699 100644 GIT binary patch delta 22 dcmZoMXffC@hmq6R+|oiv!Nk;j^HRomF#t_j22=n5 delta 22 dcmZoMXffC@hmq6B(A+{t!PMMn^HRomF#t^621x(_ diff --git a/config/.DS_Store b/config/.DS_Store index 0f229af3af0d8cc0a08ddcb32f2699cf13cfd167..bede552f5c58337f0e35991618b3077e517dbca6 100644 GIT binary patch delta 22 dcmZoMXffC@gOStN+|oiv!Nk;j^FqdCF#t^s22cP1 delta 22 dcmZoMXffC@gOSt7(A+{t!PMMn^FqdCF#t@F21Nh> diff --git a/config/cesm/.DS_Store b/config/cesm/.DS_Store index 7a6cfeca5125765e4a49e382e6ba29749fc00c59..1eed0c63f2f48fc0274b6ccb7ef8e921dc054fb5 100644 GIT binary patch delta 22 dcmZoMXffEZfQi%C+|oiv!Nk;j^Gc>@Q2@Q2 Date: Tue, 23 Jan 2024 17:20:11 -0700 Subject: [PATCH 8/9] clean up python link step, add link flag --- config/cesm/machines/config_compilers.xml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index f4263ce8f92..53df7d7a46b 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -125,10 +125,10 @@ using a fortran linker. -fconvert=big-endian -ffree-line-length-none -ffixed-line-length-none -fopenmp + least with some versions of gfortran (confirmed with 5.4.0, 6.3.0 and + 7.1.0), gfortran's isnan (which is called in cime via the + CPRGNU-specific shr_infnan_isnan) causes a floating point exception + when called on a signaling NaN. --> -g -Wall -Og -fbacktrace -ffpe-trap=zero,overflow -fcheck=bounds -O @@ -748,8 +748,7 @@ using a fortran linker. $ENV{NETCDF} $ENV{PNETCDF} - -lnetcdff -lnetcdf - -L/glade/work/wchapman/miniconda3.1/envs/cesmTORCH/lib -lpython3.11 -Wl,-rpath /glade/work/wchapman/miniconda3.1/envs/cesmTORCH/lib/ + -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf @@ -762,11 +761,15 @@ using a fortran linker. -march=core-avx2 -no-fma + + -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib + + - -march=core-avx2 -no-fma + -march=core-avx2 -no-fma -march=core-avx2 -no-fma -qno-opt-dynamic-align -fp-model precise -std=gnu99 @@ -1050,7 +1053,7 @@ using a fortran linker. + may require installation of the Apple Developer Tools. --> -framework Accelerate From d061e0789f1041610bdce6637cfaedbdb66b7446 Mon Sep 17 00:00:00 2001 From: James Edwards Date: Mon, 29 Jan 2024 11:23:53 -0700 Subject: [PATCH 9/9] add USE_CONDA xml variable, add machine derecho-gpu --- config/cesm/machines/config_batch.xml | 15 +++ config/cesm/machines/config_compilers.xml | 18 +++- config/cesm/machines/config_machines.xml | 101 ++++++++++++++++++ scripts/Tools/Makefile | 11 ++ src/build_scripts/buildlib.csm_share | 5 +- src/drivers/mct/cime_config/buildexe | 3 +- .../mct/cime_config/config_component.xml | 13 +++ 7 files changed, 163 insertions(+), 3 deletions(-) diff --git a/config/cesm/machines/config_batch.xml b/config/cesm/machines/config_batch.xml index 896e37cc9cf..b0eb7039927 100644 --- a/config/cesm/machines/config_batch.xml +++ b/config/cesm/machines/config_batch.xml @@ -370,6 +370,21 @@ + + qsub + + + + + -S {{ shell }} + -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:ngpus=4 + + + main + develop + + + sbatch diff --git a/config/cesm/machines/config_compilers.xml b/config/cesm/machines/config_compilers.xml index 28cfca3cba5..75fc4a7d3f7 100644 --- a/config/cesm/machines/config_compilers.xml +++ b/config/cesm/machines/config_compilers.xml @@ -761,11 +761,27 @@ using a fortran linker. -march=core-avx2 -no-fma + + + + $ENV{NETCDF} + $ENV{PNETCDF} - -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib + -Wl,--copy-dt-needed-entries -lnetcdff -lnetcdf + + + -qno-opt-dynamic-align -fp-model precise -std=gnu99 + -std=gnu89 + -march=core-avx2 -no-fma + + + -march=core-avx2 -no-fma + + + -march=core-avx2 -no-fma diff --git a/config/cesm/machines/config_machines.xml b/config/cesm/machines/config_machines.xml index 287e5152267..07ddd697a3e 100644 --- a/config/cesm/machines/config_machines.xml +++ b/config/cesm/machines/config_machines.xml @@ -1093,6 +1093,107 @@ This allows using a different mpirun command to launch unit tests + + NCAR AMD EPYC + CNL + intel + + mpich + $ENV{SCRATCH} + $ENV{CESMDATAROOT}/inputdata + $ENV{CESMDATAROOT}/inputdata/atm/datm7 + $CIME_OUTPUT_ROOT/archive/$CASE + $ENV{CESMDATAROOT}/cesm_baselines + $ENV{CESMDATAROOT}/cprnc/cprnc + 16 + pbs + cseg + 64 + 64 + TRUE + + mpiexec + + --label + --line-buffer + -n {{ total_tasks }} + + + + $LMOD_ROOT/lmod/init/perl + $LMOD_ROOT/lmod/init/env_modules_python.py + $LMOD_ROOT/lmod/init/sh + $LMOD_ROOT/lmod/init/csh + $LMOD_ROOT/lmod/libexec/lmod perl + $LMOD_ROOT/lmod/libexec/lmod python + module + module + + cesmdev/1.0 + ncarenv/23.09 + + craype + conda/latest + + + intel/2023.2.1 + mkl + spherepack/3.2 + + + intel-oneapi/2023.2.1 + mkl + + + intel-classic/2023.2.1 + mkl + + + cce/15.0.1 + cray-libsci/23.02.1.1 + + + gcc/12.2.0 + cray-libsci/23.02.1.1 + + + nvhpc/23.7 + + + ncarcompilers/1.0.0 + cmake + + + cray-mpich/8.1.27 + + + mpi-serial/2.3.0 + + + + netcdf/4.9.2 + + + + netcdf-mpi/4.9.2 + parallel-netcdf/1.12.3 + + + + esmf/8.6.0 + + + + + + 64M + hybrid + memhooks + $ENV{NCAR_ROOT_SPHEREPACK}/lib + + + + PNL IBM Xeon cluster, os is Linux (pgi), batch system is SLURM LINUX diff --git a/scripts/Tools/Makefile b/scripts/Tools/Makefile index dcf8ab3ce78..5d42f7aed35 100644 --- a/scripts/Tools/Makefile +++ b/scripts/Tools/Makefile @@ -217,6 +217,7 @@ else endif endif endif + # Set PETSc info if it is being used ifeq ($(strip $(USE_PETSC)), TRUE) ifdef PETSC_PATH @@ -546,6 +547,16 @@ ifdef LIB_MPI endif endif +# Set python info if it is being used +ifeq ($(strip $(USE_CONDA)), True) + ifdef CONDA_PREFIX + SLIBS += -L$(CONDA_PREFIX)/lib -lpython3 -Wl,-rpath,$(CONDA_PREFIX)/lib + else + $(error CONDA_PREFIX must be defined when USE_CONDA is TRUE) + endif +endif + + ifdef XIOS_PATH SLIBS += -L$(XIOS_PATH)/lib -lxios -lstdc++ endif diff --git a/src/build_scripts/buildlib.csm_share b/src/build_scripts/buildlib.csm_share index 3598a5ed18c..4859c1a8ccf 100755 --- a/src/build_scripts/buildlib.csm_share +++ b/src/build_scripts/buildlib.csm_share @@ -47,7 +47,6 @@ def buildlib(bldroot, installpath, caseroot): cimeroot = case.get_value("CIMEROOT") srcroot = case.get_value("SRCROOT") filepath = [os.path.join(caseroot,"SourceMods","src.share"), - os.path.join(srcroot, "forpy"), os.path.join(cimeroot,"src","drivers","mct","shr"), os.path.join(cimeroot,"src","share","streams"), os.path.join(cimeroot,"src","share","util"), @@ -61,6 +60,10 @@ def buildlib(bldroot, installpath, caseroot): use_esmf = "noesmf" filepath.append(os.path.join(cimeroot, "src", "share", "esmf_wrf_timemgr")) + if case.get_value("USE_CONDA"): + filepath.append(os.path.join(srcroot, "forpy")) + + comp_interface = case.get_value("COMP_INTERFACE") ninst_value = case.get_value("NINST_VALUE") diff --git a/src/drivers/mct/cime_config/buildexe b/src/drivers/mct/cime_config/buildexe index 409c7a78b09..5ee99cd3166 100755 --- a/src/drivers/mct/cime_config/buildexe +++ b/src/drivers/mct/cime_config/buildexe @@ -33,7 +33,8 @@ def _main_func(): model = case.get_value("MODEL") num_esp = case.get_value("NUM_COMP_INST_ESP") os.environ["PIO_VERSION"] = str(case.get_value("PIO_VERSION")) - + os.environ["USE_CONDA"] = str(case.get_value("USE_CONDA")) + print(f'USE_CONDA = {os.environ["USE_CONDA"]} CONDA_PREFIX = {os.environ["CONDA_PREFIX"]}') expect((num_esp is None) or (int(num_esp) == 1), "ESP component restricted to one instance") diff --git a/src/drivers/mct/cime_config/config_component.xml b/src/drivers/mct/cime_config/config_component.xml index b0fc582f855..0a574c59ac2 100644 --- a/src/drivers/mct/cime_config/config_component.xml +++ b/src/drivers/mct/cime_config/config_component.xml @@ -722,6 +722,19 @@ CLM. This is currently only supported for certain machines. + + logical + TRUE,FALSE + FALSE + build_def + env_build.xml + TRUE implies linking to a python3 library located in CONDA_PREFIX/lib + and using the file forpy_mod.F90 to allow calls between fortran and python + typical python usage might include pytorch, tensorflow, and/or numpy + + + + logical TRUE,FALSE