From 63887f383960e8a1e8a1389e1e55bac3a2299238 Mon Sep 17 00:00:00 2001 From: apcraig Date: Tue, 24 Sep 2024 09:28:41 -0600 Subject: [PATCH 1/2] Update Derecho Port - update inteloneapi, validate, use -O1, problems with -check all. - update cray to ncarenv/23.09 and cce/16.0.1, answers change - update intel to ncarenv/23.09 and intel/2023.2.1, answer bit-for-bit - update nvhpc to ncarenv/23.09 and nvhpc/23.7, answers change Add ifndef __INTEL_LLVM_COMPILER (for intel oneapi) around an OMP loop that the compiler doesn't handle properly (reported to intel) in ice_history.F90. Update QC documentation in the user guide to clarify where/how to run the cice.t-test.py script. --- cicecore/cicedyn/analysis/ice_history.F90 | 4 ++++ .../scripts/machines/Macros.derecho_inteloneapi | 9 +++++---- configuration/scripts/machines/env.derecho_cray | 10 +++++----- configuration/scripts/machines/env.derecho_intel | 12 ++++++------ .../scripts/machines/env.derecho_inteloneapi | 15 ++++++++------- configuration/scripts/machines/env.derecho_nvhpc | 8 ++++---- doc/source/user_guide/ug_testing.rst | 6 ++++-- 7 files changed, 36 insertions(+), 28 deletions(-) diff --git a/cicecore/cicedyn/analysis/ice_history.F90 b/cicecore/cicedyn/analysis/ice_history.F90 index 32f744477..a0313c56c 100644 --- a/cicecore/cicedyn/analysis/ice_history.F90 +++ b/cicecore/cicedyn/analysis/ice_history.F90 @@ -2325,9 +2325,11 @@ subroutine accum_hist (dt) ! increment field !--------------------------------------------------------------- +#ifndef __INTEL_LLVM_COMPILER !$OMP PARALLEL DO PRIVATE(iblk,i,j,ilo,ihi,jlo,jhi,this_block, & !$OMP k,n,qn,ns,sn,rho_ocn,rho_ice,Tice,Sbr,phi,rhob,dfresh,dfsalt,sicen, & !$OMP worka,workb,worka3,Tinz4d,Sinz4d,Tsnz4d) +#endif do iblk = 1, nblocks this_block = get_block(blocks_ice(iblk),iblk) @@ -3637,7 +3639,9 @@ subroutine accum_hist (dt) call accum_hist_snow (iblk) enddo ! iblk +#ifndef __INTEL_LLVM_COMPILER !$OMP END PARALLEL DO +#endif call icepack_warnings_flush(nu_diag) if (icepack_warnings_aborted()) call abort_ice(error_message=subname, & diff --git a/configuration/scripts/machines/Macros.derecho_inteloneapi b/configuration/scripts/machines/Macros.derecho_inteloneapi index ae6640388..6f5caa1f2 100644 --- a/configuration/scripts/machines/Macros.derecho_inteloneapi +++ b/configuration/scripts/machines/Macros.derecho_inteloneapi @@ -12,11 +12,12 @@ FFLAGS := -fp-model precise -convert big_endian -assume byterecl -ftz -trace FFLAGS_NOOPT:= -O0 ifeq ($(ICE_BLDDEBUG), true) - FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg -# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -link_mpi=dbg -stand f08 -# FFLAGS += -O0 -g -check all -fpe0 -ftrapuv -fp-model except -check noarg_temp_created -init=snan,arrays -link_mpi=dbg +# -check uninit is needed on the ld step but it still throws errors in 2023.* and 2024.0.*, likely compiler bug + FFLAGS += -O0 -g -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# FFLAGS += -O0 -g -check uninit -check bounds -check pointers -fpe0 -check noarg_temp_created -link_mpi=dbg +# LDFLAGS += -check uninit else - FFLAGS += -O2 + FFLAGS += -O1 endif SCC := icx diff --git a/configuration/scripts/machines/env.derecho_cray b/configuration/scripts/machines/env.derecho_cray index 47cebd5cb..4469190fe 100644 --- a/configuration/scripts/machines/env.derecho_cray +++ b/configuration/scripts/machines/env.derecho_cray @@ -10,11 +10,11 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load cce/15.0.1 +module load cce/16.0.1 module load ncarcompilers -module load cray-mpich/8.1.25 +module load cray-mpich/8.1.27 module load netcdf/4.9.2 #module load hdf5/1.12.2 #module load netcdf-mpi/4.9.2 @@ -29,7 +29,7 @@ if ($ICE_IOTYPE =~ pio*) then if ($ICE_IOTYPE == "pio1") then module load parallelio/1.10.1 else - module load parallelio/2.6.1 + module load parallelio/2.6.2 endif endif endif @@ -61,7 +61,7 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME cray -setenv ICE_MACHINE_ENVINFO "cce 15.0.1, cray-mpich 8.1.25, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.1" +setenv ICE_MACHINE_ENVINFO "Cray clang/fortran cce 16.0.1, cray-mpich 8.1.27, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg diff --git a/configuration/scripts/machines/env.derecho_intel b/configuration/scripts/machines/env.derecho_intel index 63626dc33..05c719838 100644 --- a/configuration/scripts/machines/env.derecho_intel +++ b/configuration/scripts/machines/env.derecho_intel @@ -10,16 +10,16 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load intel/2023.0.0 +module load intel/2023.2.1 module load ncarcompilers -module load cray-mpich/8.1.25 +module load cray-mpich/8.1.27 module load netcdf/4.9.2 #module load hdf5/1.12.2 #module load netcdf-mpi/4.9.2 -module load cray-libsci/23.02.1.1 +module load cray-libsci/23.09.1.1 if ($?ICE_IOTYPE) then if ($ICE_IOTYPE =~ pio*) then @@ -29,7 +29,7 @@ if ($ICE_IOTYPE =~ pio*) then if ($ICE_IOTYPE == "pio1") then module load parallelio/1.10.1 else - module load parallelio/2.6.1 + module load parallelio/2.6.2 endif endif endif @@ -61,7 +61,7 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME intel -setenv ICE_MACHINE_ENVINFO "ifort 2021.8.0 20221119, oneAPI DPC++/C++ 2023.0.0.20221201), cray-mpich 8.1.25, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.1" +setenv ICE_MACHINE_ENVINFO "ifort 2021.10.0 20230609, oneAPI DPC++/C++ 2023.2.0.20230721, cray-mpich 8.1.27, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg diff --git a/configuration/scripts/machines/env.derecho_inteloneapi b/configuration/scripts/machines/env.derecho_inteloneapi index 8f3911036..79715ba2a 100644 --- a/configuration/scripts/machines/env.derecho_inteloneapi +++ b/configuration/scripts/machines/env.derecho_inteloneapi @@ -10,16 +10,17 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load intel-oneapi/2023.0.0 -module load ncarcompilers -module load cray-mpich/8.1.25 +module load intel-oneapi/2023.2.1 +#module load mkl/2023.3.0 +module load ncarcompilers/1.0.0 +module load cray-mpich/8.1.27 module load netcdf/4.9.2 #module load hdf5/1.12.2 #module load netcdf-mpi/4.9.2 -module load cray-libsci/23.02.1.1 +module load cray-libsci/23.09.1.1 if ($?ICE_IOTYPE) then if ($ICE_IOTYPE =~ pio*) then @@ -29,7 +30,7 @@ if ($ICE_IOTYPE =~ pio*) then if ($ICE_IOTYPE == "pio1") then module load parallelio/1.10.1 else - module load parallelio/2.6.1 + module load parallelio/2.6.2 endif endif endif @@ -61,7 +62,7 @@ setenv OMP_STACKSIZE 64M setenv ICE_MACHINE_MACHNAME derecho setenv ICE_MACHINE_MACHINFO "HPE Cray EX Milan Slingshot 11" setenv ICE_MACHINE_ENVNAME inteloneapi -setenv ICE_MACHINE_ENVINFO "ifx 2023.0.0 20221201, oneAPI DPC++/C++ 2023.0.0.20221201, cray-mpich 8.1.25, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.1" +setenv ICE_MACHINE_ENVINFO "oneAPI DPC++/C++/ifx 2023.2.0 20230721, cray-mpich 8.1.27, netcdf4.9.2, pnetcdf1.12.3, pio1.10.1, pio2.6.2" setenv ICE_MACHINE_MAKE gmake setenv ICE_MACHINE_WKDIR /glade/derecho/scratch/$user/CICE_RUNS setenv ICE_MACHINE_INPUTDATA /glade/campaign/cesm/development/pcwg diff --git a/configuration/scripts/machines/env.derecho_nvhpc b/configuration/scripts/machines/env.derecho_nvhpc index 34342769c..eb7eee3f0 100644 --- a/configuration/scripts/machines/env.derecho_nvhpc +++ b/configuration/scripts/machines/env.derecho_nvhpc @@ -10,11 +10,11 @@ if ("$inp" != "-nomodules") then source ${MODULESHOME}/init/csh module --force purge -module load ncarenv/23.06 +module load ncarenv/23.09 module load craype -module load nvhpc/23.5 +module load nvhpc/23.7 module load ncarcompilers -module load cray-mpich/8.1.25 +module load cray-mpich/8.1.27 module load netcdf/4.9.2 #module load hdf5/1.12.2 #module load netcdf-mpi/4.9.2 @@ -29,7 +29,7 @@ if ($ICE_IOTYPE =~ pio*) then if ($ICE_IOTYPE == "pio1") then module load parallelio/1.10.1 else - module load parallelio/2.6.0 + module load parallelio/2.6.2 endif endif endif diff --git a/doc/source/user_guide/ug_testing.rst b/doc/source/user_guide/ug_testing.rst index 6867214b5..4fea329fb 100644 --- a/doc/source/user_guide/ug_testing.rst +++ b/doc/source/user_guide/ug_testing.rst @@ -1107,11 +1107,13 @@ You can also setup a conda env with the same utitities To run the validation test, setup a baseline run with the original baseline model and then a perturbation run based on recent model changes. Use ``--set qc`` in both runs in addition -to other settings needed. Then use the QC script to compare history output, +to other settings needed. Then use the QC script to compare history output. The QC script should +be run from the ``configuration/scripts/tests/QC`` directory because other files from that +directory are required for the script. .. code-block:: bash - cp configuration/scripts/tests/QC/cice.t-test.py . + cd configuration/scripts/tests/QC ./cice.t-test.py /path/to/baseline/history /path/to/test/history The script will produce output similar to: From 51e3ee76c49a8d95b902e7ed7f8745f6a15df68d Mon Sep 17 00:00:00 2001 From: Nicholas Szapiro <149816583+NickSzapiro-NOAA@users.noreply.github.com> Date: Wed, 25 Sep 2024 17:56:13 -0400 Subject: [PATCH 2/2] No ncat,ntrcr in call icepack_aggregate in ice_prescribed_mod.F90 --- cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90 | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90 b/cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90 index 2436783bc..f27c34fdd 100644 --- a/cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90 +++ b/cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90 @@ -446,8 +446,7 @@ subroutine ice_prescribed_phys() !-------------------------------------------------------------------- ! compute aggregate ice state and open water area !-------------------------------------------------------------------- - call icepack_aggregate(ncat = ncat, & - aicen = aicen(i,j,:,iblk), & + call icepack_aggregate(aicen = aicen(i,j,:,iblk), & trcrn = trcrn(i,j,1:ntrcr,:,iblk), & vicen = vicen(i,j,:,iblk), & vsnon = vsnon(i,j,:,iblk), & @@ -456,7 +455,6 @@ subroutine ice_prescribed_phys() vice = vice (i,j, iblk), & vsno = vsno (i,j, iblk), & aice0 = aice0(i,j, iblk), & - ntrcr = ntrcr, & trcr_depend = trcr_depend(1:ntrcr), & trcr_base = trcr_base(1:ntrcr,:), & n_trcr_strata = n_trcr_strata(1:ntrcr), &