From 88824e6f93f0cf25136e89eb7118dd49e61a0f38 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Wed, 12 Jan 2022 07:43:10 -0700
Subject: [PATCH 1/4] add a pio 2.5.5 module on cheyenne

---
 machines/config_machines.xml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 1180f26f..861fb237 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -761,6 +761,9 @@ This allows using a different mpirun command to launch unit tests
       <modules>
         <command name="load">ncarcompilers/0.5.0</command>
       </modules>
+      <modules mpilib="mpi-serial">
+        <command name="load">mpi-serial/2.11.0</command>
+      </modules>
       <modules compiler="gnu" mpilib="mpi-serial">
         <command name="load">netcdf/4.7.4</command>
       </modules>
@@ -770,6 +773,9 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="pgi" mpilib="mpi-serial">
         <command name="load">netcdf/4.7.4</command>
       </modules>
+      <modules>
+        <command name="load">pio/2.5.5</command>
+      </modules>
     </module_system>
     <environment_variables>
       <env name="OMP_STACKSIZE">1024M</env>

From c192a00a21b68bea0f7af00a208d8ce90e8a74df Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Wed, 12 Jan 2022 13:24:47 -0700
Subject: [PATCH 2/4] update machines def for cheyenne

---
 machines/config_machines.xml | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index 861fb237..e7e77660 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -582,9 +582,10 @@ This allows using a different mpirun command to launch unit tests
     <MPIRUN_RETRY_REGEX>MPT: Launcher network accept (MPI_LAUNCH_TIMEOUT) timed out</MPIRUN_RETRY_REGEX>
     <MPIRUN_RETRY_COUNT>10</MPIRUN_RETRY_COUNT>
     <OS>LINUX</OS>
-    <COMPILERS>intel,gnu,pgi</COMPILERS>
+    <COMPILERS>intel,gnu,nvhpc,pgi</COMPILERS>
     <MPILIBS compiler="intel" >mpt,openmpi</MPILIBS>
     <MPILIBS compiler="pgi" >openmpi,mpt</MPILIBS>
+    <MPILIBS compiler="nvhpc" >openmpi,mpt</MPILIBS>
     <MPILIBS compiler="gnu" >mpt,openmpi</MPILIBS>
     <CIME_OUTPUT_ROOT>/glade/scratch/$USER</CIME_OUTPUT_ROOT>
     <DIN_LOC_ROOT>$ENV{CESMDATAROOT}/inputdata</DIN_LOC_ROOT>
@@ -670,6 +671,9 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="pgi">
         <command name="load">pgi/20.4</command>
       </modules>
+      <modules compiler="nvhpc">
+        <command name="load">nvhpc/21.11</command>
+      </modules>
       <modules compiler="intel" mpilib="!mpi-serial" DEBUG="TRUE">
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/intel/19.1.1/</command>
         <command name="load">esmf-8.2.0b23-ncdfio-mpt-g</command>
@@ -736,9 +740,9 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">esmf-8.2.0b23-ncdfio-mpiuni-O</command>
       </modules>
       <modules mpilib="mpt" compiler="gnu">
-        <command name="load">mpt/2.21</command>
-        <command name="load">netcdf-mpi/4.7.3</command>
-        <command name="load">pnetcdf/1.12.1</command>
+        <command name="load">mpt/2.25</command>
+        <command name="load">netcdf-mpi/4.8.1</command>
+        <command name="load">pnetcdf/1.12.2</command>
       </modules>
       <modules mpilib="mpt" compiler="intel">
         <command name="load">mpt/2.22</command>
@@ -746,10 +750,20 @@ This allows using a different mpirun command to launch unit tests
         <command name="load">pnetcdf/1.12.2</command>
       </modules>
       <modules mpilib="mpt" compiler="pgi">
-        <command name="load">mpt/2.19</command>
+        <command name="load">mpt/2.22</command>
         <command name="load">netcdf-mpi/4.7.4</command>
         <command name="load">pnetcdf/1.12.1</command>
       </modules>
+      <modules mpilib="mpt" compiler="nvhpc">
+        <command name="load">mpt/2.25</command>
+        <command name="load">netcdf-mpi/4.8.1</command>
+        <command name="load">pnetcdf/1.12.2</command>
+      </modules>
+      <modules mpilib="openmpi" compiler="nvhpc">
+        <command name="load">openmpi/4.1.1</command>
+        <command name="load">netcdf-mpi/4.8.1</command>
+        <command name="load">pnetcdf/1.12.2</command>
+      </modules>
       <modules mpilib="openmpi" compiler="pgi">
         <command name="load">openmpi/4.0.5</command>
         <command name="load">netcdf-mpi/4.7.4</command>
@@ -773,7 +787,7 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="pgi" mpilib="mpi-serial">
         <command name="load">netcdf/4.7.4</command>
       </modules>
-      <modules>
+      <modules compiler="!pgi">
         <command name="load">pio/2.5.5</command>
       </modules>
     </module_system>

From 752d79d4dc4eb37b46dc9dd3f45b8911532e97a8 Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 13 Jan 2022 13:16:47 -0700
Subject: [PATCH 3/4] add esmf for nvhpc on cheyenne

---
 machines/config_machines.xml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index e7e77660..dc822fd0 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -723,6 +723,14 @@ This allows using a different mpirun command to launch unit tests
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
         <command name="load">esmf-8.2.0b23-ncdfio-mpt-O</command>
       </modules>
+      <modules compiler="nvhpc" mpilib="mpt" DEBUG="FALSE">
+        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
+        <command name="load">esmf-8.3.0b05-ncdfio-mpt-O</command>
+      </modules>
+      <modules compiler="nvhpc" mpilib="mpt" DEBUG="TRUE">
+        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
+        <command name="load">esmf-8.3.0b05-ncdfio-mpt-g</command>
+      </modules>
       <modules compiler="pgi" mpilib="openmpi" DEBUG="TRUE">
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
         <command name="load">esmf-8.2.0b23-ncdfio-openmpi-g</command>

From 719521e3f623093eeb107ea5c48e706029b83bfe Mon Sep 17 00:00:00 2001
From: Jim Edwards <jedwards@ucar.edu>
Date: Thu, 13 Jan 2022 15:20:21 -0700
Subject: [PATCH 4/4] need a cmake macro to load lapack with nvhpc

---
 machines/cmake_macros/nvhpc_cheyenne.cmake |  4 ++++
 machines/config_machines.xml               | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+)
 create mode 100644 machines/cmake_macros/nvhpc_cheyenne.cmake

diff --git a/machines/cmake_macros/nvhpc_cheyenne.cmake b/machines/cmake_macros/nvhpc_cheyenne.cmake
new file mode 100644
index 00000000..e3769dd9
--- /dev/null
+++ b/machines/cmake_macros/nvhpc_cheyenne.cmake
@@ -0,0 +1,4 @@
+string(APPEND SLIBS " -llapack -lblas")
+if (MPILIB STREQUAL mpi-serial)
+  string(APPEND SLIBS " -ldl")
+endif()
diff --git a/machines/config_machines.xml b/machines/config_machines.xml
index dc822fd0..6a8336fd 100644
--- a/machines/config_machines.xml
+++ b/machines/config_machines.xml
@@ -731,6 +731,14 @@ This allows using a different mpirun command to launch unit tests
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
         <command name="load">esmf-8.3.0b05-ncdfio-mpt-g</command>
       </modules>
+      <modules compiler="nvhpc" mpilib="openmpi" DEBUG="FALSE">
+        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
+        <command name="load">esmf-8.3.0b05-ncdfio-openmpi-O</command>
+      </modules>
+      <modules compiler="nvhpc" mpilib="openmpi" DEBUG="TRUE">
+        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
+        <command name="load">esmf-8.3.0b05-ncdfio-openmpi-g</command>
+      </modules>
       <modules compiler="pgi" mpilib="openmpi" DEBUG="TRUE">
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
         <command name="load">esmf-8.2.0b23-ncdfio-openmpi-g</command>
@@ -747,6 +755,14 @@ This allows using a different mpirun command to launch unit tests
         <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/pgi/20.4/</command>
         <command name="load">esmf-8.2.0b23-ncdfio-mpiuni-O</command>
       </modules>
+      <modules compiler="nvhpc" mpilib="mpi-serial" DEBUG="TRUE">
+        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
+        <command name="load">esmf-8.3.0b05-ncdfio-mpiuni-O</command>
+      </modules>
+      <modules compiler="nvhpc" mpilib="mpi-serial" DEBUG="FALSE">
+        <command name="use">/glade/p/cesmdata/cseg/PROGS/modulefiles/esmfpkgs/nvhpc/21.11</command>
+        <command name="load">esmf-8.3.0b05-ncdfio-mpiuni-O</command>
+      </modules>
       <modules mpilib="mpt" compiler="gnu">
         <command name="load">mpt/2.25</command>
         <command name="load">netcdf-mpi/4.8.1</command>
@@ -795,6 +811,9 @@ This allows using a different mpirun command to launch unit tests
       <modules compiler="pgi" mpilib="mpi-serial">
         <command name="load">netcdf/4.7.4</command>
       </modules>
+      <modules compiler="nvhpc" mpilib="mpi-serial">
+        <command name="load">netcdf/4.8.1</command>
+      </modules>
       <modules compiler="!pgi">
         <command name="load">pio/2.5.5</command>
       </modules>