wip

WeiqunZhang · Oct 24, 2023 · e2216b6 · e2216b6
1 parent b98d4b3
commit e2216b6
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 25 deletions.
diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs
@@ -446,6 +446,8 @@ else ifeq ($(USE_CUDA),TRUE)
  AMREX_CCOMP = $(NVCC_HOST_COMP)
  else ifeq ($(COMP),cray)
  AMREX_CCOMP = cray
+ else ifeq ($(COMP),nvhpc)
+ AMREX_CCOMP = nvhpc
  else
  AMREX_CCOMP = gnu
  endif
@@ -776,8 +778,10 @@ else ifeq ($(USE_CUDA),TRUE)
  LINK_WITH_FORTRAN_COMPILER=TRUE
  endif
 
- $(info Loading $(AMREX_HOME)/Tools/GNUMake/comps/nvcc.mak...)
- include $(AMREX_HOME)/Tools/GNUMake/comps/nvcc.mak
+ ifneq ($(lowercase_comp),nvhpc)
+ $(info Loading $(AMREX_HOME)/Tools/GNUMake/comps/nvcc.mak...)
+ include $(AMREX_HOME)/Tools/GNUMake/comps/nvcc.mak
+ endif
 
  ifeq ($(USE_MPI),TRUE)
  # Make sure that the C/C++ MPI

diff --git a/Tools/GNUMake/comps/nvhpc.mak b/Tools/GNUMake/comps/nvhpc.mak
@@ -52,9 +52,19 @@ ifeq ($(USE_ACC),TRUE)
  endif
 endif
 
-# Note that -O2 is the default optimization level for NVHPC
+ifeq ($(USE_CUDA),TRUE)
+ GENERIC_NVHPC_FLAGS += -cuda
+ ifneq ($(CUDA_ARCH),)
+ GENERIC_NVHPC_FLAGS += -gpu=cc$(CUDA_ARCH)
+ endif
+ ifeq ($(USE_GPU_RDC),TRUE)
+ GENERIC_NVHPC_FLAGS += -gpu=rdc
+ endif
+endif
+
+# Note that -O3 is the default optimization level for NVHPC
 
-NVHPC_OPT := -O2 -fast
+NVHPC_OPT := -O3 -fast
 
 ########################################################################
 ########################################################################
@@ -127,6 +137,7 @@ endif # AMREX_CCOMP == nvhpc
 ########################################################################
 ########################################################################
 
+ifneq ($(BL_NO_FORT),TRUE)
 ifeq ($(AMREX_FCOMP),nvhpc)
 
 #
@@ -220,3 +231,4 @@ override XTRALIBS += -lstdc++ -latomic -lnvf -lrt
 LINK_WITH_FORTRAN_COMPILER ?= $(USE_F_INTERFACES)
 
 endif # AMREX_FCOMP == nvhpc
+endif # not BL_NO_FORT
diff --git a/Tools/GNUMake/sites/Make.nersc b/Tools/GNUMake/sites/Make.nersc
@@ -10,6 +10,8 @@ endif
 
 ifeq ($(which_computer),$(filter $(which_computer),perlmutter))
 
+ifneq ($(AMREX_CCOMP),nvhpc)
+
  ifdef PE_ENV
  ifneq ($(USE_GPU),TRUE)
  lowercase_peenv := $(shell echo $(PE_ENV) | tr A-Z a-z)
@@ -25,28 +27,20 @@ ifeq ($(which_computer),$(filter $(which_computer),perlmutter))
  endif
 
  ifeq ($(USE_CUDA),TRUE)
- ifdef MPICH_ROOT
- CFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicc -show 2> /dev/null)))'
- CXXFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicxx -show 2> /dev/null)))'
- else ifdef OPENMPI_ROOT
- CFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicc -show 2> /dev/null)))'
- CXXFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicxx -show 2> /dev/null)))'
+ ifeq ($(AMREX_CCOMP),nvhpc)
+ #CFLAGS += $(wordlist 2,1024,$(shell cc -craype-verbose 2> /dev/null))
+ #CXXFLAGS += $(wordlist 2,1024,$(shell CC -craype-verbose 2> /dev/null))
  else
- CFLAGS += -Xcompiler='$(wordlist 2,1024,$(shell cc -craype-verbose 2> /dev/null))'
- CXXFLAGS += -Xcompiler='$(wordlist 2,1024,$(shell CC -craype-verbose 2> /dev/null))'
- endif
- else ifeq ($(USE_MPI),FALSE)
- CFLAGS += $(wordlist 2,1024,$(shell cc -craype-verbose 2> /dev/null))
- CXXFLAGS += $(wordlist 2,1024,$(shell CC -craype-verbose 2> /dev/null))
- endif
-
- ifeq ($(USE_CUDA),TRUE)
- ifdef NPE_VERSION
- CFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicc -show 2> /dev/null)))'
- CXXFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicxx -show 2> /dev/null)))'
- else
- CFLAGS += -Xcompiler='$(wordlist 2,1024,$(shell cc -craype-verbose 2> /dev/null))'
- CXXFLAGS += -Xcompiler='$(wordlist 2,1024,$(shell CC -craype-verbose 2> /dev/null))'
+ ifdef MPICH_ROOT
+ CFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicc -show 2> /dev/null)))'
+ CXXFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicxx -show 2> /dev/null)))'
+ else ifdef OPENMPI_ROOT
+ CFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicc -show 2> /dev/null)))'
+ CXXFLAGS += -Xcompiler='$(filter-out -Wl%, $(wordlist 2,1024,$(shell mpicxx -show 2> /dev/null)))'
+ else
+ CFLAGS += -Xcompiler='$(wordlist 2,1024,$(shell cc -craype-verbose 2> /dev/null))'
+ CXXFLAGS += -Xcompiler='$(wordlist 2,1024,$(shell CC -craype-verbose 2> /dev/null))'
+ endif
  endif
  else ifeq ($(USE_MPI),FALSE)
  CFLAGS += $(wordlist 2,1024,$(shell cc -craype-verbose 2> /dev/null))
@@ -119,6 +113,7 @@ ifeq ($(which_computer),$(filter $(which_computer),perlmutter))
  endif
 
 endif
+endif
 
 ifeq ($(which_computer),$(filter $(which_computer),cori))