Merge pull request #87 from EXP-code/fix-cuda-16

Updates for CUDA versions >= 12
EXP-code · Oct 23, 2024 · 181b8d1 · 181b8d1
2 parents 922d165 + 92c1f8b
commit 181b8d1
Show file tree

Hide file tree

Showing 22 changed files with 116 additions and 47 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.21) # Needed for CUDA, MPI, and CTest features
+cmake_minimum_required(VERSION 3.25) # Needed for CUDA, MPI, and CTest features
 
 project(
   EXP
@@ -121,6 +121,9 @@ if(OpenMP_FOUND)
   OPTION (USE_OpenMP "Use OpenMP" ON)
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+  if(ENABLE_CUDA)
+  set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler='${OpenMP_CXX_FLAGS}'")
+  endif()
 endif()
 # Slurm support
 if(SLURM_FOUND)

diff --git a/expui/CMakeLists.txt b/expui/CMakeLists.txt
@@ -16,8 +16,12 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
 
 
 if(ENABLE_CUDA)
-  list(APPEND common_INCLUDE ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(SLURM_FOUND)

diff --git a/exputil/CMakeLists.txt b/exputil/CMakeLists.txt
@@ -51,13 +51,18 @@ set(common_INCLUDE_DIRS $<INSTALL_INTERFACE:include>
   ${DEP_INC} ${EIGEN3_INCLUDE_DIR} ${HDF5_INCLUDE_DIRS}
   ${FFTW_INCLUDE_DIRS})
 
-set(common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
+set(common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
   yaml-cpp ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}
   ${FFTW_DOUBLE_LIB})
 
 if(ENABLE_CUDA)
-  list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+    set_source_files_properties(${BIORTH_SRC} ${PARTICLE_SRC} PROPERTIES LANGUAGE CUDA)
+  else ()
+    list(APPEND common_LINKLIB CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(ENABLE_XDR AND TIRPC_FOUND)
@@ -70,7 +75,7 @@ endif()
 add_library(exputil ${exputil_SOURCES})
 set_target_properties(exputil PROPERTIES OUTPUT_NAME exputil)
 target_include_directories(exputil PUBLIC ${common_INCLUDE_DIRS})
-target_link_libraries(exputil PUBLIC ${common_LINKLIBS})
+target_link_libraries(exputil PUBLIC ${common_LINKLIB})
 
 install(TARGETS exputil DESTINATION lib)
 

diff --git a/pyEXP/CMakeLists.txt b/pyEXP/CMakeLists.txt
@@ -17,7 +17,12 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
   ${HDF5_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR})
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(SLURM_FOUND)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -1,12 +1,3 @@
-
-set(CUDA_SRC)
-if (ENABLE_CUDA)
-  list(APPEND CUDA_SRC cudaPolarBasis.cu cudaSphericalBasis.cu
-    cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc
-    cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu
-    cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu)
-endif()
-
 set(exp_SOURCES Basis.cc Bessel.cc Component.cc
   Cube.cc Cylinder.cc ExternalCollection.cc
   ExternalForce.cc Orient.cc PotAccel.cc ScatterMFP.cc
@@ -19,9 +10,16 @@ set(exp_SOURCES Basis.cc Bessel.cc Component.cc
   OutMulti.cc OutRelaxation.cc OrbTrace.cc OutDiag.cc OutLog.cc
   OutVel.cc OutCoef.cc multistep.cc parse.cc SlabSL.cc step.cc
   tidalField.cc ultra.cc ultrasphere.cc MPL.cc OutFrac.cc OutCalbr.cc
-  ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc ${CUDA_SRC}
+  ParticleFerry.cc chkSlurm.c chkTimer.cc GravKernel.cc
   CenterFile.cc PolarBasis.cc FlatDisk.cc signals.cc)
 
+if (ENABLE_CUDA)
+  list(APPEND exp_SOURCES cudaPolarBasis.cu cudaSphericalBasis.cu
+    cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc
+    cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu
+    cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu)
+endif()
+
 set(common_INCLUDE_DIRS 
   $<INSTALL_INTERFACE:include>
   $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/include/>
@@ -39,8 +37,13 @@ if(PNG_FOUND)
 endif()
 
 if(ENABLE_CUDA)
-  list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+    set_source_files_properties(${exp_SOURCES} PROPERTIES LANGUAGE CUDA)
+  else ()
+    list(APPEND common_LINKLIB CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(SLURM_FOUND)
@@ -64,5 +67,9 @@ add_executable(exp expand.cc)
 target_include_directories(exp PUBLIC ${common_INCLUDE_DIRS})
 target_link_libraries(exp PUBLIC ${common_LINKLIB} EXPlib)
 
+if (ENABLE_CUDA)
+  set_target_properties(exp PROPERTIES LINKER_LANGUAGE CUDA)
+endif ()
+
 install(TARGETS EXPlib DESTINATION lib)
 install(TARGETS exp DESTINATION bin)
diff --git a/src/Cylinder.H b/src/Cylinder.H
@@ -14,7 +14,6 @@
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
-#include <NVTX.H>
 
 #include <CoefContainer.H>
 
@@ -155,15 +154,7 @@ private:
       a complete set of coefficients for force evaluation at an
       intermediate time step
   */
-  void compute_multistep_coefficients() 
-  {
-    if (play_back and not play_cnew) return;
-
-    nvTracerPtr tPtr;
-    if (cuda_prof)
-      tPtr = nvTracerPtr(new nvTracer("Cylinder::compute_multistep_coefficients"));
-    ortho->compute_multistep_coefficients(mfirst[mstep]);
-  }
+  void compute_multistep_coefficients();
 
   //! Reset used particle counter and mass registers
   virtual void multistep_reset();

diff --git a/src/Cylinder.cc b/src/Cylinder.cc
@@ -12,6 +12,7 @@
 #include <MixtureBasis.H>
 #include <Timer.H>
 #include <exputils.H>
+#include <NVTX.H>
 
 Timer timer_debug;
 
@@ -1659,6 +1660,16 @@ void Cylinder::multistep_update(int from, int to, Component* c, int i, int id)
 #endif
 }
 
+void Cylinder::compute_multistep_coefficients() 
+{
+  if (play_back and not play_cnew) return;
+
+  nvTracerPtr tPtr;
+  if (cuda_prof)
+    tPtr = nvTracerPtr(new nvTracer("Cylinder::compute_multistep_coefficients"));
+  ortho->compute_multistep_coefficients(mfirst[mstep]);
+}
+
 
 void Cylinder::multistep_reset() 
 { 

diff --git a/src/ExternalForce.H b/src/ExternalForce.H
@@ -53,6 +53,7 @@ public:
   //! Finish and clean-up (caching data necessary for restart)
   virtual void finish() {}
 
+  // #if HAVE_LIBCUDA==1
 #if HAVE_LIBCUDA==1
   //! Copy particles from device for non-cuda forces
   void getParticlesCuda(Component *c);

diff --git a/src/NVTX.H b/src/NVTX.H
@@ -7,7 +7,11 @@
 
 #if HAVE_LIBCUDA==1
 
+#if __CUDACC_VER_MAJOR__ < 12
 #include <nvToolsExt.h>
+#else
+#include <nvtx3/nvToolsExt.h>
+#endif 
 
 class nvTracer {
 private:

diff --git a/src/NVTX.cc b/src/NVTX.cc
@@ -1,4 +1,4 @@
-#include <NVTX.H>
+#include "NVTX.H"
 
 #if HAVE_LIBCUDA==1
 

diff --git a/src/Orient.H b/src/Orient.H
@@ -1,5 +1,3 @@
-// This may look like C code, but it is really -*- C++ -*-
-
 #ifndef _Orient_H
 #define _Orient_H
 

diff --git a/src/cudaCylinder.cu b/src/cudaCylinder.cu
@@ -3,6 +3,7 @@
 #include <Component.H>
 #include <Cylinder.H>
 #include <cudaReduce.cuH>
+#include <float.h>
 #include "expand.H"
 
 // Define for debugging

diff --git a/src/cudaPolarBasis.cu b/src/cudaPolarBasis.cu
@@ -3,6 +3,7 @@
 #include <Component.H>
 #include <PolarBasis.H>
 #include <cudaReduce.cuH>
+#include <float.h>
 #include "expand.H"
 
 // Define for debugging

diff --git a/src/global.H b/src/global.H
@@ -345,9 +345,11 @@ extern bool cuda_prof;
 #include "Species.H"
 
 #if HAVE_LIBCUDA==1
+extern int cudaGlobalDevice;
+#if defined (__NVCC__)
 #include <cudaUtil.cuH>
 extern thrust::device_vector<int> cuDstepL, cuDstepN;
-extern int cudaGlobalDevice;
+#endif
 #endif
 
 #endif
diff --git a/src/user/CMakeLists.txt b/src/user/CMakeLists.txt
@@ -9,12 +9,19 @@ set (common_INCLUDE_DIRS
   ${CMAKE_CURRENT_SOURCE_DIR}
   ${CMAKE_CURRENT_SOURCE_DIR}/..)
 
-set (common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
+set (common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
   exputil EXPlib yaml-cpp)
 
 if(ENABLE_CUDA)
-  list(APPEND common_INCLUDE_DIRS ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES} ${CUDAToolkit_INCLUDE_DIRS})
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
+  # set_source_files_properties(UserBar.cc UserDisk.cc UserHalo.cc 
+  #   UserLogPot.cc UserMNdisk.cc UserMW.cc UserTest.cc UserTestCuda.cc 
+  #   PROPERTIES LANGUAGE CUDA)
 endif()
 
 set(user_SRC      UserTest.cc)
@@ -33,7 +40,7 @@ foreach(mlib ${USER_MODULES})
   add_library(${mlib} ${${mlib}_SRC})
   set_target_properties(${mlib} PROPERTIES OUTPUT_NAME ${mlib})
   target_include_directories(${mlib} PUBLIC ${common_INCLUDE_DIRS})
-  target_link_libraries(${mlib} PUBLIC ${common_LINKLIBS})
+  target_link_libraries(${mlib} PUBLIC ${common_LINKLIB})
   install(TARGETS ${mlib} DESTINATION lib/user)
 endforeach()
 
diff --git a/utils/Analysis/CMakeLists.txt b/utils/Analysis/CMakeLists.txt
@@ -13,7 +13,12 @@ if(PNG_FOUND)
 endif()
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(SLURM_FOUND)

diff --git a/utils/ICs/CMakeLists.txt b/utils/ICs/CMakeLists.txt
@@ -9,7 +9,12 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil
   ${HDF5_CXX_LIBRARIES})
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(ENABLE_XDR AND TIRPC_FOUND)

diff --git a/utils/ICs/initial.cc b/utils/ICs/initial.cc
@@ -360,15 +360,15 @@ main(int ac, char **av)
   double       RMIN, RCYLMIN, RCYLMAX, SCSPH, RSPHSL, DMFAC, RFACTOR, SHFAC;
   double       X0, Y0, Z0, U0, V0, W0;
   int          RNUM, PNUM, TNUM, VFLAG, DFLAG;
-  bool         expcond, LOGR, CHEBY, SELECT, DUMPCOEF;
+  bool         expcond, LOGR, CHEBY, DUMPCOEF;
   int          CMAPR, CMAPZ, NCHEB, TCHEB, CMTYPE, NDR, NDZ, NHR, NHT, NDP;
   int          LMAX, NMAXH, NMAXD, MMAX, NUMX, NUMY, NOUT, NMAXLIM, NODD, DF;
   int          DIVERGE, DIVERGE2, SEED, itmax;
   double       DIVERGE_RFAC, DIVERGE_RFAC2;
   double       PPower, R_DF, DR_DF;
   double       Hratio, scale_height, scale_length, scale_lenfkN;
   double       disk_mass, gas_mass, gscal_length, ToomreQ, Temp, Tmin;
-  bool         const_height, images, multi, SVD, basis, zeropos, zerovel;
+  bool         const_height, images, multi, basis, zeropos, zerovel;
   bool         report, ignore, evolved, diskmodel;
   int          nhalo, ndisk, ngas, ngparam;
   std::string  hbods, dbods, gbods, suffix, centerfile, halofile1, halofile2;
@@ -870,7 +870,6 @@ main(int ac, char **av)
   EmpCylSL::CMAPZ       = CMAPZ;
   EmpCylSL::VFLAG       = VFLAG;
   EmpCylSL::logarithmic = LOGR;
-  EmpCylSL::PCAVAR      = SELECT;
 
                                 // Create expansion only if needed . . .
   std::shared_ptr<EmpCylSL> expandd;

diff --git a/utils/MSSA/CMakeLists.txt b/utils/MSSA/CMakeLists.txt
@@ -17,7 +17,12 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
   ${HDF5_INCLUDE_DIRS})
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(SLURM_FOUND)

diff --git a/utils/PhaseSpace/CMakeLists.txt b/utils/PhaseSpace/CMakeLists.txt
@@ -21,7 +21,12 @@ if(PNG_FOUND)
 endif()
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(ENABLE_XDR AND TIRPC_FOUND)

diff --git a/utils/SL/CMakeLists.txt b/utils/SL/CMakeLists.txt
@@ -6,7 +6,12 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil
   ${VTK_LIBRARIES})
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(ENABLE_XDR AND TIRPC_FOUND)

diff --git a/utils/Test/CMakeLists.txt b/utils/Test/CMakeLists.txt
@@ -17,7 +17,12 @@ set(common_INCLUDE
   ${CMAKE_CURRENT_SOURCE_DIR}/..)
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart CUDA::nvToolsExt )
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
+  if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
+    list(APPEND common_LINKLIB CUDA::nvtx3)
+  else ()
+    list(APPEND common_LINKLIB  CUDA::nvToolsExt)
+  endif ()
 endif()
 
 if(ENABLE_XDR AND TIRPC_FOUND)