WIP: GPU-aware MPI on AMD. Runs but may hang

ecmwf-ifs · Aug 25, 2023 · 2aa7729 · 2aa7729
1 parent ab807d2
commit 2aa7729
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 5 deletions.
diff --git a/src/trans/gpu/internal/trgtol_mod.F90 b/src/trans/gpu/internal/trgtol_mod.F90
@@ -707,7 +707,11 @@ SUBROUTINE TRGTOL_CUDAAWARE(PGLAT,KF_FS,KF_GP,KF_SCALARS_G,KVSET,KPTRGP,&
   CALL GSTATS(413,0)
 
 #ifdef ACCGPU
+#if defined(__NVCOMPILER) || defined(__PGI)
   !$ACC HOST_DATA USE_DEVICE(ZCOMBUFR,ZCOMBUFS)
+#elif defined(_CRAYFTN)
+  !$ACC HOST_DATA USE_DEVICE(ZCOMBUFR)
+#endif
 #endif
 #ifdef OMPGPU
   !$OMP TARGET DATA USE_DEVICE_PTR(ZCOMBUFR,ZCOMBUFS)

diff --git a/src/trans/gpu/internal/trltog_mod.F90 b/src/trans/gpu/internal/trltog_mod.F90
@@ -649,7 +649,11 @@ SUBROUTINE TRLTOG_CUDAAWARE(PGLAT,KF_FS,KF_GP,KF_SCALARS_G,KVSET,KPTRGP,&
 
   !...Receive loop.........................................................
 #ifdef ACCGPU
-  !$ACC HOST_DATA USE_DEVICE(ZCOMBUFS,ZCOMBUFR)
+#if defined(__NVCOMPILER) || defined(__PGI)
+  !$ACC HOST_DATA USE_DEVICE(ZCOMBUFR,ZCOMBUFS)
+#elif defined(_CRAYFTN)
+  !$ACC HOST_DATA USE_DEVICE(ZCOMBUFR)
+#endif
 #endif
 #ifdef OMPGPU
   !$OMP TARGET DATA USE_DEVICE_PTR(ZCOMBUFS,ZCOMBUFR)

diff --git a/src/trans/gpu/internal/trltom_mod.F90 b/src/trans/gpu/internal/trltom_mod.F90
@@ -183,7 +183,11 @@ END FUNCTION ALLTOALLV_CUDAIPC
 
     CALL GSTATS(411,0)
 #ifdef ACCGPU
-    !$ACC HOST_DATA USE_DEVICE(PFBUF_IN, PFBUF)
+#if defined(__NVCOMPILER) || defined(__PGI)
+  !$ACC HOST_DATA USE_DEVICE(PFBUF,PFBUF_IN)
+#elif defined(_CRAYFTN)
+  !$ACC HOST_DATA USE_DEVICE(PFBUF)
+#endif
 #endif
 #ifdef OMPGPU
     !$OMP TARGET DATA USE_DEVICE_PTR(PFBUF_IN, PFBUF)

diff --git a/src/trans/gpu/internal/trmtol_mod.F90 b/src/trans/gpu/internal/trmtol_mod.F90
@@ -159,8 +159,11 @@ SUBROUTINE TRMTOL_CUDAAWARE(PFBUF_IN,PFBUF,KFIELD)
   CALL GSTATS(410,0)
 
 #ifdef ACCGPU
-  !$ACC DATA PRESENT(PFBUF_IN,PFBUF) COPYIN(ILENS,ILENR,IOFFS,IOFFR)
-  !$ACC HOST_DATA USE_DEVICE(PFBUF_IN,PFBUF)
+#if defined(__NVCOMPILER) || defined(__PGI)
+  !$ACC HOST_DATA USE_DEVICE(PFBUF,PFBUF_IN)
+#elif defined(_CRAYFTN)
+  !$ACC HOST_DATA USE_DEVICE(PFBUF)
+#endif
 #endif
 #ifdef OMPGPU
   !$OMP TARGET DATA MAP(PRESENT,ALLOC:PFBUF_IN, PFBUF)
@@ -177,7 +180,6 @@ SUBROUTINE TRMTOL_CUDAAWARE(PFBUF_IN,PFBUF,KFIELD)
 #endif
 #ifdef ACCGPU
   !$ACC END HOST_DATA
-  !$ACC END DATA
 #endif
   CALL GSTATS(410,1)
   CALL GSTATS(807,1)