amrex::FFT

Add parallel FFT capability to AMReX. It relies on FFTW3, cuFFT, rocFFT and oneMKL, for CPU, CUDA, HIP and SYCL builds, respectively.
WeiqunZhang · Oct 17, 2024 · 01d9620 · 01d9620
1 parent 62c2a81
commit 01d9620
Show file tree

Hide file tree

Showing 29 changed files with 1,014 additions and 7 deletions.
diff --git a/.github/workflows/clang.yml b/.github/workflows/clang.yml
@@ -44,6 +44,7 @@ jobs:
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DCMAKE_INSTALL_PREFIX=/tmp/my-amrex \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_FORTRAN=ON \
  -DAMReX_MPI=OFF \
@@ -104,6 +105,7 @@ jobs:
  cmake .. \
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -158,6 +160,7 @@ jobs:
  cmake .. \
  -DCMAKE_BUILD_TYPE=RelWithDebInfo \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=OFF \
@@ -200,7 +203,7 @@ jobs:
  export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
  ccache -z
 
- ./configure --dim 2 --with-fortran no --comp llvm --with-mpi no
+ ./configure --dim 2 --with-fortran no --comp llvm --with-mpi no --enable-fft yes
  make -j4 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS="-fno-operator-names" \
  CCACHE=ccache
  make install

diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml
@@ -38,6 +38,7 @@ jobs:
 
  cmake -S . -B build \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=OFF \
@@ -97,6 +98,7 @@ jobs:
  cmake -S . -B build \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DAMReX_MPI=OFF \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=OFF \
@@ -153,6 +155,7 @@ jobs:
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_TEST_TYPE=Small \
+ -DAMReX_FFT=ON \
  -DAMReX_FORTRAN=ON \
  -DAMReX_FORTRAN_INTERFACES=ON \
  -DAMReX_GPU_BACKEND=CUDA \
@@ -196,7 +199,7 @@ jobs:
  ccache -z
 
  export PATH=/usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH}
- ./configure --dim 3 --with-cuda yes --enable-eb yes --enable-xsdk-defaults yes --with-fortran no
+ ./configure --dim 3 --with-cuda yes --enable-eb yes --enable-xsdk-defaults yes --with-fortran no --enable-fft yes
  #
  # /home/runner/work/amrex/amrex/Src/Base/AMReX_GpuLaunchGlobal.H:16:41: error: unused parameter ‘f0’ [-Werror=unused-parameter]
  # 16 | AMREX_GPU_GLOBAL void launch_global (L f0) { f0(); }

diff --git a/.github/workflows/dependencies/dependencies.sh b/.github/workflows/dependencies/dependencies.sh
@@ -16,6 +16,7 @@ sudo apt-get update
 
 sudo apt-get install -y --no-install-recommends\
  build-essential \
+ libfftw3-dev \
  g++ gfortran \
  libopenmpi-dev \
  openmpi-bin
diff --git a/.github/workflows/dependencies/dependencies_clang.sh b/.github/workflows/dependencies/dependencies_clang.sh
@@ -16,5 +16,6 @@ sudo apt-get update
 
 sudo apt-get install -y --no-install-recommends \
  build-essential \
+ libfftw3-dev \
  gfortran \
  clang-$1
diff --git a/.github/workflows/dependencies/dependencies_gcc.sh b/.github/workflows/dependencies/dependencies_gcc.sh
@@ -17,6 +17,7 @@ sudo apt-get update
 
 sudo apt-get install -y --no-install-recommends \
  build-essential \
+ libfftw3-dev \
  g++-$1 gfortran-$1 \
  libopenmpi-dev \
  openmpi-bin
diff --git a/.github/workflows/dependencies/dependencies_hip.sh b/.github/workflows/dependencies/dependencies_hip.sh
@@ -56,6 +56,7 @@ sudo apt-get install -y --no-install-recommends \
  roctracer-dev \
  rocprofiler-dev \
  rocrand-dev \
+ rocfft-dev \
  rocprim-dev
 
 # hiprand-dev is a new package that does not exist in old versions

diff --git a/.github/workflows/gcc.yml b/.github/workflows/gcc.yml
@@ -42,6 +42,7 @@ jobs:
  mkdir build
  cd build
  cmake .. \
+ -DAMReX_FFT=ON \
  -DAMReX_FORTRAN=ON \
  -DAMReX_PLOTFILE_TOOLS=ON \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
@@ -99,6 +100,7 @@ jobs:
  cmake -S . -B build \
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -147,6 +149,7 @@ jobs:
  cmake -S . -B build \
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -196,6 +199,7 @@ jobs:
  cmake -S . -B build \
  -DCMAKE_BUILD_TYPE=Debug \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=OFF \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -248,6 +252,7 @@ jobs:
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DAMReX_ASSERTIONS=ON \
  -DAMReX_TESTING=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=OFF \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_BOUND_CHECK=ON \
@@ -310,6 +315,7 @@ jobs:
  -DAMReX_TESTING=ON \
  -DAMReX_BOUND_CHECK=ON \
  -DAMReX_FPE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -371,6 +377,7 @@ jobs:
  -DAMReX_TESTING=ON \
  -DAMReX_BOUND_CHECK=ON \
  -DAMReX_FPE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=OFF \
@@ -457,7 +464,7 @@ jobs:
  export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
  ccache -z
 
- ./configure --dim 3 --enable-eb yes --enable-xsdk-defaults yes
+ ./configure --dim 3 --enable-eb yes --enable-xsdk-defaults yes --enable-fft yes
  make -j4 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS=-fno-operator-names \
  CCACHE=ccache
  make install
@@ -497,7 +504,8 @@ jobs:
  export CCACHE_LOGFILE=${{ github.workspace }}/ccache.log.txt
  ccache -z
 
- ./configure --dim 3 --enable-eb no --enable-xsdk-defaults no --single-precision yes --single-precision-particles yes --enable-tiny-profile yes
+ ./configure --dim 3 --enable-eb no --enable-xsdk-defaults no --single-precision yes \
+ --single-precision-particles yes --enable-tiny-profile yes --enable-fft yes
  make -j4 WARN_ALL=TRUE WARN_ERROR=TRUE XTRA_CXXFLAGS=-fno-operator-names \
  CCACHE=ccache
  make install
@@ -623,6 +631,7 @@ jobs:
  -DAMReX_OMP=ON \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DAMReX_ENABLE_TESTS=ON \
+ -DAMReX_FFT=ON \
  -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
  make -j 4
 

diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml
@@ -48,6 +48,7 @@ jobs:
 
  cmake -S . -B build \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -103,6 +104,7 @@ jobs:
 
  cmake -S . -B build_full_legacywrapper \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=OFF \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -145,7 +147,9 @@ jobs:
  export CCACHE_MAXSIZE=100M
  ccache -z
 
- ./configure --dim 2 --with-hip yes --enable-eb yes --enable-xsdk-defaults yes --with-mpi no --with-omp no --single-precision yes --single-precision-particles yes
+ ./configure --dim 2 --with-hip yes --enable-eb yes --enable-xsdk-defaults yes \
+ --with-mpi no --with-omp no --single-precision yes \
+ --single-precision-particles yes --enable-fft yes
  make -j4 WARN_ALL=TRUE AMD_ARCH=gfx90a CCACHE=ccache
  make install
 

diff --git a/.github/workflows/intel.yml b/.github/workflows/intel.yml
@@ -41,6 +41,7 @@ jobs:
  set -e
  cmake -S . -B build \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=OFF \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=ON \
@@ -89,6 +90,7 @@ jobs:
  set -e
  cmake -S . -B build \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
+ -DAMReX_FFT=ON \
  -DAMReX_EB=ON \
  -DAMReX_ENABLE_TESTS=ON \
  -DAMReX_FORTRAN=OFF \

diff --git a/GNUmakefile.in b/GNUmakefile.in
@@ -26,6 +26,9 @@ ifeq ($(USE_LINEAR_SOLVERS),TRUE)
  Pdirs += F_Interfaces/LinearSolvers
  endif
 endif
+ifeq ($(USE_FFT),TRUE)
+ Pdirs += FFT
+endif
 ifeq ($(USE_EB),TRUE)
  Pdirs += EB
 endif

diff --git a/Src/Base/AMReX_FabArray.H b/Src/Base/AMReX_FabArray.H
@@ -13,6 +13,7 @@
 #include <AMReX_FabFactory.H>
 #include <AMReX_DistributionMapping.H>
 #include <AMReX_Geometry.H>
+#include <AMReX_GpuComplex.H>
 #include <AMReX_ParallelDescriptor.H>
 #include <AMReX_Utility.H>
 #include <AMReX_ccse-mpi.H>
@@ -3679,6 +3680,8 @@ FabArray<FAB>::norminf (FabArray<IFAB> const& mask, int comp, int ncomp,
  return nm0;
 }
 
+using cMultiFab = FabArray<BaseFab<GpuComplex<Real> > >;
+
 }
 
 #endif /*BL_FABARRAY_H*/
diff --git a/Src/Base/AMReX_GpuError.H b/Src/Base/AMReX_GpuError.H
@@ -84,6 +84,16 @@ namespace Gpu {
  std::string errStr(std::string("CURAND error in file ") + __FILE__ \
  + " line " + std::to_string(__LINE__)); \
  amrex::Abort(errStr); }} while(0)
+
+#define AMREX_CUFFT_SAFE_CALL(call) { \
+ cufftResult_t amrex_i_err = call; \
+ if (CUFFT_SUCCESS != amrex_i_err) { \
+ std::string errStr(std::string("CUFFT error ")+std::to_string(amrex_i_err) \
+ + std::string(" in file ") + __FILE__ \
+ + " line " + std::to_string(__LINE__)); \
+ amrex::Abort(errStr); \
+ }}
+
 #endif
 
 #ifdef AMREX_USE_HIP
@@ -100,6 +110,16 @@ namespace Gpu {
  std::string errStr(std::string("HIPRAND error in file ") + __FILE__ \
  + " line " + std::to_string(__LINE__)); \
  amrex::Abort(errStr); }} while(0)
+
+#define AMREX_ROCFFT_SAFE_CALL(call) { \
+ auto amrex_i_err = call; \
+ if (rocfft_status_success != amrex_i_err) { \
+ std::string errStr(std::string("rocFFT error ")+std::to_string(amrex_i_err) \
+ + std::string(" in file ") + __FILE__ \
+ + " line " + std::to_string(__LINE__)); \
+ amrex::Abort(errStr); \
+ }}
+
 #endif
 
 #define AMREX_GPU_ERROR_CHECK() amrex::Gpu::ErrorCheck(__FILE__, __LINE__)

diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt
@@ -136,6 +136,10 @@ if (AMReX_PARTICLES)
  add_subdirectory(Particle)
 endif ()
 
+if (AMReX_FFT)
+ add_subdirectory(FFT)
+endif ()
+
 #
 # Optional external components
 #