Build WarpX with HEFFTE=ON

[titoiride]

Hi all,
when I try to build warpx with -DWarpX_HEFFTE=ON, I get an error during the cmake configure stage:

CMake Error at ${SWDIR}/heffte/cmake/HeffteConfig.cmake:2:
  Parse error.  Expected a command name, got unquoted argument with text
  "@PACKAGE_INIT@".
Call Stack (most recent call first):
  CMakeLists.txt:244 (find_package)


CMake Error at CMakeLists.txt:244 (find_package):
  find_package Error reading CMake code from
  "${SWDIR}/heffte/cmake/HeffteConfig.cmake".

Opening the file HeffteConfig.cmake I find that the first few lines are in fact

@PACKAGE_INIT@

if (TARGET Heffte::Heffte OR Heffte_FIND_QUIETLY)
    # respect the Heffte_FIND_QUIETLY and don't show outputs on second search
    set(Heffte_be_silent ON)
endif()

include("${CMAKE_CURRENT_LIST_DIR}/HeffteTargets.cmake")

if (@Heffte_ENABLE_FFTW@ AND NOT TARGET Heffte::FFTW)
    add_library(Heffte::FFTW INTERFACE IMPORTED GLOBAL)
    target_link_libraries(Heffte::FFTW INTERFACE @FFTW_LIBRARIES@)
    set_target_properties(Heffte::FFTW PROPERTIES INTERFACE_INCLUDE_DIRECTORIES @FFTW_INCLUDES@)
endif()

if (@Heffte_ENABLE_MKL@ AND NOT TARGET Heffte::MKL)
    add_library(Heffte::MKL INTERFACE IMPORTED GLOBAL)
    target_link_libraries(Heffte::MKL INTERFACE @Heffte_MKL_LIBRARIES@)
    set_target_properties(Heffte::MKL PROPERTIES INTERFACE_INCLUDE_DIRECTORIES @Heffte_MKL_INCLUDES@)
endif()

if (@Heffte_ENABLE_ROCM@ AND NOT TARGET roc::rocfft)
    if (EXISTS /opt/rocm)
        list (APPEND CMAKE_PREFIX_PATH /opt/rocm/hip /opt/rocm)
    endif()
    find_package(rocfft REQUIRED)
endif()

Is this an error where I am passing an incorrect flag and cmake can't substitute the @PACKAGE_INIT@ variable?
I build WarpX with

echo "Install WarpX application"
cmake -S . -B $BUILD_FOLDER -DCMAKE_BUILD_TYPE="Release" -DWarpX_DIMS="2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_IPO=ON \
    -DWarpX_MPI=ON -DWarpX_MPI_THREAD_MULTIPLE=ON -DWarpX_OPENPMD=ON -DWarpX_FFT=ON -DWarpX_HEFFTE=ON \
    -DWarpX_QED=OFF -DCMAKE_INSTALL_PREFIX=$HOME/Codes/WarpX
cmake --build $BUILD_FOLDER --target install -j 16

[ax3l]

Hi @titoiride,

This looks like an install issue when heFFTe was installed. How did you build/install heFFTe itself? On which system is this, Perlmutter?

At the moment, heFFTe is just a preview and not on by default. We do not always see better performance with it. You can turn it off.

[titoiride]

Thanks for the note! This is the compilation script, which I took from the recommended script and only slightly adapted. Yes, it's on Perlmutter

# heFFTe
if [ -d ${SW_DIR}/heffte ]
then
  cd ${SW_DIR}/heffte
  git fetch --prune
  git checkout v2.4.0
  cd -
else
  git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${SW_DIR}/heffte
fi
if [ -d ${build_dir}/heffte-pm-gpu-build ]
then
  rm -rf ${build_dir}/heffte-pm-gpu-build
fi
cmake \
    -S ${SW_DIR}/heffte               \
    -B ${build_dir}/heffte-pm-gpu-build \
    -DBUILD_SHARED_LIBS=ON              \
    -DCMAKE_BUILD_TYPE=Release          \
    -DCMAKE_CXX_STANDARD=17             \
    -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON  \
    -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte  \
    -DHeffte_DISABLE_GPU_AWARE_MPI=OFF  \
    -DHeffte_ENABLE_AVX=OFF             \
    -DHeffte_ENABLE_AVX512=OFF          \
    -DHeffte_ENABLE_FFTW=OFF            \
    -DHeffte_ENABLE_CUDA=ON             \
    -DHeffte_ENABLE_ROCM=OFF            \
    -DHeffte_ENABLE_ONEAPI=OFF          \
    -DHeffte_ENABLE_MKL=OFF             \
    -DHeffte_ENABLE_DOXYGEN=OFF         \
    -DHeffte_SEQUENTIAL_TESTING=OFF     \
    -DHeffte_ENABLE_TESTING=OFF         \
    -DHeffte_ENABLE_TRACING=OFF         \
    -DHeffte_ENABLE_PYTHON=OFF          \
    -DHeffte_ENABLE_FORTRAN=OFF         \
    -DHeffte_ENABLE_SWIG=OFF            \
    -DHeffte_ENABLE_MAGMA=OFF
cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16
rm -rf ${build_dir}/heffte-pm-gpu-build

# work-around for heFFTe 2.4.0 bug with NVCC
# https://github.com/icl-utk-edu/heffte/pull/54
sed -i 's/__AVX__/NOTDEFINED_DONOTUSE/g' ${SW_DIR}/heffte/include/stock_fft/heffte_stock_vec_types.h

[ax3l]

Ok, thanks @titoiride!

For now, please use -DWarpX_HEFFTE=OFF (which is the default). This feature is really not yet needed, I updated the PM docs now.

@aeriforme and I need to check what is wrong with the heFFTe instructions on Perlmutter and we will update them.

[aeriforme]

I have just tried this

cmake -S . -B build -DCMAKE_BUILD_TYPE="Release" -DWarpX_DIMS="2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_IPO=ON -DWarpX_MPI=ON -DWarpX_MPI_THREAD_MULTIPLE=ON -DWarpX_OPENPMD=ON -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED=OFF
cmake --build build -j 16

with the development branch and it worked fine.

What WarpX version are you using, @titoiride?
Mine is cf619b3c39c845f5110547887ccb1f95a355adc6