diff --git a/.github/workflows/ci-github-actions-self-hosted.yaml b/.github/workflows/ci-github-actions-self-hosted.yaml index dbf8fb3bf2..f9a36b1949 100644 --- a/.github/workflows/ci-github-actions-self-hosted.yaml +++ b/.github/workflows/ci-github-actions-self-hosted.yaml @@ -21,16 +21,16 @@ jobs: fail-fast: false matrix: jobname: [ - gcc-real-gpu-cuda-mixed, # mixed precision - gcc-complex-gpu-cuda-mixed, - gcc-real-gpu-cuda-full, # full precision - gcc-complex-gpu-cuda-full, - gcc-real-gpu-enable-cuda-afqmc-mixed, # auxiliary field, requires MPI - gcc-complex-gpu-enable-cuda-afqmc-mixed, - gcc-real-gpu-enable-cuda-afqmc-full, - gcc-complex-gpu-enable-cuda-afqmc-full, - clang14_dev-real-gpu-enable-cuda-afqmc-offload-mixed, # auxiliary field, offload requires development llvm14 - clang14_dev-real-gpu-enable-cuda-afqmc-offload-full, # TODO: update to llvm/clang14 when available + GCC8-NoMPI-Legacy-CUDA-Real-Mixed, # mixed precision + GCC8-NoMPI-Legacy-CUDA-Complex-Mixed, + GCC8-NoMPI-Legacy-CUDA-Real, # full precision + GCC8-NoMPI-Legacy-CUDA-Complex, + GCC8-MPI-CUDA-AFQMC-Real-Mixed, # auxiliary field, requires MPI + GCC8-MPI-CUDA-AFQMC-Complex-Mixed, + GCC8-MPI-CUDA-AFQMC-Real, + GCC8-MPI-CUDA-AFQMC-Complex, + Clang14Dev-MPI-CUDA-AFQMC-Offload-Real-Mixed, # auxiliary field, offload requires development llvm14 + Clang14Dev-MPI-CUDA-AFQMC-Offload-Real, ] steps: diff --git a/.github/workflows/ci-github-actions.yaml b/.github/workflows/ci-github-actions.yaml index cbe19125af..14febe9d1a 100644 --- a/.github/workflows/ci-github-actions.yaml +++ b/.github/workflows/ci-github-actions.yaml @@ -20,52 +20,52 @@ jobs: matrix: jobname: [ - gcc-openmpi-real-coverage, - gcc-openmpi-complex-coverage, - gcc11-real-werror, - gcc11-complex-werror, - clang-real-asan, - clang-complex-asan, - clang-openmpi-real-ubsan, - clang-latest-openmp-offload, + GCC9-MPI-Gcov-Real, + GCC9-MPI-Gcov-Complex, + GCC11-NoMPI-Werror-Real, + GCC11-NoMPI-Werror-Complex, + Clang10-NoMPI-ASan-Real, + Clang10-NoMPI-ASan-Complex, + Clang10-NoMPI-UBSan-Real, + Clang12-NoMPI-Offload-Real, ] include: - - jobname: gcc-openmpi-real-coverage + - jobname: GCC9-MPI-Gcov-Real container: image: williamfgc/qmcpack-ci:ubuntu20-openmpi options: -u 1001 - - jobname: gcc-openmpi-complex-coverage + - jobname: GCC9-MPI-Gcov-Complex container: image: williamfgc/qmcpack-ci:ubuntu20-openmpi options: -u 1001 - - jobname: gcc11-real-werror + - jobname: GCC11-NoMPI-Werror-Real container: image: williamfgc/qmcpack-ci:ubuntu2110-serial options: -u 1001 - - jobname: gcc11-complex-werror + - jobname: GCC11-NoMPI-Werror-Complex container: image: williamfgc/qmcpack-ci:ubuntu2110-serial options: -u 1001 - - jobname: clang-real-asan + - jobname: Clang10-NoMPI-ASan-Real container: image: williamfgc/qmcpack-ci:ubuntu20-openmpi options: -u 1001 - - jobname: clang-complex-asan + - jobname: Clang10-NoMPI-ASan-Complex container: image: williamfgc/qmcpack-ci:ubuntu20-openmpi options: -u 1001 - - jobname: clang-openmpi-real-ubsan + - jobname: Clang10-NoMPI-UBSan-Real container: image: williamfgc/qmcpack-ci:ubuntu20-openmpi options: -u 1001 - - jobname: clang-latest-openmp-offload + - jobname: Clang12-NoMPI-Offload-Real container: image: williamfgc/qmcpack-ci:ubuntu20-clang-latest options: -u 1001 @@ -105,7 +105,7 @@ jobs: strategy: fail-fast: false matrix: - jobname: [macOS-gcc11-real] + jobname: [macOS-GCC11-NoMPI-Real] steps: - name: Checkout Action @@ -113,8 +113,8 @@ jobs: - name: Setup Dependencies run: | - brew install ninja hdf5 fftw boost - pip3 install numpy h5py pandas + brew install ninja hdf5 fftw boost + pip3 install numpy h5py pandas - name: Configure run: tests/test_automation/github-actions/ci/run_step.sh configure diff --git a/tests/test_automation/github-actions/ci/run_step.sh b/tests/test_automation/github-actions/ci/run_step.sh index ab0dc3a10c..adaa7888ee 100755 --- a/tests/test_automation/github-actions/ci/run_step.sh +++ b/tests/test_automation/github-actions/ci/run_step.sh @@ -18,137 +18,143 @@ case "$1" in mkdir qmcpack-build cd qmcpack-build - # Real or Complex + # Real or Complex configuration case "${GH_JOBNAME}" in - *"real"*) + *"Real"*) echo 'Configure for real build -DQMC_COMPLEX=0' IS_COMPLEX=0 ;; - *"complex"*) + *"Complex"*) echo 'Configure for complex build -DQMC_COMPLEX=1' IS_COMPLEX=1 ;; esac - - # Mixed of Full precision, used in GPU code (cuda jobs) as it's more mature + + # Sanitizer case "${GH_JOBNAME}" in - *"full"*) - echo 'Configure for real build -DQMC_MIXED_PRECISION=0' - IS_MIXED_PRECISION=0 + *"ASan"*) + echo 'Configure for address sanitizer including leak sanitizer (lsan) -DENABLE_SANITIZER=asan' + IS_SANITIZER=asan ;; - *"mixed"*) - echo 'Configure for complex build -DQMC_MIXED_PRECISION=1' + *"UBSan"*) + echo 'Configure for undefined behavior sanitizer -DENABLE_SANITIZER=ubsan' + IS_SANITIZER=ubsan + ;; + *"TSan"*) + echo 'Configure for thread sanitizer -DENABLE_SANITIZER=tsan' + IS_SANITIZER=tsan + ;; + esac + + # Mixed or Non-Mixed (default, full) precision, used with GPU code + case "${GH_JOBNAME}" in + *"Mixed"*) + echo 'Configure for mixed precision build -DQMC_MIXED_PRECISION=1' IS_MIXED_PRECISION=1 ;; + *) + IS_MIXED_PRECISION=0 + ;; esac case "${GH_JOBNAME}" in - # Sanitize with clang compilers - *"asan"*) - echo 'Configure for address sanitizer asan including lsan (leaks)' - CC=clang CXX=clang++ \ - cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_SANITIZER=asan \ - -DQMC_MPI=0 \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} - ;; - *"ubsan"*) - echo 'Configure for undefined behavior sanitizer ubsan' - CC=clang CXX=clang++ \ - cmake -GNinja -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_SANITIZER=ubsan \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"GCC9-MPI-Gcov-"*) + echo 'Configure for code coverage with gcc and gcovr -DENABLE_GCOV=TRUE and upload reports to Codecov' + cmake -GNinja \ + -DMPI_C_COMPILER=mpicc \ + -DMPI_CXX_COMPILER=mpicxx \ + -DENABLE_GCOV=TRUE \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - *"tsan"*) - echo 'Configure for thread sanitizer tsan' - CC=clang CXX=clang++ \ - cmake -GNinja -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_SANITIZER=tsan \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"GCC11-NoMPI-Werror-"*) + echo 'Configure for building with gcc -Werror flag enabled' + cmake -GNinja \ + -DCMAKE_C_COMPILER=gcc \ + -DCMAKE_CXX_COMPILER=g++ \ + -DQMC_MPI=0 \ + -DCMAKE_CXX_FLAGS=-Werror \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - *"msan"*) - echo 'Configure for (uninitialized) memory sanitizer msan' - CC=clang CXX=clang++ \ - cmake -GNinja -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_SANITIZER=msan \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"San-"*) # Sanitize with clang compilers + cmake -GNinja \ + -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DQMC_MPI=0 \ + -DENABLE_SANITIZER=$IS_SANITIZER \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - *"coverage"*) - echo 'Configure for code coverage with gcc and gcovr' - cmake -GNinja -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo -DENABLE_GCOV=TRUE \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"Clang12-NoMPI-Offload-Real"*) + echo 'Configure for building OpenMP offload with clang12 on x86_64 target' + cmake -GNinja \ + -DCMAKE_C_COMPILER=clang-12 \ + -DCMAKE_CXX_COMPILER=clang++-12 \ + -DQMC_MPI=0 \ + -DENABLE_OFFLOAD=ON \ + -DOFFLOAD_TARGET=x86_64-pc-linux-gnu \ + -DUSE_OBJECT_TARGET=ON \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - *"clang-latest-openmp-offload"*) - echo 'Configure for building OpenMP offload with clang-12 on x86_64' - cmake -GNinja -DCMAKE_C_COMPILER=clang-12 -DCMAKE_CXX_COMPILER=clang++-12 \ - -DENABLE_OFFLOAD=ON -DOFFLOAD_TARGET=x86_64-pc-linux-gnu \ - -DUSE_OBJECT_TARGET=ON -DQMC_MPI=0 \ - ${GITHUB_WORKSPACE} - ;; - *"gpu-enable-cuda-afqmc-offload"*) - echo 'Configure for building OpenMP offload with llvm development commit 01d59c0de822 on x86_64, upgrade to llvm14 clang14 when available' - cmake -GNinja -DCMAKE_C_COMPILER=/opt/llvm/01d59c0de822/bin/clang \ - -DCMAKE_CXX_COMPILER=/opt/llvm/01d59c0de822/bin/clang++ \ - -DMPI_C_COMPILER=/usr/lib64/openmpi/bin/mpicc \ - -DMPI_CXX_COMPILER=/usr/lib64/openmpi/bin/mpicxx \ - -DMPIEXEC_EXECUTABLE=/usr/lib64/openmpi/bin/mpirun \ - -DBUILD_AFQMC=ON \ - -DENABLE_CUDA=ON \ - -DENABLE_OFFLOAD=ON \ - -DUSE_OBJECT_TARGET=ON \ - -DCMAKE_PREFIX_PATH="/opt/OpenBLAS/0.3.18" \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DQMC_COMPLEX=$IS_COMPLEX \ - -DQMC_MIXED_PRECISION=$IS_MIXED_PRECISION \ - ${GITHUB_WORKSPACE} - ;; - *"gpu-enable-cuda-afqmc"*) - echo 'Configure for building with ENABLE CUDA and AFQMC, need recent OpenBLAS' - cmake -GNinja -DCMAKE_C_COMPILER=/usr/lib64/openmpi/bin/mpicc \ - -DCMAKE_CXX_COMPILER=/usr/lib64/openmpi/bin/mpicxx \ - -DMPIEXEC_EXECUTABLE=/usr/lib64/openmpi/bin/mpirun \ - -DBUILD_AFQMC=ON \ - -DENABLE_CUDA=ON \ - -DCMAKE_PREFIX_PATH="/opt/OpenBLAS/0.3.18" \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DQMC_COMPLEX=$IS_COMPLEX \ - -DQMC_MIXED_PRECISION=$IS_MIXED_PRECISION \ - ${GITHUB_WORKSPACE} - ;; - *"gpu-cuda"*) - echo 'Configure for building GPU CUDA legacy' - cmake -GNinja -DQMC_CUDA=1 \ - -DQMC_MPI=0 \ - -DQMC_COMPLEX=$IS_COMPLEX \ - -DQMC_MIXED_PRECISION=$IS_MIXED_PRECISION \ - ${GITHUB_WORKSPACE} + *"Clang14Dev-MPI-CUDA-AFQMC-Offload"*) + echo "Configure for building with ENABLE_CUDA and AFQMC using OpenMP offload on x86_64 " \ + "with llvm development commit 01d59c0de822, need built-from-source OpenBLAS due to bug in rpm" + # TODO: upgrade to llvm14 clang14 when available + export OMPI_CC=/opt/llvm/01d59c0de822/bin/clang + export OMPI_CXX=/opt/llvm/01d59c0de822/bin/clang++ + + cmake -GNinja \ + -DCMAKE_C_COMPILER=/usr/lib64/openmpi/bin/mpicc \ + -DCMAKE_CXX_COMPILER=/usr/lib64/openmpi/bin/mpicxx \ + -DMPIEXEC_EXECUTABLE=/usr/lib64/openmpi/bin/mpirun \ + -DBUILD_AFQMC=ON \ + -DENABLE_CUDA=ON \ + -DENABLE_OFFLOAD=ON \ + -DUSE_OBJECT_TARGET=ON \ + -DCMAKE_PREFIX_PATH="/opt/OpenBLAS/0.3.18" \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DQMC_MIXED_PRECISION=$IS_MIXED_PRECISION \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - *"werror"*) - echo 'Configure for building with -Werror flag enabled' - cmake -GNinja -DCMAKE_C_COMPILER=gcc -DCMAKE_CXX_COMPILER=g++ \ - -DCMAKE_CXX_FLAGS=-Werror \ - -DQMC_MPI=0 \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"GCC8-MPI-CUDA-AFQMC"*) + echo 'Configure for building with ENABLE_CUDA and AFQMC, need built-from-source OpenBLAS due to bug in rpm' + cmake -GNinja \ + -DCMAKE_C_COMPILER=/usr/lib64/openmpi/bin/mpicc \ + -DCMAKE_CXX_COMPILER=/usr/lib64/openmpi/bin/mpicxx \ + -DMPIEXEC_EXECUTABLE=/usr/lib64/openmpi/bin/mpirun \ + -DBUILD_AFQMC=ON \ + -DENABLE_CUDA=ON \ + -DCMAKE_PREFIX_PATH="/opt/OpenBLAS/0.3.18" \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DQMC_MIXED_PRECISION=$IS_MIXED_PRECISION \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - *"macOS-gcc11"*) - echo 'Configure for building on macOS using gcc-11' - cmake -GNinja -DCMAKE_C_COMPILER=gcc-11 -DCMAKE_CXX_COMPILER=g++-11 \ - -DQMC_MPI=0 \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"GCC8-NoMPI-Legacy-CUDA"*) + echo 'Configure for building with Legacy CUDA' + cmake -GNinja \ + -DQMC_CUDA=1 \ + -DQMC_MPI=0 \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DQMC_MIXED_PRECISION=$IS_MIXED_PRECISION \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; - # Configure with default compilers - *) - echo 'Configure for default system compilers and options' - cmake -GNinja -DMPI_C_COMPILER=mpicc -DMPI_CXX_COMPILER=mpicxx \ - -DQMC_COMPLEX=$IS_COMPLEX \ - ${GITHUB_WORKSPACE} + *"macOS-GCC11-NoMPI-Real"*) + echo 'Configure for building on macOS using gcc11' + cmake -GNinja \ + -DCMAKE_C_COMPILER=gcc-11 \ + -DCMAKE_CXX_COMPILER=g++-11 \ + -DQMC_MPI=0 \ + -DQMC_COMPLEX=$IS_COMPLEX \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + ${GITHUB_WORKSPACE} ;; esac ;; @@ -161,45 +167,46 @@ case "$1" in # Run deterministic tests test) + + # Run only deterministic tests (reasonable for CI) by default + TEST_LABEL="-L deterministic" + cd ${GITHUB_WORKSPACE}/../qmcpack-build # Enable oversubscription in OpenMPI - if [[ "${GH_JOBNAME}" =~ (openmpi) ]] + if [[ "${GH_JOBNAME}" =~ (-MPI-) ]] then echo "Enabling OpenMPI oversubscription" export OMPI_MCA_rmaps_base_oversubscribe=1 export OMPI_MCA_hwloc_base_binding_policy=none fi - # Run only deterministic tests (reasonable for CI) by default - TEST_LABEL="-L deterministic" - - if [[ "${GH_JOBNAME}" =~ (clang-latest-openmp-offload) ]] + if [[ "${GH_JOBNAME}" =~ (Clang12-NoMPI-Offload) ]] then echo "Adding /usr/lib/llvm-12/lib/ to LD_LIBRARY_PATH to enable libomptarget.so" export LD_LIBRARY_PATH=/usr/lib/llvm-12/lib/:${LD_LIBRARY_PATH} - # Run only unit tests (reasonable for CI using openmp-offload) + # Run only unit tests (reasonable for CI) TEST_LABEL="-L unit" fi - if [[ "${GH_JOBNAME}" =~ (cuda) ]] + if [[ "${GH_JOBNAME}" =~ (CUDA) ]] then export LD_LIBRARY_PATH=/usr/local/cuda/lib/:/usr/local/cuda/lib64/:${LD_LIBRARY_PATH} fi - if [[ "${GH_JOBNAME}" =~ (afqmc) ]] + if [[ "${GH_JOBNAME}" =~ (AFQMC) ]] then # Avoid polluting the stderr output with libfabric error message export OMPI_MCA_btl=self fi - if [[ "${GH_JOBNAME}" =~ (offload) ]] + if [[ "${GH_JOBNAME}" =~ (Offload) ]] then # Clang helper threads used by target nowait is very broken. Disable this feature export LIBOMP_USE_HIDDEN_HELPER_TASK=0 fi - if [[ "${GH_JOBNAME}" =~ (afqmc-offload) ]] + if [[ "${GH_JOBNAME}" =~ (AFQMC-Offload) ]] then export LD_LIBRARY_PATH=/opt/llvm/01d59c0de822/lib:/usr/lib64/openmpi/lib/:${LD_LIBRARY_PATH} fi diff --git a/tests/test_automation/nightly_test_scripts/nightly_olcf_spock.sh b/tests/test_automation/nightly_test_scripts/nightly_olcf_spock.sh index f130b726c2..07be9a44fc 100644 --- a/tests/test_automation/nightly_test_scripts/nightly_olcf_spock.sh +++ b/tests/test_automation/nightly_test_scripts/nightly_olcf_spock.sh @@ -46,7 +46,7 @@ cd qmcpack/build # Start real build test echo "Start GCC10-NoMPI-CUDA2HIP-Release-Real test" -export QMCPACK_TEST_SUBMIT_NAME=GCC10-NoMPI-CUDA2HIP-Release-Real +export QMCPACK_TEST_SUBMIT_NAME=GCC10-NoMPI-CUDA2HIP-Real-Release CTEST_FLAGS="-DCMAKE_C_COMPILER=gcc \ -DCMAKE_CXX_COMPILER=g++ \ @@ -65,7 +65,7 @@ unset QMCPACK_TEST_SUBMIT_NAME # Start complex build test echo "Start GCC10-NoMPI-CUDA2HIP-Release-Complex test" -export QMCPACK_TEST_SUBMIT_NAME=GCC10-NoMPI-CUDA2HIP-Release-Complex +export QMCPACK_TEST_SUBMIT_NAME=GCC10-NoMPI-CUDA2HIP-Complex-Release cd ${base_dir}/qmcpack/build rm -fr *