Switch to using native support for CUDA instead of FindCUDA macros. (Q…

…iskit#859)

* Switch to using native support for CUDA instead of FindCUDA macros.

* Move check for AER_THRUST_BACKEND being communicated via an environment variable to the top of the file.  That is because there is a check added in this branch to see if CUDA support should be enabled.  This check is needed to avoid probing for nvcc unless needed, because that check may fail and abort the build.

* Add x86_64 checks for issue 856, pull Qiskit#864.

* Avoid compiling SIMD_SOURCE_FILE if there is not one.

* Use AMD64 as a synonym for x86_64.

* Add AMD64 as a synonym for x86_64.

* Pull fixes from vvilpas's issue-856-build-ppc

* Unconditinally use add_library even with CUDA (not cuda_add_library).

* Unconditinally use add_library, not cuda_add_library for CUDA.  Transform arguments for --compiler-options.

* pulled from revive_CUDA_windows

* no-op commit to force rebuild

* note some places to refactor

* Fix all but the last suggested changes by vvilpas

* Still need AER_COMPILER_FLAGS initialized.

* Use functions for prepending --compiler-options to pass default inherited options to NVCC.  One is for when a list is prepared, and one is for when a list is not prepared.

* Move include of nvcc_add_compiler_options to be adjacent to the FindCUDA call.

* Likewise include nvcc_add_compiler_options inside the CUDA conditional.

* Include nvcc_add_compiler_options.cmake after CMAKE environment variable assignments.

* Added file.

* space -> tab tweaks for local consistency

* fix typo in argument name

* Use an intermediate temporary variable, and an explicit set.

* Remove an indirect

* Follow change to remove an indirect from nvcc functions.

* More quoting needed for list conversion.

* More quoting needed for list conversion.

* Enable gpu wheel to check that wheel building works

* Do not include the whole FindCUDA, just the architecture selection bits.  Assume that CUDA will be found.

* force CMAKE_CUDA_COMPILER to nvcc.  Trying to figure out what is wrong with qiskit-aer-gpu builds.

* force CMAKE_CUDA_COMPILER to /usr/local/cuda/nvcc.  Trying to figure out what is wrong with qiskit-aer-gpu builds.

* remove hardcoded /usr/local/cuda/bin/nvcc in CMakeLists.txt and put a path extension in the wheels.yml workflow.

* Nope, try CUDACXX instead.

* Nope, try adding -- -DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc

* Nope, try adding PATH in CIBW_ENVIRONMENT.

* do not run verify_wheels for GPU build

* return to standard file

* Fix terminate call. Enable CI test

* disable the CONAN patch to see if that is the cause of the build failure

* reapply the CONAN patch -- there is something else wrong

* add this back

* Set CUDACXX for CUDA wheel build

* Set AER_CUDA_ARCH to 5.2 to make a quick test on CI wheel build

* Fix gpu wheel build

There is a bug in the code that makes the verify_wheel.py script
run forever when we build the gpu wheel. The problem is the check
of memory available on CUDA devices when there are no CUDA devices
available, as is the case for our GPU wheel build action.

* Enable gpu wheel build temporarily to check that it works

* Revert "Enable gpu wheel build temporarily to check that it works"

This reverts commit 7a9b5f6.

* Set cuda devices to 0 in case of error for consistency

* Recover cmake-optiona-conan release note

* Default CUDA archs to Auto again

* disable gpu build test

Co-authored-by: vvilpas <[email protected]>
Co-authored-by: Victor Villar <[email protected]>
Co-authored-by: Christopher J. Wood <[email protected]>
(cherry picked from commit 796a12f)

.github/workflows/wheels.yml

@@ -59,7 +59,7 @@ jobs:
  CIBW_SKIP: "cp27-* cp34-* cp35-* *-manylinux_i686 pp*"
  CIBW_MANYLINUX_X86_64_IMAGE: "manylinux2010"
  CIBW_MANYLINUX_I686_IMAGE: "manylinux2010"
- CIBW_ENVIRONMENT: QISKIT_AER_PACKAGE_NAME=qiskit-aer-gpu AER_THRUST_BACKEND=CUDA
+ CIBW_ENVIRONMENT: QISKIT_AER_PACKAGE_NAME=qiskit-aer-gpu AER_THRUST_BACKEND=CUDA CUDACXX=/usr/local/cuda/bin/nvcc
  CIBW_TEST_COMMAND: "python3 {project}/tools/verify_wheels.py"
  CIBW_TEST_REQUIRES: "git+https://github.com/Qiskit/qiskit-terra.git"
  run: |

CMakeLists.txt

@@ -8,8 +8,10 @@
 # For Mac, statically linking only happens with user libraries, system libraries cannot
 # be linked statically per Apple's indications.
 
-cmake_minimum_required(VERSION 3.6)
+cmake_minimum_required(VERSION 3.8 FATAL_ERROR)
 file(STRINGS "qiskit/providers/aer/VERSION.txt" VERSION_NUM)
+
+include(CheckLanguage)
 project(qasm_simulator VERSION ${VERSION_NUM} LANGUAGES CXX C)
 
 list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake)
@@ -23,6 +25,17 @@ if(NOT DEFINED AER_THRUST_BACKEND AND DEFINED ENV{AER_THRUST_BACKEND})
  set(AER_THRUST_BACKEND $ENV{AER_THRUST_BACKEND})
 endif()
 
+if(AER_THRUST_BACKEND STREQUAL "CUDA")
+ include(nvcc_add_compiler_options)
+ set(CUDA_FOUND TRUE)
+ #include(FindCUDA) # for cuda_select_nvcc_arch_flags, CUDA_FOUND
+ include(FindCUDA/select_compute_arch)
+ enable_language(CUDA)
+else()
+ # idiosyncrasy of CMake that it still creates a reference to this
+ set(CMAKE_CUDA_COMPILE_WHOLE_COMPILATION "")
+endif()
+
 
 # Warning: Because of a bug on CMake's FindBLAS or (it's not clear who's fault is)
 # libopenblas.a for Ubuntu (maybe others) we need to copy the file:
@@ -209,6 +222,14 @@ if(NOT MSVC)
  endif()
 endif()
 
+if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "AMD64")
+ if(APPLE OR UNIX)
+ set(SIMD_FLAGS_LIST "-mfma;-mavx2")
+ elseif(MSVC)
+ set(SIMD_FLAGS_LIST "/arch:AVX2")
+ endif()
+endif()
+
 set(AER_THRUST_SUPPORTED TRUE)
 if(AER_THRUST_SUPPORTED)
  if(AER_THRUST_BACKEND STREQUAL "CUDA")
@@ -223,18 +244,10 @@ if(AER_THRUST_SUPPORTED)
  endif()
  endif()
  cuda_select_nvcc_arch_flags(AER_CUDA_ARCH_FLAGS ${AER_CUDA_ARCH})
- set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -ccbin \"${CMAKE_CXX_COMPILER}\" ${AER_CUDA_ARCH_FLAGS} -DAER_THRUST_CUDA -std=c++14 -I${AER_SIMULATOR_CPP_SRC_DIR} -isystem ${AER_SIMULATOR_CPP_SRC_DIR}/third-party/headers -use_fast_math --expt-extended-lambda")
- # We have to set SIMD flags globally because there seems to be a bug in FindCUDA which doesn't allow us to set per-file compilation flags.
- # The implications is that users downloading the PyPi wheel package for the GPU, need to have a CPU with AVX2 support otherwise
- # Aer will crash with: "Unknow instruction" exception.
- # This will be fixed here: https://github.com/Qiskit/qiskit-aer/
- if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "AMD64")
- if(APPLE OR UNIX)
- set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS}; --compiler-options;-mfma,-mavx2")
- elseif(MSVC)
- set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} /arch:AVX2")
- endif()
- endif()
+
+ string(REPLACE ";" " " AER_CUDA_ARCH_FLAGS_EXPAND "${AER_CUDA_ARCH_FLAGS}")
+ set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -ccbin \"${CMAKE_CXX_COMPILER}\" ${AER_CUDA_ARCH_FLAGS_EXPAND} -DAER_THRUST_CUDA -I${AER_SIMULATOR_CPP_SRC_DIR} -isystem ${AER_SIMULATOR_CPP_SRC_DIR}/third-party/headers -use_fast_math --expt-extended-lambda")
+
  set(AER_COMPILER_DEFINITIONS ${AER_COMPILER_DEFINITIONS} THRUST_DEVICE_SYSTEM=THRUST_DEVICE_SYSTEM_CUDA)
  set(THRUST_DEPENDANT_LIBS "")
  elseif(AER_THRUST_BACKEND STREQUAL "TBB")
@@ -283,36 +296,46 @@ if(SKBUILD) # Terra Addon build
  add_subdirectory(src/open_pulse)
 else() # Standalone build
 
- # We build SIMD filed separately, because they will be reached only if the
- # machine running the code has SIMD support
  if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "AMD64")
- set(SIMD_SOURCE_FILE "${PROJECT_SOURCE_DIR}/src/simulators/statevector/qv_avx2.cpp")
- if(APPLE OR UNIX)
- set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "-mfma -mavx2")
- elseif(MSVC)
- set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "/arch:AVX2")
- endif()
- endif()
-
- set(AER_SIMULATOR_SOURCES "${PROJECT_SOURCE_DIR}/contrib/standalone/qasm_simulator.cpp"
- "${SIMD_SOURCE_FILE}")
- set_source_files_properties(${AER_SIMULATOR_SOURCES} PROPERTIES CUDA_SOURCE_PROPERTY_FORMAT OBJ)
+ # We build SIMD filed separately, because they will be reached only if the
+ # machine running the code has SIMD support
+ set(SIMD_SOURCE_FILE "${PROJECT_SOURCE_DIR}/src/simulators/statevector/qv_avx2.cpp")
+ endif()
 
+ set(AER_SIMULATOR_SOURCES "${PROJECT_SOURCE_DIR}/contrib/standalone/qasm_simulator.cpp")
  set(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
- if(CUDA_FOUND)
- cuda_add_executable(qasm_simulator ${AER_SIMULATOR_SOURCES})
+ if(CUDA_FOUND AND AER_THRUST_BACKEND STREQUAL "CUDA")
+ set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES LANGUAGE CUDA)
+ set_source_files_properties(${AER_SIMULATOR_SOURCES} PROPERTIES LANGUAGE CUDA)
+ set_source_files_properties(${AER_SIMULATOR_SOURCES} PROPERTIES COMPILE_FLAGS "${CUDA_NVCC_FLAGS}")
+ nvcc_add_compiler_options_list("${SIMD_FLAGS_LIST}" SIMD_FLAGS)
+ set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "${CUDA_NVCC_FLAGS} ${SIMD_FLAGS}")
+
+ add_executable(qasm_simulator ${AER_SIMULATOR_SOURCES} ${SIMD_SOURCE_FILE})
  target_link_libraries(qasm_simulator ${AER_LIBRARIES})
- else(CUDA_FOUND)
- add_executable(qasm_simulator ${AER_SIMULATOR_SOURCES})
+ string(STRIP ${AER_COMPILER_FLAGS} AER_COMPILER_FLAGS_STRIPPED)
+ nvcc_add_compiler_options(${AER_COMPILER_FLAGS_STRIPPED} AER_COMPILER_FLAGS_OUT)
+
+ set_target_properties(qasm_simulator PROPERTIES
+ LINKER_LANGUAGE CXX
+ CXX_STANDARD 14
+ COMPILE_FLAGS ${AER_COMPILER_FLAGS_OUT}
+ LINK_FLAGS ${AER_LINKER_FLAGS}
+ RUNTIME_OUTPUT_DIRECTORY_DEBUG Debug
+ RUNTIME_OUTPUT_DIRECTORY_RELEASE Release)
+ else()
+ string(REPLACE ";" " " SIMD_FLAGS "${SIMD_FLAGS_LIST}")
+ set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "${SIMD_FLAGS}")
+ add_executable(qasm_simulator ${AER_SIMULATOR_SOURCES} ${SIMD_SOURCE_FILE})
  target_link_libraries(qasm_simulator PRIVATE ${AER_LIBRARIES})
- endif(CUDA_FOUND)
- set_target_properties(qasm_simulator PROPERTIES
- LINKER_LANGUAGE CXX
- CXX_STANDARD 14
- COMPILE_FLAGS ${AER_COMPILER_FLAGS}
- LINK_FLAGS ${AER_LINKER_FLAGS}
- RUNTIME_OUTPUT_DIRECTORY_DEBUG Debug
-  RUNTIME_OUTPUT_DIRECTORY_RELEASE Release)
+  set_target_properties(qasm_simulator PROPERTIES
+  LINKER_LANGUAGE CXX
+  CXX_STANDARD 14
+  COMPILE_FLAGS ${AER_COMPILER_FLAGS}
+  LINK_FLAGS ${AER_LINKER_FLAGS}
+  RUNTIME_OUTPUT_DIRECTORY_DEBUG Debug
+  RUNTIME_OUTPUT_DIRECTORY_RELEASE Release)
+ endif()
  target_include_directories(qasm_simulator
  PRIVATE ${AER_SIMULATOR_CPP_SRC_DIR}
  PRIVATE ${AER_SIMULATOR_CPP_EXTERNAL_LIBS})

cmake/FindPybind11.cmake

@@ -40,11 +40,7 @@ function(basic_pybind11_add_module target_name)
  set(exclude_from_all EXCLUDE_FROM_ALL)
  endif()
 
- if(CUDA_FOUND)
- cuda_add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS})
- else()
- add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS})
- endif()
+ add_library(${target_name} ${lib_type} ${exclude_from_all} ${ARG_UNPARSED_ARGUMENTS})
 
  # This sets various properties (python include dirs) and links to python libs
  target_include_directories(${target_name} PRIVATE ${PYTHON_INCLUDE_DIRS})

cmake/cython_utils.cmake

@@ -26,16 +26,26 @@ set(CYTHON_USER_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR})
 unset(CYTHON_USER_LIB_DIRS)
 set(CYTHON_INSTALL_DIR "qiskit/providers/aer/backends")
 
+include(nvcc_add_compiler_options)
+
 function(add_cython_module module)
  add_cython_target(${module} ${module}.pyx CXX)
 
  # Avoid warnings in cython cpp generated code
  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
- set_source_files_properties(${module}.cxx PROPERTIES COMPILE_FLAGS -Wno-everything)
+ set(MODULE_COMPILE_FLAGS "-Wno-everything")
  elseif(CMAKE_CXX_COMPILER_ID MATCHES "GNU")
- set_source_files_properties(${module}.cxx PROPERTIES COMPILE_FLAGS -w)
+ set(MODULE_COMPILE_FLAGS "-w")
  elseif(CMAKE_CXX_COMPILER_ID MATCHES "MSVC")
- set_source_files_properties(${module}.cxx PROPERTIES COMPILE_FLAGS /w)
+ set(MODULE_COMPILE_FLAGS "/w")
+ endif()
+
+ if(CUDA_FOUND AND AER_THRUST_BACKEND STREQUAL "CUDA")
+ string(STRIP ${MODULE_COMPILE_FLAGS} MODULE_COMPILE_FLAGS_STRIPPED)
+ nvcc_add_compiler_options(${MODULE_COMPILE_FLAGS_STRIPPED} MODULE_COMPILE_FLAGS_OUT)
+ set_source_files_properties(${module}.cxx PROPERTIES COMPILE_FLAGS ${MODULE_COMPILE_FLAGS_OUT})
+ else()
+ set_source_files_properties(${module}.cxx PROPERTIES COMPILE_FLAGS ${MODULE_COMPILE_FLAGS})
  endif()
 
  set(options MODULE SHARED EXCLUDE_FROM_ALL NO_EXTRAS SYSTEM THIN_LTO)
@@ -52,13 +62,15 @@ function(add_cython_module module)
  set(exclude_from_all EXCLUDE_FROM_ALL)
  endif()
 
- if(CUDA_FOUND)
- cuda_add_library(${module} ${lib_type} ${exclude_from_all} ${module} ${ARG_UNPARSED_ARGUMENTS})
- set_source_files_properties(${module} PROPERTIES
- CUDA_SOURCE_PROPERTY_FORMAT OBJ)
+ if(CUDA_FOUND AND AER_THRUST_BACKEND STREQUAL "CUDA")
+ set_source_files_properties(${module}.cxx PROPERTIES LANGUAGE CUDA)
+
+ string(STRIP ${AER_COMPILER_FLAGS} AER_COMPILER_FLAGS_STRIPPED)
+ nvcc_add_compiler_options(${AER_COMPILER_FLAGS_STRIPPED} AER_COMPILER_FLAGS_OUT)
  else()
- add_library(${module} ${lib_type} ${exclude_from_all} ${module} ${ARG_UNPARSED_ARGUMENTS})
+ set(AER_COMPILER_FLAGS_OUT ${AER_COMPILER_FLAGS})
  endif()
+ add_library(${module} ${lib_type} ${exclude_from_all} ${module} ${ARG_UNPARSED_ARGUMENTS})
 
 
  # We only need to pass the linter once, as the codebase is the same for
@@ -112,7 +124,7 @@ function(add_cython_module module)
  # Warning: Do not merge PROPERTIES when one of the variables can be empty, it breaks
  # the rest of the properties so they are not properly added.
  set_target_properties(${module} PROPERTIES LINK_FLAGS ${AER_LINKER_FLAGS})
- set_target_properties(${module} PROPERTIES COMPILE_FLAGS ${AER_COMPILER_FLAGS})
+ set_target_properties(${module} PROPERTIES COMPILE_FLAGS ${AER_COMPILER_FLAGS_OUT})
  target_compile_definitions(${module} PRIVATE ${AER_COMPILER_DEFINITIONS})
 
  python_extension_module(${module}

cmake/nvcc_add_compiler_options.cmake

@@ -0,0 +1,13 @@
+function(nvcc_add_compiler_options_list inList outVarName)
+ set(L "${inList}")
+ list(TRANSFORM L PREPEND " --compiler-options ")
+ string(REPLACE ";" " " TMP ${L})
+ set(${outVarName} ${TMP} PARENT_SCOPE)
+endfunction()
+
+function(nvcc_add_compiler_options inStr outVarName)
+ string(REPLACE " " ";" L "${inStr}")
+ list(TRANSFORM L PREPEND " --compiler-options ")
+ string(REPLACE ";" " " TMP ${L})
+ set(${outVarName} ${TMP} PARENT_SCOPE)
+endfunction()

qiskit/providers/aer/backends/wrappers/CMakeLists.txt

@@ -6,26 +6,37 @@ find_package(Pybind11 REQUIRED)
 # dependencies we can, so some of these dependencies are linked statically into our
 # shared library.
 string(REPLACE " -static " "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
+
 if(CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL "AMD64")
- # We build SIMD file separately, because they will be reached only if the
- # machine running the code has SIMD support
- set(SIMD_SOURCE_FILE "../../../../../src/simulators/statevector/qv_avx2.cpp")
- if(APPLE OR UNIX)
- set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "-mfma -mavx2")
- elseif(MSVC)
- set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "/arch:AVX2")
- endif()
+ # We build SIMD filed separately, because they will be reached only if the
+ # machine running the code has SIMD support
+ set(SIMD_SOURCE_FILE "../../../../../src/simulators/statevector/qv_avx2.cpp")
 endif()
 
-set(AER_SIMULATOR_SOURCES "bindings.cc"
- "${SIMD_SOURCE_FILE}")
+set(AER_SIMULATOR_SOURCES "bindings.cc" "${SIMD_SOURCE_FILE}")
+basic_pybind11_add_module(controller_wrappers "${AER_SIMULATOR_SOURCES}")
 
-if(CUDA_FOUND)
- set_source_files_properties(${AER_SIMULATOR_SOURCES} PROPERTIES
-  CUDA_SOURCE_PROPERTY_FORMAT OBJ)
-endif()
+if(AER_THRUST_BACKEND STREQUAL "CUDA")
+ include(nvcc_add_compiler_options)
+ set_source_files_properties(bindings.cc PROPERTIES LANGUAGE CUDA)
+ set_source_files_properties(bindings.cc PROPERTIES COMPILE_FLAGS "${CUDA_NVCC_FLAGS}")
 
-basic_pybind11_add_module(controller_wrappers ${AER_SIMULATOR_SOURCES})
+ if(DEFINED SIMD_SOURCE_FILE)
+ set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES LANGUAGE CUDA)
+ nvcc_add_compiler_options_list("${SIMD_FLAGS_LIST}" SIMD_FLAGS_OUT)
+ set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "${CUDA_NVCC_FLAGS} ${SIMD_FLAGS_OUT}")
+ endif()
+
+ string(STRIP ${AER_COMPILER_FLAGS} AER_COMPILER_FLAGS_STRIPPED)
+ nvcc_add_compiler_options(${AER_COMPILER_FLAGS_STRIPPED} AER_COMPILER_FLAGS_OUT)
+ set_target_properties(controller_wrappers PROPERTIES COMPILE_FLAGS "${AER_COMPILER_FLAGS_OUT}")
+else()
+ if(DEFINED SIMD_SOURCE_FILE)
+ string(REPLACE ";" " " SIMD_FLAGS "${SIMD_FLAGS_LIST}")
+ set_source_files_properties(${SIMD_SOURCE_FILE} PROPERTIES COMPILE_FLAGS "${SIMD_FLAGS}")
+ endif()
+ set_target_properties(controller_wrappers PROPERTIES COMPILE_FLAGS "${AER_COMPILER_FLAGS}")
+endif()
 target_include_directories(controller_wrappers PRIVATE ${AER_SIMULATOR_CPP_SRC_DIR}
  PRIVATE ${AER_SIMULATOR_CPP_EXTERNAL_LIBS})
 target_link_libraries(controller_wrappers ${AER_LIBRARIES})

src/simulators/statevector/qubitvector_thrust.hpp

@@ -1678,7 +1678,7 @@ void QubitVectorThrust<data_t>::set_num_qubits(size_t num_qubits)
  tid = omp_get_thread_num();
  m_nDev = 1;
 #ifdef AER_THRUST_CUDA
- cudaGetDeviceCount(&m_nDev);
+ if(cudaGetDeviceCount(&m_nDev) != cudaSuccess) m_nDev = 0;
 #endif
 
  m_iDev = 0;