Offload matrix based interpolation

hic/CMakeLists.txt

@@ -35,6 +35,7 @@ if( HAVE_CUDA )
   find_package(CUDAToolkit REQUIRED)
 elseif( HAVE_HIP )
   find_package(hip CONFIG REQUIRED)
+  find_package(hipsparse CONFIG REQUIRED)
 endif()
 
 add_subdirectory( src )

hic/src/CMakeLists.txt

@@ -30,10 +30,17 @@ ecbuild_add_library( TARGET hic
 install( FILES ${PROJECT_BINARY_DIR}/src/hic/hic_config.h DESTINATION include/hic )
 install( FILES hic/hic.h                                  DESTINATION include/hic )
 install( FILES hic/hic_runtime.h                          DESTINATION include/hic )
+install( FILES hic/hic_namespace_macro.h                  DESTINATION include/hic )
+install( FILES hic/hic_library_types.h                    DESTINATION include/hic )
+install( FILES hic/hicsparse.h                            DESTINATION include/hic )
+install( FILES hic/hic_dummy/dummyShouldNotBeCalled.h     DESTINATION include/hic/hic_dummy )
 install( FILES hic/hic_dummy/hic_dummy_runtime.h          DESTINATION include/hic/hic_dummy )
+install( FILES hic/hic_dummy/hicsparse_dummy.h            DESTINATION include/hic/hic_dummy )
 
 if( HAVE_CUDA )
   target_link_libraries( hic INTERFACE CUDA::cudart )
+  target_link_libraries( hic INTERFACE CUDA::cusparse )
 elseif( HAVE_HIP )
   target_link_libraries( hic INTERFACE hip::host )
+  target_link_libraries( hic INTERFACE roc::hipsparse )
 endif()

hic/src/hic/hic_dummy/dummyShouldNotBeCalled.h

@@ -0,0 +1,21 @@
+/*
+ * (C) Copyright 2024- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+#pragma once
+
+#include <stdexcept>
+#include <string>
+
+namespace {
+
+[[noreturn]] void dummyShouldNotBeCalled(const char* symbol) {
+    throw std::runtime_error(std::string(symbol)+" is using the dummy backend and should not be called");
+}
+
+}

hic/src/hic/hic_dummy/hic_dummy_runtime.h

@@ -8,8 +8,7 @@
  * nor does it submit to any jurisdiction.
  */
 
-#include <stdexcept>
-#include <string>
+#include "hic/hic_dummy/dummyShouldNotBeCalled.h"
 
 #define DUMMY_SHOULD_NOT_BE_CALLED(SYMBOL) dummyShouldNotBeCalled( #SYMBOL )
 #define DUMMY_FUNCTION(SYMBOL) \
@@ -23,10 +22,6 @@
 
 namespace {
 
-[[noreturn]] void dummyShouldNotBeCalled(const char* symbol) {
-    throw std::runtime_error(std::string(symbol)+" is using the dummy backend and should not be called");
-}
-
 using dummyError_t  = int;
 using dummyEvent_t  = void*;
 using dummyStream_t = void*;

hic/src/hic/hic_dummy/hicsparse_dummy.h

@@ -0,0 +1,78 @@
+/*
+ * (C) Copyright 2024- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+
+#include "hic/hic_dummy/dummyShouldNotBeCalled.h"
+
+#define DUMMY_SHOULD_NOT_BE_CALLED(SYMBOL) dummyShouldNotBeCalled( #SYMBOL )
+#define DUMMY_FUNCTION(SYMBOL) \
+    template <typename... Args> inline \
+    dummysparseStatus_t dummy##SYMBOL(Args&&... args) { \
+        DUMMY_SHOULD_NOT_BE_CALLED( hic##SYMBOL ); \
+        return dummysparseStatus_t{0}; \
+    }
+#define DUMMY_VALUE(SYMBOL) \
+    constexpr int dummy##SYMBOL = 0;
+
+namespace {
+
+using dummysparseHandle_t = int;
+using dummysparseStatus_t = int;
+using dummysparseIndexType_t = int;
+using dummysparseOrder_t = int;
+using dummysparseConstDnVecDescr_t = void*;
+using dummysparseDnVecDescr_t = void*;
+using dummysparseConstDnMatDescr_t = void*;
+using dummysparseDnMatDescr_t = void*;
+using dummysparseConstSpVecDescr_t = void*;
+using dummysparseSpVecDescr_t = void*;
+using dummysparseConstSpMatDescr_t = void*;
+using dummysparseSpMatDescr_t = void*;
+using dummysparseSpMVAlg_t = int;
+using dummysparseSpMMAlg_t = int;
+
+DUMMY_FUNCTION(sparseGetErrorString)
+DUMMY_FUNCTION(sparseCreate)
+DUMMY_FUNCTION(sparseDestroy)
+DUMMY_FUNCTION(sparseCreateConstDnVec)
+DUMMY_FUNCTION(sparseCreateDnVec)
+DUMMY_FUNCTION(sparseDestroyDnVec)
+DUMMY_FUNCTION(sparseCreateConstDnMat)
+DUMMY_FUNCTION(sparseCreateDnMat)
+DUMMY_FUNCTION(sparseDestroyDnMat)
+DUMMY_FUNCTION(sparseCreateConstSpVec)
+DUMMY_FUNCTION(sparseCreateSpVec)
+DUMMY_FUNCTION(sparseDestroySpVec)
+DUMMY_FUNCTION(sparseCreateConstCsr)
+DUMMY_FUNCTION(sparseDestroySpMat)
+DUMMY_FUNCTION(sparseSpMV_bufferSize)
+DUMMY_FUNCTION(sparseSpMV_preprocess)
+DUMMY_FUNCTION(sparseSpMV)
+DUMMY_FUNCTION(sparseSpMM_bufferSize)
+DUMMY_FUNCTION(sparseSpMM_preprocess)
+DUMMY_FUNCTION(sparseSpMM)
+
+DUMMY_VALUE(SPARSE_STATUS_SUCCESS)
+DUMMY_VALUE(SPARSE_ORDER_COL)
+DUMMY_VALUE(SPARSE_ORDER_ROW)
+DUMMY_VALUE(SPARSE_INDEX_32I)
+DUMMY_VALUE(SPARSE_INDEX_64I)
+DUMMY_VALUE(SPARSE_INDEX_BASE_ZERO)
+DUMMY_VALUE(SPARSE_INDEX_BASE_ONE)
+DUMMY_VALUE(SPARSE_SPMV_ALG_DEFAULT)
+DUMMY_VALUE(SPARSE_SPMM_ALG_DEFAULT)
+DUMMY_VALUE(SPARSE_OPERATION_NON_TRANSPOSE)
+DUMMY_VALUE(SPARSE_OPERATION_TRANSPOSE)
+DUMMY_VALUE(SPARSE_OPERATION_CONJUGATE_TRANSPOSE)
+
+}
+
+#undef DUMMY_FUNCTION
+#undef DUMMY_VALUE
+#undef DUMMY_SHOULD_NOT_BE_CALLED

hic/src/hic/hic_library_types.h

@@ -0,0 +1,43 @@
+/*
+ * (C) Copyright 2024- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+#pragma once
+
+#include "hic/hic_namespace_macro.h"
+
+#if HIC_BACKEND_CUDA
+  #include <library_types.h>
+#elif HIC_BACKEND_HIP
+  #include <hip/library_types.h>
+#elif HIC_BACKEND_DUMMY
+  // intentionally blank
+#else
+  #error Unsupported hic backend. Please define HIC_BACKEND_CUDA or HIC_BACKEND_HIP or HIC_BACKEND_DUMMY
+#endif
+
+//------------------------------------------------
+HIC_NAMESPACE_BEGIN
+//------------------------------------------------
+
+#if HIC_BACKEND_CUDA
+  constexpr decltype(CUDA_R_32F) HIC_R_32F = CUDA_R_32F;
+  constexpr decltype(CUDA_R_64F) HIC_R_64F = CUDA_R_64F;
+#elif HIC_BACKEND_HIP
+  constexpr decltype(HIP_R_32F) HIC_R_32F = HIP_R_32F;
+  constexpr decltype(HIP_R_64F) HIC_R_64F = HIP_R_64F;
+#elif HIC_BACKEND_DUMMY
+  constexpr int HIC_R_32F = 0;
+  constexpr int HIC_R_64F = 0;
+#else
+  #error Unsupported hic backend. Please define HIC_BACKEND_CUDA or HIC_BACKEND_HIP or HIC_BACKEND_DUMMY
+#endif
+
+//------------------------------------------------
+HIC_NAMESPACE_END
+//------------------------------------------------

hic/src/hic/hic_namespace_macro.h

@@ -0,0 +1,19 @@
+/*
+ * (C) Copyright 2024- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+#pragma once
+
+#if !defined(HIC_NAMESPACE)
+  #define HIC_NAMESPACE
+  #define HIC_NAMESPACE_BEGIN
+  #define HIC_NAMESPACE_END
+#else
+  #define HIC_NAMESPACE_BEGIN namespace HIC_NAMESPACE {
+  #define HIC_NAMESPACE_END }
+#endif

hic/src/hic/hic_runtime.h

@@ -9,14 +9,7 @@
  */
 #pragma once
 
-#if !defined(HIC_NAMESPACE)
-  #define HIC_NAMESPACE
-  #define HIC_NAMESPACE_BEGIN
-  #define HIC_NAMESPACE_END
-#else
-  #define HIC_NAMESPACE_BEGIN namespace HIC_NAMESPACE {
-  #define HIC_NAMESPACE_END }
-#endif
+#include "hic/hic_namespace_macro.h"
 
 #if HIC_BACKEND_CUDA
   #define HIC_BACKEND cuda

hic/src/hic/hicsparse.h

@@ -0,0 +1,135 @@
+/*
+ * (C) Copyright 2024- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation
+ * nor does it submit to any jurisdiction.
+ */
+#pragma once
+
+#include "hic/hic_namespace_macro.h"
+#include "hic/hic_library_types.h"
+
+#if HIC_BACKEND_CUDA
+  #define HICSPARSE_BACKEND_PREFIX cu
+  #define HICSPARSE_BACKEND_PREFIX_CAPS CU
+  #include <cusparse.h>
+#elif HIC_BACKEND_HIP
+  #define HICSPARSE_BACKEND_PREFIX hip
+  #define HICSPARSE_BACKEND_PREFIX_CAPS HIP
+  #include <hipsparse/hipsparse.h>
+#elif HIC_BACKEND_DUMMY
+  #define HICSPARSE_BACKEND_PREFIX dummy
+  #define HICSPARSE_BACKEND_PREFIX_CAPS dummy
+  #include "hic/hic_dummy/hicsparse_dummy.h"
+#else
+  #error Unsupported hic backend. Please define HIC_BACKEND_CUDA or HIC_BACKEND_HIP or HIC_BACKEND_DUMMY
+#endif
+
+#define HIC_PREFIX hic
+#define HIC_PREFIX_CAPS HIC
+#define HIC_CONCAT_(A, B) A ## B
+#define HIC_CONCAT(A, B) HIC_CONCAT_(A, B)
+#define HIC_SYMBOL(API) HIC_CONCAT(HIC_PREFIX, API)
+#define HIC_SYMBOL_CAPS(API) HIC_CONCAT(HIC_PREFIX_CAPS, API)
+
+#define HIC_TYPE(TYPE) \
+    using HIC_SYMBOL(TYPE) = HIC_CONCAT(HICSPARSE_BACKEND_PREFIX, TYPE);
+
+#define HIC_FUNCTION(FUNCTION) \
+    template <typename... Args> inline \
+    auto HIC_SYMBOL(FUNCTION)(Args&&... args) -> decltype(HIC_CONCAT(HICSPARSE_BACKEND_PREFIX, FUNCTION)(std::forward<Args>(args)...)) { \
+      return HIC_CONCAT(HICSPARSE_BACKEND_PREFIX, FUNCTION)(std::forward<Args>(args)...); \
+    }
+
+#define HIC_VALUE(VALUE) \
+    constexpr decltype(HIC_CONCAT(HICSPARSE_BACKEND_PREFIX_CAPS, VALUE)) HIC_SYMBOL_CAPS(VALUE) = HIC_CONCAT(HICSPARSE_BACKEND_PREFIX_CAPS, VALUE);
+
+//------------------------------------------------
+HIC_NAMESPACE_BEGIN
+//------------------------------------------------
+
+HIC_TYPE(sparseHandle_t)
+HIC_TYPE(sparseStatus_t)
+HIC_TYPE(sparseIndexType_t)
+HIC_TYPE(sparseOrder_t)
+HIC_TYPE(sparseConstDnVecDescr_t)
+HIC_TYPE(sparseDnVecDescr_t)
+HIC_TYPE(sparseConstDnMatDescr_t)
+HIC_TYPE(sparseDnMatDescr_t)
+HIC_TYPE(sparseConstSpVecDescr_t)
+HIC_TYPE(sparseSpVecDescr_t)
+HIC_TYPE(sparseConstSpMatDescr_t)
+HIC_TYPE(sparseSpMatDescr_t)
+HIC_TYPE(sparseSpMVAlg_t)
+HIC_TYPE(sparseSpMMAlg_t)
+
+HIC_FUNCTION(sparseGetErrorString)
+HIC_FUNCTION(sparseCreate)
+HIC_FUNCTION(sparseDestroy)
+HIC_FUNCTION(sparseCreateConstDnVec)
+HIC_FUNCTION(sparseCreateDnVec)
+HIC_FUNCTION(sparseDestroyDnVec)
+HIC_FUNCTION(sparseCreateConstDnMat)
+HIC_FUNCTION(sparseCreateDnMat)
+HIC_FUNCTION(sparseDestroyDnMat)
+HIC_FUNCTION(sparseCreateConstSpVec)
+HIC_FUNCTION(sparseCreateSpVec)
+HIC_FUNCTION(sparseDestroySpVec)
+HIC_FUNCTION(sparseCreateConstCsr)
+HIC_FUNCTION(sparseDestroySpMat)
+HIC_FUNCTION(sparseSpMV_bufferSize)
+HIC_FUNCTION(sparseSpMV_preprocess)
+HIC_FUNCTION(sparseSpMV)
+HIC_FUNCTION(sparseSpMM_bufferSize)
+HIC_FUNCTION(sparseSpMM_preprocess)
+HIC_FUNCTION(sparseSpMM)
+
+HIC_VALUE(SPARSE_STATUS_SUCCESS)
+HIC_VALUE(SPARSE_ORDER_COL)
+HIC_VALUE(SPARSE_ORDER_ROW)
+HIC_VALUE(SPARSE_INDEX_32I)
+HIC_VALUE(SPARSE_INDEX_64I)
+HIC_VALUE(SPARSE_INDEX_BASE_ZERO)
+HIC_VALUE(SPARSE_INDEX_BASE_ONE)
+HIC_VALUE(SPARSE_SPMV_ALG_DEFAULT)
+HIC_VALUE(SPARSE_SPMM_ALG_DEFAULT)
+HIC_VALUE(SPARSE_OPERATION_NON_TRANSPOSE)
+HIC_VALUE(SPARSE_OPERATION_TRANSPOSE)
+HIC_VALUE(SPARSE_OPERATION_CONJUGATE_TRANSPOSE)
+
+#if HIC_BACKEND_DUMMY
+#define HICSPARSE_CALL(val)
+#else
+#define HICSPARSE_CALL(val) hicsparse_assert((val), #val, __FILE__, __LINE__)
+#endif
+
+inline void hicsparse_assert(hicsparseStatus_t status, const char* const func, const char* const file, const int line) {
+    // Ignore errors when HIP/CUDA runtime is unloaded or deinitialized.
+    // This happens when calling HIP/CUDA after main has ended, e.g. in teardown of static variables calling `hicFree`
+    //   --> ignore hicErrorDeinitialized (a.k.a. cudaErrorCudartUnloading / hipErrorDeinitialized)
+    if (status != HICSPARSE_STATUS_SUCCESS) {
+        std::ostringstream msg;
+        msg << "HIC Runtime Error [code="<<status<<"] at: " << file << " + " << line << " : " << func << "\n";
+        msg << "  Reason: " << hicsparseGetErrorString(status);
+        throw std::runtime_error(msg.str());
+    }
+}
+
+//------------------------------------------------
+HIC_NAMESPACE_END
+//------------------------------------------------
+
+#undef HIC_FUNCTION
+#undef HIC_TYPE
+#undef HIC_VALUE
+#undef HIC_PREFIX
+#undef HIC_PREFIX_CAPS
+#undef HIC_CONCAT
+#undef HIC_CONCAT_
+#undef HIC_SYMBOL
+#undef HIC_SYMBOL_CAPS
+#undef HICSPARSE_BACKEND_PREFIX
+#undef HICSPARSE_BACKEND_PREFIX_CAPS

hic/tests/CMakeLists.txt

@@ -17,6 +17,11 @@ if( HAVE_HIP OR HAVE_CUDA )
     SOURCES   test_hic.cc
     LIBS      hic
   )
+
+  ecbuild_add_test( TARGET hic_test_hicsparse
+    SOURCES   test_hicsparse.cc
+    LIBS      hic
+  )
 endif()
 
 add_subdirectory(test_find_hic)