Add a memory space abstraction

benchmark/utils/loggers.hpp

@@ -48,41 +48,43 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 // A logger that accumulates the time of all operations
 struct OperationLogger : gko::log::Logger {
-    void on_allocation_started(const gko::Executor *exec,
+    void on_allocation_started(const gko::MemorySpace *mem_space,
                                const gko::size_type &) const override
     {
-        this->start_operation(exec, "allocate");
+        this->start_operation(mem_space, "allocate");
     }
 
-    void on_allocation_completed(const gko::Executor *exec,
+    void on_allocation_completed(const gko::MemorySpace *mem_space,
                                  const gko::size_type &,
                                  const gko::uintptr &) const override
     {
-        this->end_operation(exec, "allocate");
+        this->end_operation(mem_space, "allocate");
     }
 
-    void on_free_started(const gko::Executor *exec,
+    void on_free_started(const gko::MemorySpace *mem_space,
                          const gko::uintptr &) const override
     {
-        this->start_operation(exec, "free");
+        this->start_operation(mem_space, "free");
     }
 
-    void on_free_completed(const gko::Executor *exec,
+    void on_free_completed(const gko::MemorySpace *mem_space,
                            const gko::uintptr &) const override
     {
-        this->end_operation(exec, "free");
+        this->end_operation(mem_space, "free");
     }
 
-    void on_copy_started(const gko::Executor *from, const gko::Executor *to,
-                         const gko::uintptr &, const gko::uintptr &,
+    void on_copy_started(const gko::MemorySpace *from,
+                         const gko::MemorySpace *to, const gko::uintptr &,
+                         const gko::uintptr &,
                          const gko::size_type &) const override
     {
         from->synchronize();
         this->start_operation(to, "copy");
     }
 
-    void on_copy_completed(const gko::Executor *from, const gko::Executor *to,
-                           const gko::uintptr &, const gko::uintptr &,
+    void on_copy_completed(const gko::MemorySpace *from,
+                           const gko::MemorySpace *to, const gko::uintptr &,
+                           const gko::uintptr &,
                            const gko::size_type &) const override
     {
         from->synchronize();
@@ -118,14 +120,15 @@ struct OperationLogger : gko::log::Logger {
     }
 
     OperationLogger(std::shared_ptr<const gko::Executor> exec, bool nested_name)
-        : gko::log::Logger(exec), use_nested_name{nested_name}
+        : gko::log::Logger(exec, exec->get_mem_space()),
+          use_nested_name{nested_name}
     {}
 
 private:
-    void start_operation(const gko::Executor *exec,
-                         const std::string &name) const
+    template <typename LogObject>
+    void start_operation(const LogObject *obj, const std::string &name) const
     {
-        exec->synchronize();
+        obj->synchronize();
         const std::lock_guard<std::mutex> lock(mutex);
         auto nested_name = nested.empty() || !use_nested_name
                                ? name
@@ -134,9 +137,12 @@ struct OperationLogger : gko::log::Logger {
         start[nested_name] = std::chrono::steady_clock::now();
     }
 
-    void end_operation(const gko::Executor *exec, const std::string &name) const
+    // Helper to compute the end time and store the operation's time at its
+    // end. Also time nested operations.
+    template <typename LogObject>
+    void end_operation(const LogObject *obj, const std::string &name) const
     {
-        exec->synchronize();
+        obj->synchronize();
         const std::lock_guard<std::mutex> lock(mutex);
         // if operations are properly nested, nested_name now ends with name
         auto nested_name = nested.back().first;
@@ -163,15 +169,15 @@ struct OperationLogger : gko::log::Logger {
 
 
 struct StorageLogger : gko::log::Logger {
-    void on_allocation_completed(const gko::Executor *,
+    void on_allocation_completed(const gko::MemorySpace *,
                                  const gko::size_type &num_bytes,
                                  const gko::uintptr &location) const override
     {
         const std::lock_guard<std::mutex> lock(mutex);
         storage[location] = num_bytes;
     }
 
-    void on_free_completed(const gko::Executor *,
+    void on_free_completed(const gko::MemorySpace *,
                            const gko::uintptr &location) const override
     {
         const std::lock_guard<std::mutex> lock(mutex);
@@ -190,7 +196,7 @@ struct StorageLogger : gko::log::Logger {
     }
 
     StorageLogger(std::shared_ptr<const gko::Executor> exec)
-        : gko::log::Logger(exec)
+        : gko::log::Logger(exec, exec->get_mem_space())
     {}
 
 private:
@@ -235,7 +241,8 @@ struct ResidualLogger : gko::log::Logger {
                    rapidjson::Value &true_res_norms,
                    rapidjson::Value &timestamps,
                    rapidjson::MemoryPoolAllocator<> &alloc)
-        : gko::log::Logger(exec, gko::log::Logger::iteration_complete_mask),
+        : gko::log::Logger(exec, exec->get_mem_space(),
+                           gko::log::Logger::iteration_complete_mask),
           matrix{matrix},
           b{b},
           start{std::chrono::steady_clock::now()},
@@ -267,7 +274,8 @@ struct IterationLogger : gko::log::Logger {
     }
 
     IterationLogger(std::shared_ptr<const gko::Executor> exec)
-        : gko::log::Logger(exec, gko::log::Logger::iteration_complete_mask)
+        : gko::log::Logger(exec, exec->get_mem_space(),
+                           gko::log::Logger::iteration_complete_mask)
     {}
 
     void write_data(rapidjson::Value &output,

core/CMakeLists.txt

@@ -8,6 +8,7 @@ target_sources(ginkgo
     base/combination.cpp
     base/composition.cpp
     base/executor.cpp
+    base/memory_space.cpp
     base/mtx_io.cpp
     base/perturbation.cpp
     base/version.cpp

core/base/allocator.hpp

@@ -99,7 +99,10 @@ class ExecutorAllocator {
      * @param n  the number of elements to allocate
      * @return  the pointer to a newly allocated memory area of `n` elements.
      */
-    T *allocate(std::size_t n) const { return exec_->alloc<T>(n); }
+    T *allocate(std::size_t n) const
+    {
+        return exec_->get_mem_space()->template alloc<T>(n);
+    }
 
     /**
      * Frees a memory area that was allocated by this allocator.
@@ -108,7 +111,10 @@ class ExecutorAllocator {
      *
      * @note  The second parameter is unused.
      */
-    void deallocate(T *ptr, std::size_t) const { exec_->free(ptr); }
+    void deallocate(T *ptr, std::size_t) const
+    {
+        exec_->get_mem_space()->free(ptr);
+    }
 
     /**
      * Compares two ExecutorAllocators for equality
@@ -172,4 +178,4 @@ using unordered_map =
 
 }  // namespace gko
 
-#endif  // GKO_CORE_BASE_ALLOCATOR_HPP_
+#endif  // GKO_CORE_BASE_ALLOCATOR_HPP_

core/base/memory_space.cpp

@@ -0,0 +1,69 @@
+/*******************************<GINKGO LICENSE>******************************
+Copyright (c) 2017-2020, the Ginkgo authors
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+******************************<GINKGO LICENSE>*******************************/
+
+#include <ginkgo/core/base/memory_space.hpp>
+
+
+#include <cstdlib>
+#include <cstring>
+
+
+#include <ginkgo/core/base/exception.hpp>
+#include <ginkgo/core/base/exception_helpers.hpp>
+
+
+namespace gko {
+
+
+void HostMemorySpace::raw_free(void *ptr) const noexcept { std::free(ptr); }
+
+
+void HostMemorySpace::synchronize() const
+{
+    // Currently a no-op
+}
+
+
+void *HostMemorySpace::raw_alloc(size_type num_bytes) const
+{
+    return GKO_ENSURE_ALLOCATED(std::malloc(num_bytes), "Host", num_bytes);
+}
+
+
+void HostMemorySpace::raw_copy_to(const HostMemorySpace *, size_type num_bytes,
+                                  const void *src_ptr, void *dest_ptr) const
+{
+    std::memcpy(dest_ptr, src_ptr, num_bytes);
+}
+
+
+}  // namespace gko

core/device_hooks/cuda_hooks.cpp

@@ -36,6 +36,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 #include <ginkgo/core/base/exception_helpers.hpp>
 #include <ginkgo/core/base/executor.hpp>
+#include <ginkgo/core/base/memory_space.hpp>
 #include <ginkgo/core/base/types.hpp>
 #include <ginkgo/core/base/version.hpp>
 
@@ -62,40 +63,98 @@ std::shared_ptr<CudaExecutor> CudaExecutor::create(
 }
 
 
-void OmpExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
-                              const void *src_ptr, void *dest_ptr) const
+std::shared_ptr<CudaExecutor> CudaExecutor::create(
+    int device_id, std::shared_ptr<MemorySpace> mem_space,
+    std::shared_ptr<Executor> master, bool device_reset)
+{
+    return std::shared_ptr<CudaExecutor>(new CudaExecutor(
+        device_id, mem_space, std::move(master), device_reset));
+}
+
+
+void HostMemorySpace::raw_copy_to(const CudaMemorySpace *, size_type num_bytes,
+                                  const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(cuda);
 
 
-void CudaExecutor::raw_free(void *ptr) const noexcept
+void CudaMemorySpace::raw_free(void *ptr) const noexcept
 {
     // Free must never fail, as it can be called in destructors.
     // If the nvidia module was not compiled, the library couldn't have
     // allocated the memory, so there is no need to deallocate it.
 }
 
 
-void *CudaExecutor::raw_alloc(size_type num_bytes) const GKO_NOT_COMPILED(cuda);
+void CudaUVMSpace::raw_free(void *ptr) const noexcept
+{
+    // Free must never fail, as it can be called in destructors.
+    // If the nvidia module was not compiled, the library couldn't have
+    // allocated the memory, so there is no need to deallocate it.
+}
+
+
+void *CudaMemorySpace::raw_alloc(size_type num_bytes) const
+    GKO_NOT_COMPILED(nvidia);
+
 
+void *CudaUVMSpace::raw_alloc(size_type num_bytes) const
+    GKO_NOT_COMPILED(nvidia);
 
-void CudaExecutor::raw_copy_to(const OmpExecutor *, size_type num_bytes,
-                               const void *src_ptr, void *dest_ptr) const
+
+void CudaMemorySpace::raw_copy_to(const HostMemorySpace *, size_type num_bytes,
+                                  const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(cuda);
 
 
-void CudaExecutor::raw_copy_to(const CudaExecutor *, size_type num_bytes,
-                               const void *src_ptr, void *dest_ptr) const
+void CudaMemorySpace::raw_copy_to(const CudaMemorySpace *, size_type num_bytes,
+                                  const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(cuda);
 
 
-void CudaExecutor::raw_copy_to(const HipExecutor *, size_type num_bytes,
-                               const void *src_ptr, void *dest_ptr) const
+void CudaMemorySpace::raw_copy_to(const HipMemorySpace *, size_type num_bytes,
+                                  const void *src_ptr, void *dest_ptr) const
     GKO_NOT_COMPILED(cuda);
 
 
+void CudaMemorySpace::raw_copy_to(const CudaUVMSpace *dest_mem_space,
+                                  size_type n_bytes, const void *src_ptr,
+                                  void *dest_ptr) const GKO_NOT_COMPILED(cuda);
+
+
+void CudaUVMSpace::raw_copy_to(const CudaUVMSpace *dest_mem_space,
+                               size_type n_bytes, const void *src_ptr,
+                               void *dest_ptr) const GKO_NOT_COMPILED(cuda);
+
+
+void CudaUVMSpace::raw_copy_to(const CudaMemorySpace *dest_mem_space,
+                               size_type n_bytes, const void *src_ptr,
+                               void *dest_ptr) const GKO_NOT_COMPILED(cuda);
+
+
+void CudaUVMSpace::raw_copy_to(const HipMemorySpace *dest_mem_space,
+                               size_type n_bytes, const void *src_ptr,
+                               void *dest_ptr) const GKO_NOT_COMPILED(cuda);
+
+
+void HostMemorySpace::raw_copy_to(const CudaUVMSpace *dest_mem_space,
+                                  size_type n_bytes, const void *src_ptr,
+                                  void *dest_ptr) const GKO_NOT_COMPILED(cuda);
+
+
+void CudaUVMSpace::raw_copy_to(const HostMemorySpace *dest_mem_space,
+                               size_type n_bytes, const void *src_ptr,
+                               void *dest_ptr) const GKO_NOT_COMPILED(cuda);
+
+
 void CudaExecutor::synchronize() const GKO_NOT_COMPILED(cuda);
 
 
+void CudaMemorySpace::synchronize() const GKO_NOT_COMPILED(cuda);
+
+
+void CudaUVMSpace::synchronize() const GKO_NOT_COMPILED(cuda);
+
+
 void CudaExecutor::run(const Operation &op) const
 {
     op.run(
@@ -124,6 +183,12 @@ std::string CusparseError::get_error(int64)
 int CudaExecutor::get_num_devices() { return 0; }
 
 
+int CudaMemorySpace::get_num_devices() { return 0; }
+
+
+int CudaUVMSpace::get_num_devices() { return 0; }
+
+
 void CudaExecutor::set_gpu_property() {}