PODVector Updates

Remove deprecated and unused PolymorphicAllocator. It has been replaced by
PolymorphicArenaAllocator.

Restrict PODVector's Allocator to std::allocator and AMReX's various Arena
based allocators. This simplifies the implementation of PODVector, because
std::allocator is stateless and Arena based allocators are simple even when
it's polymorphic.

Fix a few issues of PODVectors with a PolymorphicArenaAllocator. For
example, copy assignment operator should copy the Allocator. Copy
constructor should consider the possibility that other PODVector has a
different type of Arena.

Add placeholders for potentially growing and shrinking memory allocation
in-place that will be implemented in a follow-up PR.

Update PODVector's growth strategy. Hopefully this helps to reduce the
memory consumption.

  * Always try to grow in-place.

  * For assign, operator=, resize & reserve, allocate the specified size
    without capacity.

  * For push_back & emplace_back, grow the capacity by a factor that is 1.5
    by default.

  * For insert, the capacity grows either by a factor that is 1.5 by default
    or to the new size, whichever is greater.

Src/Base/AMReX_Arena.H

@@ -7,6 +7,7 @@
 #include <cstddef>
 #include <cstdlib>
 #include <limits>
+#include <utility>
 
 namespace amrex {
 
@@ -100,6 +101,26 @@ public:
  * \return a pointer to the allocated memory
  */
  [[nodiscard]] virtual void* alloc (std::size_t sz) = 0;
+
+ /**
+ * Try to allocate in-place by extending the capacity of given pointer.
+ */
+ [[nodiscard]] virtual std::pair<void*,std::size_t>
+ alloc_in_place (void* /*pt*/, std::size_t /*szmin*/, std::size_t szmax)
+ {
+ auto* p = alloc(szmax);
+ return std::make_pair(p, szmax);
+ }
+
+ /**
+ * Try to shrink in-place
+ */
+ [[nodiscard]] virtual void*
+ shrink_in_place (void* /*pt*/, std::size_t sz)
+ {
+ return alloc(sz);
+ }
+
  /**
  * \brief A pure virtual function for deleting the arena pointed to by pt
  */

Src/Base/AMReX_GpuAllocators.H

@@ -25,201 +25,255 @@ namespace amrex {
  template <typename T>
  struct IsPolymorphicArenaAllocator : std::false_type {};
 
- struct ArenaAllocatorTraits {
- typedef std::true_type propagate_on_container_copy_assignment;
- typedef std::true_type propagate_on_container_move_assignment;
- typedef std::true_type propagate_on_container_swap;
- typedef std::true_type is_always_equal;
+ struct ArenaAllocatorBase {};
+
+ template <class T, class Enable = void>
+ struct IsArenaAllocator : std::false_type {};
+ //
+ template <class T>
+ struct IsArenaAllocator
+ <T,std::enable_if_t<std::is_base_of<ArenaAllocatorBase,T>::value>>
+ : std::true_type {};
+
+ template <typename T>
+ struct FatPtr
+ {
+ T* m_ptr = nullptr;
+ std::size_t m_size = 0;
+ constexpr T* ptr () const noexcept { return m_ptr; }
+ constexpr std::size_t size () const noexcept { return m_size; }
  };
 
+ template <typename T>
+ [[nodiscard]] FatPtr<T>
+ allocateInPlace (T* p, std::size_t nmin, std::size_t nmax, Arena* ar)
+ {
+ auto pn = ar->alloc_in_place(p, nmin*sizeof(T), nmax*sizeof(T));
+ return FatPtr<T>{(T*)pn.first, pn.second/sizeof(T)};
+ }
+
  template<typename T>
  class ArenaAllocator
- : public ArenaAllocatorTraits
+ : public ArenaAllocatorBase
  {
  public :
 
  using value_type = T;
 
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T* allocate (std::size_t n)
+ {
+ return (T*) arena()->alloc(n * sizeof(T));
+ }
+
+ [[nodiscard]] FatPtr<T>
+ allocate_in_place (T* p, std::size_t nmin, std::size_t nmax)
+ {
+ return allocateInPlace(p, nmin, nmax, arena());
+ }
+
+ [[nodiscard]] T*
+ shrink_in_place (T* p, std::size_t n)
  {
- value_type* result = nullptr;
- result = (value_type*) The_Arena()->alloc(n * sizeof(T));
- return result;
+ return (T*) arena()->shrink_in_place(p,n*sizeof(T));
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ void deallocate (T* ptr, std::size_t)
  {
- if (ptr != nullptr) { The_Arena()->free(ptr); }
+ if (ptr != nullptr) { arena()->free(ptr); }
+ }
+
+ [[nodiscard]] Arena* arena () const noexcept {
+ return The_Arena();
  }
  };
 
  template<typename T>
  class DeviceArenaAllocator
- : public ArenaAllocatorTraits
+ : public ArenaAllocatorBase
  {
  public :
 
  using value_type = T;
 
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T* allocate(std::size_t n)
  {
- value_type* result = nullptr;
- result = (value_type*) The_Device_Arena()->alloc(n * sizeof(T));
- return result;
+ return (T*) arena()->alloc(n * sizeof(T));
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ [[nodiscard]] FatPtr<T>
+ allocate_in_place (T* p, std::size_t nmin, std::size_t nmax)
  {
- if (ptr != nullptr) { The_Device_Arena()->free(ptr); }
+ return allocateInPlace(p, nmin, nmax, arena());
+ }
+
+ [[nodiscard]] T*
+ shrink_in_place (T* p, std::size_t n)
+ {
+ return (T*) arena()->shrink_in_place(p,n*sizeof(T));
+ }
+
+ void deallocate(T* ptr, std::size_t)
+ {
+ if (ptr != nullptr) { arena()->free(ptr); }
+ }
+
+ [[nodiscard]] Arena* arena () const noexcept {
+ return The_Device_Arena();
  }
  };
 
  template<typename T>
  class PinnedArenaAllocator
- : public ArenaAllocatorTraits
+ : public ArenaAllocatorBase
  {
  public :
 
  using value_type = T;
 
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T* allocate(std::size_t n)
  {
- value_type* result = nullptr;
- result = (value_type*) The_Pinned_Arena()->alloc(n * sizeof(T));
- return result;
+ return (T*) arena()->alloc(n * sizeof(T));
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ [[nodiscard]] FatPtr<T>
+ allocate_in_place (T* p, std::size_t nmin, std::size_t nmax)
  {
- if (ptr != nullptr) { The_Pinned_Arena()->free(ptr); }
+ return allocateInPlace(p, nmin, nmax, arena());
+ }
+
+ [[nodiscard]] T*
+ shrink_in_place (T* p, std::size_t n)
+ {
+ return (T*) arena()->shrink_in_place(p,n*sizeof(T));
+ }
+
+ void deallocate(T* ptr, std::size_t)
+ {
+ if (ptr != nullptr) { arena()->free(ptr); }
+ }
+
+ [[nodiscard]] Arena* arena () const noexcept {
+ return The_Pinned_Arena();
  }
  };
 
  template<typename T>
  class ManagedArenaAllocator
- : public ArenaAllocatorTraits
+ : public ArenaAllocatorBase
  {
  public :
 
  using value_type = T;
 
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T* allocate(std::size_t n)
+ {
+ return (T*) arena()->alloc(n * sizeof(T));
+ }
+
+ [[nodiscard]] FatPtr<T>
+ allocate_in_place (T* p, std::size_t nmin, std::size_t nmax)
  {
- value_type* result = nullptr;
- result = (value_type*) The_Managed_Arena()->alloc(n * sizeof(T));
- return result;
+ return allocateInPlace(p, nmin, nmax, arena());
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ [[nodiscard]] T*
+ shrink_in_place (T* p, std::size_t n)
  {
- if (ptr != nullptr) { The_Managed_Arena()->free(ptr); }
+ return (T*) arena()->shrink_in_place(p,n*sizeof(T));
+ }
+
+ void deallocate(T* ptr, std::size_t)
+ {
+ if (ptr != nullptr) { arena()->free(ptr); }
+ }
+
+ [[nodiscard]] Arena* arena () const noexcept {
+ return The_Managed_Arena();
  }
  };
 
  template<typename T>
  class AsyncArenaAllocator
- : public ArenaAllocatorTraits
+ : public ArenaAllocatorBase
  {
  public :
 
  using value_type = T;
 
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T* allocate(std::size_t n)
  {
- value_type* result = nullptr;
- result = (value_type*) The_Async_Arena()->alloc(n * sizeof(T));
- return result;
+ return (T*) arena()->alloc(n * sizeof(T));
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ [[nodiscard]] FatPtr<T>
+ allocate_in_place (T* p, std::size_t nmin, std::size_t nmax)
  {
- if (ptr != nullptr) { The_Async_Arena()->free(ptr); }
+ return allocateInPlace(p, nmin, nmax, arena());
  }
- };
-
- template<typename T>
- class PolymorphicArenaAllocator
- : public ArenaAllocatorTraits
- {
- public :
 
- using value_type = T;
-
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T*
+ shrink_in_place (T* p, std::size_t n)
  {
- value_type* result = nullptr;
- result = (value_type*) arena()->alloc(n * sizeof(T));
- return result;
+ return (T*) arena()->shrink_in_place(p,n*sizeof(T));
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ void deallocate(T* ptr, std::size_t)
  {
  if (ptr != nullptr) { arena()->free(ptr); }
  }
 
  [[nodiscard]] Arena* arena () const noexcept {
- return (m_arena) ? m_arena : The_Arena();
+ return The_Async_Arena();
  }
-
- Arena* m_arena = nullptr;
  };
 
  template<typename T>
- class PolymorphicAllocator
+ class PolymorphicArenaAllocator
+ : public ArenaAllocatorBase
  {
  public :
 
  using value_type = T;
 
- PolymorphicAllocator () : m_use_gpu_aware_mpi(ParallelDescriptor::UseGpuAwareMpi()) {}
-
- inline value_type* allocate(std::size_t n)
+ [[nodiscard]] T* allocate(std::size_t n)
  {
- value_type* result = nullptr;
- if (m_use_gpu_aware_mpi)
- {
- result = (value_type*) The_Arena()->alloc(n * sizeof(T));
- }
- else
- {
- result = (value_type*) The_Pinned_Arena()->alloc(n * sizeof(T));
- }
- return result;
+ return (T*) arena()->alloc(n * sizeof(T));
  }
 
- inline void deallocate(value_type* ptr, std::size_t)
+ [[nodiscard]] FatPtr<T>
+ allocate_in_place (T* p, std::size_t nmin, std::size_t nmax)
  {
- if (ptr != nullptr)
- {
- if (m_use_gpu_aware_mpi)
- {
- The_Arena()->free(ptr);
- }
- else
- {
- The_Pinned_Arena()->free(ptr);
- }
- }
+ return allocateInPlace(p, nmin, nmax, arena());
  }
 
- bool m_use_gpu_aware_mpi;
-
- template <class U, class V>
- friend bool
- operator== (PolymorphicAllocator<U> const& a, PolymorphicAllocator<V> const& b) noexcept
+ [[nodiscard]] T*
+ shrink_in_place (T* p, std::size_t n)
  {
- return a.m_use_gpu_aware_mpi == b.m_use_gpu_aware_mpi;
+ return (T*) arena()->shrink_in_place(p,n*sizeof(T));
  }
 
- template <class U, class V>
- friend bool
- operator!= (PolymorphicAllocator<U> const& a, PolymorphicAllocator<V> const& b) noexcept
+ void deallocate(T* ptr, std::size_t)
  {
- return a.m_use_gpu_aware_mpi != b.m_use_gpu_aware_mpi;
+ if (ptr != nullptr) { arena()->free(ptr); }
  }
 
+ [[nodiscard]] Arena* arena () const noexcept {
+ return (m_arena) ? m_arena : The_Arena();
+ }
+
+ void arena (Arena* a_arena) noexcept { m_arena = a_arena; }
+
+ Arena* m_arena = nullptr;
  };
 
+ template <typename A1, typename A2,
+ std::enable_if_t<IsArenaAllocator<A1>::value &&
+ IsArenaAllocator<A2>::value, int> = 0>
+ bool operator== (A1 const& a1, A2 const& a2)
+ {
+ return a1.arena() == a2.arena();
+ }
+
 #ifdef AMREX_USE_GPU
  template <typename T>
  struct RunOnGpu<ArenaAllocator<T> > : std::true_type {};