Padding issues with kokkos

[s-sajid-ali]

Currently we are unable to use Kokkos padded arrays on GPUs due to an underlying bug in Kokkos.

In the meantime, this issue is used to document the following fix as communicated by the Kokkos developers:

template<class ViewT>
typename ViewT::HostMirror create_padded_host_mirror(ViewT v) {
  bool use_padding = false;
  if(std::is_same<typename ViewT::array_layout, Kokkos::LayoutLeft>::value)
    if(v.stride(1)!=v.extent(0)) use_padding = true;
  if(std::is_same<typename ViewT::array_layout, Kokkos::LayoutRight>::value)
    if(v.stride(ViewT::rank-2)!=v.extent(ViewT::rank-1)) use_padding = true;

  typename ViewT::HostMirror h_v;
  if(use_padding) {
    auto alloc = Kokkos::view_alloc(Kokkos::AllowPadding, "deviceview");
    h_v = typename ViewT::HostMirror(alloc, v.layout());
  } else
    h_v = Kokkos::create_mirror_view(v);
  for(int r=0; r<ViewT::rank; r++)
    if(h_v.stride(r) != v.stride(r)) throw std::runtime_error("Was not able to create correct padded host mirror.");
  return h_v;
}

template<class view_t_1, class view_t_2>
void padded_deep_copy(view_t_1 a, view_t_2 b) {
  Kokkos::deep_copy(Kokkos::View<typename view_t_1::value_type*,
                                 typename view_t_1::device_type>(a.data(), a.span()),
                    Kokkos::View<typename view_t_2::value_type*,
                                 typename view_t_2::device_type>(b.data(), b.span()));
}