MPI resource handling

fsgrid.hpp

@@ -512,17 +512,125 @@ template <typename T, int stencil> class FsGrid : public FsGridTools{
  this->data = other.getData(); // Copy assignment
  }
 
- /*! Finalize instead of destructor, as the MPI calls fail after the main program called MPI_Finalize().
- * Cleans up the cartesian communicator
+ /*! 
+ * MPI calls fail after the main program called MPI_Finalize(),
+ * so this can be used instead of the destructor
+ * Cleans up the cartesian communicator and datatypes
  */
- void finalize() {
- for(int i=0;i<27;i++){
- if(neighbourReceiveType[i] != MPI_DATATYPE_NULL)
- MPI_Type_free(&(neighbourReceiveType[i]));
- if(neighbourSendType[i] != MPI_DATATYPE_NULL)
- MPI_Type_free(&(neighbourSendType[i]));
+ void finalize() noexcept {
+ if (comm3d != MPI_COMM_NULL) {
+ MPI_Comm_free(&comm3d);
+ comm3d = MPI_COMM_NULL;
+ }
+ for (auto& type : neighbourReceiveType) {
+ if (type != MPI_DATATYPE_NULL) {
+ MPI_Type_free(&type);
+ type = MPI_DATATYPE_NULL;
+ }
+ }
+ for (auto& type : neighbourSendType) {
+ if (type != MPI_DATATYPE_NULL) {
+ MPI_Type_free(&type);
+ type = MPI_DATATYPE_NULL;
+ }
+ }
+ }
+
+ /*!
+ * If finalize() isn't called manually, object should be destroyed before MPI finalization
+ */
+ ~FsGrid() {
+ finalize();
+ }
+
+ friend void swap (FsGrid& first, FsGrid& second) noexcept {
+ using std::swap;
+ swap(first.comm3d, second.comm3d);
+ swap(first.neighbourSendType, second.neighbourSendType);
+ swap(first.neighbourReceiveType, second.neighbourReceiveType);
+ swap(first.DX, second.DX);
+ swap(first.DY, second.DY);
+ swap(first.DZ, second.DZ);
+ swap(first.physicalGlobalStart, second.physicalGlobalStart);
+ swap(first.rank, second.rank);
+ swap(first.requests, second.requests);
+ swap(first.numRequests, second.numRequests);
+ swap(first.neighbour, second.neighbour);
+ swap(first.neighbour_index, second.neighbour_index);
+ swap(first.ntasksPerDim, second.ntasksPerDim);
+ swap(first.taskPosition, second.taskPosition);
+ swap(first.periodic, second.periodic);
+ swap(first.globalSize, second.globalSize);
+ swap(first.localSize, second.localSize);
+ swap(first.storageSize, second.storageSize);
+ swap(first.localStart, second.localStart);
+ swap(first.coupling, second.coupling);
+ swap(first.data, second.data);
+ }
+
+ // Copy constructor
+ FsGrid(const FsGrid& other) : 
+ comm3d {MPI_COMM_NULL},
+ neighbourSendType {},
+ neighbourReceiveType {},
+ rank {other.rank}, 
+ DX {other.DX},
+ DY {other.DY},
+ DZ {other.DZ},
+ physicalGlobalStart {other.physicalGlobalStart},
+ requests {}, 
+ numRequests {0}, 
+ neighbour {other.neighbour},
+ neighbour_index {other.neighbour_index},
+ ntasksPerDim {other.ntasksPerDim},
+ taskPosition {other.taskPosition},
+ periodic {other.periodic},
+ globalSize {other.globalSize},
+ localSize {other.localSize},
+ storageSize {other.storageSize},
+ localStart {other.localStart},
+ coupling {other.coupling},
+ data {other.data}
+ {
+ if (other.comm3d != MPI_COMM_NULL) {
+ MPI_Comm_dup(other.comm3d, &comm3d);
+ }
+
+ neighbourSendType.fill(MPI_DATATYPE_NULL);
+ neighbourReceiveType.fill(MPI_DATATYPE_NULL);
+ for (int i = 0; i < neighbourSendType.size(); ++i) {
+ if (other.neighbourSendType[i] != MPI_DATATYPE_NULL) {
+ MPI_Type_dup(other.neighbourSendType[i], neighbourSendType.data() + i);
+ }
+ if (other.neighbourReceiveType[i] != MPI_DATATYPE_NULL) {
+ MPI_Type_dup(other.neighbourReceiveType[i], neighbourReceiveType.data() + i);
+ }
  }
- MPI_Comm_free(&comm3d);
+ }
+
+ // Move constructor
+ // We don't have a default constructor, so just set the MPI stuff NULL
+ FsGrid(FsGrid&& other) noexcept : 
+ comm3d {MPI_COMM_NULL},
+ neighbourSendType {},
+ neighbourReceiveType {}
+ {
+ // NULL all the MPI stuff so they won't get freed if destroyed
+ neighbourSendType.fill(MPI_DATATYPE_NULL);
+ neighbourReceiveType.fill(MPI_DATATYPE_NULL);
+
+ using std::swap;
+ swap(*this, other);
+ }
+
+ // Copy assignment
+ FsGrid& operator=(FsGrid other) = delete;
+
+ // Move assignment
+ FsGrid& operator=(FsGrid&& other) noexcept {
+ using std::swap;
+ swap(*this, other);
+ return *this;
  }
 
  /*! Returns the task responsible, and its localID for handling the cell with the given GlobalID
@@ -1005,12 +1113,6 @@ template <typename T, int stencil> class FsGrid : public FsGridTools{
  return &data[id];
  }
 
- /*! Physical grid spacing and physical coordinate space start.
- * TODO: Should this be private and have accesor-functions?
- */
- double DX,DY,DZ;
- std::array<double,3> physicalGlobalStart;
-
  /*! Get the physical coordinates in the global simulation space for
  * the given cell.
  *
@@ -1085,6 +1187,12 @@ template <typename T, int stencil> class FsGrid : public FsGridTools{
  return ntasksPerDim;
  }
 
+ /*! Physical grid spacing and physical coordinate space start.
+ * TODO: Should this be private and have accesor-functions?
+ */
+ double DX,DY,DZ;
+ std::array<double,3> physicalGlobalStart;
+
  private:
  //! MPI Cartesian communicator used in this grid
  MPI_Comm comm3d;