From 0e71adf7eee806da3751bf6b56b9284103510c28 Mon Sep 17 00:00:00 2001 From: atmyers Date: Wed, 17 Jul 2024 21:37:03 +0000 Subject: [PATCH] =?UTF-8?q?Deploying=20to=20main=20from=20@=20AMReX-Codes/?= =?UTF-8?q?amrex@fdea28f16727bd0b1a63fcb189c2521e0fc13dc2=20=F0=9F=9A=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../amrex.pdf | Bin 8751540 -> 8751540 bytes .../AMReX__ParticleHDF5_8H_source.html | 1455 +++++++++-------- .../doxygen/AMReX__ParticleHDF5_8H.xml | 1455 +++++++++-------- .../doxygen/AMReX__PlotFileUtilHDF5_8cpp.xml | 4 +- 4 files changed, 1458 insertions(+), 1456 deletions(-) diff --git a/amrex/docs_html/_downloads/008eb6dbfab802633dff40122ece848c/amrex.pdf b/amrex/docs_html/_downloads/008eb6dbfab802633dff40122ece848c/amrex.pdf index e983e421be5afbf9acdea12aa416ecad09ccc53a..fbaf82c6a0d67fa534e252d5fff9dfd19481c157 100644 GIT binary patch delta 376 zcmajV+fq#d0D$2^QdA^_N>R>a?cG|tl~V0Pl=JzxE2ko7N=FjDebtp|MsMM2&-5Za zfSF!|3->PmneXB+F8wSi{%yfJ>upeCqf(ocDYw}c6)IKPYMW}?)!1RDU25&N$6j^n zHQ1+7ll=~8*5aT;4m;wgV~#uFq*kW{5|Y}4A}Q@+9Zu`i<&17UdZlG#<@7o0oPGn& zyWpZrF1zBYYp%QDrdw{i6O>scD<-&A#Q`i5NmDRDr@5VoMWrrC6 delta 376 zcmajVSyPPx0D$31i=|bTC@MQCedjw1M~gbA^VzZ!>X4lXDU?FRbFN$&Gx-;;&P;#A z4`{|8$wl{Fyfe?mTl}?DR{Yu^G+KujnuAt~jgQ_@cB(yd3YGy3$)$QqCnIqRJB zF1TpWC6`@s)iu}MaMLYAZoA{IVI%VHxo^}14?XhO6Hh(!+zVq~dS%>eZ@e|(ok>&P z`{1K#Gd`I$=d&-qnzvxlH{U51e&j1+=`b0_e*erD8rHo0Ctvtim5riw7z9xm1woL^ f4aDMMJdsKSxkP_F%m$fMZ)^8@%ktl`!b;03a+rqW diff --git a/amrex/docs_html/doxygen/AMReX__ParticleHDF5_8H_source.html b/amrex/docs_html/doxygen/AMReX__ParticleHDF5_8H_source.html index 17db9e52d8..5e5d4de013 100644 --- a/amrex/docs_html/doxygen/AMReX__ParticleHDF5_8H_source.html +++ b/amrex/docs_html/doxygen/AMReX__ParticleHDF5_8H_source.html @@ -868,7 +868,7 @@
767  hsize_t chunk_size;
768  if (H5Pget_layout(dcpl_int) == H5D_CHUNKED) {
769  if (H5Pget_chunk(dcpl_int, 1, &chunk_size) > -1) {
-
770  if ((int)chunk_size > total_int_size) {
+
770  if (chunk_size > total_int_size) {
771  H5Pset_chunk(dcpl_int, 1, &total_int_size);
772  }
773  }
@@ -909,733 +909,734 @@
808  }
809  }
810 #endif
-
811  if (H5Pget_layout(dcpl_real) == H5D_CHUNKED) {
-
812  if (H5Pget_chunk(dcpl_real, 1, &chunk_size) > -1) {
-
813  if ((int)chunk_size > total_real_size) {
-
814  H5Pset_chunk(dcpl_real, 1, &total_real_size);
-
815  }
-
816  }
-
817  }
-
818 
-
819  real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
-
820  if (sizeof(typename ParticleType::RealType) == 4) {
-
821 #ifdef AMREX_USE_HDF5_ASYNC
-
822  real_dset_id = H5Dcreate_async(grp, "data:datatype=1", H5T_NATIVE_FLOAT, real_dset_space,
-
823  H5P_DEFAULT, dcpl_real, H5P_DEFAULT, es_par_g);
-
824 #else
-
825  real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_FLOAT, real_dset_space,
-
826  H5P_DEFAULT, dcpl_real, H5P_DEFAULT);
-
827 #endif
-
828  }
-
829  else {
-
830 #ifdef AMREX_USE_HDF5_ASYNC
-
831  real_dset_id = H5Dcreate_async(grp, "data:datatype=1", H5T_NATIVE_DOUBLE, real_dset_space,
-
832  H5P_DEFAULT, dcpl_real, H5P_DEFAULT, es_par_g);
-
833 #else
-
834  real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_DOUBLE, real_dset_space,
-
835  H5P_DEFAULT, dcpl_real, H5P_DEFAULT);
-
836 #endif
-
837  }
-
838  H5Sclose(real_dset_space);
-
839 
-
840  real_file_offset = 0;
-
841  for (int i = 0; i < ParallelDescriptor::MyProc(); i++) {
-
842  real_file_offset += all_mfi_real_total_size[i];
-
843  }
-
844  my_real_offset = real_file_offset;
-
845  my_real_count = 0;
-
846 
-
847  int max_mfi_count = 0, write_count = 0;
-
848  for (int i = 0; i < ParallelDescriptor::NProcs(); i++) {
-
849  if (max_mfi_count < all_mfi_cnt[i]) {
-
850  max_mfi_count = all_mfi_cnt[i];
-
851  }
-
852  }
-
853 
+
811 
+
812  if (H5Pget_layout(dcpl_real) == H5D_CHUNKED) {
+
813  if (H5Pget_chunk(dcpl_real, 1, &chunk_size) > -1) {
+
814  if (chunk_size > total_real_size) {
+
815  H5Pset_chunk(dcpl_real, 1, &total_real_size);
+
816  }
+
817  }
+
818  }
+
819 
+
820  real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
+
821  if (sizeof(typename ParticleType::RealType) == 4) {
+
822 #ifdef AMREX_USE_HDF5_ASYNC
+
823  real_dset_id = H5Dcreate_async(grp, "data:datatype=1", H5T_NATIVE_FLOAT, real_dset_space,
+
824  H5P_DEFAULT, dcpl_real, H5P_DEFAULT, es_par_g);
+
825 #else
+
826  real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_FLOAT, real_dset_space,
+
827  H5P_DEFAULT, dcpl_real, H5P_DEFAULT);
+
828 #endif
+
829  }
+
830  else {
+
831 #ifdef AMREX_USE_HDF5_ASYNC
+
832  real_dset_id = H5Dcreate_async(grp, "data:datatype=1", H5T_NATIVE_DOUBLE, real_dset_space,
+
833  H5P_DEFAULT, dcpl_real, H5P_DEFAULT, es_par_g);
+
834 #else
+
835  real_dset_id = H5Dcreate(grp, "data:datatype=1", H5T_NATIVE_DOUBLE, real_dset_space,
+
836  H5P_DEFAULT, dcpl_real, H5P_DEFAULT);
+
837 #endif
+
838  }
+
839  H5Sclose(real_dset_space);
+
840 
+
841  real_file_offset = 0;
+
842  for (int i = 0; i < ParallelDescriptor::MyProc(); i++) {
+
843  real_file_offset += all_mfi_real_total_size[i];
+
844  }
+
845  my_real_offset = real_file_offset;
+
846  my_real_count = 0;
+
847 
+
848  int max_mfi_count = 0, write_count = 0;
+
849  for (int i = 0; i < ParallelDescriptor::NProcs(); i++) {
+
850  if (max_mfi_count < all_mfi_cnt[i]) {
+
851  max_mfi_count = all_mfi_cnt[i];
+
852  }
+
853  }
854 
-
855  for (MFIter mfi(state); mfi.isValid(); ++mfi)
-
856  {
-
857  const int grid = mfi.index();
-
858 
-
859  if (count[grid] == 0) { continue; }
-
860 
-
861  Vector<int> istuff;
-
862  Vector<ParticleReal> rstuff;
-
863  particle_detail::packIOData(istuff, rstuff, *this, lev, grid,
-
864  write_real_comp, write_int_comp,
-
865  particle_io_flags, tile_map[grid], count[grid],
-
866  is_checkpoint);
-
867 
-
868  my_int_count = istuff.size();
-
869  int_mem_space = H5Screate_simple(1, &my_int_count, NULL);
-
870  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */
-
871  /* my_int_offset << ", my_int_count = " << my_int_count << ", total_int_size = " << total_int_size << std::endl; */
-
872  int_dset_space = H5Screate_simple(1, &total_int_size, NULL);
-
873  H5Sselect_hyperslab (int_dset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL);
-
874 
-
875 #ifdef AMREX_USE_HDF5_ASYNC
-
876  ret = H5Dwrite_async(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl_col, istuff.dataPtr(), es_par_g);
-
877 #else
-
878  ret = H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl_col, istuff.dataPtr());
-
879 #endif
-
880  if (ret < 0) { amrex::Abort("H5Dwrite int_dset failed!"); }
-
881 
-
882  H5Sclose(int_dset_space);
-
883  H5Sclose(int_mem_space);
-
884 
-
885  my_int_offset += my_int_count;
-
886 
-
887  my_real_count = rstuff.size();
-
888  real_mem_space = H5Screate_simple(1, &my_real_count, NULL);
-
889  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_real_offset = " << */
-
890  /* my_real_offset << ", my_real_count = " << my_real_count << ", total_real_size = " << total_real_size << '\n'; */
-
891  real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
-
892  H5Sselect_hyperslab (real_dset_space, H5S_SELECT_SET, &my_real_offset, NULL, &my_real_count, NULL);
-
893  if (sizeof(typename ParticleType::RealType) == 4) {
-
894 #ifdef AMREX_USE_HDF5_ASYNC
-
895  ret = H5Dwrite_async(real_dset_id, H5T_NATIVE_FLOAT, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr(), es_par_g);
-
896 #else
-
897  ret = H5Dwrite(real_dset_id, H5T_NATIVE_FLOAT, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr());
-
898 #endif
-
899  } else {
-
900 #ifdef AMREX_USE_HDF5_ASYNC
-
901  ret = H5Dwrite_async(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr(), es_par_g);
-
902 #else
-
903  ret = H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr());
-
904 #endif
-
905  }
-
906 
-
907  if (ret < 0) { amrex::Abort("H5Dwrite real_dset failed!"); }
-
908 
-
909  H5Sclose(real_mem_space);
-
910  H5Sclose(real_dset_space);
-
911 
-
912  my_real_offset += my_real_count;
-
913  write_count++;
-
914 
-
915  } // end for (mfi)
-
916 
-
917  // Dummy writes so that every rank participates to every possible H5Dwrite (collective)
-
918  while (write_count < max_mfi_count) {
-
919  int_dset_space = H5Screate_simple(1, &total_int_size, NULL);
-
920  real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
-
921  H5Sselect_none(int_dset_space);
-
922  H5Sselect_none(real_dset_space);
-
923 
-
924 #ifdef AMREX_USE_HDF5_ASYNC
-
925  H5Dwrite_async(int_dset_id, H5T_NATIVE_INT, int_dset_space, int_dset_space, dxpl_col, NULL, es_par_g);
-
926  if (sizeof(typename ParticleType::RealType) == 4) {
-
927  H5Dwrite_async(real_dset_id, H5T_NATIVE_FLOAT, real_dset_space, real_dset_space, dxpl_col, NULL, es_par_g);
-
928  } else {
-
929  H5Dwrite_async(real_dset_id, H5T_NATIVE_DOUBLE, real_dset_space, real_dset_space, dxpl_col, NULL, es_par_g);
-
930  }
-
931 #else
-
932  H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_dset_space, int_dset_space, dxpl_col, NULL);
-
933  if (sizeof(typename ParticleType::RealType) == 4) {
-
934  H5Dwrite(real_dset_id, H5T_NATIVE_FLOAT, real_dset_space, real_dset_space, dxpl_col, NULL);
-
935  } else {
-
936  H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_dset_space, real_dset_space, dxpl_col, NULL);
-
937  }
-
938 #endif
-
939 
-
940  H5Sclose(int_dset_space);
-
941  H5Sclose(real_dset_space);
-
942 
-
943  write_count++;
-
944  }
-
945 
-
946 #ifdef AMREX_USE_HDF5_ASYNC
-
947  H5Dclose_async(real_dset_id, es_par_g);
-
948  H5Dclose_async(int_dset_id, es_par_g);
-
949 #else
-
950  H5Dclose(real_dset_id);
-
951  H5Dclose(int_dset_id);
-
952 #endif
-
953 
-
954  // Create and write the size dataset
-
955  offset_space = H5Screate_simple(1, &total_mfi, NULL);
-
956 #ifdef AMREX_USE_HDF5_ASYNC
-
957  offset_id = H5Dcreate_async(grp, "nparticles_grid", H5T_NATIVE_INT, offset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, es_par_g);
-
958 #else
-
959  offset_id = H5Dcreate(grp, "nparticles_grid", H5T_NATIVE_INT, offset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
-
960 #endif
-
961 
-
962  my_int_offset = 0;
-
963  for (int i = 0; i < ParallelDescriptor::MyProc(); i++) {
-
964  my_int_offset += all_mfi_cnt[i];
-
965  }
-
966  my_int_count = my_mfi_cnt;
-
967  int_mem_space = H5Screate_simple(1, &my_int_count, NULL);
-
968  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */
-
969  /* my_int_offset << ", my_int_count = " << my_int_count << ", total_mfi = " << total_mfi << '\n'; */
-
970  H5Sselect_hyperslab (offset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL);
-
971 
-
972 #ifdef AMREX_USE_HDF5_ASYNC
-
973  ret = H5Dwrite_async(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0]), es_par_g);
-
974 #else
-
975  ret = H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0]));
-
976 #endif
-
977  if (ret < 0) { amrex::Abort("H5Dwrite offset failed!"); }
-
978 
-
979  H5Pclose(dcpl_int);
-
980  H5Pclose(dcpl_real);
-
981  H5Pclose(dxpl_col);
-
982  H5Pclose(dxpl_ind);
-
983 
-
984  H5Sclose(int_mem_space);
-
985  H5Sclose(offset_space);
-
986 
-
987 #ifdef AMREX_USE_HDF5_ASYNC
-
988  H5Dclose_async(offset_id, es_par_g);
-
989 #else
-
990  H5Dclose(offset_id);
-
991 #endif
-
992 
-
993  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": done WriteParticlesHDF5" << std::endl; */
-
994  return;
-
995 } // End WriteParticlesHDF5
-
996 
-
997 template <typename ParticleType, int NArrayReal, int NArrayInt,
-
998  template<class> class Allocator, class CellAssignor>
-
999 void
-
1000 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
-
1001 ::RestartHDF5 (const std::string& dir, const std::string& file, bool /*is_checkpoint*/)
-
1002 {
-
1003  RestartHDF5(dir, file);
-
1004 }
-
1005 
-
1006 template <typename ParticleType, int NArrayReal, int NArrayInt,
-
1007  template<class> class Allocator, class CellAssignor>
-
1008 void
-
1009 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
-
1010 ::RestartHDF5 (const std::string& dir, const std::string& file)
-
1011 {
-
1012  BL_PROFILE("ParticleContainer::RestartHDF5()");
-
1013  AMREX_ASSERT(!dir.empty());
-
1014  AMREX_ASSERT(!file.empty());
-
1015 
-
1016  const auto strttime = amrex::second();
-
1017 
-
1018  std::string fullname = dir;
-
1019  if (!fullname.empty() && fullname[fullname.size()-1] != '/') {
-
1020  fullname += '/';
-
1021  }
-
1022 
-
1023  fullname += file;
-
1024  fullname += ".h5";
-
1025 
-
1026  hid_t fid, dset, grp, fapl, attr, atype, dspace, int_dset, real_dset;
-
1027  int ret;
-
1028 
-
1029  fapl = H5Pcreate (H5P_FILE_ACCESS);
-
1030 #ifdef AMREX_USE_MPI
-
1031  SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
-
1032 #else
-
1033  SetHDF5fapl(fapl);
-
1034 #endif
-
1035 
-
1036  fid = H5Fopen(fullname.c_str(), H5F_ACC_RDONLY, fapl);
-
1037  if (fid < 0) {
-
1038  std::string msg("ParticleContainer::RestartHDF5(): unable to open file: ");
-
1039  msg += fullname;
-
1040  amrex::Abort(msg.c_str());
-
1041  }
-
1042 
-
1043  attr = H5Aopen(fid, "version_name", H5P_DEFAULT);
-
1044  if (attr < 0) {
-
1045  std::string msg("ParticleContainer::RestartHDF5(): unable to open version attribute ");
-
1046  amrex::Abort(msg.c_str());
-
1047  }
-
1048 
-
1049  atype = H5Aget_type(attr);
-
1050  if (atype < 0) {
-
1051  std::string msg("ParticleContainer::RestartHDF5(): unable to get type of attribute ");
-
1052  amrex::Abort(msg.c_str());
-
1053  }
-
1054 
-
1055  size_t attr_len = H5Tget_size(atype);
-
1056  if (attr_len == 0) {
-
1057  std::string msg("ParticleContainer::RestartHDF5(): unable to get size of attribute ");
-
1058  amrex::Abort(msg.c_str());
-
1059  }
-
1060 
-
1061  std::string version;
-
1062  version.resize(attr_len+1);
-
1063 
-
1064  ret = H5Aread(attr, atype, &version[0]);
-
1065  H5Tclose(atype);
-
1066 
-
1067  H5Aclose(attr);
-
1068 
-
1069  AMREX_ASSERT(!version.empty());
-
1070 
-
1071  // What do our version strings mean?
-
1072  // "Version_One_Dot_Zero" -- hard-wired to write out in double precision.
-
1073  // "Version_One_Dot_One" -- can write out either as either single or double precision.
-
1074  // Appended to the latter version string are either "_single" or "_double" to
-
1075  // indicate how the particles were written.
-
1076  // "Version_Two_Dot_Zero" -- this is the AMReX particle file format
-
1077  // "Version_Two_Dot_One" -- expanded particle ids to allow for 2**39-1 per proc
-
1078  std::string how;
-
1079  bool convert_ids = false;
-
1080  if (version.find("Version_Two_Dot_One") != std::string::npos) {
-
1081  convert_ids = true;
-
1082  }
-
1083  if (version.find("_single") != std::string::npos) {
-
1084  how = "single";
-
1085  }
-
1086  else if (version.find("_double") != std::string::npos) {
-
1087  how = "double";
-
1088  }
-
1089  else {
-
1090  std::string msg("ParticleContainer::Restart(): bad version string: ");
-
1091  msg += version;
-
1092  amrex::Error(version.c_str());
-
1093  }
-
1094 
-
1095  std::string gname = "Chombo_global";
-
1096  std::string aname = "SpaceDim";
-
1097  int dm;
-
1098  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
-
1099  if (grp < 0) {
-
1100  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
-
1101  msg += gname;
-
1102  amrex::Abort(msg.c_str());
-
1103  }
-
1104  ret = ReadHDF5Attr(grp, aname.c_str(), &dm, H5T_NATIVE_INT);
-
1105  if (ret < 0) {
-
1106  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1107  msg += aname;
-
1108  amrex::Abort(msg.c_str());
-
1109  }
-
1110  H5Gclose(grp);
-
1111 
-
1112  aname = "num_component_real";
-
1113  int nr;
-
1114  ret = ReadHDF5Attr(fid, aname.c_str(), &nr, H5T_NATIVE_INT);
-
1115  if (ret < 0) {
-
1116  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1117  msg += aname;
-
1118  amrex::Abort(msg.c_str());
-
1119  }
-
1120  if (nr != NStructReal + NumRealComps())
-
1121  amrex::Abort("ParticleContainer::Restart(): nr != NStructReal + NumRealComps()");
-
1122 
-
1123  aname = "num_component_int";
-
1124  int ni;
-
1125  ret = ReadHDF5Attr(fid, aname.c_str(), &ni, H5T_NATIVE_INT);
-
1126  if (ret < 0) {
-
1127  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1128  msg += aname;
-
1129  amrex::Abort(msg.c_str());
-
1130  }
-
1131  if (ni != NStructInt + NumIntComps()) {
-
1132  amrex::Abort("ParticleContainer::Restart(): ni != NStructInt");
-
1133  }
-
1134 
-
1135  aname = "nparticles";
-
1136  Long nparticles;
-
1137  ret = ReadHDF5Attr(fid, aname.c_str(), &nparticles, H5T_NATIVE_LLONG);
-
1138  if (ret < 0) {
-
1139  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1140  msg += aname;
-
1141  amrex::Abort(msg.c_str());
-
1142  }
-
1143  AMREX_ASSERT(nparticles >= 0);
-
1144 
-
1145  aname = "maxnextid";
-
1146  Long maxnextid;
-
1147  ret = ReadHDF5Attr(fid, aname.c_str(), &maxnextid, H5T_NATIVE_LLONG);
-
1148  if (ret < 0) {
-
1149  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1150  msg += aname;
-
1151  amrex::Abort(msg.c_str());
-
1152  }
-
1153  AMREX_ASSERT(maxnextid > 0);
-
1154  ParticleType::NextID(maxnextid);
-
1155 
-
1156  aname = "finest_level";
-
1157  int finest_level_in_file;
-
1158  ret = ReadHDF5Attr(fid, aname.c_str(), &finest_level_in_file, H5T_NATIVE_INT);
-
1159  if (ret < 0) {
-
1160  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1161  msg += aname;
-
1162  amrex::Abort(msg.c_str());
-
1163  }
-
1164  AMREX_ASSERT(finest_level_in_file >= 0);
-
1165 
-
1166  hid_t comp_dtype = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int));
-
1167  if (1 == AMREX_SPACEDIM) {
-
1168  H5Tinsert (comp_dtype, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
-
1169  H5Tinsert (comp_dtype, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT);
-
1170  }
-
1171  else if (2 == AMREX_SPACEDIM) {
-
1172  H5Tinsert (comp_dtype, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
-
1173  H5Tinsert (comp_dtype, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
-
1174  H5Tinsert (comp_dtype, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT);
-
1175  H5Tinsert (comp_dtype, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT);
-
1176  }
-
1177  else if (3 == AMREX_SPACEDIM) {
-
1178  H5Tinsert (comp_dtype, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
-
1179  H5Tinsert (comp_dtype, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
-
1180  H5Tinsert (comp_dtype, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT);
-
1181  H5Tinsert (comp_dtype, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT);
-
1182  H5Tinsert (comp_dtype, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT);
-
1183  H5Tinsert (comp_dtype, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT);
-
1184  }
-
1185 
-
1186  // Determine whether this is a dual-grid restart or not.
-
1187  Vector<BoxArray> particle_box_arrays(finest_level_in_file + 1);
-
1188  bool dual_grid = false;
-
1189 
-
1190  for (int lev = 0; lev <= finest_level_in_file; lev++)
-
1191  {
-
1192  if (lev > finestLevel())
-
1193  {
-
1194  dual_grid = true;
-
1195  break;
-
1196  }
-
1197 
-
1198  gname = "level_" + std::to_string(lev);
-
1199  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
-
1200  if (grp < 0) {
-
1201  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
-
1202  msg += gname;
-
1203  amrex::Abort(msg.c_str());
-
1204  }
-
1205 
-
1206  dset = H5Dopen(grp, "boxes", H5P_DEFAULT);
-
1207  dspace = H5Dget_space(dset);
-
1208  hsize_t ngrid;
-
1209  H5Sget_simple_extent_dims(dspace, &ngrid, NULL);
-
1210 
-
1211  Vector<int> boxes(ngrid*AMREX_SPACEDIM*2);
-
1212  ret = H5Dread(dset, comp_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &(boxes[0]));
-
1213  if (ret < 0) {
-
1214  std::string msg("ParticleContainer::RestartHDF5(): unable to read nparticles_grid dataset");
-
1215  amrex::Abort(msg.c_str());
-
1216  }
-
1217  H5Sclose(dspace);
-
1218  H5Dclose(dset);
-
1219  H5Gclose(grp);
-
1220 
-
1221  /* particle_box_arrays[lev].readFrom(phdr_file); */
-
1222  particle_box_arrays[lev] = ParticleBoxArray(lev);
-
1223  for (int i = 0; i < (int)ngrid; i++) {
-
1224  Box tmp (IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM], boxes[i*AMREX_SPACEDIM+1], boxes[i*AMREX_SPACEDIM+2])},
-
1225  IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM*2], boxes[i*AMREX_SPACEDIM*2+1], boxes[i*AMREX_SPACEDIM*2+2])});
-
1226  particle_box_arrays[lev][i] = tmp;
-
1227  }
-
1228 
-
1229  if (! particle_box_arrays[lev].CellEqual(ParticleBoxArray(lev)))
-
1230  dual_grid = true;
-
1231  }
-
1232 
-
1233  if (dual_grid) {
-
1234  for (int lev = 0; lev <= finestLevel(); lev++) {
-
1235  SetParticleBoxArray(lev, particle_box_arrays[lev]);
-
1236  DistributionMapping pdm(particle_box_arrays[lev]);
-
1237  SetParticleDistributionMap(lev, pdm);
-
1238  }
-
1239  }
-
1240 
-
1241  Vector<int> ngrids(finest_level_in_file+1);
-
1242  for (int lev = 0; lev <= finest_level_in_file; lev++) {
-
1243  gname = "level_" + std::to_string(lev);
-
1244  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
-
1245  if (grp < 0) {
-
1246  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
-
1247  msg += gname;
-
1248  amrex::Abort(msg.c_str());
-
1249  }
-
1250 
-
1251  aname = "ngrids";
-
1252  ret = ReadHDF5Attr(grp, aname.c_str(), &ngrids[lev], H5T_NATIVE_INT);
-
1253  if (ret < 0) {
-
1254  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
-
1255  msg += aname;
-
1256  amrex::Abort(msg.c_str());
-
1257  }
-
1258 
-
1259  AMREX_ASSERT(ngrids[lev] > 0);
-
1260  if (lev <= finestLevel()) {
-
1261  AMREX_ASSERT(ngrids[lev] == int(ParticleBoxArray(lev).size()));
-
1262  }
-
1263 
-
1264  H5Gclose(grp);
-
1265  }
-
1266 
-
1267  resizeData();
-
1268 
-
1269  if (finest_level_in_file > finestLevel()) {
-
1270  m_particles.resize(finest_level_in_file+1);
-
1271  }
-
1272 
-
1273  for (int lev = 0; lev <= finest_level_in_file; lev++) {
-
1274 
-
1275  gname = "level_" + std::to_string(lev);
-
1276  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
-
1277  if (grp < 0) {
-
1278  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
-
1279  msg += gname;
-
1280  amrex::Abort(msg.c_str());
-
1281  }
-
1282 
-
1283  dset = H5Dopen(grp, "nparticles_grid", H5P_DEFAULT);
-
1284  Vector<int> count(ngrids[lev]);
-
1285  ret = H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &(count[0]));
-
1286  if (ret < 0) {
-
1287  std::string msg("ParticleContainer::RestartHDF5(): unable to read nparticles_grid dataset");
-
1288  amrex::Abort(msg.c_str());
-
1289  }
-
1290  H5Dclose(dset);
-
1291 
-
1292  Vector<hsize_t> offset(ngrids[lev]);
-
1293  offset[0] = 0;
-
1294  for (int i = 1; i < ngrids[lev]; i++) {
-
1295  offset[i] = offset[i-1] + count[i-1];
-
1296  }
-
1297 
-
1298  int_dset = H5Dopen(grp, "data:datatype=0", H5P_DEFAULT);
-
1299  if (int_dset < 0) {
-
1300  std::string msg("ParticleContainer::RestartHDF5(): unable to open int dataset");
-
1301  amrex::Abort(msg.c_str());
-
1302  }
-
1303  real_dset = H5Dopen(grp, "data:datatype=1", H5P_DEFAULT);
-
1304  if (real_dset < 0) {
-
1305  std::string msg("ParticleContainer::RestartHDF5(): unable to open int dataset");
-
1306  amrex::Abort(msg.c_str());
-
1307  }
-
1308 
-
1309  Vector<int> grids_to_read;
-
1310  if (lev <= finestLevel()) {
-
1311  for (MFIter mfi(*m_dummy_mf[lev]); mfi.isValid(); ++mfi) {
-
1312  grids_to_read.push_back(mfi.index());
-
1313  }
-
1314  } else {
-
1315 
-
1316  // we lost a level on restart. we still need to read in particles
-
1317  // on finer levels, and put them in the right place via Redistribute()
-
1318 
-
1319  const int rank = ParallelDescriptor::MyProc();
-
1320  const int NReaders = MaxReaders();
-
1321  if (rank >= NReaders) { return; }
-
1322 
-
1323  const int Navg = ngrids[lev] / NReaders;
-
1324  const int Nleft = ngrids[lev] - Navg * NReaders;
-
1325 
-
1326  int lo, hi;
-
1327  if (rank < Nleft) {
-
1328  lo = rank*(Navg + 1);
-
1329  hi = lo + Navg + 1;
-
1330  }
-
1331  else {
-
1332  lo = rank * Navg + Nleft;
-
1333  hi = lo + Navg;
-
1334  }
-
1335 
-
1336  for (int i = lo; i < hi; ++i) {
-
1337  grids_to_read.push_back(i);
-
1338  }
-
1339  }
-
1340 
-
1341  for(int igrid = 0; igrid < static_cast<int>(grids_to_read.size()); ++igrid) {
-
1342  const int grid = grids_to_read[igrid];
-
1343 
-
1344  if (how == "single") {
-
1345  ReadParticlesHDF5<float>(offset[grid], count[grid], grid, lev, int_dset, real_dset, finest_level_in_file, convert_ids);
-
1346  }
-
1347  else if (how == "double") {
-
1348  ReadParticlesHDF5<double>(offset[grid], count[grid], grid, lev, int_dset, real_dset, finest_level_in_file, convert_ids);
-
1349  }
-
1350  else {
-
1351  std::string msg("ParticleContainer::Restart(): bad parameter: ");
-
1352  msg += how;
-
1353  amrex::Error(msg.c_str());
-
1354  }
-
1355  }
-
1356 
-
1357  H5Dclose(int_dset);
-
1358  H5Dclose(real_dset);
-
1359  H5Gclose(grp);
-
1360  } // end for level
-
1361 
-
1362  H5Fclose(fid);
-
1363 
-
1364  Redistribute();
-
1365 
-
1366  AMREX_ASSERT(OK());
-
1367 
-
1368  if (m_verbose > 1) {
-
1369  auto stoptime = amrex::second() - strttime;
-
1370  ParallelDescriptor::ReduceRealMax(stoptime, ParallelDescriptor::IOProcessorNumber());
-
1371  amrex::Print() << "ParticleContainer::Restart() time: " << stoptime << '\n';
-
1372  }
-
1373 }
-
1374 
-
1375 // Read a batch of particles from the checkpoint file
-
1376 template <typename ParticleType, int NArrayReal, int NArrayInt,
-
1377  template<class> class Allocator, class CellAssignor>
-
1378 template <class RTYPE>
-
1379 void
-
1380 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
-
1381 ::ReadParticlesHDF5 (hsize_t offset, hsize_t cnt, int grd, int lev,
-
1382  hid_t int_dset, hid_t real_dset, int finest_level_in_file,
-
1383  bool convert_ids)
-
1384 {
-
1385  BL_PROFILE("ParticleContainer::ReadParticlesHDF5()");
-
1386  AMREX_ASSERT(cnt > 0);
-
1387  AMREX_ASSERT(lev < int(m_particles.size()));
-
1388 
-
1389  hid_t int_dspace, int_fspace, real_dspace, real_fspace;
-
1390 
-
1391  // First read in the integer data in binary. We do not store
-
1392  // the m_lev and m_grid data on disk. We can easily recreate
-
1393  // that given the structure of the checkpoint file.
-
1394  const int iChunkSize = 2 + NStructInt + NumIntComps();
-
1395  Vector<int> istuff(cnt*iChunkSize);
-
1396  int_fspace = H5Dget_space(int_dset);
-
1397  hsize_t int_cnt = cnt*iChunkSize;
-
1398  hsize_t int_offset = offset*iChunkSize;
-
1399  int_dspace = H5Screate_simple(1, &int_cnt, NULL);
-
1400  H5Sselect_hyperslab (int_fspace, H5S_SELECT_SET, &int_offset, NULL, &int_cnt, NULL);
-
1401  H5Dread(int_dset, H5T_NATIVE_INT, int_dspace, int_fspace, H5P_DEFAULT, istuff.dataPtr());
-
1402 
-
1403  H5Sclose(int_fspace);
-
1404  H5Sclose(int_dspace);
-
1405 
-
1406  // Then the real data in binary.
-
1407  const int rChunkSize = AMREX_SPACEDIM + NStructReal + NumRealComps();
-
1408  Vector<RTYPE> rstuff(cnt*rChunkSize);
-
1409  real_fspace = H5Dget_space(real_dset);
-
1410  hsize_t real_cnt = cnt*rChunkSize;
-
1411  hsize_t real_offset = offset*rChunkSize;
-
1412  real_dspace = H5Screate_simple(1, &real_cnt, NULL);
-
1413  H5Sselect_hyperslab (real_fspace, H5S_SELECT_SET, &real_offset, NULL, &real_cnt, NULL);
-
1414  if (sizeof(RTYPE) == 4) {
-
1415  H5Dread(real_dset, H5T_NATIVE_FLOAT, real_dspace, real_fspace, H5P_DEFAULT, rstuff.dataPtr());
-
1416  } else {
-
1417  H5Dread(real_dset, H5T_NATIVE_DOUBLE, real_dspace, real_fspace, H5P_DEFAULT, rstuff.dataPtr());
-
1418  }
-
1419 
-
1420  H5Sclose(real_fspace);
-
1421  H5Sclose(real_dspace);
-
1422 
-
1423  // Now reassemble the particles.
-
1424  int* iptr = istuff.dataPtr();
-
1425  RTYPE* rptr = rstuff.dataPtr();
-
1426 
-
1427  ParticleType p;
-
1428  ParticleLocData pld;
-
1429 
-
1430  Vector<std::map<std::pair<int, int>, Gpu::HostVector<ParticleType> > > host_particles;
-
1431  host_particles.reserve(15);
-
1432  host_particles.resize(finest_level_in_file+1);
-
1433 
-
1434  Vector<std::map<std::pair<int, int>,
-
1435  std::vector<Gpu::HostVector<ParticleReal> > > > host_real_attribs;
-
1436  host_real_attribs.reserve(15);
-
1437  host_real_attribs.resize(finest_level_in_file+1);
-
1438 
-
1439  Vector<std::map<std::pair<int, int>,
-
1440  std::vector<Gpu::HostVector<int> > > > host_int_attribs;
-
1441  host_int_attribs.reserve(15);
-
1442  host_int_attribs.resize(finest_level_in_file+1);
-
1443 
-
1444  for (hsize_t i = 0; i < cnt; i++) {
-
1445  if (convert_ids) {
-
1446  std::int32_t xi, yi;
-
1447  std::uint32_t xu, yu;
-
1448  xi = iptr[0];
-
1449  yi = iptr[1];
-
1450  std::memcpy(&xu, &xi, sizeof(xi));
-
1451  std::memcpy(&yu, &yi, sizeof(yi));
-
1452  p.m_idcpu = ((std::uint64_t)xu) << 32 | yu;
-
1453  } else {
-
1454  p.id() = iptr[0];
-
1455  p.cpu() = iptr[1];
-
1456  }
-
1457  iptr += 2;
-
1458 
-
1459  for (int j = 0; j < NStructInt; j++)
-
1460  {
-
1461  p.idata(j) = *iptr;
-
1462  ++iptr;
-
1463  }
-
1464 
-
1465  AMREX_ASSERT(p.id() > 0);
-
1466 
-
1467  AMREX_D_TERM(p.pos(0) = ParticleReal(rptr[0]);,
-
1468  p.pos(1) = ParticleReal(rptr[1]);,
-
1469  p.pos(2) = ParticleReal(rptr[2]););
-
1470 
-
1471  rptr += AMREX_SPACEDIM;
-
1472 
-
1473  for (int j = 0; j < NStructReal; j++)
-
1474  {
-
1475  p.rdata(j) = ParticleReal(*rptr);
-
1476  ++rptr;
-
1477  }
-
1478 
-
1479  locateParticle(p, pld, 0, finestLevel(), 0);
-
1480 
-
1481  std::pair<int, int> ind(grd, pld.m_tile);
-
1482 
-
1483  host_real_attribs[lev][ind].resize(NumRealComps());
-
1484  host_int_attribs[lev][ind].resize(NumIntComps());
-
1485 
-
1486  // add the struct
-
1487  host_particles[lev][ind].push_back(p);
-
1488 
-
1489  // add the real...
-
1490  for (int icomp = 0; icomp < NumRealComps(); icomp++) {
-
1491  host_real_attribs[lev][ind][icomp].push_back(ParticleReal(*rptr));
-
1492  ++rptr;
-
1493  }
-
1494 
-
1495  // ... and int array data
-
1496  for (int icomp = 0; icomp < NumIntComps(); icomp++) {
-
1497  host_int_attribs[lev][ind][icomp].push_back(*iptr);
-
1498  ++iptr;
-
1499  }
-
1500  }
-
1501 
-
1502  for (int host_lev = 0; host_lev < static_cast<int>(host_particles.size()); ++host_lev)
-
1503  {
-
1504  for (auto& kv : host_particles[host_lev]) {
-
1505  auto grid = kv.first.first;
-
1506  auto tile = kv.first.second;
-
1507  const auto& src_tile = kv.second;
-
1508 
-
1509  auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile);
-
1510  auto old_size = dst_tile.GetArrayOfStructs().size();
-
1511  auto new_size = old_size + src_tile.size();
-
1512  dst_tile.resize(new_size);
-
1513 
-
1514  Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(),
-
1515  dst_tile.GetArrayOfStructs().begin() + old_size);
-
1516 
-
1517  for (int i = 0; i < NumRealComps(); ++i) {
-
1518  Gpu::copyAsync(Gpu::hostToDevice,
-
1519  host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(),
-
1520  host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(),
-
1521  dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size);
-
1522  }
-
1523 
-
1524  for (int i = 0; i < NumIntComps(); ++i) {
-
1525  Gpu::copyAsync(Gpu::hostToDevice,
-
1526  host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(),
-
1527  host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(),
-
1528  dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size);
-
1529  }
-
1530  }
-
1531  }
-
1532 
-
1533  Gpu::streamSynchronize();
-
1534 }
-
1535 
-
1536 #endif /*AMREX_USE_HDF5*/
-
1537 #endif /*AMREX_PARTICLEHDF5_H*/
+
855 
+
856  for (MFIter mfi(state); mfi.isValid(); ++mfi)
+
857  {
+
858  const int grid = mfi.index();
+
859 
+
860  if (count[grid] == 0) { continue; }
+
861 
+
862  Vector<int> istuff;
+
863  Vector<ParticleReal> rstuff;
+
864  particle_detail::packIOData(istuff, rstuff, *this, lev, grid,
+
865  write_real_comp, write_int_comp,
+
866  particle_io_flags, tile_map[grid], count[grid],
+
867  is_checkpoint);
+
868 
+
869  my_int_count = istuff.size();
+
870  int_mem_space = H5Screate_simple(1, &my_int_count, NULL);
+
871  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */
+
872  /* my_int_offset << ", my_int_count = " << my_int_count << ", total_int_size = " << total_int_size << std::endl; */
+
873  int_dset_space = H5Screate_simple(1, &total_int_size, NULL);
+
874  H5Sselect_hyperslab (int_dset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL);
+
875 
+
876 #ifdef AMREX_USE_HDF5_ASYNC
+
877  ret = H5Dwrite_async(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl_col, istuff.dataPtr(), es_par_g);
+
878 #else
+
879  ret = H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_mem_space, int_dset_space, dxpl_col, istuff.dataPtr());
+
880 #endif
+
881  if (ret < 0) { amrex::Abort("H5Dwrite int_dset failed!"); }
+
882 
+
883  H5Sclose(int_dset_space);
+
884  H5Sclose(int_mem_space);
+
885 
+
886  my_int_offset += my_int_count;
+
887 
+
888  my_real_count = rstuff.size();
+
889  real_mem_space = H5Screate_simple(1, &my_real_count, NULL);
+
890  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_real_offset = " << */
+
891  /* my_real_offset << ", my_real_count = " << my_real_count << ", total_real_size = " << total_real_size << '\n'; */
+
892  real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
+
893  H5Sselect_hyperslab (real_dset_space, H5S_SELECT_SET, &my_real_offset, NULL, &my_real_count, NULL);
+
894  if (sizeof(typename ParticleType::RealType) == 4) {
+
895 #ifdef AMREX_USE_HDF5_ASYNC
+
896  ret = H5Dwrite_async(real_dset_id, H5T_NATIVE_FLOAT, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr(), es_par_g);
+
897 #else
+
898  ret = H5Dwrite(real_dset_id, H5T_NATIVE_FLOAT, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr());
+
899 #endif
+
900  } else {
+
901 #ifdef AMREX_USE_HDF5_ASYNC
+
902  ret = H5Dwrite_async(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr(), es_par_g);
+
903 #else
+
904  ret = H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_mem_space, real_dset_space, dxpl_col, rstuff.dataPtr());
+
905 #endif
+
906  }
+
907 
+
908  if (ret < 0) { amrex::Abort("H5Dwrite real_dset failed!"); }
+
909 
+
910  H5Sclose(real_mem_space);
+
911  H5Sclose(real_dset_space);
+
912 
+
913  my_real_offset += my_real_count;
+
914  write_count++;
+
915 
+
916  } // end for (mfi)
+
917 
+
918  // Dummy writes so that every rank participates to every possible H5Dwrite (collective)
+
919  while (write_count < max_mfi_count) {
+
920  int_dset_space = H5Screate_simple(1, &total_int_size, NULL);
+
921  real_dset_space = H5Screate_simple(1, &total_real_size, NULL);
+
922  H5Sselect_none(int_dset_space);
+
923  H5Sselect_none(real_dset_space);
+
924 
+
925 #ifdef AMREX_USE_HDF5_ASYNC
+
926  H5Dwrite_async(int_dset_id, H5T_NATIVE_INT, int_dset_space, int_dset_space, dxpl_col, NULL, es_par_g);
+
927  if (sizeof(typename ParticleType::RealType) == 4) {
+
928  H5Dwrite_async(real_dset_id, H5T_NATIVE_FLOAT, real_dset_space, real_dset_space, dxpl_col, NULL, es_par_g);
+
929  } else {
+
930  H5Dwrite_async(real_dset_id, H5T_NATIVE_DOUBLE, real_dset_space, real_dset_space, dxpl_col, NULL, es_par_g);
+
931  }
+
932 #else
+
933  H5Dwrite(int_dset_id, H5T_NATIVE_INT, int_dset_space, int_dset_space, dxpl_col, NULL);
+
934  if (sizeof(typename ParticleType::RealType) == 4) {
+
935  H5Dwrite(real_dset_id, H5T_NATIVE_FLOAT, real_dset_space, real_dset_space, dxpl_col, NULL);
+
936  } else {
+
937  H5Dwrite(real_dset_id, H5T_NATIVE_DOUBLE, real_dset_space, real_dset_space, dxpl_col, NULL);
+
938  }
+
939 #endif
+
940 
+
941  H5Sclose(int_dset_space);
+
942  H5Sclose(real_dset_space);
+
943 
+
944  write_count++;
+
945  }
+
946 
+
947 #ifdef AMREX_USE_HDF5_ASYNC
+
948  H5Dclose_async(real_dset_id, es_par_g);
+
949  H5Dclose_async(int_dset_id, es_par_g);
+
950 #else
+
951  H5Dclose(real_dset_id);
+
952  H5Dclose(int_dset_id);
+
953 #endif
+
954 
+
955  // Create and write the size dataset
+
956  offset_space = H5Screate_simple(1, &total_mfi, NULL);
+
957 #ifdef AMREX_USE_HDF5_ASYNC
+
958  offset_id = H5Dcreate_async(grp, "nparticles_grid", H5T_NATIVE_INT, offset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT, es_par_g);
+
959 #else
+
960  offset_id = H5Dcreate(grp, "nparticles_grid", H5T_NATIVE_INT, offset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+
961 #endif
+
962 
+
963  my_int_offset = 0;
+
964  for (int i = 0; i < ParallelDescriptor::MyProc(); i++) {
+
965  my_int_offset += all_mfi_cnt[i];
+
966  }
+
967  my_int_count = my_mfi_cnt;
+
968  int_mem_space = H5Screate_simple(1, &my_int_count, NULL);
+
969  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": my_int_offset = " << */
+
970  /* my_int_offset << ", my_int_count = " << my_int_count << ", total_mfi = " << total_mfi << '\n'; */
+
971  H5Sselect_hyperslab (offset_space, H5S_SELECT_SET, &my_int_offset, NULL, &my_int_count, NULL);
+
972 
+
973 #ifdef AMREX_USE_HDF5_ASYNC
+
974  ret = H5Dwrite_async(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0]), es_par_g);
+
975 #else
+
976  ret = H5Dwrite(offset_id, H5T_NATIVE_INT, int_mem_space, offset_space, dxpl_col, &(my_nparticles[0]));
+
977 #endif
+
978  if (ret < 0) { amrex::Abort("H5Dwrite offset failed!"); }
+
979 
+
980  H5Pclose(dcpl_int);
+
981  H5Pclose(dcpl_real);
+
982  H5Pclose(dxpl_col);
+
983  H5Pclose(dxpl_ind);
+
984 
+
985  H5Sclose(int_mem_space);
+
986  H5Sclose(offset_space);
+
987 
+
988 #ifdef AMREX_USE_HDF5_ASYNC
+
989  H5Dclose_async(offset_id, es_par_g);
+
990 #else
+
991  H5Dclose(offset_id);
+
992 #endif
+
993 
+
994  /* std::cout << "Rank " << ParallelDescriptor::MyProc() << ": done WriteParticlesHDF5" << std::endl; */
+
995  return;
+
996 } // End WriteParticlesHDF5
+
997 
+
998 template <typename ParticleType, int NArrayReal, int NArrayInt,
+
999  template<class> class Allocator, class CellAssignor>
+
1000 void
+
1001 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
+
1002 ::RestartHDF5 (const std::string& dir, const std::string& file, bool /*is_checkpoint*/)
+
1003 {
+
1004  RestartHDF5(dir, file);
+
1005 }
+
1006 
+
1007 template <typename ParticleType, int NArrayReal, int NArrayInt,
+
1008  template<class> class Allocator, class CellAssignor>
+
1009 void
+
1010 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
+
1011 ::RestartHDF5 (const std::string& dir, const std::string& file)
+
1012 {
+
1013  BL_PROFILE("ParticleContainer::RestartHDF5()");
+
1014  AMREX_ASSERT(!dir.empty());
+
1015  AMREX_ASSERT(!file.empty());
+
1016 
+
1017  const auto strttime = amrex::second();
+
1018 
+
1019  std::string fullname = dir;
+
1020  if (!fullname.empty() && fullname[fullname.size()-1] != '/') {
+
1021  fullname += '/';
+
1022  }
+
1023 
+
1024  fullname += file;
+
1025  fullname += ".h5";
+
1026 
+
1027  hid_t fid, dset, grp, fapl, attr, atype, dspace, int_dset, real_dset;
+
1028  int ret;
+
1029 
+
1030  fapl = H5Pcreate (H5P_FILE_ACCESS);
+
1031 #ifdef AMREX_USE_MPI
+
1032  SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
+
1033 #else
+
1034  SetHDF5fapl(fapl);
+
1035 #endif
+
1036 
+
1037  fid = H5Fopen(fullname.c_str(), H5F_ACC_RDONLY, fapl);
+
1038  if (fid < 0) {
+
1039  std::string msg("ParticleContainer::RestartHDF5(): unable to open file: ");
+
1040  msg += fullname;
+
1041  amrex::Abort(msg.c_str());
+
1042  }
+
1043 
+
1044  attr = H5Aopen(fid, "version_name", H5P_DEFAULT);
+
1045  if (attr < 0) {
+
1046  std::string msg("ParticleContainer::RestartHDF5(): unable to open version attribute ");
+
1047  amrex::Abort(msg.c_str());
+
1048  }
+
1049 
+
1050  atype = H5Aget_type(attr);
+
1051  if (atype < 0) {
+
1052  std::string msg("ParticleContainer::RestartHDF5(): unable to get type of attribute ");
+
1053  amrex::Abort(msg.c_str());
+
1054  }
+
1055 
+
1056  size_t attr_len = H5Tget_size(atype);
+
1057  if (attr_len == 0) {
+
1058  std::string msg("ParticleContainer::RestartHDF5(): unable to get size of attribute ");
+
1059  amrex::Abort(msg.c_str());
+
1060  }
+
1061 
+
1062  std::string version;
+
1063  version.resize(attr_len+1);
+
1064 
+
1065  ret = H5Aread(attr, atype, &version[0]);
+
1066  H5Tclose(atype);
+
1067 
+
1068  H5Aclose(attr);
+
1069 
+
1070  AMREX_ASSERT(!version.empty());
+
1071 
+
1072  // What do our version strings mean?
+
1073  // "Version_One_Dot_Zero" -- hard-wired to write out in double precision.
+
1074  // "Version_One_Dot_One" -- can write out either as either single or double precision.
+
1075  // Appended to the latter version string are either "_single" or "_double" to
+
1076  // indicate how the particles were written.
+
1077  // "Version_Two_Dot_Zero" -- this is the AMReX particle file format
+
1078  // "Version_Two_Dot_One" -- expanded particle ids to allow for 2**39-1 per proc
+
1079  std::string how;
+
1080  bool convert_ids = false;
+
1081  if (version.find("Version_Two_Dot_One") != std::string::npos) {
+
1082  convert_ids = true;
+
1083  }
+
1084  if (version.find("_single") != std::string::npos) {
+
1085  how = "single";
+
1086  }
+
1087  else if (version.find("_double") != std::string::npos) {
+
1088  how = "double";
+
1089  }
+
1090  else {
+
1091  std::string msg("ParticleContainer::Restart(): bad version string: ");
+
1092  msg += version;
+
1093  amrex::Error(version.c_str());
+
1094  }
+
1095 
+
1096  std::string gname = "Chombo_global";
+
1097  std::string aname = "SpaceDim";
+
1098  int dm;
+
1099  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
+
1100  if (grp < 0) {
+
1101  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
+
1102  msg += gname;
+
1103  amrex::Abort(msg.c_str());
+
1104  }
+
1105  ret = ReadHDF5Attr(grp, aname.c_str(), &dm, H5T_NATIVE_INT);
+
1106  if (ret < 0) {
+
1107  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1108  msg += aname;
+
1109  amrex::Abort(msg.c_str());
+
1110  }
+
1111  H5Gclose(grp);
+
1112 
+
1113  aname = "num_component_real";
+
1114  int nr;
+
1115  ret = ReadHDF5Attr(fid, aname.c_str(), &nr, H5T_NATIVE_INT);
+
1116  if (ret < 0) {
+
1117  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1118  msg += aname;
+
1119  amrex::Abort(msg.c_str());
+
1120  }
+
1121  if (nr != NStructReal + NumRealComps())
+
1122  amrex::Abort("ParticleContainer::Restart(): nr != NStructReal + NumRealComps()");
+
1123 
+
1124  aname = "num_component_int";
+
1125  int ni;
+
1126  ret = ReadHDF5Attr(fid, aname.c_str(), &ni, H5T_NATIVE_INT);
+
1127  if (ret < 0) {
+
1128  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1129  msg += aname;
+
1130  amrex::Abort(msg.c_str());
+
1131  }
+
1132  if (ni != NStructInt + NumIntComps()) {
+
1133  amrex::Abort("ParticleContainer::Restart(): ni != NStructInt");
+
1134  }
+
1135 
+
1136  aname = "nparticles";
+
1137  Long nparticles;
+
1138  ret = ReadHDF5Attr(fid, aname.c_str(), &nparticles, H5T_NATIVE_LLONG);
+
1139  if (ret < 0) {
+
1140  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1141  msg += aname;
+
1142  amrex::Abort(msg.c_str());
+
1143  }
+
1144  AMREX_ASSERT(nparticles >= 0);
+
1145 
+
1146  aname = "maxnextid";
+
1147  Long maxnextid;
+
1148  ret = ReadHDF5Attr(fid, aname.c_str(), &maxnextid, H5T_NATIVE_LLONG);
+
1149  if (ret < 0) {
+
1150  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1151  msg += aname;
+
1152  amrex::Abort(msg.c_str());
+
1153  }
+
1154  AMREX_ASSERT(maxnextid > 0);
+
1155  ParticleType::NextID(maxnextid);
+
1156 
+
1157  aname = "finest_level";
+
1158  int finest_level_in_file;
+
1159  ret = ReadHDF5Attr(fid, aname.c_str(), &finest_level_in_file, H5T_NATIVE_INT);
+
1160  if (ret < 0) {
+
1161  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1162  msg += aname;
+
1163  amrex::Abort(msg.c_str());
+
1164  }
+
1165  AMREX_ASSERT(finest_level_in_file >= 0);
+
1166 
+
1167  hid_t comp_dtype = H5Tcreate (H5T_COMPOUND, 2 * AMREX_SPACEDIM * sizeof(int));
+
1168  if (1 == AMREX_SPACEDIM) {
+
1169  H5Tinsert (comp_dtype, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
+
1170  H5Tinsert (comp_dtype, "hi_i", 1 * sizeof(int), H5T_NATIVE_INT);
+
1171  }
+
1172  else if (2 == AMREX_SPACEDIM) {
+
1173  H5Tinsert (comp_dtype, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
+
1174  H5Tinsert (comp_dtype, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
+
1175  H5Tinsert (comp_dtype, "hi_i", 2 * sizeof(int), H5T_NATIVE_INT);
+
1176  H5Tinsert (comp_dtype, "hi_j", 3 * sizeof(int), H5T_NATIVE_INT);
+
1177  }
+
1178  else if (3 == AMREX_SPACEDIM) {
+
1179  H5Tinsert (comp_dtype, "lo_i", 0 * sizeof(int), H5T_NATIVE_INT);
+
1180  H5Tinsert (comp_dtype, "lo_j", 1 * sizeof(int), H5T_NATIVE_INT);
+
1181  H5Tinsert (comp_dtype, "lo_k", 2 * sizeof(int), H5T_NATIVE_INT);
+
1182  H5Tinsert (comp_dtype, "hi_i", 3 * sizeof(int), H5T_NATIVE_INT);
+
1183  H5Tinsert (comp_dtype, "hi_j", 4 * sizeof(int), H5T_NATIVE_INT);
+
1184  H5Tinsert (comp_dtype, "hi_k", 5 * sizeof(int), H5T_NATIVE_INT);
+
1185  }
+
1186 
+
1187  // Determine whether this is a dual-grid restart or not.
+
1188  Vector<BoxArray> particle_box_arrays(finest_level_in_file + 1);
+
1189  bool dual_grid = false;
+
1190 
+
1191  for (int lev = 0; lev <= finest_level_in_file; lev++)
+
1192  {
+
1193  if (lev > finestLevel())
+
1194  {
+
1195  dual_grid = true;
+
1196  break;
+
1197  }
+
1198 
+
1199  gname = "level_" + std::to_string(lev);
+
1200  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
+
1201  if (grp < 0) {
+
1202  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
+
1203  msg += gname;
+
1204  amrex::Abort(msg.c_str());
+
1205  }
+
1206 
+
1207  dset = H5Dopen(grp, "boxes", H5P_DEFAULT);
+
1208  dspace = H5Dget_space(dset);
+
1209  hsize_t ngrid;
+
1210  H5Sget_simple_extent_dims(dspace, &ngrid, NULL);
+
1211 
+
1212  Vector<int> boxes(ngrid*AMREX_SPACEDIM*2);
+
1213  ret = H5Dread(dset, comp_dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, &(boxes[0]));
+
1214  if (ret < 0) {
+
1215  std::string msg("ParticleContainer::RestartHDF5(): unable to read nparticles_grid dataset");
+
1216  amrex::Abort(msg.c_str());
+
1217  }
+
1218  H5Sclose(dspace);
+
1219  H5Dclose(dset);
+
1220  H5Gclose(grp);
+
1221 
+
1222  /* particle_box_arrays[lev].readFrom(phdr_file); */
+
1223  particle_box_arrays[lev] = ParticleBoxArray(lev);
+
1224  for (int i = 0; i < (int)ngrid; i++) {
+
1225  Box tmp (IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM], boxes[i*AMREX_SPACEDIM+1], boxes[i*AMREX_SPACEDIM+2])},
+
1226  IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM*2], boxes[i*AMREX_SPACEDIM*2+1], boxes[i*AMREX_SPACEDIM*2+2])});
+
1227  particle_box_arrays[lev][i] = tmp;
+
1228  }
+
1229 
+
1230  if (! particle_box_arrays[lev].CellEqual(ParticleBoxArray(lev)))
+
1231  dual_grid = true;
+
1232  }
+
1233 
+
1234  if (dual_grid) {
+
1235  for (int lev = 0; lev <= finestLevel(); lev++) {
+
1236  SetParticleBoxArray(lev, particle_box_arrays[lev]);
+
1237  DistributionMapping pdm(particle_box_arrays[lev]);
+
1238  SetParticleDistributionMap(lev, pdm);
+
1239  }
+
1240  }
+
1241 
+
1242  Vector<int> ngrids(finest_level_in_file+1);
+
1243  for (int lev = 0; lev <= finest_level_in_file; lev++) {
+
1244  gname = "level_" + std::to_string(lev);
+
1245  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
+
1246  if (grp < 0) {
+
1247  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
+
1248  msg += gname;
+
1249  amrex::Abort(msg.c_str());
+
1250  }
+
1251 
+
1252  aname = "ngrids";
+
1253  ret = ReadHDF5Attr(grp, aname.c_str(), &ngrids[lev], H5T_NATIVE_INT);
+
1254  if (ret < 0) {
+
1255  std::string msg("ParticleContainer::RestartHDF5(): unable to read attribute ");
+
1256  msg += aname;
+
1257  amrex::Abort(msg.c_str());
+
1258  }
+
1259 
+
1260  AMREX_ASSERT(ngrids[lev] > 0);
+
1261  if (lev <= finestLevel()) {
+
1262  AMREX_ASSERT(ngrids[lev] == int(ParticleBoxArray(lev).size()));
+
1263  }
+
1264 
+
1265  H5Gclose(grp);
+
1266  }
+
1267 
+
1268  resizeData();
+
1269 
+
1270  if (finest_level_in_file > finestLevel()) {
+
1271  m_particles.resize(finest_level_in_file+1);
+
1272  }
+
1273 
+
1274  for (int lev = 0; lev <= finest_level_in_file; lev++) {
+
1275 
+
1276  gname = "level_" + std::to_string(lev);
+
1277  grp = H5Gopen(fid, gname.c_str(), H5P_DEFAULT);
+
1278  if (grp < 0) {
+
1279  std::string msg("ParticleContainer::RestartHDF5(): unable to open group ");
+
1280  msg += gname;
+
1281  amrex::Abort(msg.c_str());
+
1282  }
+
1283 
+
1284  dset = H5Dopen(grp, "nparticles_grid", H5P_DEFAULT);
+
1285  Vector<int> count(ngrids[lev]);
+
1286  ret = H5Dread(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &(count[0]));
+
1287  if (ret < 0) {
+
1288  std::string msg("ParticleContainer::RestartHDF5(): unable to read nparticles_grid dataset");
+
1289  amrex::Abort(msg.c_str());
+
1290  }
+
1291  H5Dclose(dset);
+
1292 
+
1293  Vector<hsize_t> offset(ngrids[lev]);
+
1294  offset[0] = 0;
+
1295  for (int i = 1; i < ngrids[lev]; i++) {
+
1296  offset[i] = offset[i-1] + count[i-1];
+
1297  }
+
1298 
+
1299  int_dset = H5Dopen(grp, "data:datatype=0", H5P_DEFAULT);
+
1300  if (int_dset < 0) {
+
1301  std::string msg("ParticleContainer::RestartHDF5(): unable to open int dataset");
+
1302  amrex::Abort(msg.c_str());
+
1303  }
+
1304  real_dset = H5Dopen(grp, "data:datatype=1", H5P_DEFAULT);
+
1305  if (real_dset < 0) {
+
1306  std::string msg("ParticleContainer::RestartHDF5(): unable to open int dataset");
+
1307  amrex::Abort(msg.c_str());
+
1308  }
+
1309 
+
1310  Vector<int> grids_to_read;
+
1311  if (lev <= finestLevel()) {
+
1312  for (MFIter mfi(*m_dummy_mf[lev]); mfi.isValid(); ++mfi) {
+
1313  grids_to_read.push_back(mfi.index());
+
1314  }
+
1315  } else {
+
1316 
+
1317  // we lost a level on restart. we still need to read in particles
+
1318  // on finer levels, and put them in the right place via Redistribute()
+
1319 
+
1320  const int rank = ParallelDescriptor::MyProc();
+
1321  const int NReaders = MaxReaders();
+
1322  if (rank >= NReaders) { return; }
+
1323 
+
1324  const int Navg = ngrids[lev] / NReaders;
+
1325  const int Nleft = ngrids[lev] - Navg * NReaders;
+
1326 
+
1327  int lo, hi;
+
1328  if (rank < Nleft) {
+
1329  lo = rank*(Navg + 1);
+
1330  hi = lo + Navg + 1;
+
1331  }
+
1332  else {
+
1333  lo = rank * Navg + Nleft;
+
1334  hi = lo + Navg;
+
1335  }
+
1336 
+
1337  for (int i = lo; i < hi; ++i) {
+
1338  grids_to_read.push_back(i);
+
1339  }
+
1340  }
+
1341 
+
1342  for(int igrid = 0; igrid < static_cast<int>(grids_to_read.size()); ++igrid) {
+
1343  const int grid = grids_to_read[igrid];
+
1344 
+
1345  if (how == "single") {
+
1346  ReadParticlesHDF5<float>(offset[grid], count[grid], grid, lev, int_dset, real_dset, finest_level_in_file, convert_ids);
+
1347  }
+
1348  else if (how == "double") {
+
1349  ReadParticlesHDF5<double>(offset[grid], count[grid], grid, lev, int_dset, real_dset, finest_level_in_file, convert_ids);
+
1350  }
+
1351  else {
+
1352  std::string msg("ParticleContainer::Restart(): bad parameter: ");
+
1353  msg += how;
+
1354  amrex::Error(msg.c_str());
+
1355  }
+
1356  }
+
1357 
+
1358  H5Dclose(int_dset);
+
1359  H5Dclose(real_dset);
+
1360  H5Gclose(grp);
+
1361  } // end for level
+
1362 
+
1363  H5Fclose(fid);
+
1364 
+
1365  Redistribute();
+
1366 
+
1367  AMREX_ASSERT(OK());
+
1368 
+
1369  if (m_verbose > 1) {
+
1370  auto stoptime = amrex::second() - strttime;
+
1371  ParallelDescriptor::ReduceRealMax(stoptime, ParallelDescriptor::IOProcessorNumber());
+
1372  amrex::Print() << "ParticleContainer::Restart() time: " << stoptime << '\n';
+
1373  }
+
1374 }
+
1375 
+
1376 // Read a batch of particles from the checkpoint file
+
1377 template <typename ParticleType, int NArrayReal, int NArrayInt,
+
1378  template<class> class Allocator, class CellAssignor>
+
1379 template <class RTYPE>
+
1380 void
+
1381 ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
+
1382 ::ReadParticlesHDF5 (hsize_t offset, hsize_t cnt, int grd, int lev,
+
1383  hid_t int_dset, hid_t real_dset, int finest_level_in_file,
+
1384  bool convert_ids)
+
1385 {
+
1386  BL_PROFILE("ParticleContainer::ReadParticlesHDF5()");
+
1387  AMREX_ASSERT(cnt > 0);
+
1388  AMREX_ASSERT(lev < int(m_particles.size()));
+
1389 
+
1390  hid_t int_dspace, int_fspace, real_dspace, real_fspace;
+
1391 
+
1392  // First read in the integer data in binary. We do not store
+
1393  // the m_lev and m_grid data on disk. We can easily recreate
+
1394  // that given the structure of the checkpoint file.
+
1395  const int iChunkSize = 2 + NStructInt + NumIntComps();
+
1396  Vector<int> istuff(cnt*iChunkSize);
+
1397  int_fspace = H5Dget_space(int_dset);
+
1398  hsize_t int_cnt = cnt*iChunkSize;
+
1399  hsize_t int_offset = offset*iChunkSize;
+
1400  int_dspace = H5Screate_simple(1, &int_cnt, NULL);
+
1401  H5Sselect_hyperslab (int_fspace, H5S_SELECT_SET, &int_offset, NULL, &int_cnt, NULL);
+
1402  H5Dread(int_dset, H5T_NATIVE_INT, int_dspace, int_fspace, H5P_DEFAULT, istuff.dataPtr());
+
1403 
+
1404  H5Sclose(int_fspace);
+
1405  H5Sclose(int_dspace);
+
1406 
+
1407  // Then the real data in binary.
+
1408  const int rChunkSize = AMREX_SPACEDIM + NStructReal + NumRealComps();
+
1409  Vector<RTYPE> rstuff(cnt*rChunkSize);
+
1410  real_fspace = H5Dget_space(real_dset);
+
1411  hsize_t real_cnt = cnt*rChunkSize;
+
1412  hsize_t real_offset = offset*rChunkSize;
+
1413  real_dspace = H5Screate_simple(1, &real_cnt, NULL);
+
1414  H5Sselect_hyperslab (real_fspace, H5S_SELECT_SET, &real_offset, NULL, &real_cnt, NULL);
+
1415  if (sizeof(RTYPE) == 4) {
+
1416  H5Dread(real_dset, H5T_NATIVE_FLOAT, real_dspace, real_fspace, H5P_DEFAULT, rstuff.dataPtr());
+
1417  } else {
+
1418  H5Dread(real_dset, H5T_NATIVE_DOUBLE, real_dspace, real_fspace, H5P_DEFAULT, rstuff.dataPtr());
+
1419  }
+
1420 
+
1421  H5Sclose(real_fspace);
+
1422  H5Sclose(real_dspace);
+
1423 
+
1424  // Now reassemble the particles.
+
1425  int* iptr = istuff.dataPtr();
+
1426  RTYPE* rptr = rstuff.dataPtr();
+
1427 
+
1428  ParticleType p;
+
1429  ParticleLocData pld;
+
1430 
+
1431  Vector<std::map<std::pair<int, int>, Gpu::HostVector<ParticleType> > > host_particles;
+
1432  host_particles.reserve(15);
+
1433  host_particles.resize(finest_level_in_file+1);
+
1434 
+
1435  Vector<std::map<std::pair<int, int>,
+
1436  std::vector<Gpu::HostVector<ParticleReal> > > > host_real_attribs;
+
1437  host_real_attribs.reserve(15);
+
1438  host_real_attribs.resize(finest_level_in_file+1);
+
1439 
+
1440  Vector<std::map<std::pair<int, int>,
+
1441  std::vector<Gpu::HostVector<int> > > > host_int_attribs;
+
1442  host_int_attribs.reserve(15);
+
1443  host_int_attribs.resize(finest_level_in_file+1);
+
1444 
+
1445  for (hsize_t i = 0; i < cnt; i++) {
+
1446  if (convert_ids) {
+
1447  std::int32_t xi, yi;
+
1448  std::uint32_t xu, yu;
+
1449  xi = iptr[0];
+
1450  yi = iptr[1];
+
1451  std::memcpy(&xu, &xi, sizeof(xi));
+
1452  std::memcpy(&yu, &yi, sizeof(yi));
+
1453  p.m_idcpu = ((std::uint64_t)xu) << 32 | yu;
+
1454  } else {
+
1455  p.id() = iptr[0];
+
1456  p.cpu() = iptr[1];
+
1457  }
+
1458  iptr += 2;
+
1459 
+
1460  for (int j = 0; j < NStructInt; j++)
+
1461  {
+
1462  p.idata(j) = *iptr;
+
1463  ++iptr;
+
1464  }
+
1465 
+
1466  AMREX_ASSERT(p.id() > 0);
+
1467 
+
1468  AMREX_D_TERM(p.pos(0) = ParticleReal(rptr[0]);,
+
1469  p.pos(1) = ParticleReal(rptr[1]);,
+
1470  p.pos(2) = ParticleReal(rptr[2]););
+
1471 
+
1472  rptr += AMREX_SPACEDIM;
+
1473 
+
1474  for (int j = 0; j < NStructReal; j++)
+
1475  {
+
1476  p.rdata(j) = ParticleReal(*rptr);
+
1477  ++rptr;
+
1478  }
+
1479 
+
1480  locateParticle(p, pld, 0, finestLevel(), 0);
+
1481 
+
1482  std::pair<int, int> ind(grd, pld.m_tile);
+
1483 
+
1484  host_real_attribs[lev][ind].resize(NumRealComps());
+
1485  host_int_attribs[lev][ind].resize(NumIntComps());
+
1486 
+
1487  // add the struct
+
1488  host_particles[lev][ind].push_back(p);
+
1489 
+
1490  // add the real...
+
1491  for (int icomp = 0; icomp < NumRealComps(); icomp++) {
+
1492  host_real_attribs[lev][ind][icomp].push_back(ParticleReal(*rptr));
+
1493  ++rptr;
+
1494  }
+
1495 
+
1496  // ... and int array data
+
1497  for (int icomp = 0; icomp < NumIntComps(); icomp++) {
+
1498  host_int_attribs[lev][ind][icomp].push_back(*iptr);
+
1499  ++iptr;
+
1500  }
+
1501  }
+
1502 
+
1503  for (int host_lev = 0; host_lev < static_cast<int>(host_particles.size()); ++host_lev)
+
1504  {
+
1505  for (auto& kv : host_particles[host_lev]) {
+
1506  auto grid = kv.first.first;
+
1507  auto tile = kv.first.second;
+
1508  const auto& src_tile = kv.second;
+
1509 
+
1510  auto& dst_tile = DefineAndReturnParticleTile(host_lev, grid, tile);
+
1511  auto old_size = dst_tile.GetArrayOfStructs().size();
+
1512  auto new_size = old_size + src_tile.size();
+
1513  dst_tile.resize(new_size);
+
1514 
+
1515  Gpu::copyAsync(Gpu::hostToDevice, src_tile.begin(), src_tile.end(),
+
1516  dst_tile.GetArrayOfStructs().begin() + old_size);
+
1517 
+
1518  for (int i = 0; i < NumRealComps(); ++i) {
+
1519  Gpu::copyAsync(Gpu::hostToDevice,
+
1520  host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(),
+
1521  host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(),
+
1522  dst_tile.GetStructOfArrays().GetRealData(i).begin() + old_size);
+
1523  }
+
1524 
+
1525  for (int i = 0; i < NumIntComps(); ++i) {
+
1526  Gpu::copyAsync(Gpu::hostToDevice,
+
1527  host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(),
+
1528  host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(),
+
1529  dst_tile.GetStructOfArrays().GetIntData(i).begin() + old_size);
+
1530  }
+
1531  }
+
1532  }
+
1533 
+
1534  Gpu::streamSynchronize();
+
1535 }
+
1536 
+
1537 #endif /*AMREX_USE_HDF5*/
+
1538 #endif /*AMREX_PARTICLEHDF5_H*/
BL_PROFILE
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:551
AMREX_ASSERT
#define AMREX_ASSERT(EX)
Definition: AMReX_BLassert.H:38
diff --git a/amrex/docs_xml/doxygen/AMReX__ParticleHDF5_8H.xml b/amrex/docs_xml/doxygen/AMReX__ParticleHDF5_8H.xml index 523e012b31..fff353ca44 100644 --- a/amrex/docs_xml/doxygen/AMReX__ParticleHDF5_8H.xml +++ b/amrex/docs_xml/doxygen/AMReX__ParticleHDF5_8H.xml @@ -2949,7 +2949,7 @@ hsize_tchunk_size; if(H5Pget_layout(dcpl_int)==H5D_CHUNKED){ if(H5Pget_chunk(dcpl_int,1,&chunk_size)>-1){ -if((int)chunk_size>total_int_size){ +if(chunk_size>total_int_size){ H5Pset_chunk(dcpl_int,1,&total_int_size); } } @@ -2990,733 +2990,734 @@ } } #endif -if(H5Pget_layout(dcpl_real)==H5D_CHUNKED){ -if(H5Pget_chunk(dcpl_real,1,&chunk_size)>-1){ -if((int)chunk_size>total_real_size){ -H5Pset_chunk(dcpl_real,1,&total_real_size); -} -} -} - -real_dset_space=H5Screate_simple(1,&total_real_size,NULL); -if(sizeof(typenameParticleType::RealType)==4){ -#ifdefAMREX_USE_HDF5_ASYNC -real_dset_id=H5Dcreate_async(grp,"data:datatype=1",H5T_NATIVE_FLOAT,real_dset_space, -H5P_DEFAULT,dcpl_real,H5P_DEFAULT,es_par_g); -#else -real_dset_id=H5Dcreate(grp,"data:datatype=1",H5T_NATIVE_FLOAT,real_dset_space, -H5P_DEFAULT,dcpl_real,H5P_DEFAULT); -#endif -} -else{ -#ifdefAMREX_USE_HDF5_ASYNC -real_dset_id=H5Dcreate_async(grp,"data:datatype=1",H5T_NATIVE_DOUBLE,real_dset_space, -H5P_DEFAULT,dcpl_real,H5P_DEFAULT,es_par_g); -#else -real_dset_id=H5Dcreate(grp,"data:datatype=1",H5T_NATIVE_DOUBLE,real_dset_space, -H5P_DEFAULT,dcpl_real,H5P_DEFAULT); -#endif -} -H5Sclose(real_dset_space); - -real_file_offset=0; -for(inti=0;i<ParallelDescriptor::MyProc();i++){ -real_file_offset+=all_mfi_real_total_size[i]; -} -my_real_offset=real_file_offset; -my_real_count=0; - -intmax_mfi_count=0,write_count=0; -for(inti=0;i<ParallelDescriptor::NProcs();i++){ -if(max_mfi_count<all_mfi_cnt[i]){ -max_mfi_count=all_mfi_cnt[i]; -} -} - + +if(H5Pget_layout(dcpl_real)==H5D_CHUNKED){ +if(H5Pget_chunk(dcpl_real,1,&chunk_size)>-1){ +if(chunk_size>total_real_size){ +H5Pset_chunk(dcpl_real,1,&total_real_size); +} +} +} + +real_dset_space=H5Screate_simple(1,&total_real_size,NULL); +if(sizeof(typenameParticleType::RealType)==4){ +#ifdefAMREX_USE_HDF5_ASYNC +real_dset_id=H5Dcreate_async(grp,"data:datatype=1",H5T_NATIVE_FLOAT,real_dset_space, +H5P_DEFAULT,dcpl_real,H5P_DEFAULT,es_par_g); +#else +real_dset_id=H5Dcreate(grp,"data:datatype=1",H5T_NATIVE_FLOAT,real_dset_space, +H5P_DEFAULT,dcpl_real,H5P_DEFAULT); +#endif +} +else{ +#ifdefAMREX_USE_HDF5_ASYNC +real_dset_id=H5Dcreate_async(grp,"data:datatype=1",H5T_NATIVE_DOUBLE,real_dset_space, +H5P_DEFAULT,dcpl_real,H5P_DEFAULT,es_par_g); +#else +real_dset_id=H5Dcreate(grp,"data:datatype=1",H5T_NATIVE_DOUBLE,real_dset_space, +H5P_DEFAULT,dcpl_real,H5P_DEFAULT); +#endif +} +H5Sclose(real_dset_space); + +real_file_offset=0; +for(inti=0;i<ParallelDescriptor::MyProc();i++){ +real_file_offset+=all_mfi_real_total_size[i]; +} +my_real_offset=real_file_offset; +my_real_count=0; + +intmax_mfi_count=0,write_count=0; +for(inti=0;i<ParallelDescriptor::NProcs();i++){ +if(max_mfi_count<all_mfi_cnt[i]){ +max_mfi_count=all_mfi_cnt[i]; +} +} -for(MFItermfi(state);mfi.isValid();++mfi) -{ -constintgrid=mfi.index(); - -if(count[grid]==0){continue;} - -Vector<int>istuff; -Vector<ParticleReal>rstuff; -particle_detail::packIOData(istuff,rstuff,*this,lev,grid, -write_real_comp,write_int_comp, -particle_io_flags,tile_map[grid],count[grid], -is_checkpoint); - -my_int_count=istuff.size(); -int_mem_space=H5Screate_simple(1,&my_int_count,NULL); -/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":my_int_offset="<<*/ -/*my_int_offset<<",my_int_count="<<my_int_count<<",total_int_size="<<total_int_size<<std::endl;*/ -int_dset_space=H5Screate_simple(1,&total_int_size,NULL); -H5Sselect_hyperslab(int_dset_space,H5S_SELECT_SET,&my_int_offset,NULL,&my_int_count,NULL); - -#ifdefAMREX_USE_HDF5_ASYNC -ret=H5Dwrite_async(int_dset_id,H5T_NATIVE_INT,int_mem_space,int_dset_space,dxpl_col,istuff.dataPtr(),es_par_g); -#else -ret=H5Dwrite(int_dset_id,H5T_NATIVE_INT,int_mem_space,int_dset_space,dxpl_col,istuff.dataPtr()); -#endif -if(ret<0){amrex::Abort("H5Dwriteint_dsetfailed!");} - -H5Sclose(int_dset_space); -H5Sclose(int_mem_space); - -my_int_offset+=my_int_count; - -my_real_count=rstuff.size(); -real_mem_space=H5Screate_simple(1,&my_real_count,NULL); -/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":my_real_offset="<<*/ -/*my_real_offset<<",my_real_count="<<my_real_count<<",total_real_size="<<total_real_size<<'\n';*/ -real_dset_space=H5Screate_simple(1,&total_real_size,NULL); -H5Sselect_hyperslab(real_dset_space,H5S_SELECT_SET,&my_real_offset,NULL,&my_real_count,NULL); -if(sizeof(typenameParticleType::RealType)==4){ -#ifdefAMREX_USE_HDF5_ASYNC -ret=H5Dwrite_async(real_dset_id,H5T_NATIVE_FLOAT,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr(),es_par_g); -#else -ret=H5Dwrite(real_dset_id,H5T_NATIVE_FLOAT,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr()); -#endif -}else{ -#ifdefAMREX_USE_HDF5_ASYNC -ret=H5Dwrite_async(real_dset_id,H5T_NATIVE_DOUBLE,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr(),es_par_g); -#else -ret=H5Dwrite(real_dset_id,H5T_NATIVE_DOUBLE,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr()); -#endif -} - -if(ret<0){amrex::Abort("H5Dwritereal_dsetfailed!");} - -H5Sclose(real_mem_space); -H5Sclose(real_dset_space); - -my_real_offset+=my_real_count; -write_count++; - -}//endfor(mfi) - -//DummywritessothateveryrankparticipatestoeverypossibleH5Dwrite(collective) -while(write_count<max_mfi_count){ -int_dset_space=H5Screate_simple(1,&total_int_size,NULL); -real_dset_space=H5Screate_simple(1,&total_real_size,NULL); -H5Sselect_none(int_dset_space); -H5Sselect_none(real_dset_space); - -#ifdefAMREX_USE_HDF5_ASYNC -H5Dwrite_async(int_dset_id,H5T_NATIVE_INT,int_dset_space,int_dset_space,dxpl_col,NULL,es_par_g); -if(sizeof(typenameParticleType::RealType)==4){ -H5Dwrite_async(real_dset_id,H5T_NATIVE_FLOAT,real_dset_space,real_dset_space,dxpl_col,NULL,es_par_g); -}else{ -H5Dwrite_async(real_dset_id,H5T_NATIVE_DOUBLE,real_dset_space,real_dset_space,dxpl_col,NULL,es_par_g); -} -#else -H5Dwrite(int_dset_id,H5T_NATIVE_INT,int_dset_space,int_dset_space,dxpl_col,NULL); -if(sizeof(typenameParticleType::RealType)==4){ -H5Dwrite(real_dset_id,H5T_NATIVE_FLOAT,real_dset_space,real_dset_space,dxpl_col,NULL); -}else{ -H5Dwrite(real_dset_id,H5T_NATIVE_DOUBLE,real_dset_space,real_dset_space,dxpl_col,NULL); -} -#endif - -H5Sclose(int_dset_space); -H5Sclose(real_dset_space); - -write_count++; -} - -#ifdefAMREX_USE_HDF5_ASYNC -H5Dclose_async(real_dset_id,es_par_g); -H5Dclose_async(int_dset_id,es_par_g); -#else -H5Dclose(real_dset_id); -H5Dclose(int_dset_id); -#endif - -//Createandwritethesizedataset -offset_space=H5Screate_simple(1,&total_mfi,NULL); -#ifdefAMREX_USE_HDF5_ASYNC -offset_id=H5Dcreate_async(grp,"nparticles_grid",H5T_NATIVE_INT,offset_space,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT,es_par_g); -#else -offset_id=H5Dcreate(grp,"nparticles_grid",H5T_NATIVE_INT,offset_space,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT); -#endif - -my_int_offset=0; -for(inti=0;i<ParallelDescriptor::MyProc();i++){ -my_int_offset+=all_mfi_cnt[i]; -} -my_int_count=my_mfi_cnt; -int_mem_space=H5Screate_simple(1,&my_int_count,NULL); -/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":my_int_offset="<<*/ -/*my_int_offset<<",my_int_count="<<my_int_count<<",total_mfi="<<total_mfi<<'\n';*/ -H5Sselect_hyperslab(offset_space,H5S_SELECT_SET,&my_int_offset,NULL,&my_int_count,NULL); - -#ifdefAMREX_USE_HDF5_ASYNC -ret=H5Dwrite_async(offset_id,H5T_NATIVE_INT,int_mem_space,offset_space,dxpl_col,&(my_nparticles[0]),es_par_g); -#else -ret=H5Dwrite(offset_id,H5T_NATIVE_INT,int_mem_space,offset_space,dxpl_col,&(my_nparticles[0])); -#endif -if(ret<0){amrex::Abort("H5Dwriteoffsetfailed!");} - -H5Pclose(dcpl_int); -H5Pclose(dcpl_real); -H5Pclose(dxpl_col); -H5Pclose(dxpl_ind); - -H5Sclose(int_mem_space); -H5Sclose(offset_space); - -#ifdefAMREX_USE_HDF5_ASYNC -H5Dclose_async(offset_id,es_par_g); -#else -H5Dclose(offset_id); -#endif - -/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":doneWriteParticlesHDF5"<<std::endl;*/ -return; -}//EndWriteParticlesHDF5 - -template<typenameParticleType,intNArrayReal,intNArrayInt, -template<class>classAllocator,classCellAssignor> -void -ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> -::RestartHDF5(conststd::string&dir,conststd::string&file,bool/*is_checkpoint*/) -{ -RestartHDF5(dir,file); -} - -template<typenameParticleType,intNArrayReal,intNArrayInt, -template<class>classAllocator,classCellAssignor> -void -ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> -::RestartHDF5(conststd::string&dir,conststd::string&file) -{ -BL_PROFILE("ParticleContainer::RestartHDF5()"); -AMREX_ASSERT(!dir.empty()); -AMREX_ASSERT(!file.empty()); - -constautostrttime=amrex::second(); - -std::stringfullname=dir; -if(!fullname.empty()&&fullname[fullname.size()-1]!='/'){ -fullname+='/'; -} - -fullname+=file; -fullname+=".h5"; - -hid_tfid,dset,grp,fapl,attr,atype,dspace,int_dset,real_dset; -intret; - -fapl=H5Pcreate(H5P_FILE_ACCESS); -#ifdefAMREX_USE_MPI -SetHDF5fapl(fapl,ParallelDescriptor::Communicator()); -#else -SetHDF5fapl(fapl); -#endif - -fid=H5Fopen(fullname.c_str(),H5F_ACC_RDONLY,fapl); -if(fid<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenfile:"); -msg+=fullname; -amrex::Abort(msg.c_str()); -} - -attr=H5Aopen(fid,"version_name",H5P_DEFAULT); -if(attr<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenversionattribute"); -amrex::Abort(msg.c_str()); -} - -atype=H5Aget_type(attr); -if(atype<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletogettypeofattribute"); -amrex::Abort(msg.c_str()); -} - -size_tattr_len=H5Tget_size(atype); -if(attr_len==0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletogetsizeofattribute"); -amrex::Abort(msg.c_str()); -} - -std::stringversion; -version.resize(attr_len+1); - -ret=H5Aread(attr,atype,&version[0]); -H5Tclose(atype); - -H5Aclose(attr); - -AMREX_ASSERT(!version.empty()); - -//Whatdoourversionstringsmean? -//"Version_One_Dot_Zero"--hard-wiredtowriteoutindoubleprecision. -//"Version_One_Dot_One"--canwriteouteitheraseithersingleordoubleprecision. -//Appendedtothelatterversionstringareeither"_single"or"_double"to -//indicatehowtheparticleswerewritten. -//"Version_Two_Dot_Zero"--thisistheAMReXparticlefileformat -//"Version_Two_Dot_One"--expandedparticleidstoallowfor2**39-1perproc -std::stringhow; -boolconvert_ids=false; -if(version.find("Version_Two_Dot_One")!=std::string::npos){ -convert_ids=true; -} -if(version.find("_single")!=std::string::npos){ -how="single"; -} -elseif(version.find("_double")!=std::string::npos){ -how="double"; -} -else{ -std::stringmsg("ParticleContainer::Restart():badversionstring:"); -msg+=version; -amrex::Error(version.c_str()); -} - -std::stringgname="Chombo_global"; -std::stringaname="SpaceDim"; -intdm; -grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); -if(grp<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); -msg+=gname; -amrex::Abort(msg.c_str()); -} -ret=ReadHDF5Attr(grp,aname.c_str(),&dm,H5T_NATIVE_INT); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} -H5Gclose(grp); - -aname="num_component_real"; -intnr; -ret=ReadHDF5Attr(fid,aname.c_str(),&nr,H5T_NATIVE_INT); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} -if(nr!=NStructReal+NumRealComps()) -amrex::Abort("ParticleContainer::Restart():nr!=NStructReal+NumRealComps()"); - -aname="num_component_int"; -intni; -ret=ReadHDF5Attr(fid,aname.c_str(),&ni,H5T_NATIVE_INT); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} -if(ni!=NStructInt+NumIntComps()){ -amrex::Abort("ParticleContainer::Restart():ni!=NStructInt"); -} - -aname="nparticles"; -Longnparticles; -ret=ReadHDF5Attr(fid,aname.c_str(),&nparticles,H5T_NATIVE_LLONG); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} -AMREX_ASSERT(nparticles>=0); - -aname="maxnextid"; -Longmaxnextid; -ret=ReadHDF5Attr(fid,aname.c_str(),&maxnextid,H5T_NATIVE_LLONG); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} -AMREX_ASSERT(maxnextid>0); -ParticleType::NextID(maxnextid); - -aname="finest_level"; -intfinest_level_in_file; -ret=ReadHDF5Attr(fid,aname.c_str(),&finest_level_in_file,H5T_NATIVE_INT); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} -AMREX_ASSERT(finest_level_in_file>=0); - -hid_tcomp_dtype=H5Tcreate(H5T_COMPOUND,2*AMREX_SPACEDIM*sizeof(int)); -if(1==AMREX_SPACEDIM){ -H5Tinsert(comp_dtype,"lo_i",0*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"hi_i",1*sizeof(int),H5T_NATIVE_INT); -} -elseif(2==AMREX_SPACEDIM){ -H5Tinsert(comp_dtype,"lo_i",0*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"lo_j",1*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"hi_i",2*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"hi_j",3*sizeof(int),H5T_NATIVE_INT); -} -elseif(3==AMREX_SPACEDIM){ -H5Tinsert(comp_dtype,"lo_i",0*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"lo_j",1*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"lo_k",2*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"hi_i",3*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"hi_j",4*sizeof(int),H5T_NATIVE_INT); -H5Tinsert(comp_dtype,"hi_k",5*sizeof(int),H5T_NATIVE_INT); -} - -//Determinewhetherthisisadual-gridrestartornot. -Vector<BoxArray>particle_box_arrays(finest_level_in_file+1); -booldual_grid=false; - -for(intlev=0;lev<=finest_level_in_file;lev++) -{ -if(lev>finestLevel()) -{ -dual_grid=true; -break; -} - -gname="level_"+std::to_string(lev); -grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); -if(grp<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); -msg+=gname; -amrex::Abort(msg.c_str()); -} - -dset=H5Dopen(grp,"boxes",H5P_DEFAULT); -dspace=H5Dget_space(dset); -hsize_tngrid; -H5Sget_simple_extent_dims(dspace,&ngrid,NULL); - -Vector<int>boxes(ngrid*AMREX_SPACEDIM*2); -ret=H5Dread(dset,comp_dtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&(boxes[0])); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadnparticles_griddataset"); -amrex::Abort(msg.c_str()); -} -H5Sclose(dspace); -H5Dclose(dset); -H5Gclose(grp); - -/*particle_box_arrays[lev].readFrom(phdr_file);*/ -particle_box_arrays[lev]=ParticleBoxArray(lev); -for(inti=0;i<(int)ngrid;i++){ -Boxtmp(IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM],boxes[i*AMREX_SPACEDIM+1],boxes[i*AMREX_SPACEDIM+2])}, -IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM*2],boxes[i*AMREX_SPACEDIM*2+1],boxes[i*AMREX_SPACEDIM*2+2])}); -particle_box_arrays[lev][i]=tmp; -} - -if(!particle_box_arrays[lev].CellEqual(ParticleBoxArray(lev))) -dual_grid=true; -} - -if(dual_grid){ -for(intlev=0;lev<=finestLevel();lev++){ -SetParticleBoxArray(lev,particle_box_arrays[lev]); -DistributionMappingpdm(particle_box_arrays[lev]); -SetParticleDistributionMap(lev,pdm); -} -} - -Vector<int>ngrids(finest_level_in_file+1); -for(intlev=0;lev<=finest_level_in_file;lev++){ -gname="level_"+std::to_string(lev); -grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); -if(grp<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); -msg+=gname; -amrex::Abort(msg.c_str()); -} - -aname="ngrids"; -ret=ReadHDF5Attr(grp,aname.c_str(),&ngrids[lev],H5T_NATIVE_INT); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); -msg+=aname; -amrex::Abort(msg.c_str()); -} - -AMREX_ASSERT(ngrids[lev]>0); -if(lev<=finestLevel()){ -AMREX_ASSERT(ngrids[lev]==int(ParticleBoxArray(lev).size())); -} - -H5Gclose(grp); -} - -resizeData(); - -if(finest_level_in_file>finestLevel()){ -m_particles.resize(finest_level_in_file+1); -} - -for(intlev=0;lev<=finest_level_in_file;lev++){ - -gname="level_"+std::to_string(lev); -grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); -if(grp<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); -msg+=gname; -amrex::Abort(msg.c_str()); -} - -dset=H5Dopen(grp,"nparticles_grid",H5P_DEFAULT); -Vector<int>count(ngrids[lev]); -ret=H5Dread(dset,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,&(count[0])); -if(ret<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadnparticles_griddataset"); -amrex::Abort(msg.c_str()); -} -H5Dclose(dset); - -Vector<hsize_t>offset(ngrids[lev]); -offset[0]=0; -for(inti=1;i<ngrids[lev];i++){ -offset[i]=offset[i-1]+count[i-1]; -} - -int_dset=H5Dopen(grp,"data:datatype=0",H5P_DEFAULT); -if(int_dset<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenintdataset"); -amrex::Abort(msg.c_str()); -} -real_dset=H5Dopen(grp,"data:datatype=1",H5P_DEFAULT); -if(real_dset<0){ -std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenintdataset"); -amrex::Abort(msg.c_str()); -} - -Vector<int>grids_to_read; -if(lev<=finestLevel()){ -for(MFItermfi(*m_dummy_mf[lev]);mfi.isValid();++mfi){ -grids_to_read.push_back(mfi.index()); -} -}else{ - -//welostalevelonrestart.westillneedtoreadinparticles -//onfinerlevels,andputthemintherightplaceviaRedistribute() - -constintrank=ParallelDescriptor::MyProc(); -constintNReaders=MaxReaders(); -if(rank>=NReaders){return;} - -constintNavg=ngrids[lev]/NReaders; -constintNleft=ngrids[lev]-Navg*NReaders; - -intlo,hi; -if(rank<Nleft){ -lo=rank*(Navg+1); -hi=lo+Navg+1; -} -else{ -lo=rank*Navg+Nleft; -hi=lo+Navg; -} - -for(inti=lo;i<hi;++i){ -grids_to_read.push_back(i); -} -} - -for(intigrid=0;igrid<static_cast<int>(grids_to_read.size());++igrid){ -constintgrid=grids_to_read[igrid]; - -if(how=="single"){ -ReadParticlesHDF5<float>(offset[grid],count[grid],grid,lev,int_dset,real_dset,finest_level_in_file,convert_ids); -} -elseif(how=="double"){ -ReadParticlesHDF5<double>(offset[grid],count[grid],grid,lev,int_dset,real_dset,finest_level_in_file,convert_ids); -} -else{ -std::stringmsg("ParticleContainer::Restart():badparameter:"); -msg+=how; -amrex::Error(msg.c_str()); -} -} - -H5Dclose(int_dset); -H5Dclose(real_dset); -H5Gclose(grp); -}//endforlevel - -H5Fclose(fid); - -Redistribute(); - -AMREX_ASSERT(OK()); - -if(m_verbose>1){ -autostoptime=amrex::second()-strttime; -ParallelDescriptor::ReduceRealMax(stoptime,ParallelDescriptor::IOProcessorNumber()); -amrex::Print()<<"ParticleContainer::Restart()time:"<<stoptime<<'\n'; -} -} - -//Readabatchofparticlesfromthecheckpointfile -template<typenameParticleType,intNArrayReal,intNArrayInt, -template<class>classAllocator,classCellAssignor> -template<classRTYPE> -void -ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> -::ReadParticlesHDF5(hsize_toffset,hsize_tcnt,intgrd,intlev, -hid_tint_dset,hid_treal_dset,intfinest_level_in_file, -boolconvert_ids) -{ -BL_PROFILE("ParticleContainer::ReadParticlesHDF5()"); -AMREX_ASSERT(cnt>0); -AMREX_ASSERT(lev<int(m_particles.size())); - -hid_tint_dspace,int_fspace,real_dspace,real_fspace; - -//Firstreadintheintegerdatainbinary.Wedonotstore -//them_levandm_griddataondisk.Wecaneasilyrecreate -//thatgiventhestructureofthecheckpointfile. -constintiChunkSize=2+NStructInt+NumIntComps(); -Vector<int>istuff(cnt*iChunkSize); -int_fspace=H5Dget_space(int_dset); -hsize_tint_cnt=cnt*iChunkSize; -hsize_tint_offset=offset*iChunkSize; -int_dspace=H5Screate_simple(1,&int_cnt,NULL); -H5Sselect_hyperslab(int_fspace,H5S_SELECT_SET,&int_offset,NULL,&int_cnt,NULL); -H5Dread(int_dset,H5T_NATIVE_INT,int_dspace,int_fspace,H5P_DEFAULT,istuff.dataPtr()); - -H5Sclose(int_fspace); -H5Sclose(int_dspace); - -//Thentherealdatainbinary. -constintrChunkSize=AMREX_SPACEDIM+NStructReal+NumRealComps(); -Vector<RTYPE>rstuff(cnt*rChunkSize); -real_fspace=H5Dget_space(real_dset); -hsize_treal_cnt=cnt*rChunkSize; -hsize_treal_offset=offset*rChunkSize; -real_dspace=H5Screate_simple(1,&real_cnt,NULL); -H5Sselect_hyperslab(real_fspace,H5S_SELECT_SET,&real_offset,NULL,&real_cnt,NULL); -if(sizeof(RTYPE)==4){ -H5Dread(real_dset,H5T_NATIVE_FLOAT,real_dspace,real_fspace,H5P_DEFAULT,rstuff.dataPtr()); -}else{ -H5Dread(real_dset,H5T_NATIVE_DOUBLE,real_dspace,real_fspace,H5P_DEFAULT,rstuff.dataPtr()); -} - -H5Sclose(real_fspace); -H5Sclose(real_dspace); - -//Nowreassembletheparticles. -int*iptr=istuff.dataPtr(); -RTYPE*rptr=rstuff.dataPtr(); - -ParticleTypep; -ParticleLocDatapld; - -Vector<std::map<std::pair<int,int>,Gpu::HostVector<ParticleType>>>host_particles; -host_particles.reserve(15); -host_particles.resize(finest_level_in_file+1); - -Vector<std::map<std::pair<int,int>, -std::vector<Gpu::HostVector<ParticleReal>>>>host_real_attribs; -host_real_attribs.reserve(15); -host_real_attribs.resize(finest_level_in_file+1); - -Vector<std::map<std::pair<int,int>, -std::vector<Gpu::HostVector<int>>>>host_int_attribs; -host_int_attribs.reserve(15); -host_int_attribs.resize(finest_level_in_file+1); - -for(hsize_ti=0;i<cnt;i++){ -if(convert_ids){ -std::int32_txi,yi; -std::uint32_txu,yu; -xi=iptr[0]; -yi=iptr[1]; -std::memcpy(&xu,&xi,sizeof(xi)); -std::memcpy(&yu,&yi,sizeof(yi)); -p.m_idcpu=((std::uint64_t)xu)<<32|yu; -}else{ -p.id()=iptr[0]; -p.cpu()=iptr[1]; -} -iptr+=2; - -for(intj=0;j<NStructInt;j++) -{ -p.idata(j)=*iptr; -++iptr; -} - -AMREX_ASSERT(p.id()>0); - -AMREX_D_TERM(p.pos(0)=ParticleReal(rptr[0]);, -p.pos(1)=ParticleReal(rptr[1]);, -p.pos(2)=ParticleReal(rptr[2]);); - -rptr+=AMREX_SPACEDIM; - -for(intj=0;j<NStructReal;j++) -{ -p.rdata(j)=ParticleReal(*rptr); -++rptr; -} - -locateParticle(p,pld,0,finestLevel(),0); - -std::pair<int,int>ind(grd,pld.m_tile); - -host_real_attribs[lev][ind].resize(NumRealComps()); -host_int_attribs[lev][ind].resize(NumIntComps()); - -//addthestruct -host_particles[lev][ind].push_back(p); - -//addthereal... -for(inticomp=0;icomp<NumRealComps();icomp++){ -host_real_attribs[lev][ind][icomp].push_back(ParticleReal(*rptr)); -++rptr; -} - -//...andintarraydata -for(inticomp=0;icomp<NumIntComps();icomp++){ -host_int_attribs[lev][ind][icomp].push_back(*iptr); -++iptr; -} -} - -for(inthost_lev=0;host_lev<static_cast<int>(host_particles.size());++host_lev) -{ -for(auto&kv:host_particles[host_lev]){ -autogrid=kv.first.first; -autotile=kv.first.second; -constauto&src_tile=kv.second; - -auto&dst_tile=DefineAndReturnParticleTile(host_lev,grid,tile); -autoold_size=dst_tile.GetArrayOfStructs().size(); -autonew_size=old_size+src_tile.size(); -dst_tile.resize(new_size); - -Gpu::copyAsync(Gpu::hostToDevice,src_tile.begin(),src_tile.end(), -dst_tile.GetArrayOfStructs().begin()+old_size); - -for(inti=0;i<NumRealComps();++i){ -Gpu::copyAsync(Gpu::hostToDevice, -host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), -host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), -dst_tile.GetStructOfArrays().GetRealData(i).begin()+old_size); -} - -for(inti=0;i<NumIntComps();++i){ -Gpu::copyAsync(Gpu::hostToDevice, -host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), -host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), -dst_tile.GetStructOfArrays().GetIntData(i).begin()+old_size); -} -} -} - -Gpu::streamSynchronize(); -} - -#endif/*AMREX_USE_HDF5*/ -#endif/*AMREX_PARTICLEHDF5_H*/ + +for(MFItermfi(state);mfi.isValid();++mfi) +{ +constintgrid=mfi.index(); + +if(count[grid]==0){continue;} + +Vector<int>istuff; +Vector<ParticleReal>rstuff; +particle_detail::packIOData(istuff,rstuff,*this,lev,grid, +write_real_comp,write_int_comp, +particle_io_flags,tile_map[grid],count[grid], +is_checkpoint); + +my_int_count=istuff.size(); +int_mem_space=H5Screate_simple(1,&my_int_count,NULL); +/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":my_int_offset="<<*/ +/*my_int_offset<<",my_int_count="<<my_int_count<<",total_int_size="<<total_int_size<<std::endl;*/ +int_dset_space=H5Screate_simple(1,&total_int_size,NULL); +H5Sselect_hyperslab(int_dset_space,H5S_SELECT_SET,&my_int_offset,NULL,&my_int_count,NULL); + +#ifdefAMREX_USE_HDF5_ASYNC +ret=H5Dwrite_async(int_dset_id,H5T_NATIVE_INT,int_mem_space,int_dset_space,dxpl_col,istuff.dataPtr(),es_par_g); +#else +ret=H5Dwrite(int_dset_id,H5T_NATIVE_INT,int_mem_space,int_dset_space,dxpl_col,istuff.dataPtr()); +#endif +if(ret<0){amrex::Abort("H5Dwriteint_dsetfailed!");} + +H5Sclose(int_dset_space); +H5Sclose(int_mem_space); + +my_int_offset+=my_int_count; + +my_real_count=rstuff.size(); +real_mem_space=H5Screate_simple(1,&my_real_count,NULL); +/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":my_real_offset="<<*/ +/*my_real_offset<<",my_real_count="<<my_real_count<<",total_real_size="<<total_real_size<<'\n';*/ +real_dset_space=H5Screate_simple(1,&total_real_size,NULL); +H5Sselect_hyperslab(real_dset_space,H5S_SELECT_SET,&my_real_offset,NULL,&my_real_count,NULL); +if(sizeof(typenameParticleType::RealType)==4){ +#ifdefAMREX_USE_HDF5_ASYNC +ret=H5Dwrite_async(real_dset_id,H5T_NATIVE_FLOAT,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr(),es_par_g); +#else +ret=H5Dwrite(real_dset_id,H5T_NATIVE_FLOAT,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr()); +#endif +}else{ +#ifdefAMREX_USE_HDF5_ASYNC +ret=H5Dwrite_async(real_dset_id,H5T_NATIVE_DOUBLE,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr(),es_par_g); +#else +ret=H5Dwrite(real_dset_id,H5T_NATIVE_DOUBLE,real_mem_space,real_dset_space,dxpl_col,rstuff.dataPtr()); +#endif +} + +if(ret<0){amrex::Abort("H5Dwritereal_dsetfailed!");} + +H5Sclose(real_mem_space); +H5Sclose(real_dset_space); + +my_real_offset+=my_real_count; +write_count++; + +}//endfor(mfi) + +//DummywritessothateveryrankparticipatestoeverypossibleH5Dwrite(collective) +while(write_count<max_mfi_count){ +int_dset_space=H5Screate_simple(1,&total_int_size,NULL); +real_dset_space=H5Screate_simple(1,&total_real_size,NULL); +H5Sselect_none(int_dset_space); +H5Sselect_none(real_dset_space); + +#ifdefAMREX_USE_HDF5_ASYNC +H5Dwrite_async(int_dset_id,H5T_NATIVE_INT,int_dset_space,int_dset_space,dxpl_col,NULL,es_par_g); +if(sizeof(typenameParticleType::RealType)==4){ +H5Dwrite_async(real_dset_id,H5T_NATIVE_FLOAT,real_dset_space,real_dset_space,dxpl_col,NULL,es_par_g); +}else{ +H5Dwrite_async(real_dset_id,H5T_NATIVE_DOUBLE,real_dset_space,real_dset_space,dxpl_col,NULL,es_par_g); +} +#else +H5Dwrite(int_dset_id,H5T_NATIVE_INT,int_dset_space,int_dset_space,dxpl_col,NULL); +if(sizeof(typenameParticleType::RealType)==4){ +H5Dwrite(real_dset_id,H5T_NATIVE_FLOAT,real_dset_space,real_dset_space,dxpl_col,NULL); +}else{ +H5Dwrite(real_dset_id,H5T_NATIVE_DOUBLE,real_dset_space,real_dset_space,dxpl_col,NULL); +} +#endif + +H5Sclose(int_dset_space); +H5Sclose(real_dset_space); + +write_count++; +} + +#ifdefAMREX_USE_HDF5_ASYNC +H5Dclose_async(real_dset_id,es_par_g); +H5Dclose_async(int_dset_id,es_par_g); +#else +H5Dclose(real_dset_id); +H5Dclose(int_dset_id); +#endif + +//Createandwritethesizedataset +offset_space=H5Screate_simple(1,&total_mfi,NULL); +#ifdefAMREX_USE_HDF5_ASYNC +offset_id=H5Dcreate_async(grp,"nparticles_grid",H5T_NATIVE_INT,offset_space,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT,es_par_g); +#else +offset_id=H5Dcreate(grp,"nparticles_grid",H5T_NATIVE_INT,offset_space,H5P_DEFAULT,H5P_DEFAULT,H5P_DEFAULT); +#endif + +my_int_offset=0; +for(inti=0;i<ParallelDescriptor::MyProc();i++){ +my_int_offset+=all_mfi_cnt[i]; +} +my_int_count=my_mfi_cnt; +int_mem_space=H5Screate_simple(1,&my_int_count,NULL); +/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":my_int_offset="<<*/ +/*my_int_offset<<",my_int_count="<<my_int_count<<",total_mfi="<<total_mfi<<'\n';*/ +H5Sselect_hyperslab(offset_space,H5S_SELECT_SET,&my_int_offset,NULL,&my_int_count,NULL); + +#ifdefAMREX_USE_HDF5_ASYNC +ret=H5Dwrite_async(offset_id,H5T_NATIVE_INT,int_mem_space,offset_space,dxpl_col,&(my_nparticles[0]),es_par_g); +#else +ret=H5Dwrite(offset_id,H5T_NATIVE_INT,int_mem_space,offset_space,dxpl_col,&(my_nparticles[0])); +#endif +if(ret<0){amrex::Abort("H5Dwriteoffsetfailed!");} + +H5Pclose(dcpl_int); +H5Pclose(dcpl_real); +H5Pclose(dxpl_col); +H5Pclose(dxpl_ind); + +H5Sclose(int_mem_space); +H5Sclose(offset_space); + +#ifdefAMREX_USE_HDF5_ASYNC +H5Dclose_async(offset_id,es_par_g); +#else +H5Dclose(offset_id); +#endif + +/*std::cout<<"Rank"<<ParallelDescriptor::MyProc()<<":doneWriteParticlesHDF5"<<std::endl;*/ +return; +}//EndWriteParticlesHDF5 + +template<typenameParticleType,intNArrayReal,intNArrayInt, +template<class>classAllocator,classCellAssignor> +void +ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> +::RestartHDF5(conststd::string&dir,conststd::string&file,bool/*is_checkpoint*/) +{ +RestartHDF5(dir,file); +} + +template<typenameParticleType,intNArrayReal,intNArrayInt, +template<class>classAllocator,classCellAssignor> +void +ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> +::RestartHDF5(conststd::string&dir,conststd::string&file) +{ +BL_PROFILE("ParticleContainer::RestartHDF5()"); +AMREX_ASSERT(!dir.empty()); +AMREX_ASSERT(!file.empty()); + +constautostrttime=amrex::second(); + +std::stringfullname=dir; +if(!fullname.empty()&&fullname[fullname.size()-1]!='/'){ +fullname+='/'; +} + +fullname+=file; +fullname+=".h5"; + +hid_tfid,dset,grp,fapl,attr,atype,dspace,int_dset,real_dset; +intret; + +fapl=H5Pcreate(H5P_FILE_ACCESS); +#ifdefAMREX_USE_MPI +SetHDF5fapl(fapl,ParallelDescriptor::Communicator()); +#else +SetHDF5fapl(fapl); +#endif + +fid=H5Fopen(fullname.c_str(),H5F_ACC_RDONLY,fapl); +if(fid<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenfile:"); +msg+=fullname; +amrex::Abort(msg.c_str()); +} + +attr=H5Aopen(fid,"version_name",H5P_DEFAULT); +if(attr<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenversionattribute"); +amrex::Abort(msg.c_str()); +} + +atype=H5Aget_type(attr); +if(atype<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletogettypeofattribute"); +amrex::Abort(msg.c_str()); +} + +size_tattr_len=H5Tget_size(atype); +if(attr_len==0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletogetsizeofattribute"); +amrex::Abort(msg.c_str()); +} + +std::stringversion; +version.resize(attr_len+1); + +ret=H5Aread(attr,atype,&version[0]); +H5Tclose(atype); + +H5Aclose(attr); + +AMREX_ASSERT(!version.empty()); + +//Whatdoourversionstringsmean? +//"Version_One_Dot_Zero"--hard-wiredtowriteoutindoubleprecision. +//"Version_One_Dot_One"--canwriteouteitheraseithersingleordoubleprecision. +//Appendedtothelatterversionstringareeither"_single"or"_double"to +//indicatehowtheparticleswerewritten. +//"Version_Two_Dot_Zero"--thisistheAMReXparticlefileformat +//"Version_Two_Dot_One"--expandedparticleidstoallowfor2**39-1perproc +std::stringhow; +boolconvert_ids=false; +if(version.find("Version_Two_Dot_One")!=std::string::npos){ +convert_ids=true; +} +if(version.find("_single")!=std::string::npos){ +how="single"; +} +elseif(version.find("_double")!=std::string::npos){ +how="double"; +} +else{ +std::stringmsg("ParticleContainer::Restart():badversionstring:"); +msg+=version; +amrex::Error(version.c_str()); +} + +std::stringgname="Chombo_global"; +std::stringaname="SpaceDim"; +intdm; +grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); +if(grp<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); +msg+=gname; +amrex::Abort(msg.c_str()); +} +ret=ReadHDF5Attr(grp,aname.c_str(),&dm,H5T_NATIVE_INT); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} +H5Gclose(grp); + +aname="num_component_real"; +intnr; +ret=ReadHDF5Attr(fid,aname.c_str(),&nr,H5T_NATIVE_INT); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} +if(nr!=NStructReal+NumRealComps()) +amrex::Abort("ParticleContainer::Restart():nr!=NStructReal+NumRealComps()"); + +aname="num_component_int"; +intni; +ret=ReadHDF5Attr(fid,aname.c_str(),&ni,H5T_NATIVE_INT); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} +if(ni!=NStructInt+NumIntComps()){ +amrex::Abort("ParticleContainer::Restart():ni!=NStructInt"); +} + +aname="nparticles"; +Longnparticles; +ret=ReadHDF5Attr(fid,aname.c_str(),&nparticles,H5T_NATIVE_LLONG); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} +AMREX_ASSERT(nparticles>=0); + +aname="maxnextid"; +Longmaxnextid; +ret=ReadHDF5Attr(fid,aname.c_str(),&maxnextid,H5T_NATIVE_LLONG); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} +AMREX_ASSERT(maxnextid>0); +ParticleType::NextID(maxnextid); + +aname="finest_level"; +intfinest_level_in_file; +ret=ReadHDF5Attr(fid,aname.c_str(),&finest_level_in_file,H5T_NATIVE_INT); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} +AMREX_ASSERT(finest_level_in_file>=0); + +hid_tcomp_dtype=H5Tcreate(H5T_COMPOUND,2*AMREX_SPACEDIM*sizeof(int)); +if(1==AMREX_SPACEDIM){ +H5Tinsert(comp_dtype,"lo_i",0*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"hi_i",1*sizeof(int),H5T_NATIVE_INT); +} +elseif(2==AMREX_SPACEDIM){ +H5Tinsert(comp_dtype,"lo_i",0*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"lo_j",1*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"hi_i",2*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"hi_j",3*sizeof(int),H5T_NATIVE_INT); +} +elseif(3==AMREX_SPACEDIM){ +H5Tinsert(comp_dtype,"lo_i",0*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"lo_j",1*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"lo_k",2*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"hi_i",3*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"hi_j",4*sizeof(int),H5T_NATIVE_INT); +H5Tinsert(comp_dtype,"hi_k",5*sizeof(int),H5T_NATIVE_INT); +} + +//Determinewhetherthisisadual-gridrestartornot. +Vector<BoxArray>particle_box_arrays(finest_level_in_file+1); +booldual_grid=false; + +for(intlev=0;lev<=finest_level_in_file;lev++) +{ +if(lev>finestLevel()) +{ +dual_grid=true; +break; +} + +gname="level_"+std::to_string(lev); +grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); +if(grp<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); +msg+=gname; +amrex::Abort(msg.c_str()); +} + +dset=H5Dopen(grp,"boxes",H5P_DEFAULT); +dspace=H5Dget_space(dset); +hsize_tngrid; +H5Sget_simple_extent_dims(dspace,&ngrid,NULL); + +Vector<int>boxes(ngrid*AMREX_SPACEDIM*2); +ret=H5Dread(dset,comp_dtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&(boxes[0])); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadnparticles_griddataset"); +amrex::Abort(msg.c_str()); +} +H5Sclose(dspace); +H5Dclose(dset); +H5Gclose(grp); + +/*particle_box_arrays[lev].readFrom(phdr_file);*/ +particle_box_arrays[lev]=ParticleBoxArray(lev); +for(inti=0;i<(int)ngrid;i++){ +Boxtmp(IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM],boxes[i*AMREX_SPACEDIM+1],boxes[i*AMREX_SPACEDIM+2])}, +IntVect{AMREX_D_DECL(boxes[i*AMREX_SPACEDIM*2],boxes[i*AMREX_SPACEDIM*2+1],boxes[i*AMREX_SPACEDIM*2+2])}); +particle_box_arrays[lev][i]=tmp; +} + +if(!particle_box_arrays[lev].CellEqual(ParticleBoxArray(lev))) +dual_grid=true; +} + +if(dual_grid){ +for(intlev=0;lev<=finestLevel();lev++){ +SetParticleBoxArray(lev,particle_box_arrays[lev]); +DistributionMappingpdm(particle_box_arrays[lev]); +SetParticleDistributionMap(lev,pdm); +} +} + +Vector<int>ngrids(finest_level_in_file+1); +for(intlev=0;lev<=finest_level_in_file;lev++){ +gname="level_"+std::to_string(lev); +grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); +if(grp<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); +msg+=gname; +amrex::Abort(msg.c_str()); +} + +aname="ngrids"; +ret=ReadHDF5Attr(grp,aname.c_str(),&ngrids[lev],H5T_NATIVE_INT); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadattribute"); +msg+=aname; +amrex::Abort(msg.c_str()); +} + +AMREX_ASSERT(ngrids[lev]>0); +if(lev<=finestLevel()){ +AMREX_ASSERT(ngrids[lev]==int(ParticleBoxArray(lev).size())); +} + +H5Gclose(grp); +} + +resizeData(); + +if(finest_level_in_file>finestLevel()){ +m_particles.resize(finest_level_in_file+1); +} + +for(intlev=0;lev<=finest_level_in_file;lev++){ + +gname="level_"+std::to_string(lev); +grp=H5Gopen(fid,gname.c_str(),H5P_DEFAULT); +if(grp<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopengroup"); +msg+=gname; +amrex::Abort(msg.c_str()); +} + +dset=H5Dopen(grp,"nparticles_grid",H5P_DEFAULT); +Vector<int>count(ngrids[lev]); +ret=H5Dread(dset,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,&(count[0])); +if(ret<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoreadnparticles_griddataset"); +amrex::Abort(msg.c_str()); +} +H5Dclose(dset); + +Vector<hsize_t>offset(ngrids[lev]); +offset[0]=0; +for(inti=1;i<ngrids[lev];i++){ +offset[i]=offset[i-1]+count[i-1]; +} + +int_dset=H5Dopen(grp,"data:datatype=0",H5P_DEFAULT); +if(int_dset<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenintdataset"); +amrex::Abort(msg.c_str()); +} +real_dset=H5Dopen(grp,"data:datatype=1",H5P_DEFAULT); +if(real_dset<0){ +std::stringmsg("ParticleContainer::RestartHDF5():unabletoopenintdataset"); +amrex::Abort(msg.c_str()); +} + +Vector<int>grids_to_read; +if(lev<=finestLevel()){ +for(MFItermfi(*m_dummy_mf[lev]);mfi.isValid();++mfi){ +grids_to_read.push_back(mfi.index()); +} +}else{ + +//welostalevelonrestart.westillneedtoreadinparticles +//onfinerlevels,andputthemintherightplaceviaRedistribute() + +constintrank=ParallelDescriptor::MyProc(); +constintNReaders=MaxReaders(); +if(rank>=NReaders){return;} + +constintNavg=ngrids[lev]/NReaders; +constintNleft=ngrids[lev]-Navg*NReaders; + +intlo,hi; +if(rank<Nleft){ +lo=rank*(Navg+1); +hi=lo+Navg+1; +} +else{ +lo=rank*Navg+Nleft; +hi=lo+Navg; +} + +for(inti=lo;i<hi;++i){ +grids_to_read.push_back(i); +} +} + +for(intigrid=0;igrid<static_cast<int>(grids_to_read.size());++igrid){ +constintgrid=grids_to_read[igrid]; + +if(how=="single"){ +ReadParticlesHDF5<float>(offset[grid],count[grid],grid,lev,int_dset,real_dset,finest_level_in_file,convert_ids); +} +elseif(how=="double"){ +ReadParticlesHDF5<double>(offset[grid],count[grid],grid,lev,int_dset,real_dset,finest_level_in_file,convert_ids); +} +else{ +std::stringmsg("ParticleContainer::Restart():badparameter:"); +msg+=how; +amrex::Error(msg.c_str()); +} +} + +H5Dclose(int_dset); +H5Dclose(real_dset); +H5Gclose(grp); +}//endforlevel + +H5Fclose(fid); + +Redistribute(); + +AMREX_ASSERT(OK()); + +if(m_verbose>1){ +autostoptime=amrex::second()-strttime; +ParallelDescriptor::ReduceRealMax(stoptime,ParallelDescriptor::IOProcessorNumber()); +amrex::Print()<<"ParticleContainer::Restart()time:"<<stoptime<<'\n'; +} +} + +//Readabatchofparticlesfromthecheckpointfile +template<typenameParticleType,intNArrayReal,intNArrayInt, +template<class>classAllocator,classCellAssignor> +template<classRTYPE> +void +ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor> +::ReadParticlesHDF5(hsize_toffset,hsize_tcnt,intgrd,intlev, +hid_tint_dset,hid_treal_dset,intfinest_level_in_file, +boolconvert_ids) +{ +BL_PROFILE("ParticleContainer::ReadParticlesHDF5()"); +AMREX_ASSERT(cnt>0); +AMREX_ASSERT(lev<int(m_particles.size())); + +hid_tint_dspace,int_fspace,real_dspace,real_fspace; + +//Firstreadintheintegerdatainbinary.Wedonotstore +//them_levandm_griddataondisk.Wecaneasilyrecreate +//thatgiventhestructureofthecheckpointfile. +constintiChunkSize=2+NStructInt+NumIntComps(); +Vector<int>istuff(cnt*iChunkSize); +int_fspace=H5Dget_space(int_dset); +hsize_tint_cnt=cnt*iChunkSize; +hsize_tint_offset=offset*iChunkSize; +int_dspace=H5Screate_simple(1,&int_cnt,NULL); +H5Sselect_hyperslab(int_fspace,H5S_SELECT_SET,&int_offset,NULL,&int_cnt,NULL); +H5Dread(int_dset,H5T_NATIVE_INT,int_dspace,int_fspace,H5P_DEFAULT,istuff.dataPtr()); + +H5Sclose(int_fspace); +H5Sclose(int_dspace); + +//Thentherealdatainbinary. +constintrChunkSize=AMREX_SPACEDIM+NStructReal+NumRealComps(); +Vector<RTYPE>rstuff(cnt*rChunkSize); +real_fspace=H5Dget_space(real_dset); +hsize_treal_cnt=cnt*rChunkSize; +hsize_treal_offset=offset*rChunkSize; +real_dspace=H5Screate_simple(1,&real_cnt,NULL); +H5Sselect_hyperslab(real_fspace,H5S_SELECT_SET,&real_offset,NULL,&real_cnt,NULL); +if(sizeof(RTYPE)==4){ +H5Dread(real_dset,H5T_NATIVE_FLOAT,real_dspace,real_fspace,H5P_DEFAULT,rstuff.dataPtr()); +}else{ +H5Dread(real_dset,H5T_NATIVE_DOUBLE,real_dspace,real_fspace,H5P_DEFAULT,rstuff.dataPtr()); +} + +H5Sclose(real_fspace); +H5Sclose(real_dspace); + +//Nowreassembletheparticles. +int*iptr=istuff.dataPtr(); +RTYPE*rptr=rstuff.dataPtr(); + +ParticleTypep; +ParticleLocDatapld; + +Vector<std::map<std::pair<int,int>,Gpu::HostVector<ParticleType>>>host_particles; +host_particles.reserve(15); +host_particles.resize(finest_level_in_file+1); + +Vector<std::map<std::pair<int,int>, +std::vector<Gpu::HostVector<ParticleReal>>>>host_real_attribs; +host_real_attribs.reserve(15); +host_real_attribs.resize(finest_level_in_file+1); + +Vector<std::map<std::pair<int,int>, +std::vector<Gpu::HostVector<int>>>>host_int_attribs; +host_int_attribs.reserve(15); +host_int_attribs.resize(finest_level_in_file+1); + +for(hsize_ti=0;i<cnt;i++){ +if(convert_ids){ +std::int32_txi,yi; +std::uint32_txu,yu; +xi=iptr[0]; +yi=iptr[1]; +std::memcpy(&xu,&xi,sizeof(xi)); +std::memcpy(&yu,&yi,sizeof(yi)); +p.m_idcpu=((std::uint64_t)xu)<<32|yu; +}else{ +p.id()=iptr[0]; +p.cpu()=iptr[1]; +} +iptr+=2; + +for(intj=0;j<NStructInt;j++) +{ +p.idata(j)=*iptr; +++iptr; +} + +AMREX_ASSERT(p.id()>0); + +AMREX_D_TERM(p.pos(0)=ParticleReal(rptr[0]);, +p.pos(1)=ParticleReal(rptr[1]);, +p.pos(2)=ParticleReal(rptr[2]);); + +rptr+=AMREX_SPACEDIM; + +for(intj=0;j<NStructReal;j++) +{ +p.rdata(j)=ParticleReal(*rptr); +++rptr; +} + +locateParticle(p,pld,0,finestLevel(),0); + +std::pair<int,int>ind(grd,pld.m_tile); + +host_real_attribs[lev][ind].resize(NumRealComps()); +host_int_attribs[lev][ind].resize(NumIntComps()); + +//addthestruct +host_particles[lev][ind].push_back(p); + +//addthereal... +for(inticomp=0;icomp<NumRealComps();icomp++){ +host_real_attribs[lev][ind][icomp].push_back(ParticleReal(*rptr)); +++rptr; +} + +//...andintarraydata +for(inticomp=0;icomp<NumIntComps();icomp++){ +host_int_attribs[lev][ind][icomp].push_back(*iptr); +++iptr; +} +} + +for(inthost_lev=0;host_lev<static_cast<int>(host_particles.size());++host_lev) +{ +for(auto&kv:host_particles[host_lev]){ +autogrid=kv.first.first; +autotile=kv.first.second; +constauto&src_tile=kv.second; + +auto&dst_tile=DefineAndReturnParticleTile(host_lev,grid,tile); +autoold_size=dst_tile.GetArrayOfStructs().size(); +autonew_size=old_size+src_tile.size(); +dst_tile.resize(new_size); + +Gpu::copyAsync(Gpu::hostToDevice,src_tile.begin(),src_tile.end(), +dst_tile.GetArrayOfStructs().begin()+old_size); + +for(inti=0;i<NumRealComps();++i){ +Gpu::copyAsync(Gpu::hostToDevice, +host_real_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), +host_real_attribs[host_lev][std::make_pair(grid,tile)][i].end(), +dst_tile.GetStructOfArrays().GetRealData(i).begin()+old_size); +} + +for(inti=0;i<NumIntComps();++i){ +Gpu::copyAsync(Gpu::hostToDevice, +host_int_attribs[host_lev][std::make_pair(grid,tile)][i].begin(), +host_int_attribs[host_lev][std::make_pair(grid,tile)][i].end(), +dst_tile.GetStructOfArrays().GetIntData(i).begin()+old_size); +} +} +} + +Gpu::streamSynchronize(); +} + +#endif/*AMREX_USE_HDF5*/ +#endif/*AMREX_PARTICLEHDF5_H*/ diff --git a/amrex/docs_xml/doxygen/AMReX__PlotFileUtilHDF5_8cpp.xml b/amrex/docs_xml/doxygen/AMReX__PlotFileUtilHDF5_8cpp.xml index f17401ed28..fa69ec6bf4 100644 --- a/amrex/docs_xml/doxygen/AMReX__PlotFileUtilHDF5_8cpp.xml +++ b/amrex/docs_xml/doxygen/AMReX__PlotFileUtilHDF5_8cpp.xml @@ -2868,7 +2868,7 @@ if(H5Pget_layout(lev_dcpl_id)==H5D_CHUNKED){ hsize_tchunk_size; if(H5Pget_chunk(lev_dcpl_id,1,&chunk_size)>-1){ -if((int)chunk_size>hs_allprocsize[0]){ +if(chunk_size>hs_allprocsize[0]){ H5Pset_chunk(lev_dcpl_id,1,hs_allprocsize); } } @@ -3340,7 +3340,7 @@ if(H5Pget_layout(lev_dcpl_id)==H5D_CHUNKED){ hsize_tchunk_size; if(H5Pget_chunk(lev_dcpl_id,1,&chunk_size)>-1){ -if((int)chunk_size>hs_allprocsize[0]){ +if(chunk_size>hs_allprocsize[0]){ H5Pset_chunk(lev_dcpl_id,1,hs_allprocsize); } }