diff --git a/amrex/docs_html/_downloads/008eb6dbfab802633dff40122ece848c/amrex.pdf b/amrex/docs_html/_downloads/008eb6dbfab802633dff40122ece848c/amrex.pdf
index 442f3a284c..3e7c81a2d8 100644
Binary files a/amrex/docs_html/_downloads/008eb6dbfab802633dff40122ece848c/amrex.pdf and b/amrex/docs_html/_downloads/008eb6dbfab802633dff40122ece848c/amrex.pdf differ
diff --git a/amrex/docs_html/doxygen/AMReX__ParticleIO_8H_source.html b/amrex/docs_html/doxygen/AMReX__ParticleIO_8H_source.html
index 1e7a2a6877..ef224c9554 100644
--- a/amrex/docs_html/doxygen/AMReX__ParticleIO_8H_source.html
+++ b/amrex/docs_html/doxygen/AMReX__ParticleIO_8H_source.html
@@ -1398,8 +1398,8 @@
amrex::ParmParse pp
Input file parser instance for the given namespace.
Definition: AMReX_HypreIJIface.cpp:15
#define AMREX_D_TERM(a, b, c)
Definition: AMReX_SPACE.H:129
-void WriteBinaryParticleDataAsync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:683
-void WriteBinaryParticleDataSync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, F const &f, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:391
+void WriteBinaryParticleDataAsync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:684
+void WriteBinaryParticleDataSync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, F const &f, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:392
A collection of Boxes stored in an Array.
Definition: AMReX_BoxArray.H:529
Calculates the distribution of FABs to MPI processes.
Definition: AMReX_DistributionMapping.H:41
Definition: AMReX_MFIter.H:57
diff --git a/amrex/docs_html/doxygen/AMReX__WriteBinaryParticleData_8H_source.html b/amrex/docs_html/doxygen/AMReX__WriteBinaryParticleData_8H_source.html
index 941912f298..3ee1b6348a 100644
--- a/amrex/docs_html/doxygen/AMReX__WriteBinaryParticleData_8H_source.html
+++ b/amrex/docs_html/doxygen/AMReX__WriteBinaryParticleData_8H_source.html
@@ -290,977 +290,978 @@
189 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
190 rdata.resize(np*rChunkSize);
- 192 typename PC::IntVector idata_d(idata.size());
- 193 typename PC::RealVector rdata_d(rdata.size());
-
- 195 typename PC::IntVector write_int_comp_d(write_int_comp.size());
- 196 typename PC::IntVector write_real_comp_d(write_real_comp.size());
-
- 198 write_int_comp_d.begin());
-
- 200 write_real_comp_d.begin());
-
-
- 203 const auto write_int_comp_d_ptr = write_int_comp_d.data();
- 204 const auto write_real_comp_d_ptr = write_real_comp_d.data();
-
- 206 std::size_t poffset = 0;
- 207 for (
int tile : tiles) {
- 208 const auto& ptile = pc.ParticlesAt(lev, grid, tile);
- 209 const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tile));
- 210 int np_tile = ptile.numParticles();
- 211 typename PC::IntVector offsets(np_tile);
-
-
- 214 const auto flag_ptr = pflags.data();
-
- 216 auto idata_d_ptr = idata_d.data();
- 217 auto rdata_d_ptr = rdata_d.data();
-
- 219 const auto ptd = ptile.getConstParticleTileData();
-
-
-
-
- 224 const auto p = ptd.getSuperParticle(pindex);
-
- 226 if (flag_ptr[pindex]) {
- 227 std::size_t iout_index = (pindex+poffset)*iChunkSize;
-
-
-
- 231 std::size_t rout_index = (pindex+poffset)*rChunkSize;
- 232 for (
int j = 0; j < AMREX_SPACEDIM; j++) {
- 233 rdata_d_ptr[rout_index] = p.pos(j);
-
-
-
- 237 for (
int j = 0; j < PC::SuperParticleType::NInt; j++) {
- 238 if (write_int_comp_d_ptr[j]) {
- 239 idata_d_ptr[iout_index] = p.idata(j);
-
-
-
-
- 244 for (
int j = 0; j < ptd.m_num_runtime_int; j++) {
- 245 if (write_int_comp_d_ptr[PC::SuperParticleType::NInt + j]) {
- 246 idata_d_ptr[iout_index] = ptd.m_runtime_idata[j][pindex];
-
-
-
-
-
- 252 const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
- 253 for (
int j = real_start_offset; j < PC::SuperParticleType::NReal; j++) {
- 254 const int write_comp_index = j-real_start_offset;
- 255 if (write_real_comp_d_ptr[write_comp_index]) {
- 256 rdata_d_ptr[rout_index] = p.rdata(j);
-
-
-
-
- 261 for (
int j = 0; j < ptd.m_num_runtime_real; j++) {
- 262 if (write_real_comp_d_ptr[PC::SuperParticleType::NReal+j-real_start_offset]) {
- 263 rdata_d_ptr[rout_index] = ptd.m_runtime_rdata[j][pindex];
-
-
-
-
-
-
- 270 poffset += num_copies;
-
-
-
-
-
-
-
-
- 279 std::enable_if_t<!RunOnGpu<typename PC::template AllocatorType<int>>::value>
- 280 packIOData (Vector<int>& idata, Vector<ParticleReal>& rdata,
const PC& pc,
int lev,
int grid,
- 281 const Vector<int>& write_real_comp,
const Vector<int>& write_int_comp,
- 282 const Vector<std::map<std::pair<int, int>,
typename PC::IntVector>>& particle_io_flags,
- 283 const Vector<int>& tiles,
int np,
bool is_checkpoint)
-
- 285 int num_output_int = 0;
- 286 for (
int i = 0; i < pc.NumIntComps() + PC::NStructInt; ++i) {
- 287 if (write_int_comp[i]) { ++num_output_int; }
-
-
- 290 const Long iChunkSize = 2 + num_output_int;
- 291 idata.resize(np*iChunkSize);
-
- 293 int num_output_real = 0;
- 294 for (
int i : write_real_comp) {
- 295 if (i) { ++num_output_real; }
-
-
- 298 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
- 299 rdata.resize(np*rChunkSize);
-
- 301 int* iptr = idata.dataPtr();
- 302 ParticleReal* rptr = rdata.dataPtr();
- 303 for (
int tile : tiles) {
- 304 const auto& ptile = pc.ParticlesAt(lev, grid, tile);
- 305 const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tile));
- 306 for (
int pindex = 0; pindex < ptile.numParticles(); ++pindex) {
- 307 if (pflags[pindex]) {
- 308 const auto& soa = ptile.GetStructOfArrays();
-
-
-
- 312 if constexpr(!PC::ParticleType::is_soa_particle)
-
- 314 const auto& aos = ptile.GetArrayOfStructs();
- 315 const auto& p = aos[pindex];
-
-
-
-
-
-
- 322 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = p.pos(j); }
- 323 rptr += AMREX_SPACEDIM;
-
-
- 326 for (
int j = 0; j < PC::NStructInt; j++) {
- 327 if (write_int_comp[j]) {
-
-
-
-
-
- 333 for (
int j = 0; j < PC::NStructReal; j++) {
- 334 if (write_real_comp[j]) {
-
-
-
-
-
-
- 341 uint64_t idcpu = soa.GetIdCPUData()[pindex];
-
-
- 344 std::uint32_t xu, yu;
- 345 xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32);
- 346 yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL);
-
-
-
-
-
-
-
-
- 355 *iptr = (
int) ParticleIDWrapper(idcpu);
-
- 357 *iptr = (
int) ParticleCPUWrapper(idcpu);
-
-
-
-
- 362 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = soa.GetRealData(j)[pindex]; }
- 363 rptr += AMREX_SPACEDIM;
-
-
-
- 367 const int int_start_offset = 0;
- 368 for (
int j = int_start_offset; j < pc.NumIntComps(); j++) {
- 369 if (write_int_comp[PC::NStructInt+j]) {
- 370 *iptr = soa.GetIntData(j)[pindex];
-
-
-
-
-
- 376 const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
- 377 for (
int j = real_start_offset; j < pc.NumRealComps(); j++) {
- 378 const int write_comp_index = PC::NStructReal+j-real_start_offset;
- 379 if (write_real_comp[write_comp_index]) {
- 380 *rptr = (ParticleReal) soa.GetRealData(j)[pindex];
-
-
-
-
-
-
-
+ 192 typename PC::IntVector write_int_comp_d(write_int_comp.size());
+ 193 typename PC::IntVector write_real_comp_d(write_real_comp.size());
+
+ 195 write_int_comp_d.begin());
+
+ 197 write_real_comp_d.begin());
+
+ 199 const auto write_int_comp_d_ptr = write_int_comp_d.data();
+ 200 const auto write_real_comp_d_ptr = write_real_comp_d.data();
+
+ 202 std::size_t poffset = 0;
+ 203 for (
int tile : tiles) {
+ 204 const auto& ptile = pc.ParticlesAt(lev, grid, tile);
+ 205 const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tile));
+ 206 int np_tile = ptile.numParticles();
+ 207 typename PC::IntVector offsets(np_tile);
+
+
+ 210 typename PC::IntVector idata_d(num_copies*iChunkSize);
+ 211 typename PC::RealVector rdata_d(num_copies*rChunkSize);
+
+ 213 const auto flag_ptr = pflags.data();
+
+ 215 auto idata_d_ptr = idata_d.data();
+ 216 auto rdata_d_ptr = rdata_d.data();
+
+ 218 const auto ptd = ptile.getConstParticleTileData();
+
+
+
+
+ 223 const auto p = ptd.getSuperParticle(pindex);
+
+ 225 if (flag_ptr[pindex]) {
+ 226 std::size_t iout_index = pindex*iChunkSize;
+
+
+
+ 230 std::size_t rout_index = pindex*rChunkSize;
+ 231 for (
int j = 0; j < AMREX_SPACEDIM; j++) {
+ 232 rdata_d_ptr[rout_index] = p.pos(j);
+
+
+
+ 236 for (
int j = 0; j < PC::SuperParticleType::NInt; j++) {
+ 237 if (write_int_comp_d_ptr[j]) {
+ 238 idata_d_ptr[iout_index] = p.idata(j);
+
+
+
+
+ 243 for (
int j = 0; j < ptd.m_num_runtime_int; j++) {
+ 244 if (write_int_comp_d_ptr[PC::SuperParticleType::NInt + j]) {
+ 245 idata_d_ptr[iout_index] = ptd.m_runtime_idata[j][pindex];
+
+
+
+
+
+ 251 const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+ 252 for (
int j = real_start_offset; j < PC::SuperParticleType::NReal; j++) {
+ 253 const int write_comp_index = j-real_start_offset;
+ 254 if (write_real_comp_d_ptr[write_comp_index]) {
+ 255 rdata_d_ptr[rout_index] = p.rdata(j);
+
+
+
+
+ 260 for (
int j = 0; j < ptd.m_num_runtime_real; j++) {
+ 261 if (write_real_comp_d_ptr[PC::SuperParticleType::NReal+j-real_start_offset]) {
+ 262 rdata_d_ptr[rout_index] = ptd.m_runtime_rdata[j][pindex];
+
+
+
+
+
+
+
+ 270 idata.begin() +
typename PC::IntVector::difference_type(poffset));
+
+ 272 rdata.begin() +
typename PC::RealVector::difference_type(poffset));
+
+
+ 275 poffset += num_copies;
+
+
+
+
+ 280 std::enable_if_t<!RunOnGpu<typename PC::template AllocatorType<int>>::value>
+ 281 packIOData (Vector<int>& idata, Vector<ParticleReal>& rdata,
const PC& pc,
int lev,
int grid,
+ 282 const Vector<int>& write_real_comp,
const Vector<int>& write_int_comp,
+ 283 const Vector<std::map<std::pair<int, int>,
typename PC::IntVector>>& particle_io_flags,
+ 284 const Vector<int>& tiles,
int np,
bool is_checkpoint)
+
+ 286 int num_output_int = 0;
+ 287 for (
int i = 0; i < pc.NumIntComps() + PC::NStructInt; ++i) {
+ 288 if (write_int_comp[i]) { ++num_output_int; }
+
+
+ 291 const Long iChunkSize = 2 + num_output_int;
+ 292 idata.resize(np*iChunkSize);
+
+ 294 int num_output_real = 0;
+ 295 for (
int i : write_real_comp) {
+ 296 if (i) { ++num_output_real; }
+
+
+ 299 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
+ 300 rdata.resize(np*rChunkSize);
+
+ 302 int* iptr = idata.dataPtr();
+ 303 ParticleReal* rptr = rdata.dataPtr();
+ 304 for (
int tile : tiles) {
+ 305 const auto& ptile = pc.ParticlesAt(lev, grid, tile);
+ 306 const auto& pflags = particle_io_flags[lev].at(std::make_pair(grid, tile));
+ 307 for (
int pindex = 0; pindex < ptile.numParticles(); ++pindex) {
+ 308 if (pflags[pindex]) {
+ 309 const auto& soa = ptile.GetStructOfArrays();
+
+
+
+ 313 if constexpr(!PC::ParticleType::is_soa_particle)
+
+ 315 const auto& aos = ptile.GetArrayOfStructs();
+ 316 const auto& p = aos[pindex];
+
+
+
+
+
+
+ 323 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = p.pos(j); }
+ 324 rptr += AMREX_SPACEDIM;
+
+
+ 327 for (
int j = 0; j < PC::NStructInt; j++) {
+ 328 if (write_int_comp[j]) {
+
+
+
+
+
+ 334 for (
int j = 0; j < PC::NStructReal; j++) {
+ 335 if (write_real_comp[j]) {
+
+
+
+
+
+
+ 342 uint64_t idcpu = soa.GetIdCPUData()[pindex];
+
+
+ 345 std::uint32_t xu, yu;
+ 346 xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32);
+ 347 yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL);
+
+
+
+
+
+
+
+
+ 356 *iptr = (
int) ParticleIDWrapper(idcpu);
+
+ 358 *iptr = (
int) ParticleCPUWrapper(idcpu);
+
+
+
+
+ 363 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = soa.GetRealData(j)[pindex]; }
+ 364 rptr += AMREX_SPACEDIM;
+
+
+
+ 368 const int int_start_offset = 0;
+ 369 for (
int j = int_start_offset; j < pc.NumIntComps(); j++) {
+ 370 if (write_int_comp[PC::NStructInt+j]) {
+ 371 *iptr = soa.GetIntData(j)[pindex];
+
+
+
+
+
+ 377 const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+ 378 for (
int j = real_start_offset; j < pc.NumRealComps(); j++) {
+ 379 const int write_comp_index = PC::NStructReal+j-real_start_offset;
+ 380 if (write_real_comp[write_comp_index]) {
+ 381 *rptr = (ParticleReal) soa.GetRealData(j)[pindex];
+
+
+
+
+
+
-
- 390 template <class PC, class F, std::enable_if_t<IsParticleContainer<PC>::value,
int> foo = 0>
-
- 392 const std::string& dir,
const std::string& name,
- 393 const Vector<int>& write_real_comp,
- 394 const Vector<int>& write_int_comp,
- 395 const Vector<std::string>& real_comp_names,
- 396 const Vector<std::string>& int_comp_names,
- 397 F
const&
f,
bool is_checkpoint)
-
-
-
-
- 402 AMREX_ASSERT(
sizeof(
typename PC::ParticleType::RealType) == 4 ||
- 403 sizeof(
typename PC::ParticleType::RealType) == 8);
-
- 405 constexpr
int NStructReal = PC::NStructReal;
- 406 constexpr
int NStructInt = PC::NStructInt;
-
-
-
-
- 411 if constexpr(PC::ParticleType::is_soa_particle) {
- 412 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal - AMREX_SPACEDIM);
-
-
-
-
-
- 418 std::string pdir = dir;
- 419 if ( ! pdir.empty() && pdir[pdir.size()-1] !=
'/') { pdir +=
'/'; }
-
-
- 422 if ( ! pc.GetLevelDirectoriesCreated()) {
-
-
-
-
-
-
-
-
-
-
- 433 std::ofstream HdrFile;
-
-
-
-
-
- 439 Vector<std::map<std::pair<int, int>,
typename PC::IntVector > >
- 440 particle_io_flags(pc.GetParticles().size());
- 441 for (
int lev = 0; lev < pc.GetParticles().
size(); lev++)
-
- 443 const auto& pmap = pc.GetParticles(lev);
- 444 for (
const auto& kv : pmap)
-
- 446 auto& flags = particle_io_flags[lev][kv.first];
-
-
-
-
-
-
- 453 if(pc.GetUsePrePost())
-
- 455 nparticles = pc.GetNParticlesPrePost();
- 456 maxnextid = pc.GetMaxNextIDPrePost();
-
-
-
-
- 461 maxnextid = PC::ParticleType::NextID();
-
- 463 PC::ParticleType::NextID(maxnextid);
-
-
-
-
-
- 469 std::string HdrFileName = pdir;
-
- 471 if ( ! HdrFileName.empty() && HdrFileName[HdrFileName.size()-1] !=
'/') {
-
-
-
- 475 HdrFileName +=
"Header";
- 476 pc.HdrFileNamePrePost = HdrFileName;
-
- 478 HdrFile.open(HdrFileName.c_str(), std::ios::out|std::ios::trunc);
-
-
-
-
-
-
-
-
- 487 std::string version_string = is_checkpoint ? PC::CheckpointVersion() : PC::PlotfileVersion();
- 488 if (
sizeof(
typename PC::ParticleType::RealType) == 4)
-
- 490 HdrFile << version_string <<
"_single" <<
'\n';
-
-
-
- 494 HdrFile << version_string <<
"_double" <<
'\n';
-
-
- 497 int num_output_real = 0;
- 498 for (
int i : write_real_comp) {
- 499 if (i) { ++num_output_real; }
-
-
- 502 int num_output_int = 0;
- 503 for (
int i = 0; i < pc.NumIntComps() + NStructInt; ++i) {
- 504 if (write_int_comp[i]) { ++num_output_int; }
-
-
-
- 508 HdrFile << AMREX_SPACEDIM <<
'\n';
-
-
- 511 HdrFile << num_output_real <<
'\n';
-
-
- 514 for (
int i = 0; i < (
int) real_comp_names.size(); ++i ) {
- 515 if (write_real_comp[i]) { HdrFile << real_comp_names[i] <<
'\n'; }
-
-
-
- 519 HdrFile << num_output_int <<
'\n';
-
-
- 522 for (
int i = 0; i < NStructInt + pc.NumIntComps(); ++i ) {
- 523 if (write_int_comp[i]) { HdrFile << int_comp_names[i] <<
'\n'; }
-
-
- 526 bool is_checkpoint_legacy =
true;
- 527 HdrFile << is_checkpoint_legacy <<
'\n';
-
-
- 530 HdrFile << nparticles <<
'\n';
-
-
- 533 HdrFile << maxnextid <<
'\n';
-
-
- 536 HdrFile << pc.finestLevel() <<
'\n';
-
-
- 539 for (
int lev = 0; lev <= pc.finestLevel(); lev++) {
- 540 HdrFile << pc.ParticleBoxArray(lev).size() <<
'\n';
-
-
-
-
-
-
-
- 548 ParmParse
pp(
"particles");
-
- 550 if(nOutFiles == -1) { nOutFiles =
NProcs; }
-
- 552 pc.nOutFilesPrePost = nOutFiles;
-
- 554 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
-
-
-
- 559 gotsome = (pc.nParticlesAtLevelPrePost[lev] > 0);
-
-
-
- 563 gotsome = (pc.NumberOfParticlesAtLevel(lev) > 0);
-
-
-
- 567 std::string LevelDir = pdir;
-
-
-
- 571 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] !=
'/') { LevelDir +=
'/'; }
-
-
-
- 575 if ( ! pc.GetLevelDirectoriesCreated())
-
-
-
-
-
-
-
-
-
-
-
-
- 588 std::string HeaderFileName = LevelDir;
- 589 HeaderFileName +=
"/Particle_H";
- 590 std::ofstream ParticleHeader(HeaderFileName);
-
- 592 pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
- 593 ParticleHeader <<
'\n';
-
- 595 ParticleHeader.flush();
- 596 ParticleHeader.close();
-
-
-
- 600 info.SetAlloc(
false);
- 601 MultiFab state(pc.ParticleBoxArray(lev),
- 602 pc.ParticleDistributionMap(lev),
-
-
-
-
- 607 Vector<int> which(state.size(),0);
- 608 Vector<int > count(state.size(),0);
- 609 Vector<Long> where(state.size(),0);
-
- 611 std::string filePrefix(LevelDir);
-
- 613 filePrefix += PC::DataPrefix();
-
- 615 pc.filePrefixPrePost[lev] = filePrefix;
-
- 617 bool groupSets(
false), setBuf(
true);
-
-
-
- 621 for(NFilesIter nfi(nOutFiles, filePrefix, groupSets, setBuf); nfi.ReadyToWrite(); ++nfi)
-
- 623 auto& myStream = (std::ofstream&) nfi.Stream();
- 624 pc.WriteParticles(lev, myStream, nfi.FileNumber(), which, count, where,
- 625 write_real_comp, write_int_comp, particle_io_flags, is_checkpoint);
-
-
-
- 629 pc.whichPrePost[lev] = which;
- 630 pc.countPrePost[lev] = count;
- 631 pc.wherePrePost[lev] = where;
-
-
-
-
-
-
-
-
-
- 641 if(pc.GetUsePrePost()) {
-
-
- 644 for (
int j = 0; j < state.size(); j++)
-
- 646 HdrFile << which[j] <<
' ' << count[j] <<
' ' << where[j] <<
'\n';
-
-
- 649 if (gotsome && pc.doUnlink)
-
-
- 652 Vector<Long> cnt(nOutFiles,0);
-
- 654 for (
int i = 0, N=
static_cast<int>(count.size()); i < N; i++) {
- 655 cnt[which[i]] += count[i];
-
-
- 658 for (
int i = 0, N=
static_cast<int>(cnt.size()); i < N; i++)
-
-
-
- 662 std::string FullFileName = NFilesIter::FileName(i, filePrefix);
-
-
-
-
-
-
-
-
-
-
-
-
- 675 if ( ! HdrFile.good())
-
- 677 amrex::Abort(
"ParticleContainer::Checkpoint(): problem writing HdrFile");
-
-
-
-
- 682 template <class PC, std::enable_if_t<IsParticleContainer<PC>::value,
int> foo = 0>
-
- 684 const std::string& dir,
const std::string& name,
- 685 const Vector<int>& write_real_comp,
- 686 const Vector<int>& write_int_comp,
- 687 const Vector<std::string>& real_comp_names,
- 688 const Vector<std::string>& int_comp_names,
bool is_checkpoint)
-
-
-
-
- 693 AMREX_ASSERT(
sizeof(
typename PC::ParticleType::RealType) == 4 ||
- 694 sizeof(
typename PC::ParticleType::RealType) == 8);
-
- 696 constexpr
int NStructReal = PC::NStructReal;
- 697 constexpr
int NStructInt = PC::NStructInt;
- 698 constexpr
int NArrayReal = PC::NArrayReal;
- 699 constexpr
int NArrayInt = PC::NArrayInt;
-
-
-
- 703 const int IOProcNumber =
NProcs - 1;
-
- 705 if constexpr(PC::ParticleType::is_soa_particle) {
- 706 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal - AMREX_SPACEDIM);
-
-
-
-
-
- 712 Vector<LayoutData<Long> > np_per_grid_local(pc.finestLevel()+1);
- 713 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
- 715 np_per_grid_local[lev].define(pc.ParticleBoxArray(lev), pc.ParticleDistributionMap(lev));
- 716 using ParIter =
typename PC::ParConstIterType;
- 717 for (
ParIter pti(pc, lev); pti.isValid(); ++pti)
-
- 719 int gid = pti.index();
- 720 const auto& ptile = pc.ParticlesAt(lev, pti);
- 721 const auto& ptd = ptile.getConstParticleTileData();
- 722 const int np = ptile.numParticles();
-
- 724 ReduceOps<ReduceOpSum> reduce_op;
- 725 ReduceData<int> reduce_data(reduce_op);
- 726 using ReduceTuple =
typename decltype(reduce_data)::Type;
-
- 728 reduce_op.eval(np, reduce_data,
-
-
- 731 return (ptd.id(i) > 0) ? 1 : 0;
-
-
- 734 int np_valid = amrex::get<0>(reduce_data.value(reduce_op));
- 735 np_per_grid_local[lev][gid] += np_valid;
-
-
-
- 739 Vector<Vector<Long> > np_per_grid_global(pc.finestLevel()+1);
-
- 741 Vector<Long> np_per_level(pc.finestLevel()+1);
- 742 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
- 744 np_per_grid_global[lev].resize(np_per_grid_local[lev].
size());
-
- 746 np_per_grid_global[lev],
-
- 748 np_per_level[lev] = std::accumulate(np_per_grid_global[lev].
begin(),
- 749 np_per_grid_global[lev].
end(), 0L);
- 750 total_np += np_per_level[lev];
-
-
- 753 std::string pdir = dir;
- 754 if ( ! pdir.empty() && pdir[pdir.size()-1] !=
'/') { pdir +=
'/'; }
-
-
- 757 if (
MyProc == IOProcNumber)
-
- 759 if ( ! pc.GetLevelDirectoriesCreated())
-
-
-
-
-
-
-
- 767 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
- 769 std::string LevelDir = pdir;
- 770 bool gotsome = np_per_level[lev];
-
-
-
- 774 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] !=
'/') { LevelDir +=
'/'; }
-
-
-
- 778 if ( ! pc.GetLevelDirectoriesCreated())
-
-
-
-
-
-
-
- 786 std::string HeaderFileName = LevelDir;
- 787 HeaderFileName +=
"/Particle_H";
- 788 std::ofstream ParticleHeader(HeaderFileName);
-
- 790 pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
- 791 ParticleHeader <<
'\n';
-
- 793 ParticleHeader.flush();
- 794 ParticleHeader.close();
-
-
-
-
-
- 800 Long maxnextid = PC::ParticleType::NextID();
-
-
- 803 Vector<Long> np_on_rank(
NProcs, 0L);
- 804 std::size_t psize = particle_detail::PSizeInFile<ParticleReal>(write_real_comp, write_int_comp);
- 805 Vector<int64_t> rank_start_offset(
NProcs);
- 806 if (
MyProc == IOProcNumber)
-
- 808 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
- 810 for (
int k = 0; k < pc.ParticleBoxArray(lev).
size(); ++k)
-
- 812 int rank = pc.ParticleDistributionMap(lev)[k];
- 813 np_on_rank[rank] += np_per_grid_global[lev][k];
-
-
-
- 817 for (
int ip = 0; ip <
NProcs; ++ip)
-
-
- 820 rank_start_offset[ip] = (info.ispot == 0) ? 0 :
static_cast<int64_t
>(rank_start_offset[ip-1] + np_on_rank[ip-1]*psize);
-
-
-
-
- 825 using PinnedPTile = ParticleTile<
typename PC::ParticleType, NArrayReal, NArrayInt,
- 826 PinnedArenaAllocator>;
- 827 auto myptiles = std::make_shared<Vector<std::map<std::pair<int, int>,PinnedPTile> > >();
- 828 myptiles->resize(pc.finestLevel()+1);
- 829 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
- 831 for (MFIter mfi = pc.MakeMFIter(lev); mfi.isValid(); ++mfi)
-
- 833 auto& new_ptile = (*myptiles)[lev][std::make_pair(mfi.index(),
- 834 mfi.LocalTileIndex())];
-
- 836 if (np_per_grid_local[lev][mfi.index()] > 0)
-
- 838 const auto& ptile = pc.ParticlesAt(lev, mfi);
-
- 840 const auto np = np_per_grid_local[lev][mfi.index()];
-
- 842 new_ptile.resize(np);
-
- 844 const auto runtime_real_comps = ptile.NumRuntimeRealComps();
- 845 const auto runtime_int_comps = ptile.NumRuntimeIntComps();
-
- 847 new_ptile.define(runtime_real_comps, runtime_int_comps);
-
- 849 for (
auto comp(0); comp < runtime_real_comps; ++comp) {
- 850 new_ptile.push_back_real(NArrayReal+comp, np, 0.);
-
-
- 853 for (
auto comp(0); comp < runtime_int_comps; ++comp) {
- 854 new_ptile.push_back_int(NArrayInt+comp, np, 0);
-
-
-
-
-
-
-
- 862 int finest_level = pc.finestLevel();
- 863 Vector<BoxArray> bas;
- 864 Vector<DistributionMapping> dms;
- 865 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
-
- 867 bas.push_back(pc.ParticleBoxArray(lev));
- 868 dms.push_back(pc.ParticleDistributionMap(lev));
-
-
- 871 int nrc = pc.NumRealComps();
- 872 int nic = pc.NumIntComps();
- 873 int rnames_size = (
int) real_comp_names.size();
-
- 875 auto RD = pc.ParticleRealDescriptor;
-
-
- 878 #
if defined(__GNUC__) && (__GNUC__ == 8) && (__GNUC_MINOR__ == 1)
-
-
-
- 882 if (
MyProc == IOProcNumber)
-
- 884 std::string HdrFileName = pdir;
- 885 std::ofstream HdrFile;
-
- 887 if ( ! HdrFileName.empty() && HdrFileName[HdrFileName.size()-1] !=
'/') {
-
-
-
- 891 HdrFileName +=
"Header";
-
- 893 HdrFile.open(HdrFileName.c_str(), std::ios::out|std::ios::trunc);
-
- 895 if ( ! HdrFile.good()) { amrex::FileOpenFailed(HdrFileName); }
-
- 897 std::string version_string = is_checkpoint ? PC::CheckpointVersion() : PC::PlotfileVersion();
- 898 if (
sizeof(
typename PC::ParticleType::RealType) == 4)
-
- 900 HdrFile << version_string <<
"_single" <<
'\n';
-
-
-
- 904 HdrFile << version_string <<
"_double" <<
'\n';
-
-
- 907 int num_output_real = 0;
- 908 for (
int i = 0; i < rnames_size; ++i) {
- 909 if (write_real_comp[i]) { ++num_output_real; }
-
-
- 912 int num_output_int = 0;
- 913 for (
int i = 0; i < nic + NStructInt; ++i) {
- 914 if (write_int_comp[i]) { ++num_output_int; }
-
-
-
- 918 HdrFile << AMREX_SPACEDIM <<
'\n';
-
-
- 921 HdrFile << num_output_real <<
'\n';
-
-
- 924 for (
int i = 0; i < rnames_size; ++i ) {
- 925 if (write_real_comp[i]) { HdrFile << real_comp_names[i] <<
'\n'; }
-
-
-
- 929 HdrFile << num_output_int <<
'\n';
-
-
- 932 for (
int i = 0; i < NStructInt + nic; ++i ) {
- 933 if (write_int_comp[i]) { HdrFile << int_comp_names[i] <<
'\n'; }
-
-
- 936 bool is_checkpoint_legacy =
true;
- 937 HdrFile << is_checkpoint_legacy <<
'\n';
-
-
- 940 HdrFile << total_np <<
'\n';
-
-
- 943 HdrFile << maxnextid <<
'\n';
-
-
- 946 HdrFile << finest_level <<
'\n';
-
-
- 949 for (
int lev = 0; lev <= finest_level; lev++) {
- 950 HdrFile << dms[lev].size() <<
'\n';
-
-
- 953 for (
int lev = 0; lev <= finest_level; lev++)
-
- 955 Vector<int64_t> grid_offset(
NProcs, 0);
- 956 for (
int k = 0; k < bas[lev].size(); ++k)
-
- 958 int rank = dms[lev][k];
-
- 960 HdrFile << info.ifile <<
' '
- 961 << np_per_grid_global[lev][k] <<
' '
- 962 << grid_offset[rank] + rank_start_offset[rank] <<
'\n';
- 963 grid_offset[rank] +=
static_cast<int64_t
>(np_per_grid_global[lev][k]*psize);
-
-
-
-
-
- 969 if ( ! HdrFile.good())
-
- 971 amrex::Abort(
"ParticleContainer::Checkpoint(): problem writing HdrFile");
-
-
-
-
-
- 977 for (
int lev = 0; lev <= finest_level; lev++)
-
-
- 980 std::map<int, Vector<int> > tile_map;
-
- 982 for (
const auto& kv : (*myptiles)[lev])
-
- 984 const int grid = kv.first.first;
- 985 const int tile = kv.first.second;
- 986 tile_map[grid].push_back(tile);
-
-
- 989 std::string LevelDir = pdir;
- 990 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] !=
'/') { LevelDir +=
'/'; }
-
- 992 std::string filePrefix(LevelDir);
-
- 994 filePrefix += PC::DataPrefix();
-
-
-
- 998 ofs.open(file_name.c_str(), (info.ispot == 0) ? (std::ios::binary | std::ios::trunc)
- 999 : (std::ios::binary | std::ios::app));
-
- 1001 for (
int k = 0; k < bas[lev].size(); ++k)
-
- 1003 int rank = dms[lev][k];
- 1004 if (rank !=
MyProc) {
continue; }
-
- 1006 if (np_per_grid_local[lev][grid] == 0) {
continue; }
-
-
- 1009 int num_output_int = 0;
- 1010 for (
int i = 0; i < nic + NStructInt; ++i) {
- 1011 if (write_int_comp[i]) { ++num_output_int; }
-
-
- 1014 const Long iChunkSize = 2 + num_output_int;
- 1015 Vector<int> istuff(np_per_grid_local[lev][grid]*iChunkSize);
- 1016 int* iptr = istuff.dataPtr();
-
- 1018 for (
unsigned i = 0; i < tile_map[grid].size(); i++) {
- 1019 auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
- 1020 const auto& pbox = (*myptiles)[lev][ptile_index];
- 1021 const auto ptd = pbox.getConstParticleTileData();
- 1022 for (
int pindex = 0; pindex < pbox.numParticles(); ++pindex)
-
- 1024 const auto& soa = pbox.GetStructOfArrays();
-
-
- 1027 if (p.id() <= 0) {
continue; }
-
-
-
- 1031 if constexpr(!PC::ParticleType::is_soa_particle)
-
-
-
-
-
-
- 1038 for (
int j = 0; j < NStructInt; j++)
-
- 1040 if (write_int_comp[j])
-
-
-
-
-
-
-
- 1048 uint64_t idcpu = soa.GetIdCPUData()[pindex];
- 1049 if (is_checkpoint) {
- 1050 std::int32_t xi, yi;
- 1051 std::uint32_t xu, yu;
- 1052 xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32);
- 1053 yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL);
-
-
-
-
-
-
-
-
- 1062 *iptr = (
int) ParticleIDWrapper(idcpu);
-
- 1064 *iptr = (
int) ParticleCPUWrapper(idcpu);
-
-
-
-
-
- 1070 const int int_start_offset = 0;
- 1071 for (
int j = int_start_offset; j < nic; j++)
-
- 1073 if (write_int_comp[NStructInt+j])
-
- 1075 *iptr = soa.GetIntData(j)[pindex];
-
-
-
-
-
-
-
-
-
-
- 1086 int num_output_real = 0;
- 1087 for (
int i = 0; i < rnames_size; ++i) {
- 1088 if (write_real_comp[i]) { ++num_output_real; }
-
-
- 1091 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
- 1092 Vector<typename PC::ParticleType::RealType> rstuff(np_per_grid_local[lev][grid]*rChunkSize);
- 1093 typename PC::ParticleType::RealType* rptr = rstuff.dataPtr();
-
- 1095 for (
unsigned i = 0; i < tile_map[grid].size(); i++) {
- 1096 auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
- 1097 const auto& pbox = (*myptiles)[lev][ptile_index];
- 1098 const auto ptd = pbox.getConstParticleTileData();
- 1099 for (
int pindex = 0;
- 1100 pindex < pbox.numParticles(); ++pindex)
-
- 1102 const auto& soa = pbox.GetStructOfArrays();
-
-
- 1105 if (p.id() <= 0) {
continue; }
-
- 1107 if constexpr(!PC::ParticleType::is_soa_particle)
-
-
- 1110 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = p.pos(j); }
- 1111 rptr += AMREX_SPACEDIM;
-
-
- 1114 for (
int j = 0; j < NStructReal; j++)
-
- 1116 if (write_real_comp[j])
-
-
-
-
-
-
-
-
- 1125 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = soa.GetRealData(j)[pindex]; }
- 1126 rptr += AMREX_SPACEDIM;
-
-
-
- 1130 const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
- 1131 for (
int j = real_start_offset; j < nrc; j++)
-
- 1133 const int write_comp_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
- 1134 const int write_comp_index = PC::NStructReal+j-write_comp_offset;
- 1135 if (write_real_comp[write_comp_index])
-
- 1137 *rptr = (
typename PC::ParticleType::RealType) soa.GetRealData(j)[pindex];
-
-
-
-
-
-
- 1144 if (
sizeof(
typename PC::ParticleType::RealType) == 4) {
- 1145 writeFloatData((
float*) rstuff.dataPtr(), rstuff.size(), ofs, RD);
-
- 1147 else if (
sizeof(
typename PC::ParticleType::RealType) == 8) {
-
-
-
-
-
-
-
-
-
-
- 1158 #ifdef AMREX_USE_HDF5
-
-
-
-
+
+
+ 391 template <class PC, class F, std::enable_if_t<IsParticleContainer<PC>::value,
int> foo = 0>
+
+ 393 const std::string& dir,
const std::string& name,
+ 394 const Vector<int>& write_real_comp,
+ 395 const Vector<int>& write_int_comp,
+ 396 const Vector<std::string>& real_comp_names,
+ 397 const Vector<std::string>& int_comp_names,
+ 398 F
const&
f,
bool is_checkpoint)
+
+
+
+
+ 403 AMREX_ASSERT(
sizeof(
typename PC::ParticleType::RealType) == 4 ||
+ 404 sizeof(
typename PC::ParticleType::RealType) == 8);
+
+ 406 constexpr
int NStructReal = PC::NStructReal;
+ 407 constexpr
int NStructInt = PC::NStructInt;
+
+
+
+
+ 412 if constexpr(PC::ParticleType::is_soa_particle) {
+ 413 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal - AMREX_SPACEDIM);
+
+
+
+
+
+ 419 std::string pdir = dir;
+ 420 if ( ! pdir.empty() && pdir[pdir.size()-1] !=
'/') { pdir +=
'/'; }
+
+
+ 423 if ( ! pc.GetLevelDirectoriesCreated()) {
+
+
+
+
+
+
+
+
+
+
+ 434 std::ofstream HdrFile;
+
+
+
+
+
+ 440 Vector<std::map<std::pair<int, int>,
typename PC::IntVector > >
+ 441 particle_io_flags(pc.GetParticles().size());
+ 442 for (
int lev = 0; lev < pc.GetParticles().
size(); lev++)
+
+ 444 const auto& pmap = pc.GetParticles(lev);
+ 445 for (
const auto& kv : pmap)
+
+ 447 auto& flags = particle_io_flags[lev][kv.first];
+
+
+
+
+
+
+ 454 if(pc.GetUsePrePost())
+
+ 456 nparticles = pc.GetNParticlesPrePost();
+ 457 maxnextid = pc.GetMaxNextIDPrePost();
+
+
+
+
+ 462 maxnextid = PC::ParticleType::NextID();
+
+ 464 PC::ParticleType::NextID(maxnextid);
+
+
+
+
+
+ 470 std::string HdrFileName = pdir;
+
+ 472 if ( ! HdrFileName.empty() && HdrFileName[HdrFileName.size()-1] !=
'/') {
+
+
+
+ 476 HdrFileName +=
"Header";
+ 477 pc.HdrFileNamePrePost = HdrFileName;
+
+ 479 HdrFile.open(HdrFileName.c_str(), std::ios::out|std::ios::trunc);
+
+
+
+
+
+
+
+
+ 488 std::string version_string = is_checkpoint ? PC::CheckpointVersion() : PC::PlotfileVersion();
+ 489 if (
sizeof(
typename PC::ParticleType::RealType) == 4)
+
+ 491 HdrFile << version_string <<
"_single" <<
'\n';
+
+
+
+ 495 HdrFile << version_string <<
"_double" <<
'\n';
+
+
+ 498 int num_output_real = 0;
+ 499 for (
int i : write_real_comp) {
+ 500 if (i) { ++num_output_real; }
+
+
+ 503 int num_output_int = 0;
+ 504 for (
int i = 0; i < pc.NumIntComps() + NStructInt; ++i) {
+ 505 if (write_int_comp[i]) { ++num_output_int; }
+
+
+
+ 509 HdrFile << AMREX_SPACEDIM <<
'\n';
+
+
+ 512 HdrFile << num_output_real <<
'\n';
+
+
+ 515 for (
int i = 0; i < (
int) real_comp_names.size(); ++i ) {
+ 516 if (write_real_comp[i]) { HdrFile << real_comp_names[i] <<
'\n'; }
+
+
+
+ 520 HdrFile << num_output_int <<
'\n';
+
+
+ 523 for (
int i = 0; i < NStructInt + pc.NumIntComps(); ++i ) {
+ 524 if (write_int_comp[i]) { HdrFile << int_comp_names[i] <<
'\n'; }
+
+
+ 527 bool is_checkpoint_legacy =
true;
+ 528 HdrFile << is_checkpoint_legacy <<
'\n';
+
+
+ 531 HdrFile << nparticles <<
'\n';
+
+
+ 534 HdrFile << maxnextid <<
'\n';
+
+
+ 537 HdrFile << pc.finestLevel() <<
'\n';
+
+
+ 540 for (
int lev = 0; lev <= pc.finestLevel(); lev++) {
+ 541 HdrFile << pc.ParticleBoxArray(lev).size() <<
'\n';
+
+
+
+
+
+
+
+ 549 ParmParse
pp(
"particles");
+
+ 551 if(nOutFiles == -1) { nOutFiles =
NProcs; }
+
+ 553 pc.nOutFilesPrePost = nOutFiles;
+
+ 555 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+
+
+
+ 560 gotsome = (pc.nParticlesAtLevelPrePost[lev] > 0);
+
+
+
+ 564 gotsome = (pc.NumberOfParticlesAtLevel(lev) > 0);
+
+
+
+ 568 std::string LevelDir = pdir;
+
+
+
+ 572 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] !=
'/') { LevelDir +=
'/'; }
+
+
+
+ 576 if ( ! pc.GetLevelDirectoriesCreated())
+
+
+
+
+
+
+
+
+
+
+
+
+ 589 std::string HeaderFileName = LevelDir;
+ 590 HeaderFileName +=
"/Particle_H";
+ 591 std::ofstream ParticleHeader(HeaderFileName);
+
+ 593 pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
+ 594 ParticleHeader <<
'\n';
+
+ 596 ParticleHeader.flush();
+ 597 ParticleHeader.close();
+
+
+
+ 601 info.SetAlloc(
false);
+ 602 MultiFab state(pc.ParticleBoxArray(lev),
+ 603 pc.ParticleDistributionMap(lev),
+
+
+
+
+ 608 Vector<int> which(state.size(),0);
+ 609 Vector<int > count(state.size(),0);
+ 610 Vector<Long> where(state.size(),0);
+
+ 612 std::string filePrefix(LevelDir);
+
+ 614 filePrefix += PC::DataPrefix();
+
+ 616 pc.filePrefixPrePost[lev] = filePrefix;
+
+ 618 bool groupSets(
false), setBuf(
true);
+
+
+
+ 622 for(NFilesIter nfi(nOutFiles, filePrefix, groupSets, setBuf); nfi.ReadyToWrite(); ++nfi)
+
+ 624 auto& myStream = (std::ofstream&) nfi.Stream();
+ 625 pc.WriteParticles(lev, myStream, nfi.FileNumber(), which, count, where,
+ 626 write_real_comp, write_int_comp, particle_io_flags, is_checkpoint);
+
+
+
+ 630 pc.whichPrePost[lev] = which;
+ 631 pc.countPrePost[lev] = count;
+ 632 pc.wherePrePost[lev] = where;
+
+
+
+
+
+
+
+
+
+ 642 if(pc.GetUsePrePost()) {
+
+
+ 645 for (
int j = 0; j < state.size(); j++)
+
+ 647 HdrFile << which[j] <<
' ' << count[j] <<
' ' << where[j] <<
'\n';
+
+
+ 650 if (gotsome && pc.doUnlink)
+
+
+ 653 Vector<Long> cnt(nOutFiles,0);
+
+ 655 for (
int i = 0, N=
static_cast<int>(count.size()); i < N; i++) {
+ 656 cnt[which[i]] += count[i];
+
+
+ 659 for (
int i = 0, N=
static_cast<int>(cnt.size()); i < N; i++)
+
+
+
+ 663 std::string FullFileName = NFilesIter::FileName(i, filePrefix);
+
+
+
+
+
+
+
+
+
+
+
+
+ 676 if ( ! HdrFile.good())
+
+ 678 amrex::Abort(
"ParticleContainer::Checkpoint(): problem writing HdrFile");
+
+
+
+
+ 683 template <class PC, std::enable_if_t<IsParticleContainer<PC>::value,
int> foo = 0>
+
+ 685 const std::string& dir,
const std::string& name,
+ 686 const Vector<int>& write_real_comp,
+ 687 const Vector<int>& write_int_comp,
+ 688 const Vector<std::string>& real_comp_names,
+ 689 const Vector<std::string>& int_comp_names,
bool is_checkpoint)
+
+
+
+
+ 694 AMREX_ASSERT(
sizeof(
typename PC::ParticleType::RealType) == 4 ||
+ 695 sizeof(
typename PC::ParticleType::RealType) == 8);
+
+ 697 constexpr
int NStructReal = PC::NStructReal;
+ 698 constexpr
int NStructInt = PC::NStructInt;
+ 699 constexpr
int NArrayReal = PC::NArrayReal;
+ 700 constexpr
int NArrayInt = PC::NArrayInt;
+
+
+
+ 704 const int IOProcNumber =
NProcs - 1;
+
+ 706 if constexpr(PC::ParticleType::is_soa_particle) {
+ 707 AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc.NumRealComps() + NStructReal - AMREX_SPACEDIM);
+
+
+
+
+
+ 713 Vector<LayoutData<Long> > np_per_grid_local(pc.finestLevel()+1);
+ 714 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+ 716 np_per_grid_local[lev].define(pc.ParticleBoxArray(lev), pc.ParticleDistributionMap(lev));
+ 717 using ParIter =
typename PC::ParConstIterType;
+ 718 for (
ParIter pti(pc, lev); pti.isValid(); ++pti)
+
+ 720 int gid = pti.index();
+ 721 const auto& ptile = pc.ParticlesAt(lev, pti);
+ 722 const auto& ptd = ptile.getConstParticleTileData();
+ 723 const int np = ptile.numParticles();
+
+ 725 ReduceOps<ReduceOpSum> reduce_op;
+ 726 ReduceData<int> reduce_data(reduce_op);
+ 727 using ReduceTuple =
typename decltype(reduce_data)::Type;
+
+ 729 reduce_op.eval(np, reduce_data,
+
+
+ 732 return (ptd.id(i) > 0) ? 1 : 0;
+
+
+ 735 int np_valid = amrex::get<0>(reduce_data.value(reduce_op));
+ 736 np_per_grid_local[lev][gid] += np_valid;
+
+
+
+ 740 Vector<Vector<Long> > np_per_grid_global(pc.finestLevel()+1);
+
+ 742 Vector<Long> np_per_level(pc.finestLevel()+1);
+ 743 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+ 745 np_per_grid_global[lev].resize(np_per_grid_local[lev].
size());
+
+ 747 np_per_grid_global[lev],
+
+ 749 np_per_level[lev] = std::accumulate(np_per_grid_global[lev].
begin(),
+ 750 np_per_grid_global[lev].
end(), 0L);
+ 751 total_np += np_per_level[lev];
+
+
+ 754 std::string pdir = dir;
+ 755 if ( ! pdir.empty() && pdir[pdir.size()-1] !=
'/') { pdir +=
'/'; }
+
+
+ 758 if (
MyProc == IOProcNumber)
+
+ 760 if ( ! pc.GetLevelDirectoriesCreated())
+
+
+
+
+
+
+
+ 768 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+ 770 std::string LevelDir = pdir;
+ 771 bool gotsome = np_per_level[lev];
+
+
+
+ 775 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] !=
'/') { LevelDir +=
'/'; }
+
+
+
+ 779 if ( ! pc.GetLevelDirectoriesCreated())
+
+
+
+
+
+
+
+ 787 std::string HeaderFileName = LevelDir;
+ 788 HeaderFileName +=
"/Particle_H";
+ 789 std::ofstream ParticleHeader(HeaderFileName);
+
+ 791 pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
+ 792 ParticleHeader <<
'\n';
+
+ 794 ParticleHeader.flush();
+ 795 ParticleHeader.close();
+
+
+
+
+
+ 801 Long maxnextid = PC::ParticleType::NextID();
+
+
+ 804 Vector<Long> np_on_rank(
NProcs, 0L);
+ 805 std::size_t psize = particle_detail::PSizeInFile<ParticleReal>(write_real_comp, write_int_comp);
+ 806 Vector<int64_t> rank_start_offset(
NProcs);
+ 807 if (
MyProc == IOProcNumber)
+
+ 809 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+ 811 for (
int k = 0; k < pc.ParticleBoxArray(lev).
size(); ++k)
+
+ 813 int rank = pc.ParticleDistributionMap(lev)[k];
+ 814 np_on_rank[rank] += np_per_grid_global[lev][k];
+
+
+
+ 818 for (
int ip = 0; ip <
NProcs; ++ip)
+
+
+ 821 rank_start_offset[ip] = (info.ispot == 0) ? 0 :
static_cast<int64_t
>(rank_start_offset[ip-1] + np_on_rank[ip-1]*psize);
+
+
+
+
+ 826 using PinnedPTile = ParticleTile<
typename PC::ParticleType, NArrayReal, NArrayInt,
+ 827 PinnedArenaAllocator>;
+ 828 auto myptiles = std::make_shared<Vector<std::map<std::pair<int, int>,PinnedPTile> > >();
+ 829 myptiles->resize(pc.finestLevel()+1);
+ 830 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+ 832 for (MFIter mfi = pc.MakeMFIter(lev); mfi.isValid(); ++mfi)
+
+ 834 auto& new_ptile = (*myptiles)[lev][std::make_pair(mfi.index(),
+ 835 mfi.LocalTileIndex())];
+
+ 837 if (np_per_grid_local[lev][mfi.index()] > 0)
+
+ 839 const auto& ptile = pc.ParticlesAt(lev, mfi);
+
+ 841 const auto np = np_per_grid_local[lev][mfi.index()];
+
+ 843 new_ptile.resize(np);
+
+ 845 const auto runtime_real_comps = ptile.NumRuntimeRealComps();
+ 846 const auto runtime_int_comps = ptile.NumRuntimeIntComps();
+
+ 848 new_ptile.define(runtime_real_comps, runtime_int_comps);
+
+ 850 for (
auto comp(0); comp < runtime_real_comps; ++comp) {
+ 851 new_ptile.push_back_real(NArrayReal+comp, np, 0.);
+
+
+ 854 for (
auto comp(0); comp < runtime_int_comps; ++comp) {
+ 855 new_ptile.push_back_int(NArrayInt+comp, np, 0);
+
+
+
+
+
+
+
+ 863 int finest_level = pc.finestLevel();
+ 864 Vector<BoxArray> bas;
+ 865 Vector<DistributionMapping> dms;
+ 866 for (
int lev = 0; lev <= pc.finestLevel(); lev++)
+
+ 868 bas.push_back(pc.ParticleBoxArray(lev));
+ 869 dms.push_back(pc.ParticleDistributionMap(lev));
+
+
+ 872 int nrc = pc.NumRealComps();
+ 873 int nic = pc.NumIntComps();
+ 874 int rnames_size = (
int) real_comp_names.size();
+
+ 876 auto RD = pc.ParticleRealDescriptor;
+
+
+ 879 #
if defined(__GNUC__) && (__GNUC__ == 8) && (__GNUC_MINOR__ == 1)
+
+
+
+ 883 if (
MyProc == IOProcNumber)
+
+ 885 std::string HdrFileName = pdir;
+ 886 std::ofstream HdrFile;
+
+ 888 if ( ! HdrFileName.empty() && HdrFileName[HdrFileName.size()-1] !=
'/') {
+
+
+
+ 892 HdrFileName +=
"Header";
+
+ 894 HdrFile.open(HdrFileName.c_str(), std::ios::out|std::ios::trunc);
+
+ 896 if ( ! HdrFile.good()) { amrex::FileOpenFailed(HdrFileName); }
+
+ 898 std::string version_string = is_checkpoint ? PC::CheckpointVersion() : PC::PlotfileVersion();
+ 899 if (
sizeof(
typename PC::ParticleType::RealType) == 4)
+
+ 901 HdrFile << version_string <<
"_single" <<
'\n';
+
+
+
+ 905 HdrFile << version_string <<
"_double" <<
'\n';
+
+
+ 908 int num_output_real = 0;
+ 909 for (
int i = 0; i < rnames_size; ++i) {
+ 910 if (write_real_comp[i]) { ++num_output_real; }
+
+
+ 913 int num_output_int = 0;
+ 914 for (
int i = 0; i < nic + NStructInt; ++i) {
+ 915 if (write_int_comp[i]) { ++num_output_int; }
+
+
+
+ 919 HdrFile << AMREX_SPACEDIM <<
'\n';
+
+
+ 922 HdrFile << num_output_real <<
'\n';
+
+
+ 925 for (
int i = 0; i < rnames_size; ++i ) {
+ 926 if (write_real_comp[i]) { HdrFile << real_comp_names[i] <<
'\n'; }
+
+
+
+ 930 HdrFile << num_output_int <<
'\n';
+
+
+ 933 for (
int i = 0; i < NStructInt + nic; ++i ) {
+ 934 if (write_int_comp[i]) { HdrFile << int_comp_names[i] <<
'\n'; }
+
+
+ 937 bool is_checkpoint_legacy =
true;
+ 938 HdrFile << is_checkpoint_legacy <<
'\n';
+
+
+ 941 HdrFile << total_np <<
'\n';
+
+
+ 944 HdrFile << maxnextid <<
'\n';
+
+
+ 947 HdrFile << finest_level <<
'\n';
+
+
+ 950 for (
int lev = 0; lev <= finest_level; lev++) {
+ 951 HdrFile << dms[lev].size() <<
'\n';
+
+
+ 954 for (
int lev = 0; lev <= finest_level; lev++)
+
+ 956 Vector<int64_t> grid_offset(
NProcs, 0);
+ 957 for (
int k = 0; k < bas[lev].size(); ++k)
+
+ 959 int rank = dms[lev][k];
+
+ 961 HdrFile << info.ifile <<
' '
+ 962 << np_per_grid_global[lev][k] <<
' '
+ 963 << grid_offset[rank] + rank_start_offset[rank] <<
'\n';
+ 964 grid_offset[rank] +=
static_cast<int64_t
>(np_per_grid_global[lev][k]*psize);
+
+
+
+
+
+ 970 if ( ! HdrFile.good())
+
+ 972 amrex::Abort(
"ParticleContainer::Checkpoint(): problem writing HdrFile");
+
+
+
+
+
+ 978 for (
int lev = 0; lev <= finest_level; lev++)
+
+
+ 981 std::map<int, Vector<int> > tile_map;
+
+ 983 for (
const auto& kv : (*myptiles)[lev])
+
+ 985 const int grid = kv.first.first;
+ 986 const int tile = kv.first.second;
+ 987 tile_map[grid].push_back(tile);
+
+
+ 990 std::string LevelDir = pdir;
+ 991 if ( ! LevelDir.empty() && LevelDir[LevelDir.size()-1] !=
'/') { LevelDir +=
'/'; }
+
+ 993 std::string filePrefix(LevelDir);
+
+ 995 filePrefix += PC::DataPrefix();
+
+
+
+ 999 ofs.open(file_name.c_str(), (info.ispot == 0) ? (std::ios::binary | std::ios::trunc)
+ 1000 : (std::ios::binary | std::ios::app));
+
+ 1002 for (
int k = 0; k < bas[lev].size(); ++k)
+
+ 1004 int rank = dms[lev][k];
+ 1005 if (rank !=
MyProc) {
continue; }
+
+ 1007 if (np_per_grid_local[lev][grid] == 0) {
continue; }
+
+
+ 1010 int num_output_int = 0;
+ 1011 for (
int i = 0; i < nic + NStructInt; ++i) {
+ 1012 if (write_int_comp[i]) { ++num_output_int; }
+
+
+ 1015 const Long iChunkSize = 2 + num_output_int;
+ 1016 Vector<int> istuff(np_per_grid_local[lev][grid]*iChunkSize);
+ 1017 int* iptr = istuff.dataPtr();
+
+ 1019 for (
unsigned i = 0; i < tile_map[grid].size(); i++) {
+ 1020 auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
+ 1021 const auto& pbox = (*myptiles)[lev][ptile_index];
+ 1022 const auto ptd = pbox.getConstParticleTileData();
+ 1023 for (
int pindex = 0; pindex < pbox.numParticles(); ++pindex)
+
+ 1025 const auto& soa = pbox.GetStructOfArrays();
+
+
+ 1028 if (p.id() <= 0) {
continue; }
+
+
+
+ 1032 if constexpr(!PC::ParticleType::is_soa_particle)
+
+
+
+
+
+
+ 1039 for (
int j = 0; j < NStructInt; j++)
+
+ 1041 if (write_int_comp[j])
+
+
+
+
+
+
+
+ 1049 uint64_t idcpu = soa.GetIdCPUData()[pindex];
+ 1050 if (is_checkpoint) {
+ 1051 std::int32_t xi, yi;
+ 1052 std::uint32_t xu, yu;
+ 1053 xu = (std::uint32_t)((idcpu & 0xFFFFFFFF00000000LL) >> 32);
+ 1054 yu = (std::uint32_t)( idcpu & 0xFFFFFFFFLL);
+
+
+
+
+
+
+
+
+ 1063 *iptr = (
int) ParticleIDWrapper(idcpu);
+
+ 1065 *iptr = (
int) ParticleCPUWrapper(idcpu);
+
+
+
+
+
+ 1071 const int int_start_offset = 0;
+ 1072 for (
int j = int_start_offset; j < nic; j++)
+
+ 1074 if (write_int_comp[NStructInt+j])
+
+ 1076 *iptr = soa.GetIntData(j)[pindex];
+
+
+
+
+
+
+
+
+
+
+ 1087 int num_output_real = 0;
+ 1088 for (
int i = 0; i < rnames_size; ++i) {
+ 1089 if (write_real_comp[i]) { ++num_output_real; }
+
+
+ 1092 const Long rChunkSize = AMREX_SPACEDIM + num_output_real;
+ 1093 Vector<typename PC::ParticleType::RealType> rstuff(np_per_grid_local[lev][grid]*rChunkSize);
+ 1094 typename PC::ParticleType::RealType* rptr = rstuff.dataPtr();
+
+ 1096 for (
unsigned i = 0; i < tile_map[grid].size(); i++) {
+ 1097 auto ptile_index = std::make_pair(grid, tile_map[grid][i]);
+ 1098 const auto& pbox = (*myptiles)[lev][ptile_index];
+ 1099 const auto ptd = pbox.getConstParticleTileData();
+ 1100 for (
int pindex = 0;
+ 1101 pindex < pbox.numParticles(); ++pindex)
+
+ 1103 const auto& soa = pbox.GetStructOfArrays();
+
+
+ 1106 if (p.id() <= 0) {
continue; }
+
+ 1108 if constexpr(!PC::ParticleType::is_soa_particle)
+
+
+ 1111 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = p.pos(j); }
+ 1112 rptr += AMREX_SPACEDIM;
+
+
+ 1115 for (
int j = 0; j < NStructReal; j++)
+
+ 1117 if (write_real_comp[j])
+
+
+
+
+
+
+
+
+ 1126 for (
int j = 0; j < AMREX_SPACEDIM; j++) { rptr[j] = soa.GetRealData(j)[pindex]; }
+ 1127 rptr += AMREX_SPACEDIM;
+
+
+
+ 1131 const int real_start_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+ 1132 for (
int j = real_start_offset; j < nrc; j++)
+
+ 1134 const int write_comp_offset = PC::ParticleType::is_soa_particle ? AMREX_SPACEDIM : 0;
+ 1135 const int write_comp_index = PC::NStructReal+j-write_comp_offset;
+ 1136 if (write_real_comp[write_comp_index])
+
+ 1138 *rptr = (
typename PC::ParticleType::RealType) soa.GetRealData(j)[pindex];
+
+
+
+
+
+
+ 1145 if (
sizeof(
typename PC::ParticleType::RealType) == 4) {
+ 1146 writeFloatData((
float*) rstuff.dataPtr(), rstuff.size(), ofs, RD);
+
+ 1148 else if (
sizeof(
typename PC::ParticleType::RealType) == 8) {
+
+
+
+
+
+
+
+
+
+
+ 1159 #ifdef AMREX_USE_HDF5
+
+
+
+
#define BL_PROFILE(a)
Definition: AMReX_BLProfiler.H:558
#define AMREX_ASSERT(EX)
Definition: AMReX_BLassert.H:38
#define AMREX_ALWAYS_ASSERT(EX)
Definition: AMReX_BLassert.H:50
@@ -1271,8 +1272,8 @@
-void WriteBinaryParticleDataAsync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:683
-void WriteBinaryParticleDataSync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, F const &f, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:391
+void WriteBinaryParticleDataAsync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:684
+void WriteBinaryParticleDataSync(PC const &pc, const std::string &dir, const std::string &name, const Vector< int > &write_real_comp, const Vector< int > &write_int_comp, const Vector< std::string > &real_comp_names, const Vector< std::string > &int_comp_names, F const &f, bool is_checkpoint)
Definition: AMReX_WriteBinaryParticleData.H:392
if(!(yy_init))
Definition: amrex_iparser.lex.nolint.H:924
int queryAdd(const char *name, T &ref)
If name is found, the value in the ParmParse database will be stored in the ref argument....
Definition: AMReX_ParmParse.H:954
WriteInfo GetWriteInfo(int rank)
Definition: AMReX_AsyncOut.cpp:72
diff --git a/amrex/docs_xml/doxygen/AMReX__WriteBinaryParticleData_8H.xml b/amrex/docs_xml/doxygen/AMReX__WriteBinaryParticleData_8H.xml
index 05c61a4458..fdbce5e4ca 100644
--- a/amrex/docs_xml/doxygen/AMReX__WriteBinaryParticleData_8H.xml
+++ b/amrex/docs_xml/doxygen/AMReX__WriteBinaryParticleData_8H.xml
@@ -2284,7 +2284,7 @@
-
+
@@ -2340,7 +2340,7 @@
-
+
@@ -2539,977 +2539,978 @@
constLongrChunkSize=AMREX_SPACEDIM+num_output_real;
rdata.resize(np*rChunkSize);
-typenamePC::IntVectoridata_d(idata.size());
-typenamePC::RealVectorrdata_d(rdata.size());
-
-typenamePC::IntVectorwrite_int_comp_d(write_int_comp.size());
-typenamePC::IntVectorwrite_real_comp_d(write_real_comp.size());
-[Gpu::copyAsync]([Gpu::hostToDevice],write_int_comp.begin(),write_int_comp.end(),
-write_int_comp_d.begin());
-[Gpu::copyAsync]([Gpu::hostToDevice],write_real_comp.begin(),write_real_comp.end(),
-write_real_comp_d.begin());
-[Gpu::Device::streamSynchronize]();
-
-constautowrite_int_comp_d_ptr=write_int_comp_d.data();
-constautowrite_real_comp_d_ptr=write_real_comp_d.data();
-
-std::size_tpoffset=0;
-for(inttile:tiles){
-constauto&ptile=pc.ParticlesAt(lev,grid,tile);
-constauto&pflags=particle_io_flags[lev].at(std::make_pair(grid,tile));
-intnp_tile=ptile.numParticles();
-typenamePC::IntVectoroffsets(np_tile);
-intnum_copies=[Scan::ExclusiveSum](np_tile,pflags.begin(),offsets.begin(),[Scan::retSum]);
-
-constautoflag_ptr=pflags.data();
-
-autoidata_d_ptr=idata_d.data();
-autordata_d_ptr=rdata_d.data();
-
-constautoptd=ptile.getConstParticleTileData();
-[amrex::ParallelFor](num_copies,
-[=][AMREX_GPU_DEVICE](intpindex)noexcept
-{
-
-constautop=ptd.getSuperParticle(pindex);
-
-if(flag_ptr[pindex]){
-std::size_tiout_index=(pindex+poffset)*iChunkSize;
-[packParticleIDs](&idata_d_ptr[iout_index],p,is_checkpoint);
-iout_index+=2;
-
-std::size_trout_index=(pindex+poffset)*rChunkSize;
-for(intj=0;j<AMREX_SPACEDIM;j++){
-rdata_d_ptr[rout_index]=p.pos(j);
-rout_index++;
-}
-
-for(intj=0;j<PC::SuperParticleType::NInt;j++){
-if(write_int_comp_d_ptr[j]){
-idata_d_ptr[iout_index]=p.idata(j);
-iout_index++;
-}
-}
-
-for(intj=0;j<ptd.m_num_runtime_int;j++){
-if(write_int_comp_d_ptr[PC::SuperParticleType::NInt+j]){
-idata_d_ptr[iout_index]=ptd.m_runtime_idata[j][pindex];
-iout_index++;
-}
-}
-
-
-constintreal_start_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
-for(intj=real_start_offset;j<PC::SuperParticleType::NReal;j++){
-constintwrite_comp_index=j-real_start_offset;
-if(write_real_comp_d_ptr[write_comp_index]){
-rdata_d_ptr[rout_index]=p.rdata(j);
-rout_index++;
-}
-}
-
-for(intj=0;j<ptd.m_num_runtime_real;j++){
-if(write_real_comp_d_ptr[PC::SuperParticleType::NReal+j-real_start_offset]){
-rdata_d_ptr[rout_index]=ptd.m_runtime_rdata[j][pindex];
-rout_index++;
-}
-}
-}
-});
-
-poffset+=num_copies;
-}
-
-[Gpu::copyAsync]([Gpu::deviceToHost],idata_d.begin(),idata_d.end(),idata.begin());
-[Gpu::copyAsync]([Gpu::deviceToHost],rdata_d.begin(),rdata_d.end(),rdata.begin());
-[Gpu::Device::streamSynchronize]();
-}
-
-template<classPC>
-std::enable_if_t<!RunOnGpu<typenamePC::templateAllocatorType<int>>::value>
-[packIOData](Vector<int>&idata,Vector<ParticleReal>&rdata,constPC&pc,intlev,intgrid,
-constVector<int>&write_real_comp,constVector<int>&write_int_comp,
-constVector<std::map<std::pair<int,int>,typenamePC::IntVector>>&particle_io_flags,
-constVector<int>&tiles,intnp,boolis_checkpoint)
-{
-intnum_output_int=0;
-for(inti=0;i<pc.NumIntComps()+PC::NStructInt;++i){
-if(write_int_comp[i]){++num_output_int;}
-}
-
-constLongiChunkSize=2+num_output_int;
-idata.resize(np*iChunkSize);
-
-intnum_output_real=0;
-for(inti:write_real_comp){
-if(i){++num_output_real;}
-}
-
-constLongrChunkSize=AMREX_SPACEDIM+num_output_real;
-rdata.resize(np*rChunkSize);
-
-int*iptr=idata.dataPtr();
-ParticleReal*rptr=rdata.dataPtr();
-for(inttile:tiles){
-constauto&ptile=pc.ParticlesAt(lev,grid,tile);
-constauto&pflags=particle_io_flags[lev].at(std::make_pair(grid,tile));
-for(intpindex=0;pindex<ptile.numParticles();++pindex){
-if(pflags[pindex]){
-constauto&soa=ptile.GetStructOfArrays();
-
-
-
-ifconstexpr(!PC::ParticleType::is_soa_particle)
-{
-constauto&aos=ptile.GetArrayOfStructs();
-constauto&p=aos[pindex];
-
-
-[packParticleIDs](iptr,p,is_checkpoint);
-iptr+=2;
-
-
-for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=p.pos(j);}
-rptr+=AMREX_SPACEDIM;
-
-
-for(intj=0;j<PC::NStructInt;j++){
-if(write_int_comp[j]){
-*iptr=p.idata(j);
-++iptr;
-}
-}
-
-for(intj=0;j<PC::NStructReal;j++){
-if(write_real_comp[j]){
-*rptr=p.rdata(j);
-++rptr;
-}
-}
-}
-else{
-uint64_tidcpu=soa.GetIdCPUData()[pindex];
-if(is_checkpoint){
-std::int32_txi,yi;
-std::uint32_txu,yu;
-xu=(std::uint32_t)((idcpu&0xFFFFFFFF00000000LL)>>32);
-yu=(std::uint32_t)(idcpu&0xFFFFFFFFLL);
-[std::memcpy](&xi,&xu,sizeof(xu));
-[std::memcpy](&yi,&yu,sizeof(yu));
-*iptr=xi;
-iptr+=1;
-*iptr=yi;
-iptr+=1;
-}else{
-
-*iptr=([int])ParticleIDWrapper(idcpu);
-iptr+=1;
-*iptr=([int])ParticleCPUWrapper(idcpu);
-iptr+=1;
-}
-
-
-for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=soa.GetRealData(j)[pindex];}
-rptr+=AMREX_SPACEDIM;
-}
-
-
-constintint_start_offset=0;
-for(intj=int_start_offset;j<pc.NumIntComps();j++){
-if(write_int_comp[PC::NStructInt+j]){
-*iptr=soa.GetIntData(j)[pindex];
-++iptr;
-}
-}
-
-
-constintreal_start_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
-for(intj=real_start_offset;j<pc.NumRealComps();j++){
-constintwrite_comp_index=PC::NStructReal+j-real_start_offset;
-if(write_real_comp[write_comp_index]){
-*rptr=(ParticleReal)soa.GetRealData(j)[pindex];
-++rptr;
-}
-}
-}
-}
-}
-}
+typenamePC::IntVectorwrite_int_comp_d(write_int_comp.size());
+typenamePC::IntVectorwrite_real_comp_d(write_real_comp.size());
+[Gpu::copyAsync]([Gpu::hostToDevice],write_int_comp.begin(),write_int_comp.end(),
+write_int_comp_d.begin());
+[Gpu::copyAsync]([Gpu::hostToDevice],write_real_comp.begin(),write_real_comp.end(),
+write_real_comp_d.begin());
+
+constautowrite_int_comp_d_ptr=write_int_comp_d.data();
+constautowrite_real_comp_d_ptr=write_real_comp_d.data();
+
+std::size_tpoffset=0;
+for(inttile:tiles){
+constauto&ptile=pc.ParticlesAt(lev,grid,tile);
+constauto&pflags=particle_io_flags[lev].at(std::make_pair(grid,tile));
+intnp_tile=ptile.numParticles();
+typenamePC::IntVectoroffsets(np_tile);
+intnum_copies=[Scan::ExclusiveSum](np_tile,pflags.begin(),offsets.begin(),[Scan::retSum]);
+
+typenamePC::IntVectoridata_d(num_copies*iChunkSize);
+typenamePC::RealVectorrdata_d(num_copies*rChunkSize);
+
+constautoflag_ptr=pflags.data();
+
+autoidata_d_ptr=idata_d.data();
+autordata_d_ptr=rdata_d.data();
+
+constautoptd=ptile.getConstParticleTileData();
+[amrex::ParallelFor](num_copies,
+[=][AMREX_GPU_DEVICE](intpindex)noexcept
+{
+
+constautop=ptd.getSuperParticle(pindex);
+
+if(flag_ptr[pindex]){
+std::size_tiout_index=pindex*iChunkSize;
+[packParticleIDs](&idata_d_ptr[iout_index],p,is_checkpoint);
+iout_index+=2;
+
+std::size_trout_index=pindex*rChunkSize;
+for(intj=0;j<AMREX_SPACEDIM;j++){
+rdata_d_ptr[rout_index]=p.pos(j);
+rout_index++;
+}
+
+for(intj=0;j<PC::SuperParticleType::NInt;j++){
+if(write_int_comp_d_ptr[j]){
+idata_d_ptr[iout_index]=p.idata(j);
+iout_index++;
+}
+}
+
+for(intj=0;j<ptd.m_num_runtime_int;j++){
+if(write_int_comp_d_ptr[PC::SuperParticleType::NInt+j]){
+idata_d_ptr[iout_index]=ptd.m_runtime_idata[j][pindex];
+iout_index++;
+}
+}
+
+
+constintreal_start_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
+for(intj=real_start_offset;j<PC::SuperParticleType::NReal;j++){
+constintwrite_comp_index=j-real_start_offset;
+if(write_real_comp_d_ptr[write_comp_index]){
+rdata_d_ptr[rout_index]=p.rdata(j);
+rout_index++;
+}
+}
+
+for(intj=0;j<ptd.m_num_runtime_real;j++){
+if(write_real_comp_d_ptr[PC::SuperParticleType::NReal+j-real_start_offset]){
+rdata_d_ptr[rout_index]=ptd.m_runtime_rdata[j][pindex];
+rout_index++;
+}
+}
+}
+});
+
+[Gpu::copyAsync]([Gpu::deviceToHost],idata_d.begin(),idata_d.end(),
+idata.begin()+typenamePC::IntVector::difference_type(poffset));
+[Gpu::copyAsync]([Gpu::deviceToHost],rdata_d.begin(),rdata_d.end(),
+rdata.begin()+typenamePC::RealVector::difference_type(poffset));
+[Gpu::Device::streamSynchronize]();
+
+poffset+=num_copies;
+}
+}
+
+template<classPC>
+std::enable_if_t<!RunOnGpu<typenamePC::templateAllocatorType<int>>::value>
+[packIOData](Vector<int>&idata,Vector<ParticleReal>&rdata,constPC&pc,intlev,intgrid,
+constVector<int>&write_real_comp,constVector<int>&write_int_comp,
+constVector<std::map<std::pair<int,int>,typenamePC::IntVector>>&particle_io_flags,
+constVector<int>&tiles,intnp,boolis_checkpoint)
+{
+intnum_output_int=0;
+for(inti=0;i<pc.NumIntComps()+PC::NStructInt;++i){
+if(write_int_comp[i]){++num_output_int;}
+}
+
+constLongiChunkSize=2+num_output_int;
+idata.resize(np*iChunkSize);
+
+intnum_output_real=0;
+for(inti:write_real_comp){
+if(i){++num_output_real;}
+}
+
+constLongrChunkSize=AMREX_SPACEDIM+num_output_real;
+rdata.resize(np*rChunkSize);
+
+int*iptr=idata.dataPtr();
+ParticleReal*rptr=rdata.dataPtr();
+for(inttile:tiles){
+constauto&ptile=pc.ParticlesAt(lev,grid,tile);
+constauto&pflags=particle_io_flags[lev].at(std::make_pair(grid,tile));
+for(intpindex=0;pindex<ptile.numParticles();++pindex){
+if(pflags[pindex]){
+constauto&soa=ptile.GetStructOfArrays();
+
+
+
+ifconstexpr(!PC::ParticleType::is_soa_particle)
+{
+constauto&aos=ptile.GetArrayOfStructs();
+constauto&p=aos[pindex];
+
+
+[packParticleIDs](iptr,p,is_checkpoint);
+iptr+=2;
+
+
+for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=p.pos(j);}
+rptr+=AMREX_SPACEDIM;
+
+
+for(intj=0;j<PC::NStructInt;j++){
+if(write_int_comp[j]){
+*iptr=p.idata(j);
+++iptr;
+}
+}
+
+for(intj=0;j<PC::NStructReal;j++){
+if(write_real_comp[j]){
+*rptr=p.rdata(j);
+++rptr;
+}
+}
+}
+else{
+uint64_tidcpu=soa.GetIdCPUData()[pindex];
+if(is_checkpoint){
+std::int32_txi,yi;
+std::uint32_txu,yu;
+xu=(std::uint32_t)((idcpu&0xFFFFFFFF00000000LL)>>32);
+yu=(std::uint32_t)(idcpu&0xFFFFFFFFLL);
+[std::memcpy](&xi,&xu,sizeof(xu));
+[std::memcpy](&yi,&yu,sizeof(yu));
+*iptr=xi;
+iptr+=1;
+*iptr=yi;
+iptr+=1;
+}else{
+
+*iptr=([int])ParticleIDWrapper(idcpu);
+iptr+=1;
+*iptr=([int])ParticleCPUWrapper(idcpu);
+iptr+=1;
+}
+
+
+for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=soa.GetRealData(j)[pindex];}
+rptr+=AMREX_SPACEDIM;
+}
+
+
+constintint_start_offset=0;
+for(intj=int_start_offset;j<pc.NumIntComps();j++){
+if(write_int_comp[PC::NStructInt+j]){
+*iptr=soa.GetIntData(j)[pindex];
+++iptr;
+}
+}
+
+
+constintreal_start_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
+for(intj=real_start_offset;j<pc.NumRealComps();j++){
+constintwrite_comp_index=PC::NStructReal+j-real_start_offset;
+if(write_real_comp[write_comp_index]){
+*rptr=(ParticleReal)soa.GetRealData(j)[pindex];
+++rptr;
+}
+}
+}
+}
+}
}
-
-template<classPC,classF,std::enable_if_t<IsParticleContainer<PC>::value,int>foo=0>
-void[WriteBinaryParticleDataSync](PCconst&pc,
-conststd::string&dir,conststd::string&name,
-constVector<int>&write_real_comp,
-constVector<int>&write_int_comp,
-constVector<std::string>&real_comp_names,
-constVector<std::string>&int_comp_names,
-Fconst&[f],boolis_checkpoint)
-{
-[BL_PROFILE]("WriteBinaryParticleData()");
-[AMREX_ASSERT](pc.OK());
-
-[AMREX_ASSERT](sizeof(typenamePC::ParticleType::RealType)==4||
-sizeof(typenamePC::ParticleType::RealType)==8);
-
-constexprintNStructReal=PC::NStructReal;
-constexprintNStructInt=PC::NStructInt;
-
-constint[NProcs]=[ParallelDescriptor::NProcs]();
-constintIOProcNumber=[ParallelDescriptor::IOProcessorNumber]();
-
-ifconstexpr(PC::ParticleType::is_soa_particle){
-[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal-AMREX_SPACEDIM);
-}else{
-[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal);
-}
-[AMREX_ALWAYS_ASSERT](int_comp_names.size()==pc.NumIntComps()+NStructInt);
-
-std::stringpdir=dir;
-if(!pdir.empty()&&pdir[pdir.size()-1]!='/'){pdir+='/';}
-pdir+=name;
-
-if(!pc.GetLevelDirectoriesCreated()){
-if([ParallelDescriptor::IOProcessor]())
-{
-if(![amrex::UtilCreateDirectory](pdir,0755))
-{
-[amrex::CreateDirectoryFailed](pdir);
-}
-}
-[ParallelDescriptor::Barrier]();
-}
-
-std::ofstreamHdrFile;
-
-Longnparticles=0;
-Longmaxnextid;
-
-
-Vector<std::map<std::pair<int,int>,typenamePC::IntVector>>
-particle_io_flags(pc.GetParticles().size());
-for(intlev=0;lev<pc.GetParticles().[size]();lev++)
-{
-constauto&pmap=pc.GetParticles(lev);
-for(constauto&kv:pmap)
-{
-auto&flags=particle_io_flags[lev][kv.first];
-[particle_detail::fillFlags](flags,kv.second,[f]);
-}
-}
-
-[Gpu::Device::streamSynchronize]();
-
-if(pc.GetUsePrePost())
-{
-nparticles=pc.GetNParticlesPrePost();
-maxnextid=pc.GetMaxNextIDPrePost();
-}
-else
-{
-nparticles=[particle_detail::countFlags](particle_io_flags,pc);
-maxnextid=PC::ParticleType::NextID();
-[ParallelDescriptor::ReduceLongSum](nparticles,IOProcNumber);
-PC::ParticleType::NextID(maxnextid);
-[ParallelDescriptor::ReduceLongMax](maxnextid,IOProcNumber);
-}
-
-if([ParallelDescriptor::IOProcessor]())
-{
-std::stringHdrFileName=pdir;
-
-if(!HdrFileName.empty()&&HdrFileName[HdrFileName.size()-1]!='/'){
-HdrFileName+='/';
-}
-
-HdrFileName+="Header";
-pc.HdrFileNamePrePost=HdrFileName;
-
-HdrFile.open(HdrFileName.c_str(),std::ios::out|std::ios::trunc);
-
-if(!HdrFile.good()){[amrex::FileOpenFailed](HdrFileName);}
-
-
-
-
-
-
-std::stringversion_string=is_checkpoint?PC::CheckpointVersion():PC::PlotfileVersion();
-if(sizeof(typenamePC::ParticleType::RealType)==4)
-{
-HdrFile<<version_string<<"_single"<<'\n';
-}
-else
-{
-HdrFile<<version_string<<"_double"<<'\n';
-}
-
-intnum_output_real=0;
-for(inti:write_real_comp){
-if(i){++num_output_real;}
-}
-
-intnum_output_int=0;
-for(inti=0;i<pc.NumIntComps()+NStructInt;++i){
-if(write_int_comp[i]){++num_output_int;}
-}
-
-
-HdrFile<<AMREX_SPACEDIM<<'\n';
-
-
-HdrFile<<num_output_real<<'\n';
-
-
-for(inti=0;i<([int])real_comp_names.size();++i){
-if(write_real_comp[i]){HdrFile<<real_comp_names[i]<<'\n';}
-}
-
-
-HdrFile<<num_output_int<<'\n';
-
-
-for(inti=0;i<NStructInt+pc.NumIntComps();++i){
-if(write_int_comp[i]){HdrFile<<int_comp_names[i]<<'\n';}
-}
-
-boolis_checkpoint_legacy=true;
-HdrFile<<is_checkpoint_legacy<<'\n';
-
-
-HdrFile<<nparticles<<'\n';
-
-
-HdrFile<<maxnextid<<'\n';
-
-
-HdrFile<<pc.finestLevel()<<'\n';
-
-
-for(intlev=0;lev<=pc.finestLevel();lev++){
-HdrFile<<pc.ParticleBoxArray(lev).size()<<'\n';
-}
-}
-
-
-
-intnOutFiles(256);
-
-ParmParse[pp]("particles");
-[pp].[queryAdd]("particles_nfiles",nOutFiles);
-if(nOutFiles==-1){nOutFiles=[NProcs];}
-nOutFiles=[std::max](1,[std::min](nOutFiles,[NProcs]));
-pc.nOutFilesPrePost=nOutFiles;
-
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-boolgotsome;
-if(pc.usePrePost)
-{
-gotsome=(pc.nParticlesAtLevelPrePost[lev]>0);
-}
-else
-{
-gotsome=(pc.NumberOfParticlesAtLevel(lev)>0);
-}
-
-
-std::stringLevelDir=pdir;
-
-if(gotsome)
-{
-if(!LevelDir.empty()&&LevelDir[LevelDir.size()-1]!='/'){LevelDir+='/';}
-
-LevelDir=[amrex::Concatenate](LevelDir.append("Level_"),lev,1);
-
-if(!pc.GetLevelDirectoriesCreated())
-{
-if([ParallelDescriptor::IOProcessor]()){
-if(![amrex::UtilCreateDirectory](LevelDir,0755)){
-[amrex::CreateDirectoryFailed](LevelDir);
-}
-}
-[ParallelDescriptor::Barrier]();
-}
-}
-
-
-if(gotsome&&[ParallelDescriptor::IOProcessor]()){
-std::stringHeaderFileName=LevelDir;
-HeaderFileName+="/Particle_H";
-std::ofstreamParticleHeader(HeaderFileName);
-
-pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
-ParticleHeader<<'\n';
-
-ParticleHeader.flush();
-ParticleHeader.close();
-}
-
-MFInfoinfo;
-info.SetAlloc(false);
-MultiFabstate(pc.ParticleBoxArray(lev),
-pc.ParticleDistributionMap(lev),
-1,0,info);
-
-
-
-Vector<int>which(state.size(),0);
-Vector<int>count(state.size(),0);
-Vector<Long>where(state.size(),0);
-
-std::stringfilePrefix(LevelDir);
-filePrefix+='/';
-filePrefix+=PC::DataPrefix();
-if(pc.usePrePost){
-pc.filePrefixPrePost[lev]=filePrefix;
-}
-boolgroupSets(false),setBuf(true);
-
-if(gotsome)
-{
-for(NFilesIternfi(nOutFiles,filePrefix,groupSets,setBuf);nfi.ReadyToWrite();++nfi)
-{
-auto&myStream=(std::ofstream&)nfi.Stream();
-pc.WriteParticles(lev,myStream,nfi.FileNumber(),which,count,where,
-write_real_comp,write_int_comp,particle_io_flags,is_checkpoint);
-}
-
-if(pc.usePrePost){
-pc.whichPrePost[lev]=which;
-pc.countPrePost[lev]=count;
-pc.wherePrePost[lev]=where;
-}else{
-[ParallelDescriptor::ReduceIntSum](which.dataPtr(),static_cast<int>(which.size()),IOProcNumber);
-[ParallelDescriptor::ReduceIntSum](count.dataPtr(),static_cast<int>(count.size()),IOProcNumber);
-[ParallelDescriptor::ReduceLongSum](where.dataPtr(),static_cast<int>(where.size()),IOProcNumber);
-}
-}
-
-if([ParallelDescriptor::IOProcessor]())
-{
-if(pc.GetUsePrePost()){
-
-}else{
-for(intj=0;j<state.size();j++)
-{
-HdrFile<<which[j]<<''<<count[j]<<''<<where[j]<<'\n';
-}
-
-if(gotsome&&pc.doUnlink)
-{
-
-Vector<Long>cnt(nOutFiles,0);
-
-for(inti=0,N=static_cast<int>(count.size());i<N;i++){
-cnt[which[i]]+=count[i];
-}
-
-for(inti=0,N=static_cast<int>(cnt.size());i<N;i++)
-{
-if(cnt[i]==0)
-{
-std::stringFullFileName=NFilesIter::FileName(i,filePrefix);
-[FileSystem::Remove](FullFileName);
-}
-}
-}
-}
-}
-}
-
-if([ParallelDescriptor::IOProcessor]())
-{
-HdrFile.flush();
-HdrFile.close();
-if(!HdrFile.good())
-{
-[amrex::Abort]("ParticleContainer::Checkpoint():problemwritingHdrFile");
-}
-}
-}
-
-template<classPC,std::enable_if_t<IsParticleContainer<PC>::value,int>foo=0>
-void[WriteBinaryParticleDataAsync](PCconst&pc,
-conststd::string&dir,conststd::string&name,
-constVector<int>&write_real_comp,
-constVector<int>&write_int_comp,
-constVector<std::string>&real_comp_names,
-constVector<std::string>&int_comp_names,boolis_checkpoint)
-{
-[BL_PROFILE]("WriteBinaryParticleDataAsync");
-[AMREX_ASSERT](pc.OK());
-
-[AMREX_ASSERT](sizeof(typenamePC::ParticleType::RealType)==4||
-sizeof(typenamePC::ParticleType::RealType)==8);
-
-constexprintNStructReal=PC::NStructReal;
-constexprintNStructInt=PC::NStructInt;
-constexprintNArrayReal=PC::NArrayReal;
-constexprintNArrayInt=PC::NArrayInt;
-
-constint[MyProc]=[ParallelDescriptor::MyProc]();
-constint[NProcs]=[ParallelDescriptor::NProcs]();
-constintIOProcNumber=[NProcs]-1;
-
-ifconstexpr(PC::ParticleType::is_soa_particle){
-[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal-AMREX_SPACEDIM);
-}else{
-[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal);
-}
-[AMREX_ALWAYS_ASSERT](int_comp_names.size()==pc.NumIntComps()+NStructInt);
-
-Vector<LayoutData<Long>>np_per_grid_local(pc.finestLevel()+1);
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-np_per_grid_local[lev].define(pc.ParticleBoxArray(lev),pc.ParticleDistributionMap(lev));
-using[ParIter]=typenamePC::ParConstIterType;
-for([ParIter]pti(pc,lev);pti.isValid();++pti)
-{
-intgid=pti.index();
-constauto&ptile=pc.ParticlesAt(lev,pti);
-constauto&ptd=ptile.getConstParticleTileData();
-constintnp=ptile.numParticles();
-
-ReduceOps<ReduceOpSum>reduce_op;
-ReduceData<int>reduce_data(reduce_op);
-usingReduceTuple=typenamedecltype(reduce_data)::Type;
-
-reduce_op.eval(np,reduce_data,
-[=][AMREX_GPU_DEVICE](inti)->ReduceTuple
-{
-return(ptd.id(i)>0)?1:0;
-});
-
-intnp_valid=amrex::get<0>(reduce_data.value(reduce_op));
-np_per_grid_local[lev][gid]+=np_valid;
-}
-}
-
-Vector<Vector<Long>>np_per_grid_global(pc.finestLevel()+1);
-Longtotal_np=0;
-Vector<Long>np_per_level(pc.finestLevel()+1);
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-np_per_grid_global[lev].resize(np_per_grid_local[lev].[size]());
-[ParallelDescriptor::GatherLayoutDataToVector](np_per_grid_local[lev],
-np_per_grid_global[lev],
-IOProcNumber);
-np_per_level[lev]=std::accumulate(np_per_grid_global[lev].[begin](),
-np_per_grid_global[lev].[end](),0L);
-total_np+=np_per_level[lev];
-}
-
-std::stringpdir=dir;
-if(!pdir.empty()&&pdir[pdir.size()-1]!='/'){pdir+='/';}
-pdir+=name;
-
-if([MyProc]==IOProcNumber)
-{
-if(!pc.GetLevelDirectoriesCreated())
-{
-if(![amrex::UtilCreateDirectory](pdir,0755))
-{
-[amrex::CreateDirectoryFailed](pdir);
-}
-}
-
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-std::stringLevelDir=pdir;
-boolgotsome=np_per_level[lev];
-
-if(gotsome)
-{
-if(!LevelDir.empty()&&LevelDir[LevelDir.size()-1]!='/'){LevelDir+='/';}
-
-LevelDir=[amrex::Concatenate](LevelDir.append("Level_"),lev,1);
-
-if(!pc.GetLevelDirectoriesCreated())
-{
-if(![amrex::UtilCreateDirectory](LevelDir,0755))
-{
-[amrex::CreateDirectoryFailed](LevelDir);
-}
-}
-
-std::stringHeaderFileName=LevelDir;
-HeaderFileName+="/Particle_H";
-std::ofstreamParticleHeader(HeaderFileName);
-
-pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
-ParticleHeader<<'\n';
-
-ParticleHeader.flush();
-ParticleHeader.close();
-}
-}
-}
-[ParallelDescriptor::Barrier]();
-
-Longmaxnextid=PC::ParticleType::NextID();
-[ParallelDescriptor::ReduceLongMax](maxnextid,IOProcNumber);
-
-Vector<Long>np_on_rank([NProcs],0L);
-std::size_tpsize=particle_detail::PSizeInFile<ParticleReal>(write_real_comp,write_int_comp);
-Vector<int64_t>rank_start_offset([NProcs]);
-if([MyProc]==IOProcNumber)
-{
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-for(intk=0;k<pc.ParticleBoxArray(lev).[size]();++k)
-{
-intrank=pc.ParticleDistributionMap(lev)[k];
-np_on_rank[rank]+=np_per_grid_global[lev][k];
-}
-}
-
-for(intip=0;ip<[NProcs];++ip)
-{
-autoinfo=[AsyncOut::GetWriteInfo](ip);
-rank_start_offset[ip]=(info.ispot==0)?0:static_cast<int64_t>(rank_start_offset[ip-1]+np_on_rank[ip-1]*psize);
-}
-}
-
-
-usingPinnedPTile=ParticleTile<typenamePC::ParticleType,NArrayReal,NArrayInt,
-PinnedArenaAllocator>;
-automyptiles=std::make_shared<Vector<std::map<std::pair<int,int>,PinnedPTile>>>();
-myptiles->resize(pc.finestLevel()+1);
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-for(MFItermfi=pc.MakeMFIter(lev);mfi.isValid();++mfi)
-{
-auto&new_ptile=(*myptiles)[lev][std::make_pair(mfi.index(),
-mfi.LocalTileIndex())];
-
-if(np_per_grid_local[lev][mfi.index()]>0)
-{
-constauto&ptile=pc.ParticlesAt(lev,mfi);
-
-constautonp=np_per_grid_local[lev][mfi.index()];
-
-new_ptile.resize(np);
-
-constautoruntime_real_comps=ptile.NumRuntimeRealComps();
-constautoruntime_int_comps=ptile.NumRuntimeIntComps();
-
-new_ptile.define(runtime_real_comps,runtime_int_comps);
-
-for(autocomp(0);comp<runtime_real_comps;++comp){
-new_ptile.push_back_real(NArrayReal+comp,np,0.);
-}
-
-for(autocomp(0);comp<runtime_int_comps;++comp){
-new_ptile.push_back_int(NArrayInt+comp,np,0);
-}
-
-[amrex::filterParticles](new_ptile,ptile,[KeepValidFilter]());
-}
-}
-}
-
-intfinest_level=pc.finestLevel();
-Vector<BoxArray>bas;
-Vector<DistributionMapping>dms;
-for(intlev=0;lev<=pc.finestLevel();lev++)
-{
-bas.push_back(pc.ParticleBoxArray(lev));
-dms.push_back(pc.ParticleDistributionMap(lev));
-}
-
-intnrc=pc.NumRealComps();
-intnic=pc.NumIntComps();
-intrnames_size=([int])real_comp_names.size();
-
-autoRD=pc.ParticleRealDescriptor;
-
-[AsyncOut::Submit]([=]()
-#ifdefined(__GNUC__)&&(__GNUC__==8)&&(__GNUC_MINOR__==1)
-mutable
-#endif
-{
-if([MyProc]==IOProcNumber)
-{
-std::stringHdrFileName=pdir;
-std::ofstreamHdrFile;
-
-if(!HdrFileName.empty()&&HdrFileName[HdrFileName.size()-1]!='/'){
-HdrFileName+='/';
-}
-
-HdrFileName+="Header";
-
-HdrFile.open(HdrFileName.c_str(),std::ios::out|std::ios::trunc);
-
-[if](!HdrFile.good()){amrex::FileOpenFailed(HdrFileName);}
-
-std::stringversion_string=is_checkpoint?PC::CheckpointVersion():PC::PlotfileVersion();
-[if](sizeof(typenamePC::ParticleType::RealType)==4)
-{
-HdrFile<<version_string<<"_single"<<'\n';
-}
-else
-{
-HdrFile<<version_string<<"_double"<<'\n';
-}
-
-intnum_output_real=0;
-for(inti=0;i<rnames_size;++i){
-if(write_real_comp[i]){++num_output_real;}
-}
-
-intnum_output_int=0;
-for(inti=0;i<nic+NStructInt;++i){
-if(write_int_comp[i]){++num_output_int;}
-}
-
-
-HdrFile<<AMREX_SPACEDIM<<'\n';
-
-
-HdrFile<<num_output_real<<'\n';
-
-
-for(inti=0;i<rnames_size;++i){
-if(write_real_comp[i]){HdrFile<<real_comp_names[i]<<'\n';}
-}
-
-
-HdrFile<<num_output_int<<'\n';
-
-
-for(inti=0;i<NStructInt+nic;++i){
-if(write_int_comp[i]){HdrFile<<int_comp_names[i]<<'\n';}
-}
-
-boolis_checkpoint_legacy=true;
-HdrFile<<is_checkpoint_legacy<<'\n';
-
-
-HdrFile<<total_np<<'\n';
-
-
-HdrFile<<maxnextid<<'\n';
-
-
-HdrFile<<finest_level<<'\n';
-
-
-for(intlev=0;lev<=finest_level;lev++){
-HdrFile<<dms[lev].size()<<'\n';
-}
-
-for(intlev=0;lev<=finest_level;lev++)
-{
-Vector<int64_t>grid_offset([NProcs],0);
-for(intk=0;k<bas[lev].size();++k)
-{
-intrank=dms[lev][k];
-autoinfo=[AsyncOut::GetWriteInfo](rank);
-HdrFile<<info.ifile<<''
-<<np_per_grid_global[lev][k]<<''
-<<grid_offset[rank]+rank_start_offset[rank]<<'\n';
-grid_offset[rank]+=static_cast<int64_t>(np_per_grid_global[lev][k]*psize);
-}
-}
-
-HdrFile.flush();
-HdrFile.close();
-if(!HdrFile.good())
-{
-[amrex::Abort]("ParticleContainer::Checkpoint():problemwritingHdrFile");
-}
-}
-
-[AsyncOut::Wait]();
-
-for(intlev=0;lev<=finest_level;lev++)
-{
-
-std::map<int,Vector<int>>tile_map;
-
-for(constauto&kv:(*myptiles)[lev])
-{
-constintgrid=kv.first.first;
-constinttile=kv.first.second;
-tile_map[grid].push_back(tile);
-}
-
-std::stringLevelDir=pdir;
-if(!LevelDir.empty()&&LevelDir[LevelDir.size()-1]!='/'){LevelDir+='/';}
-LevelDir=[amrex::Concatenate](LevelDir.append("Level_"),lev,1);
-std::stringfilePrefix(LevelDir);
-filePrefix+='/';
-filePrefix+=PC::DataPrefix();
-autoinfo=[AsyncOut::GetWriteInfo]([MyProc]);
-std::stringfile_name=[amrex::Concatenate](filePrefix,info.ifile,5);
-std::ofstreamofs;
-ofs.open(file_name.c_str(),(info.ispot==0)?(std::ios::binary|std::ios::trunc)
-:(std::ios::binary|std::ios::app));
-
-for(intk=0;k<bas[lev].size();++k)
-{
-intrank=dms[lev][k];
-if(rank!=[MyProc]){continue;}
-constintgrid=k;
-if(np_per_grid_local[lev][grid]==0){continue;}
-
-
-intnum_output_int=0;
-for(inti=0;i<nic+NStructInt;++i){
-if(write_int_comp[i]){++num_output_int;}
-}
-
-constLongiChunkSize=2+num_output_int;
-Vector<int>istuff(np_per_grid_local[lev][grid]*iChunkSize);
-int*iptr=istuff.dataPtr();
-
-for(unsignedi=0;i<tile_map[grid].size();i++){
-autoptile_index=std::make_pair(grid,tile_map[grid][i]);
-constauto&pbox=(*myptiles)[lev][ptile_index];
-constautoptd=pbox.getConstParticleTileData();
-for(intpindex=0;pindex<pbox.numParticles();++pindex)
-{
-constauto&soa=pbox.GetStructOfArrays();
-
-constauto&p=[make_particle<typename PC::ConstParticleType>]{}(ptd,pindex);
-if(p.id()<=0){continue;}
-
-
-
-ifconstexpr(!PC::ParticleType::is_soa_particle)
-{
-
-[particle_detail::packParticleIDs](iptr,p,is_checkpoint);
-iptr+=2;
-
-
-for(intj=0;j<NStructInt;j++)
-{
-if(write_int_comp[j])
-{
-*iptr=p.idata(j);
-++iptr;
-}
-}
-}
-else{
-uint64_tidcpu=soa.GetIdCPUData()[pindex];
-if(is_checkpoint){
-std::int32_txi,yi;
-std::uint32_txu,yu;
-xu=(std::uint32_t)((idcpu&0xFFFFFFFF00000000LL)>>32);
-yu=(std::uint32_t)(idcpu&0xFFFFFFFFLL);
-[std::memcpy](&xi,&xu,sizeof(xu));
-[std::memcpy](&yi,&yu,sizeof(yu));
-*iptr=xi;
-iptr+=1;
-*iptr=yi;
-iptr+=1;
-}else{
-
-*iptr=([int])ParticleIDWrapper(idcpu);
-iptr+=1;
-*iptr=([int])ParticleCPUWrapper(idcpu);
-iptr+=1;
-}
-}
-
-
-constintint_start_offset=0;
-for(intj=int_start_offset;j<nic;j++)
-{
-if(write_int_comp[NStructInt+j])
-{
-*iptr=soa.GetIntData(j)[pindex];
-++iptr;
-}
-}
-}
-}
-
-[writeIntData](istuff.dataPtr(),istuff.size(),ofs);
-ofs.flush();
-
-
-intnum_output_real=0;
-for(inti=0;i<rnames_size;++i){
-if(write_real_comp[i]){++num_output_real;}
-}
-
-constLongrChunkSize=AMREX_SPACEDIM+num_output_real;
-Vector<typenamePC::ParticleType::RealType>rstuff(np_per_grid_local[lev][grid]*rChunkSize);
-typenamePC::ParticleType::RealType*rptr=rstuff.dataPtr();
-
-for(unsignedi=0;i<tile_map[grid].size();i++){
-autoptile_index=std::make_pair(grid,tile_map[grid][i]);
-constauto&pbox=(*myptiles)[lev][ptile_index];
-constautoptd=pbox.getConstParticleTileData();
-for(intpindex=0;
-pindex<pbox.numParticles();++pindex)
-{
-constauto&soa=pbox.GetStructOfArrays();
-constauto&p=[make_particle<typename PC::ConstParticleType>]{}(ptd,pindex);
-
-if(p.id()<=0){continue;}
-
-ifconstexpr(!PC::ParticleType::is_soa_particle)
-{
-
-for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=p.pos(j);}
-rptr+=AMREX_SPACEDIM;
-
-
-for(intj=0;j<NStructReal;j++)
-{
-if(write_real_comp[j])
-{
-*rptr=p.rdata(j);
-++rptr;
-}
-}
-}
-else{
-
-for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=soa.GetRealData(j)[pindex];}
-rptr+=AMREX_SPACEDIM;
-}
-
-
-constintreal_start_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
-for(intj=real_start_offset;j<nrc;j++)
-{
-constintwrite_comp_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
-constintwrite_comp_index=PC::NStructReal+j-write_comp_offset;
-if(write_real_comp[write_comp_index])
-{
-*rptr=(typenamePC::ParticleType::RealType)soa.GetRealData(j)[pindex];
-++rptr;
-}
-}
-}
-}
-
-if(sizeof(typenamePC::ParticleType::RealType)==4){
-[writeFloatData]((float*)rstuff.dataPtr(),rstuff.size(),ofs,RD);
-}
-elseif(sizeof(typenamePC::ParticleType::RealType)==8){
-[writeDoubleData]((double*)rstuff.dataPtr(),rstuff.size(),ofs,RD);
-}
-
-ofs.flush();
-}
-}
-[AsyncOut::Notify]();
-});
-}
-
-#ifdefAMREX_USE_HDF5
-#include<[AMReX_WriteBinaryParticleDataHDF5.H]>
-#endif
-
-#endif
+}
+
+template<classPC,classF,std::enable_if_t<IsParticleContainer<PC>::value,int>foo=0>
+void[WriteBinaryParticleDataSync](PCconst&pc,
+conststd::string&dir,conststd::string&name,
+constVector<int>&write_real_comp,
+constVector<int>&write_int_comp,
+constVector<std::string>&real_comp_names,
+constVector<std::string>&int_comp_names,
+Fconst&[f],boolis_checkpoint)
+{
+[BL_PROFILE]("WriteBinaryParticleData()");
+[AMREX_ASSERT](pc.OK());
+
+[AMREX_ASSERT](sizeof(typenamePC::ParticleType::RealType)==4||
+sizeof(typenamePC::ParticleType::RealType)==8);
+
+constexprintNStructReal=PC::NStructReal;
+constexprintNStructInt=PC::NStructInt;
+
+constint[NProcs]=[ParallelDescriptor::NProcs]();
+constintIOProcNumber=[ParallelDescriptor::IOProcessorNumber]();
+
+ifconstexpr(PC::ParticleType::is_soa_particle){
+[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal-AMREX_SPACEDIM);
+}else{
+[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal);
+}
+[AMREX_ALWAYS_ASSERT](int_comp_names.size()==pc.NumIntComps()+NStructInt);
+
+std::stringpdir=dir;
+if(!pdir.empty()&&pdir[pdir.size()-1]!='/'){pdir+='/';}
+pdir+=name;
+
+if(!pc.GetLevelDirectoriesCreated()){
+if([ParallelDescriptor::IOProcessor]())
+{
+if(![amrex::UtilCreateDirectory](pdir,0755))
+{
+[amrex::CreateDirectoryFailed](pdir);
+}
+}
+[ParallelDescriptor::Barrier]();
+}
+
+std::ofstreamHdrFile;
+
+Longnparticles=0;
+Longmaxnextid;
+
+
+Vector<std::map<std::pair<int,int>,typenamePC::IntVector>>
+particle_io_flags(pc.GetParticles().size());
+for(intlev=0;lev<pc.GetParticles().[size]();lev++)
+{
+constauto&pmap=pc.GetParticles(lev);
+for(constauto&kv:pmap)
+{
+auto&flags=particle_io_flags[lev][kv.first];
+[particle_detail::fillFlags](flags,kv.second,[f]);
+}
+}
+
+[Gpu::Device::streamSynchronize]();
+
+if(pc.GetUsePrePost())
+{
+nparticles=pc.GetNParticlesPrePost();
+maxnextid=pc.GetMaxNextIDPrePost();
+}
+else
+{
+nparticles=[particle_detail::countFlags](particle_io_flags,pc);
+maxnextid=PC::ParticleType::NextID();
+[ParallelDescriptor::ReduceLongSum](nparticles,IOProcNumber);
+PC::ParticleType::NextID(maxnextid);
+[ParallelDescriptor::ReduceLongMax](maxnextid,IOProcNumber);
+}
+
+if([ParallelDescriptor::IOProcessor]())
+{
+std::stringHdrFileName=pdir;
+
+if(!HdrFileName.empty()&&HdrFileName[HdrFileName.size()-1]!='/'){
+HdrFileName+='/';
+}
+
+HdrFileName+="Header";
+pc.HdrFileNamePrePost=HdrFileName;
+
+HdrFile.open(HdrFileName.c_str(),std::ios::out|std::ios::trunc);
+
+if(!HdrFile.good()){[amrex::FileOpenFailed](HdrFileName);}
+
+
+
+
+
+
+std::stringversion_string=is_checkpoint?PC::CheckpointVersion():PC::PlotfileVersion();
+if(sizeof(typenamePC::ParticleType::RealType)==4)
+{
+HdrFile<<version_string<<"_single"<<'\n';
+}
+else
+{
+HdrFile<<version_string<<"_double"<<'\n';
+}
+
+intnum_output_real=0;
+for(inti:write_real_comp){
+if(i){++num_output_real;}
+}
+
+intnum_output_int=0;
+for(inti=0;i<pc.NumIntComps()+NStructInt;++i){
+if(write_int_comp[i]){++num_output_int;}
+}
+
+
+HdrFile<<AMREX_SPACEDIM<<'\n';
+
+
+HdrFile<<num_output_real<<'\n';
+
+
+for(inti=0;i<([int])real_comp_names.size();++i){
+if(write_real_comp[i]){HdrFile<<real_comp_names[i]<<'\n';}
+}
+
+
+HdrFile<<num_output_int<<'\n';
+
+
+for(inti=0;i<NStructInt+pc.NumIntComps();++i){
+if(write_int_comp[i]){HdrFile<<int_comp_names[i]<<'\n';}
+}
+
+boolis_checkpoint_legacy=true;
+HdrFile<<is_checkpoint_legacy<<'\n';
+
+
+HdrFile<<nparticles<<'\n';
+
+
+HdrFile<<maxnextid<<'\n';
+
+
+HdrFile<<pc.finestLevel()<<'\n';
+
+
+for(intlev=0;lev<=pc.finestLevel();lev++){
+HdrFile<<pc.ParticleBoxArray(lev).size()<<'\n';
+}
+}
+
+
+
+intnOutFiles(256);
+
+ParmParse[pp]("particles");
+[pp].[queryAdd]("particles_nfiles",nOutFiles);
+if(nOutFiles==-1){nOutFiles=[NProcs];}
+nOutFiles=[std::max](1,[std::min](nOutFiles,[NProcs]));
+pc.nOutFilesPrePost=nOutFiles;
+
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+boolgotsome;
+if(pc.usePrePost)
+{
+gotsome=(pc.nParticlesAtLevelPrePost[lev]>0);
+}
+else
+{
+gotsome=(pc.NumberOfParticlesAtLevel(lev)>0);
+}
+
+
+std::stringLevelDir=pdir;
+
+if(gotsome)
+{
+if(!LevelDir.empty()&&LevelDir[LevelDir.size()-1]!='/'){LevelDir+='/';}
+
+LevelDir=[amrex::Concatenate](LevelDir.append("Level_"),lev,1);
+
+if(!pc.GetLevelDirectoriesCreated())
+{
+if([ParallelDescriptor::IOProcessor]()){
+if(![amrex::UtilCreateDirectory](LevelDir,0755)){
+[amrex::CreateDirectoryFailed](LevelDir);
+}
+}
+[ParallelDescriptor::Barrier]();
+}
+}
+
+
+if(gotsome&&[ParallelDescriptor::IOProcessor]()){
+std::stringHeaderFileName=LevelDir;
+HeaderFileName+="/Particle_H";
+std::ofstreamParticleHeader(HeaderFileName);
+
+pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
+ParticleHeader<<'\n';
+
+ParticleHeader.flush();
+ParticleHeader.close();
+}
+
+MFInfoinfo;
+info.SetAlloc(false);
+MultiFabstate(pc.ParticleBoxArray(lev),
+pc.ParticleDistributionMap(lev),
+1,0,info);
+
+
+
+Vector<int>which(state.size(),0);
+Vector<int>count(state.size(),0);
+Vector<Long>where(state.size(),0);
+
+std::stringfilePrefix(LevelDir);
+filePrefix+='/';
+filePrefix+=PC::DataPrefix();
+if(pc.usePrePost){
+pc.filePrefixPrePost[lev]=filePrefix;
+}
+boolgroupSets(false),setBuf(true);
+
+if(gotsome)
+{
+for(NFilesIternfi(nOutFiles,filePrefix,groupSets,setBuf);nfi.ReadyToWrite();++nfi)
+{
+auto&myStream=(std::ofstream&)nfi.Stream();
+pc.WriteParticles(lev,myStream,nfi.FileNumber(),which,count,where,
+write_real_comp,write_int_comp,particle_io_flags,is_checkpoint);
+}
+
+if(pc.usePrePost){
+pc.whichPrePost[lev]=which;
+pc.countPrePost[lev]=count;
+pc.wherePrePost[lev]=where;
+}else{
+[ParallelDescriptor::ReduceIntSum](which.dataPtr(),static_cast<int>(which.size()),IOProcNumber);
+[ParallelDescriptor::ReduceIntSum](count.dataPtr(),static_cast<int>(count.size()),IOProcNumber);
+[ParallelDescriptor::ReduceLongSum](where.dataPtr(),static_cast<int>(where.size()),IOProcNumber);
+}
+}
+
+if([ParallelDescriptor::IOProcessor]())
+{
+if(pc.GetUsePrePost()){
+
+}else{
+for(intj=0;j<state.size();j++)
+{
+HdrFile<<which[j]<<''<<count[j]<<''<<where[j]<<'\n';
+}
+
+if(gotsome&&pc.doUnlink)
+{
+
+Vector<Long>cnt(nOutFiles,0);
+
+for(inti=0,N=static_cast<int>(count.size());i<N;i++){
+cnt[which[i]]+=count[i];
+}
+
+for(inti=0,N=static_cast<int>(cnt.size());i<N;i++)
+{
+if(cnt[i]==0)
+{
+std::stringFullFileName=NFilesIter::FileName(i,filePrefix);
+[FileSystem::Remove](FullFileName);
+}
+}
+}
+}
+}
+}
+
+if([ParallelDescriptor::IOProcessor]())
+{
+HdrFile.flush();
+HdrFile.close();
+if(!HdrFile.good())
+{
+[amrex::Abort]("ParticleContainer::Checkpoint():problemwritingHdrFile");
+}
+}
+}
+
+template<classPC,std::enable_if_t<IsParticleContainer<PC>::value,int>foo=0>
+void[WriteBinaryParticleDataAsync](PCconst&pc,
+conststd::string&dir,conststd::string&name,
+constVector<int>&write_real_comp,
+constVector<int>&write_int_comp,
+constVector<std::string>&real_comp_names,
+constVector<std::string>&int_comp_names,boolis_checkpoint)
+{
+[BL_PROFILE]("WriteBinaryParticleDataAsync");
+[AMREX_ASSERT](pc.OK());
+
+[AMREX_ASSERT](sizeof(typenamePC::ParticleType::RealType)==4||
+sizeof(typenamePC::ParticleType::RealType)==8);
+
+constexprintNStructReal=PC::NStructReal;
+constexprintNStructInt=PC::NStructInt;
+constexprintNArrayReal=PC::NArrayReal;
+constexprintNArrayInt=PC::NArrayInt;
+
+constint[MyProc]=[ParallelDescriptor::MyProc]();
+constint[NProcs]=[ParallelDescriptor::NProcs]();
+constintIOProcNumber=[NProcs]-1;
+
+ifconstexpr(PC::ParticleType::is_soa_particle){
+[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal-AMREX_SPACEDIM);
+}else{
+[AMREX_ALWAYS_ASSERT](real_comp_names.size()==pc.NumRealComps()+NStructReal);
+}
+[AMREX_ALWAYS_ASSERT](int_comp_names.size()==pc.NumIntComps()+NStructInt);
+
+Vector<LayoutData<Long>>np_per_grid_local(pc.finestLevel()+1);
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+np_per_grid_local[lev].define(pc.ParticleBoxArray(lev),pc.ParticleDistributionMap(lev));
+using[ParIter]=typenamePC::ParConstIterType;
+for([ParIter]pti(pc,lev);pti.isValid();++pti)
+{
+intgid=pti.index();
+constauto&ptile=pc.ParticlesAt(lev,pti);
+constauto&ptd=ptile.getConstParticleTileData();
+constintnp=ptile.numParticles();
+
+ReduceOps<ReduceOpSum>reduce_op;
+ReduceData<int>reduce_data(reduce_op);
+usingReduceTuple=typenamedecltype(reduce_data)::Type;
+
+reduce_op.eval(np,reduce_data,
+[=][AMREX_GPU_DEVICE](inti)->ReduceTuple
+{
+return(ptd.id(i)>0)?1:0;
+});
+
+intnp_valid=amrex::get<0>(reduce_data.value(reduce_op));
+np_per_grid_local[lev][gid]+=np_valid;
+}
+}
+
+Vector<Vector<Long>>np_per_grid_global(pc.finestLevel()+1);
+Longtotal_np=0;
+Vector<Long>np_per_level(pc.finestLevel()+1);
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+np_per_grid_global[lev].resize(np_per_grid_local[lev].[size]());
+[ParallelDescriptor::GatherLayoutDataToVector](np_per_grid_local[lev],
+np_per_grid_global[lev],
+IOProcNumber);
+np_per_level[lev]=std::accumulate(np_per_grid_global[lev].[begin](),
+np_per_grid_global[lev].[end](),0L);
+total_np+=np_per_level[lev];
+}
+
+std::stringpdir=dir;
+if(!pdir.empty()&&pdir[pdir.size()-1]!='/'){pdir+='/';}
+pdir+=name;
+
+if([MyProc]==IOProcNumber)
+{
+if(!pc.GetLevelDirectoriesCreated())
+{
+if(![amrex::UtilCreateDirectory](pdir,0755))
+{
+[amrex::CreateDirectoryFailed](pdir);
+}
+}
+
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+std::stringLevelDir=pdir;
+boolgotsome=np_per_level[lev];
+
+if(gotsome)
+{
+if(!LevelDir.empty()&&LevelDir[LevelDir.size()-1]!='/'){LevelDir+='/';}
+
+LevelDir=[amrex::Concatenate](LevelDir.append("Level_"),lev,1);
+
+if(!pc.GetLevelDirectoriesCreated())
+{
+if(![amrex::UtilCreateDirectory](LevelDir,0755))
+{
+[amrex::CreateDirectoryFailed](LevelDir);
+}
+}
+
+std::stringHeaderFileName=LevelDir;
+HeaderFileName+="/Particle_H";
+std::ofstreamParticleHeader(HeaderFileName);
+
+pc.ParticleBoxArray(lev).writeOn(ParticleHeader);
+ParticleHeader<<'\n';
+
+ParticleHeader.flush();
+ParticleHeader.close();
+}
+}
+}
+[ParallelDescriptor::Barrier]();
+
+Longmaxnextid=PC::ParticleType::NextID();
+[ParallelDescriptor::ReduceLongMax](maxnextid,IOProcNumber);
+
+Vector<Long>np_on_rank([NProcs],0L);
+std::size_tpsize=particle_detail::PSizeInFile<ParticleReal>(write_real_comp,write_int_comp);
+Vector<int64_t>rank_start_offset([NProcs]);
+if([MyProc]==IOProcNumber)
+{
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+for(intk=0;k<pc.ParticleBoxArray(lev).[size]();++k)
+{
+intrank=pc.ParticleDistributionMap(lev)[k];
+np_on_rank[rank]+=np_per_grid_global[lev][k];
+}
+}
+
+for(intip=0;ip<[NProcs];++ip)
+{
+autoinfo=[AsyncOut::GetWriteInfo](ip);
+rank_start_offset[ip]=(info.ispot==0)?0:static_cast<int64_t>(rank_start_offset[ip-1]+np_on_rank[ip-1]*psize);
+}
+}
+
+
+usingPinnedPTile=ParticleTile<typenamePC::ParticleType,NArrayReal,NArrayInt,
+PinnedArenaAllocator>;
+automyptiles=std::make_shared<Vector<std::map<std::pair<int,int>,PinnedPTile>>>();
+myptiles->resize(pc.finestLevel()+1);
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+for(MFItermfi=pc.MakeMFIter(lev);mfi.isValid();++mfi)
+{
+auto&new_ptile=(*myptiles)[lev][std::make_pair(mfi.index(),
+mfi.LocalTileIndex())];
+
+if(np_per_grid_local[lev][mfi.index()]>0)
+{
+constauto&ptile=pc.ParticlesAt(lev,mfi);
+
+constautonp=np_per_grid_local[lev][mfi.index()];
+
+new_ptile.resize(np);
+
+constautoruntime_real_comps=ptile.NumRuntimeRealComps();
+constautoruntime_int_comps=ptile.NumRuntimeIntComps();
+
+new_ptile.define(runtime_real_comps,runtime_int_comps);
+
+for(autocomp(0);comp<runtime_real_comps;++comp){
+new_ptile.push_back_real(NArrayReal+comp,np,0.);
+}
+
+for(autocomp(0);comp<runtime_int_comps;++comp){
+new_ptile.push_back_int(NArrayInt+comp,np,0);
+}
+
+[amrex::filterParticles](new_ptile,ptile,[KeepValidFilter]());
+}
+}
+}
+
+intfinest_level=pc.finestLevel();
+Vector<BoxArray>bas;
+Vector<DistributionMapping>dms;
+for(intlev=0;lev<=pc.finestLevel();lev++)
+{
+bas.push_back(pc.ParticleBoxArray(lev));
+dms.push_back(pc.ParticleDistributionMap(lev));
+}
+
+intnrc=pc.NumRealComps();
+intnic=pc.NumIntComps();
+intrnames_size=([int])real_comp_names.size();
+
+autoRD=pc.ParticleRealDescriptor;
+
+[AsyncOut::Submit]([=]()
+#ifdefined(__GNUC__)&&(__GNUC__==8)&&(__GNUC_MINOR__==1)
+mutable
+#endif
+{
+if([MyProc]==IOProcNumber)
+{
+std::stringHdrFileName=pdir;
+std::ofstreamHdrFile;
+
+if(!HdrFileName.empty()&&HdrFileName[HdrFileName.size()-1]!='/'){
+HdrFileName+='/';
+}
+
+HdrFileName+="Header";
+
+HdrFile.open(HdrFileName.c_str(),std::ios::out|std::ios::trunc);
+
+[if](!HdrFile.good()){amrex::FileOpenFailed(HdrFileName);}
+
+std::stringversion_string=is_checkpoint?PC::CheckpointVersion():PC::PlotfileVersion();
+[if](sizeof(typenamePC::ParticleType::RealType)==4)
+{
+HdrFile<<version_string<<"_single"<<'\n';
+}
+else
+{
+HdrFile<<version_string<<"_double"<<'\n';
+}
+
+intnum_output_real=0;
+for(inti=0;i<rnames_size;++i){
+if(write_real_comp[i]){++num_output_real;}
+}
+
+intnum_output_int=0;
+for(inti=0;i<nic+NStructInt;++i){
+if(write_int_comp[i]){++num_output_int;}
+}
+
+
+HdrFile<<AMREX_SPACEDIM<<'\n';
+
+
+HdrFile<<num_output_real<<'\n';
+
+
+for(inti=0;i<rnames_size;++i){
+if(write_real_comp[i]){HdrFile<<real_comp_names[i]<<'\n';}
+}
+
+
+HdrFile<<num_output_int<<'\n';
+
+
+for(inti=0;i<NStructInt+nic;++i){
+if(write_int_comp[i]){HdrFile<<int_comp_names[i]<<'\n';}
+}
+
+boolis_checkpoint_legacy=true;
+HdrFile<<is_checkpoint_legacy<<'\n';
+
+
+HdrFile<<total_np<<'\n';
+
+
+HdrFile<<maxnextid<<'\n';
+
+
+HdrFile<<finest_level<<'\n';
+
+
+for(intlev=0;lev<=finest_level;lev++){
+HdrFile<<dms[lev].size()<<'\n';
+}
+
+for(intlev=0;lev<=finest_level;lev++)
+{
+Vector<int64_t>grid_offset([NProcs],0);
+for(intk=0;k<bas[lev].size();++k)
+{
+intrank=dms[lev][k];
+autoinfo=[AsyncOut::GetWriteInfo](rank);
+HdrFile<<info.ifile<<''
+<<np_per_grid_global[lev][k]<<''
+<<grid_offset[rank]+rank_start_offset[rank]<<'\n';
+grid_offset[rank]+=static_cast<int64_t>(np_per_grid_global[lev][k]*psize);
+}
+}
+
+HdrFile.flush();
+HdrFile.close();
+if(!HdrFile.good())
+{
+[amrex::Abort]("ParticleContainer::Checkpoint():problemwritingHdrFile");
+}
+}
+
+[AsyncOut::Wait]();
+
+for(intlev=0;lev<=finest_level;lev++)
+{
+
+std::map<int,Vector<int>>tile_map;
+
+for(constauto&kv:(*myptiles)[lev])
+{
+constintgrid=kv.first.first;
+constinttile=kv.first.second;
+tile_map[grid].push_back(tile);
+}
+
+std::stringLevelDir=pdir;
+if(!LevelDir.empty()&&LevelDir[LevelDir.size()-1]!='/'){LevelDir+='/';}
+LevelDir=[amrex::Concatenate](LevelDir.append("Level_"),lev,1);
+std::stringfilePrefix(LevelDir);
+filePrefix+='/';
+filePrefix+=PC::DataPrefix();
+autoinfo=[AsyncOut::GetWriteInfo]([MyProc]);
+std::stringfile_name=[amrex::Concatenate](filePrefix,info.ifile,5);
+std::ofstreamofs;
+ofs.open(file_name.c_str(),(info.ispot==0)?(std::ios::binary|std::ios::trunc)
+:(std::ios::binary|std::ios::app));
+
+for(intk=0;k<bas[lev].size();++k)
+{
+intrank=dms[lev][k];
+if(rank!=[MyProc]){continue;}
+constintgrid=k;
+if(np_per_grid_local[lev][grid]==0){continue;}
+
+
+intnum_output_int=0;
+for(inti=0;i<nic+NStructInt;++i){
+if(write_int_comp[i]){++num_output_int;}
+}
+
+constLongiChunkSize=2+num_output_int;
+Vector<int>istuff(np_per_grid_local[lev][grid]*iChunkSize);
+int*iptr=istuff.dataPtr();
+
+for(unsignedi=0;i<tile_map[grid].size();i++){
+autoptile_index=std::make_pair(grid,tile_map[grid][i]);
+constauto&pbox=(*myptiles)[lev][ptile_index];
+constautoptd=pbox.getConstParticleTileData();
+for(intpindex=0;pindex<pbox.numParticles();++pindex)
+{
+constauto&soa=pbox.GetStructOfArrays();
+
+constauto&p=[make_particle<typename PC::ConstParticleType>]{}(ptd,pindex);
+if(p.id()<=0){continue;}
+
+
+
+ifconstexpr(!PC::ParticleType::is_soa_particle)
+{
+
+[particle_detail::packParticleIDs](iptr,p,is_checkpoint);
+iptr+=2;
+
+
+for(intj=0;j<NStructInt;j++)
+{
+if(write_int_comp[j])
+{
+*iptr=p.idata(j);
+++iptr;
+}
+}
+}
+else{
+uint64_tidcpu=soa.GetIdCPUData()[pindex];
+if(is_checkpoint){
+std::int32_txi,yi;
+std::uint32_txu,yu;
+xu=(std::uint32_t)((idcpu&0xFFFFFFFF00000000LL)>>32);
+yu=(std::uint32_t)(idcpu&0xFFFFFFFFLL);
+[std::memcpy](&xi,&xu,sizeof(xu));
+[std::memcpy](&yi,&yu,sizeof(yu));
+*iptr=xi;
+iptr+=1;
+*iptr=yi;
+iptr+=1;
+}else{
+
+*iptr=([int])ParticleIDWrapper(idcpu);
+iptr+=1;
+*iptr=([int])ParticleCPUWrapper(idcpu);
+iptr+=1;
+}
+}
+
+
+constintint_start_offset=0;
+for(intj=int_start_offset;j<nic;j++)
+{
+if(write_int_comp[NStructInt+j])
+{
+*iptr=soa.GetIntData(j)[pindex];
+++iptr;
+}
+}
+}
+}
+
+[writeIntData](istuff.dataPtr(),istuff.size(),ofs);
+ofs.flush();
+
+
+intnum_output_real=0;
+for(inti=0;i<rnames_size;++i){
+if(write_real_comp[i]){++num_output_real;}
+}
+
+constLongrChunkSize=AMREX_SPACEDIM+num_output_real;
+Vector<typenamePC::ParticleType::RealType>rstuff(np_per_grid_local[lev][grid]*rChunkSize);
+typenamePC::ParticleType::RealType*rptr=rstuff.dataPtr();
+
+for(unsignedi=0;i<tile_map[grid].size();i++){
+autoptile_index=std::make_pair(grid,tile_map[grid][i]);
+constauto&pbox=(*myptiles)[lev][ptile_index];
+constautoptd=pbox.getConstParticleTileData();
+for(intpindex=0;
+pindex<pbox.numParticles();++pindex)
+{
+constauto&soa=pbox.GetStructOfArrays();
+constauto&p=[make_particle<typename PC::ConstParticleType>]{}(ptd,pindex);
+
+if(p.id()<=0){continue;}
+
+ifconstexpr(!PC::ParticleType::is_soa_particle)
+{
+
+for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=p.pos(j);}
+rptr+=AMREX_SPACEDIM;
+
+
+for(intj=0;j<NStructReal;j++)
+{
+if(write_real_comp[j])
+{
+*rptr=p.rdata(j);
+++rptr;
+}
+}
+}
+else{
+
+for(intj=0;j<AMREX_SPACEDIM;j++){rptr[j]=soa.GetRealData(j)[pindex];}
+rptr+=AMREX_SPACEDIM;
+}
+
+
+constintreal_start_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
+for(intj=real_start_offset;j<nrc;j++)
+{
+constintwrite_comp_offset=PC::ParticleType::is_soa_particle?AMREX_SPACEDIM:0;
+constintwrite_comp_index=PC::NStructReal+j-write_comp_offset;
+if(write_real_comp[write_comp_index])
+{
+*rptr=(typenamePC::ParticleType::RealType)soa.GetRealData(j)[pindex];
+++rptr;
+}
+}
+}
+}
+
+if(sizeof(typenamePC::ParticleType::RealType)==4){
+[writeFloatData]((float*)rstuff.dataPtr(),rstuff.size(),ofs,RD);
+}
+elseif(sizeof(typenamePC::ParticleType::RealType)==8){
+[writeDoubleData]((double*)rstuff.dataPtr(),rstuff.size(),ofs,RD);
+}
+
+ofs.flush();
+}
+}
+[AsyncOut::Notify]();
+});
+}
+
+#ifdefAMREX_USE_HDF5
+#include<[AMReX_WriteBinaryParticleDataHDF5.H]>
+#endif
+
+#endif
diff --git a/amrex/docs_xml/doxygen/namespaceparticle__detail.xml b/amrex/docs_xml/doxygen/namespaceparticle__detail.xml
index 8bdfe0422a..f401706b3d 100644
--- a/amrex/docs_xml/doxygen/namespaceparticle__detail.xml
+++ b/amrex/docs_xml/doxygen/namespaceparticle__detail.xml
@@ -326,7 +326,7 @@
-
+
@@ -388,7 +388,7 @@
-
+