HDF5 bug fix for vol-async and compression (AMReX-Codes#3451)

## Summary - Fix a couple of issues with vol-async and SZ compression, regular HDF5 I/O should not be affected. - Changed BL_USE_MPI to AMREX_USE_MPI. ## Additional background ## Checklist The proposed changes: - [x] fix a bug or incorrect behavior in AMReX - [ ] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate
WeiqunZhang · Jul 26, 2023 · 49dd703 · 49dd703
1 parent b75dd9d
commit 49dd703
Show file tree

Hide file tree

Showing 4 changed files with 82 additions and 57 deletions.
diff --git a/Src/Extern/HDF5/AMReX_ParticleHDF5.H b/Src/Extern/HDF5/AMReX_ParticleHDF5.H
@@ -591,7 +591,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
  hid_t dxpl_col, dxpl_ind, dcpl_int, dcpl_real;
  dxpl_col = H5Pcreate(H5P_DATASET_XFER);
  dxpl_ind = H5Pcreate(H5P_DATASET_XFER);
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
 #endif
  dcpl_int = H5Pcreate(H5P_DATASET_CREATE);
@@ -721,7 +721,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
  my_mfi_cnt++;
  }
 
- #ifdef BL_USE_MPI
+ #ifdef AMREX_USE_MPI
  // Collect the number of mf and total size of mf from each rank
  MPI_Allgather(&my_mfi_cnt, 1, ParallelDescriptor::Mpi_typemap<int>::type(), &(all_mfi_cnt[0]), 1,
  ParallelDescriptor::Mpi_typemap<int>::type(), ParallelDescriptor::Communicator());
@@ -767,7 +767,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
  H5Sclose(int_dset_space);
 
  // Create the real data
- #ifdef BL_USE_MPI
+ #ifdef AMREX_USE_MPI
  MPI_Allgather(&my_mfi_real_total_size, 1, ParallelDescriptor::Mpi_typemap<ULong>::type(),
  &(all_mfi_real_total_size[0]), 1, ParallelDescriptor::Mpi_typemap<ULong>::type(), ParallelDescriptor::Communicator());
  #else
@@ -996,7 +996,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator>
  int ret;
 
  fapl = H5Pcreate (H5P_FILE_ACCESS);
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
 #else
  SetHDF5fapl(fapl);

diff --git a/Src/Extern/HDF5/AMReX_PlotFileUtilHDF5.cpp b/Src/Extern/HDF5/AMReX_PlotFileUtilHDF5.cpp
@@ -107,13 +107,13 @@ static int CreateWriteHDF5AttrString(hid_t loc, const char *name, const char* st
  return 1;
 }
 
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
 static void SetHDF5fapl(hid_t fapl, MPI_Comm comm)
 #else
 static void SetHDF5fapl(hid_t fapl)
 #endif
 {
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  H5Pset_fapl_mpio(fapl, comm, MPI_INFO_NULL);
 
  // Alignment and metadata block size
@@ -127,16 +127,18 @@ static void SetHDF5fapl(hid_t fapl)
  H5Pset_all_coll_metadata_ops(fapl, true);
 
  // Defer cache flush
- H5AC_cache_config_t cache_config;
- cache_config.version = H5AC__CURR_CACHE_CONFIG_VERSION;
- H5Pget_mdc_config(fapl, &cache_config);
- cache_config.set_initial_size = 1;
- cache_config.initial_size = 16 * 1024 * 1024;
- cache_config.evictions_enabled = 0;
- cache_config.incr_mode = H5C_incr__off;
- cache_config.flash_incr_mode = H5C_flash_incr__off;
- cache_config.decr_mode = H5C_decr__off;
- H5Pset_mdc_config (fapl, &cache_config);
+ // 7/19/23 Comment out the following as it is causing segfault
+ // with vol-async on Frontier
+ /* H5AC_cache_config_t cache_config; */
+ /* cache_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; */
+ /* H5Pget_mdc_config(fapl, &cache_config); */
+ /* cache_config.set_initial_size = 1; */
+ /* cache_config.initial_size = 16 * 1024 * 1024; */
+ /* cache_config.evictions_enabled = 0; */
+ /* cache_config.incr_mode = H5C_incr__off; */
+ /* cache_config.flash_incr_mode = H5C_flash_incr__off; */
+ /* cache_config.decr_mode = H5C_decr__off; */
+ /* H5Pset_mdc_config (fapl, &cache_config); */
 #else
  H5Pset_fapl_sec2(fapl);
 #endif
@@ -372,7 +374,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
  std::string filename(plotfilename + ".h5");
 
  // Write out root level metadata
- hid_t fapl, dxpl_col, dxpl_ind, dcpl_id, fid, grp;
+ hid_t fapl, dxpl_col, dxpl_ind, dcpl_id, lev_dcpl_id, fid, grp;
 
  if(ParallelDescriptor::IOProcessor()) {
  BL_PROFILE_VAR("H5writeMetadata", h5dwm);
@@ -432,7 +434,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
  dxpl_col = H5Pcreate(H5P_DATASET_XFER);
  dxpl_ind = H5Pcreate(H5P_DATASET_XFER);
 
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
  H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
 #else
@@ -441,6 +443,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
 
  dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
  H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER);
+ H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_INCR);
 
 #if (defined AMREX_USE_HDF5_ZFP) || (defined AMREX_USE_HDF5_SZ)
  const char *chunk_env = NULL;
@@ -453,7 +456,6 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
  chunk_dim = atoi(chunk_env);
 
  H5Pset_chunk(dcpl_id, 1, &chunk_dim);
- H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_LATE);
 
  std::string::size_type pos = compression.find('@');
  if (pos != std::string::npos) {
@@ -668,13 +670,6 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
  hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL);
  hid_t memdataspace = H5Screate_simple(1, hs_procsize, NULL);
 
- /* fprintf(stderr, "Rank %d: level %d, offset %ld, size %ld, all size %ld\n", myProc, level, ch_offset[0], hs_procsize[0], hs_allprocsize[0]); */
-
- if (hs_procsize[0] == 0)
- H5Sselect_none(dataspace);
- else
- H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);
-
  Vector<Real> a_buffer(procBufferSize[myProc], -1.0);
  const MultiFab* data;
  std::unique_ptr<MultiFab> mf_tmp;
@@ -705,24 +700,32 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
 
  BL_PROFILE_VAR("H5DwriteData", h5dwg);
 
+ lev_dcpl_id = H5Pcopy(dcpl_id);
 #ifdef AMREX_USE_HDF5_SZ
  if (mode_env == "SZ") {
  size_t cd_nelmts;
  unsigned int* cd_values = NULL;
  unsigned filter_config;
  SZ_metaDataToCdArray(&cd_nelmts, &cd_values, SZ_DOUBLE, 0, 0, 0, 0, hs_allprocsize[0]);
- H5Pset_filter(dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
+ H5Pset_filter(lev_dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
  }
 #endif
 
 #ifdef AMREX_USE_HDF5_ASYNC
- hid_t dataset = H5Dcreate_async(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, es_id_g);
+ hid_t dataset = H5Dcreate_async(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT, es_id_g);
 #else
- hid_t dataset = H5Dcreate(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
+ hid_t dataset = H5Dcreate(grp, dataname.c_str(), H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT);
 #endif
  if(dataset < 0)
  std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl;
 
+ /* fprintf(stderr, "Rank %d: level %d, offset %ld, size %ld, all size %ld\n", myProc, level, ch_offset[0], hs_procsize[0], hs_allprocsize[0]); */
+
+ if (hs_procsize[0] == 0)
+ H5Sselect_none(dataspace);
+ else
+ H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);
+
 #ifdef AMREX_USE_HDF5_ASYNC
  ret = H5Dwrite_async(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, a_buffer.dataPtr(), es_id_g);
 #else
@@ -732,6 +735,7 @@ void WriteMultiLevelPlotfileHDF5SingleDset (const std::string& plotfilename,
 
  BL_PROFILE_VAR_STOP(h5dwg);
 
+ H5Pclose(lev_dcpl_id);
  H5Sclose(memdataspace);
  H5Sclose(dataspace);
  H5Sclose(offsetdataspace);
@@ -812,7 +816,7 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
  std::string filename(plotfilename + ".h5");
 
  // Write out root level metadata
- hid_t fapl, dxpl_col, dxpl_ind, fid, grp, dcpl_id;
+ hid_t fapl, dxpl_col, dxpl_ind, fid, grp, dcpl_id, lev_dcpl_id;
 
  if(ParallelDescriptor::IOProcessor()) {
  BL_PROFILE_VAR("H5writeMetadata", h5dwm);
@@ -872,7 +876,7 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
  dxpl_col = H5Pcreate(H5P_DATASET_XFER);
  dxpl_ind = H5Pcreate(H5P_DATASET_XFER);
 
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
  H5Pset_dxpl_mpio(dxpl_col, H5FD_MPIO_COLLECTIVE);
 #else
@@ -881,6 +885,7 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
 
  dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
  H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_NEVER);
+ H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_INCR);
 
 #if (defined AMREX_USE_HDF5_ZFP) || (defined AMREX_USE_HDF5_SZ)
  const char *chunk_env = NULL;
@@ -893,7 +898,6 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
  chunk_dim = atoi(chunk_env);
 
  H5Pset_chunk(dcpl_id, 1, &chunk_dim);
- H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_LATE);
 
  std::string::size_type pos = compression.find('@');
  if (pos != std::string::npos) {
@@ -1097,14 +1101,8 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
  hs_procsize[0] = procBufferSize[myProc]; // ---- size of buffer on this proc
  hs_allprocsize[0] = offsets[sortedGrids.size()]; // ---- size of buffer on all procs
 
- hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL);
  hid_t memdataspace = H5Screate_simple(1, hs_procsize, NULL);
 
- if (hs_procsize[0] == 0)
- H5Sselect_none(dataspace);
- else
- H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);
-
  Vector<Real> a_buffer(procBufferSize[myProc]*ncomp, -1.0);
  Vector<Real> a_buffer_ind(procBufferSize[myProc], -1.0);
  const MultiFab* data;
@@ -1123,13 +1121,14 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
  hid_t dataset;
  char dataname[64];
 
+ lev_dcpl_id = H5Pcopy(dcpl_id);
 #ifdef AMREX_USE_HDF5_SZ
  if (mode_env == "SZ") {
  size_t cd_nelmts;
  unsigned int* cd_values = NULL;
  unsigned filter_config;
  SZ_metaDataToCdArray(&cd_nelmts, &cd_values, SZ_DOUBLE, 0, 0, 0, 0, hs_allprocsize[0]);
- H5Pset_filter(dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
+ H5Pset_filter(lev_dcpl_id, H5Z_FILTER_SZ, H5Z_FLAG_MANDATORY, cd_nelmts, cd_values);
  }
 #endif
 
@@ -1158,26 +1157,40 @@ void WriteMultiLevelPlotfileHDF5MultiDset (const std::string& plotfilename,
  writeDataSize += writeDataItems;
  }
 
+ hid_t dataspace = H5Screate_simple(1, hs_allprocsize, NULL);
  snprintf(dataname, sizeof dataname, "data:datatype=%d", jj);
 #ifdef AMREX_USE_HDF5_ASYNC
- dataset = H5Dcreate_async(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT, es_id_g);
+ dataset = H5Dcreate_async(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT, es_id_g);
  if(dataset < 0) std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl;
+
+ if (hs_procsize[0] == 0)
+ H5Sselect_none(dataspace);
+ else
+ H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);
+
  ret = H5Dwrite_async(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, a_buffer_ind.dataPtr(), es_id_g);
  if(ret < 0) { std::cout << ParallelDescriptor::MyProc() << "Write data failed! ret = " << ret << std::endl; break; }
  H5Dclose_async(dataset, es_id_g);
 #else
- dataset = H5Dcreate(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, dcpl_id, H5P_DEFAULT);
+ dataset = H5Dcreate(grp, dataname, H5T_NATIVE_DOUBLE, dataspace, H5P_DEFAULT, lev_dcpl_id, H5P_DEFAULT);
  if(dataset < 0) std::cout << ParallelDescriptor::MyProc() << "create data failed! ret = " << dataset << std::endl;
+
+ if (hs_procsize[0] == 0)
+ H5Sselect_none(dataspace);
+ else
+ H5Sselect_hyperslab(dataspace, H5S_SELECT_SET, ch_offset, NULL, hs_procsize, NULL);
+
  ret = H5Dwrite(dataset, H5T_NATIVE_DOUBLE, memdataspace, dataspace, dxpl_col, a_buffer_ind.dataPtr());
  if(ret < 0) { std::cout << ParallelDescriptor::MyProc() << "Write data failed! ret = " << ret << std::endl; break; }
  H5Dclose(dataset);
 #endif
+ H5Sclose(dataspace);
  }
 
  BL_PROFILE_VAR_STOP(h5dwg);
 
+ H5Pclose(lev_dcpl_id);
  H5Sclose(memdataspace);
- H5Sclose(dataspace);
  H5Sclose(offsetdataspace);
  H5Sclose(centerdataspace);
  H5Sclose(boxdataspace);

diff --git a/Src/Extern/HDF5/AMReX_WriteBinaryParticleDataHDF5.H b/Src/Extern/HDF5/AMReX_WriteBinaryParticleDataHDF5.H
@@ -76,13 +76,13 @@ static int ReadHDF5Attr(hid_t loc, const char *name, void *data, hid_t dtype)
  return 1;
 }
 
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
 static void SetHDF5fapl(hid_t fapl, MPI_Comm comm)
 #else
 static void SetHDF5fapl(hid_t fapl)
 #endif
 {
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  H5Pset_fapl_mpio(fapl, comm, MPI_INFO_NULL);
 
  // Alignment and metadata block size
@@ -381,7 +381,7 @@ void WriteHDF5ParticleDataSync (PC const& pc,
  pc.nOutFilesPrePost = nOutFiles;
 
  fapl = H5Pcreate (H5P_FILE_ACCESS);
-#ifdef BL_USE_MPI
+#ifdef AMREX_USE_MPI
  SetHDF5fapl(fapl, ParallelDescriptor::Communicator());
 #else
  SetHDF5fapl(fapl);

diff --git a/Tests/HDF5Benchmark/GNUmakefile b/Tests/HDF5Benchmark/GNUmakefile
@@ -23,17 +23,26 @@ USE_PARTICLES = TRUE
 EBASE = main
 
 USE_HDF5 = TRUE
-# HDF5_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/hdf5/develop_build/hdf5
 HDF5_HOME = $(OLCF_HDF5_ROOT)
+# HDF5_HOME = /lustre/orion/geo130/world-shared/gnu/hdf5.1.14.1.sync.cmake
+# HDF5_HOME = $(HDF5_ROOT)
+# HDF5_HOME = /lustre/orion/geo130/world-shared/gnu/hdf5.1.14.1.async.cmake
 
-# To use HDF5 with ZFP compression, follow the instructions at https://h5z-zfp.readthedocs.io to compile ZFP and H5Z_ZFP plugin, or use the following gcc build on Summit
+# To use HDF5 with ZFP compression, follow the instructions at https://h5z-zfp.readthedocs.io to compile ZFP and H5Z_ZFP plugin, or use the following gcc build on Frontier
 USE_HDF5_ZFP = FALSE
-H5Z_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/H5Z-ZFP/install
-ZFP_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/zfp
-
-# To use HDF5 with SZ compression, follow the instructions at https://github.com/szcompressor/SZ/tree/master/hdf5-filter/H5Z-SZ to compile SZ and H5Z_SZ plugin, or use the following gcc build on Summit
+# Frontier
+H5Z_HOME = /lustre/orion/geo130/world-shared/gnu/H5Z-ZFP
+ZFP_HOME = /lustre/orion/geo130/world-shared/gnu/zfp
+# Summit
+# H5Z_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/H5Z-ZFP/install
+# ZFP_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/zfp
+
+# To use HDF5 with SZ compression, follow the instructions at https://github.com/szcompressor/SZ/tree/master/hdf5-filter/H5Z-SZ to compile SZ and H5Z_SZ plugin, or use the following gcc build on Frontier
 USE_HDF5_SZ = FALSE
-SZ_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/SZ/install
+# Frontier
+SZ_HOME = /lustre/orion/geo130/world-shared/gnu/SZ
+# Summit
+# SZ_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/SZ/install
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
 
@@ -43,13 +52,16 @@ include $(AMREX_HOME)/Src/Particle/Make.package
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.rules
 
-# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://github.com/hpc-io/vol-async, add -DENABLE_WRITE_MEMCPY=1 to CFLAGS in src/Makefile, or use the following gcc build on Summit
+# To use HDF5 asynchronous I/O VOL connector, follow the instructions at https://github.com/hpc-io/vol-async, add -DENABLE_WRITE_MEMCPY=1 to CFLAGS in src/Makefile, or use the following gcc build on Frontier
 USE_HDF5_ASYNC = FALSE
-ABT_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/argobots/install
-ASYNC_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/vol-async-memcpy/src
+# Frontier
+ABT_HOME = /lustre/orion/geo130/world-shared/gnu/argobots
+ASYNC_HOME = /lustre/orion/geo130/world-shared/gnu/vol-async-memcpy
+# Summit
+# ABT_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/argobots/install
+# ASYNC_HOME = /gpfs/alpine/csc300/world-shared/gnu_build/vol-async-memcpy/src
 
 ifeq ($(USE_HDF5_ASYNC), TRUE)
 DEFINES += -DAMREX_USE_HDF5_ASYNC 
-INCLUDE_LOCATIONS += $(ABT_HOME)/include $(ASYNC_HOME)
-LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME) -lh5async -lasynchdf5 -labt -Wl,-rpath=$(ABT_HOME)/lib -Wl,-rpath=$(ASYNC_HOME)
+LIBRARIES += -L$(ABT_HOME)/lib -L$(ASYNC_HOME)/lib -lh5async -labt -Wl,-rpath=$(ABT_HOME)/lib -Wl,-rpath=$(ASYNC_HOME)/lib
 endif