Add ZFP compression support for HDF5 plot files (Exawind#616)

* Add ZFP compression support for HDF5 plot files.

* Update amrex.

CMakeLists.txt

@@ -35,6 +35,7 @@ option(AMR_WIND_ENABLE_TINY_PROFILE "Enable AMReX TinyProfile support" OFF)
 option(AMR_WIND_USE_INTERNAL_AMREX "Add AMReX as subproject" ON)
 option(AMR_WIND_USE_INTERNAL_AMREX_HYDRO "Add AMReX-Hydro as subproject" ON)
 option(AMR_WIND_ENABLE_HDF5 "Enable HDF5 library" OFF)
+option(AMR_WIND_ENABLE_HDF5_ZFP "Enable ZFP compression in HDF5 library" OFF)
 option(AMR_WIND_ENABLE_NETCDF "Enable NetCDF library" OFF)
 option(AMR_WIND_ENABLE_MASA "Enable MASA library" OFF)
 option(AMR_WIND_ENABLE_HYPRE "Enable HYPRE integration" OFF)
@@ -122,15 +123,15 @@ endif()
 
 if(AMR_WIND_ENABLE_HDF5)
   set(CMAKE_PREFIX_PATH ${HDF5_DIR} ${CMAKE_PREFIX_PATH})
-  if(AMR_WIND_ENABLE_MPI)
-    set(HDF5_PREFER_PARALLEL TRUE)
-  endif()
-  find_package(HDF5 1.10.4 REQUIRED)
-  if(AMR_WIND_ENABLE_MPI AND (NOT HDF5_IS_PARALLEL))
-    message(FATAL_ERROR "MPI enabled but HDF5 library does not support parallel I/O")
-  endif()
   target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_HDF5)
-  target_link_libraries_system(${amr_wind_lib_name} PUBLIC HDF5::HDF5)
+endif()
+
+if(AMR_WIND_ENABLE_HDF5_ZFP)
+  if(NOT AMR_WIND_ENABLE_HDF5)
+    message(FATAL_ERROR "HDF5 must be enabled to use ZFP compression in HDF5 plot files")
+  endif()
+  set(CMAKE_PREFIX_PATH ${H5Z_ZFP_DIR} ${CMAKE_PREFIX_PATH})
+  target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_HDF5_ZFP)
 endif()
 
 if(AMR_WIND_ENABLE_MASA)

amr-wind/utilities/IOManager.H

@@ -136,6 +136,10 @@ private:
 #ifdef AMR_WIND_USE_HDF5
     //! Flag indicating whether or not to output HDF5 plot files
     bool m_output_hdf5_plotfile{false};
+#ifdef AMR_WIND_USE_HDF5_ZFP
+    //! String for requesting a particular ZFP compression in HDF5 plot files
+    std::string m_hdf5_compression{"None@0"};
+#endif
 #endif
 };
 

amr-wind/utilities/IOManager.cpp

@@ -46,6 +46,9 @@ void IOManager::initialize_io()
     pp.query("allow_missing_restart_fields", m_allow_missing_restart_fields);
 #ifdef AMR_WIND_USE_HDF5
     pp.query("output_hdf5_plotfile", m_output_hdf5_plotfile);
+#ifdef AMR_WIND_USE_HDF5_ZFP
+    pp.query("hdf5_compression", m_hdf5_compression);
+#endif
 #endif
 
     // ParmParse requires us to read in a vector
@@ -159,9 +162,14 @@ void IOManager::write_plot_file()
                    << m_sim.time().new_time() << std::endl;
 #ifdef AMR_WIND_USE_HDF5
     if (m_output_hdf5_plotfile) {
-        amrex::WriteMultiLevelPlotfileHDF5(
+        amrex::WriteMultiLevelPlotfileHDF5SingleDset(
             plt_filename, nlevels, outfield->vec_const_ptrs(), m_plt_var_names,
-            mesh.Geom(), m_sim.time().new_time(), istep, mesh.refRatio());
+            mesh.Geom(), m_sim.time().new_time(), istep, mesh.refRatio()
+#ifdef AMR_WIND_USE_HDF5_ZFP
+                                                             ,
+            m_hdf5_compression
+#endif
+        );
     } else {
 #endif
         amrex::WriteMultiLevelPlotfile(

cmake/set_amrex_options.cmake

@@ -28,6 +28,7 @@ set(AMReX_SUNDIALS OFF)
 set(AMReX_TINY_PROFILE ${AMR_WIND_ENABLE_TINY_PROFILE})
 set(AMReX_TRACE_PROFILE OFF)
 set(AMReX_HDF5 ${AMR_WIND_ENABLE_HDF5})
+set(AMReX_HDF5_ZFP ${AMR_WIND_ENABLE_HDF5_ZFP})
 
 if (AMR_WIND_ENABLE_CUDA)
   set(AMReX_GPU_BACKEND CUDA CACHE STRING "AMReX GPU type" FORCE)

test/CMakeLists.txt

@@ -227,6 +227,9 @@ endif()
 
 if(AMR_WIND_ENABLE_HDF5)
   add_test_re(abl_stable_hdf5)
+  if(AMR_WIND_ENABLE_HDF5_ZFP)
+    add_test_re(abl_stable_hdf5_zfp)
+  endif()
 endif()
 
 #=============================================================================

test/test_files/abl_stable_hdf5_zfp/abl_stable_hdf5_zfp.i

@@ -0,0 +1,78 @@
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#            SIMULATION STOP            #
+#.......................................#
+time.stop_time               = 28800.0     # Max (simulated) time to evolve
+time.max_step                =  10          # Max number of time steps
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#         TIME STEP COMPUTATION         #
+#.......................................#
+time.fixed_dt         =   0.25        # Use this constant dt if > 0
+time.cfl              =   0.95       # CFL factor
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#            INPUT AND OUTPUT           #
+#.......................................#
+time.plot_interval            =  10       # Steps between plot files
+time.checkpoint_interval      =  -1         # Steps between checkpoint files
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#               PHYSICS                 #
+#.......................................#
+incflo.gravity        =  0.0  0.0 -9.81  # Gravitational force (3D)
+incflo.density        =  1.3223            # Reference density
+incflo.use_godunov = 1
+incflo.godunov_type = "ppm_nolim"
+transport.viscosity = 0.0
+transport.laminar_prandtl = 0.7
+transport.turbulent_prandtl = 0.3333
+turbulence.model = OneEqKsgsM84
+incflo.physics = ABL
+ICNS.source_terms = BoussinesqBuoyancy CoriolisForcing GeostrophicForcing
+TKE.source_terms = KsgsM84Src
+BoussinesqBuoyancy.reference_temperature = 263.5
+CoriolisForcing.east_vector = 1.0 0.0 0.0
+CoriolisForcing.north_vector = 0.0 1.0 0.0
+CoriolisForcing.latitude = 90.0
+CoriolisForcing.rotational_time_period = 90405.5439881955
+GeostrophicForcing.geostrophic_wind = 8.0 0.0 0.0
+incflo.velocity = 8.0 0.0 0.0
+ABL.reference_temperature = 263.5
+ABL.temperature_heights = 0.0 100 400.0
+ABL.temperature_values = 265.0 265.0 268.0
+ABL.perturb_temperature = false
+ABL.cutoff_height = 50.0
+ABL.perturb_velocity = true
+ABL.perturb_ref_height = 50.0
+ABL.Uperiods = 4.0
+ABL.Vperiods = 4.0
+ABL.deltaU = 1e-5
+ABL.deltaV = 1e-5
+ABL.kappa = .40
+ABL.mo_gamma_m = 4.8
+ABL.mo_gamma_h = 7.8
+ABL.surface_roughness_z0 = 0.1
+ABL.surface_temp_rate = -0.25
+ABL.surface_temp_init = 265.0
+ABL.normal_direction = 2
+ABL.stats_output_frequency = 100
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#        ADAPTIVE MESH REFINEMENT       #
+#.......................................#
+amr.n_cell              = 48 48 48 # Grid cells at coarsest AMRlevel
+amr.max_level           = 0           # Max AMR level in hierarchy
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#              GEOMETRY                 #
+#.......................................#
+geometry.prob_lo        =   0.       0.     0.  # Lo corner coordinates
+geometry.prob_hi        =   400.   400.   400.  # Hi corner coordinates
+geometry.is_periodic    =   1   1   0   # Periodicity x y z (0/1)
+# Boundary conditions
+zlo.type =   "wall_model"
+zlo.temperature_type = "wall_model"
+zhi.type =   "slip_wall"
+zhi.temperature_type = "fixed_gradient"
+zhi.temperature = 0.01
+zlo.tke_type = "zero_gradient"
+#zhi.tke_type = "zero_gradient"
+incflo.verbose          =   0
+
+io.output_hdf5_plotfile = true
+io.hdf5_compression="[email protected]"