Add HDF5 plot file support (Exawind#606)

CMakeLists.txt

@@ -34,6 +34,7 @@ option(AMR_WIND_ENABLE_TINY_PROFILE "Enable AMReX TinyProfile support" OFF)
 # Third party libraries
 option(AMR_WIND_USE_INTERNAL_AMREX "Add AMReX as subproject" ON)
 option(AMR_WIND_USE_INTERNAL_AMREX_HYDRO "Add AMReX-Hydro as subproject" ON)
+option(AMR_WIND_ENABLE_HDF5 "Enable HDF5 library" OFF)
 option(AMR_WIND_ENABLE_NETCDF "Enable NetCDF library" OFF)
 option(AMR_WIND_ENABLE_MASA "Enable MASA library" OFF)
 option(AMR_WIND_ENABLE_HYPRE "Enable HYPRE integration" OFF)
@@ -57,9 +58,9 @@ if(AMR_WIND_ENABLE_CUDA)
   endif()
 endif()
 
-if (AMR_WIND_ENABLE_ROCM)
+if(AMR_WIND_ENABLE_ROCM)
   find_package(HIP REQUIRED)
-  if (NOT DEFINED AMD_ARCH)
+  if(NOT DEFINED AMD_ARCH)
     # Set default AMD architectures (based on Spock)
     set(AMD_ARCH "gfx908")
   endif()
@@ -112,32 +113,36 @@ endif()
 
 include(set_compile_flags)
 
-if (AMR_WIND_ENABLE_NETCDF)
+if(AMR_WIND_ENABLE_NETCDF)
   set(CMAKE_PREFIX_PATH ${NETCDF_DIR} ${CMAKE_PREFIX_PATH})
-  find_package(NetCDF QUIET REQUIRED)
-  if(NetCDF_FOUND)
-    message(STATUS "Found NetCDF = ${NETCDF_DIR}")
-  endif()
+  find_package(NetCDF REQUIRED)
   target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_NETCDF)
   target_link_libraries_system(${amr_wind_lib_name} PUBLIC NetCDF::NetCDF)
 endif()
 
+if(AMR_WIND_ENABLE_HDF5)
+  set(CMAKE_PREFIX_PATH ${HDF5_DIR} ${CMAKE_PREFIX_PATH})
+  if(AMR_WIND_ENABLE_MPI)
+    set(HDF5_PREFER_PARALLEL TRUE)
+  endif()
+  find_package(HDF5 1.10.4 REQUIRED)
+  if(AMR_WIND_ENABLE_MPI AND (NOT HDF5_IS_PARALLEL))
+    message(FATAL_ERROR "MPI enabled but HDF5 library does not support parallel I/O")
+  endif()
+  target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_HDF5)
+  target_link_libraries_system(${amr_wind_lib_name} PUBLIC HDF5::HDF5)
+endif()
+
 if(AMR_WIND_ENABLE_MASA)
   set(CMAKE_PREFIX_PATH ${MASA_DIR} ${CMAKE_PREFIX_PATH})
-  find_package(MASA QUIET REQUIRED)
-  if(MASA_FOUND)
-    message(STATUS "Found MASA = ${MASA_DIR}")
-  endif()
+  find_package(MASA REQUIRED)
   target_compile_definitions(${amr_wind_lib_name} PUBLIC AMR_WIND_USE_MASA)
   target_link_libraries(${amr_wind_lib_name} PUBLIC MASA::MASA)
 endif()
 
-if (AMR_WIND_ENABLE_OPENFAST)
+if(AMR_WIND_ENABLE_OPENFAST)
   set(CMAKE_PREFIX_PATH ${OPENFAST_DIR} ${CMAKE_PREFIX_PATH})
-  find_package(OpenFAST QUIET REQUIRED)
-  if(OpenFAST_FOUND)
-    message(STATUS "Found OpenFAST = ${OPENFAST_DIR}")
-  endif()
+  find_package(OpenFAST 2.6.0 REQUIRED)
   include(CheckIncludeFileCXX)
   set(CMAKE_REQUIRED_INCLUDES "${OpenFAST_INCLUDE_DIRS}")
   check_include_file_cxx("SCDataEx_Types.h" USE_SCDX)
@@ -151,11 +156,8 @@ endif()
 
 if(AMR_WIND_ENABLE_ASCENT)
   set(CMAKE_PREFIX_PATH ${ASCENT_DIR} ${CMAKE_PREFIX_PATH})
-  find_package(Ascent QUIET REQUIRED)
-  if (Ascent_FOUND)
-    message(STATUS "Found Ascent = ${ASCENT_INCLUDE_DIRS}")
-  endif()
-  if (AMR_WIND_ENABLE_MPI)
+  find_package(Ascent REQUIRED)
+  if(AMR_WIND_ENABLE_MPI)
     target_link_libraries(${amr_wind_lib_name} PUBLIC ascent::ascent_mpi)
   else()
     target_link_libraries(${amr_wind_lib_name} PUBLIC ascent::ascent)
@@ -167,7 +169,7 @@ endif()
 generate_version_info()
 add_subdirectory(amr-wind)
 
-if (AMR_WIND_ENABLE_CUDA)
+if(AMR_WIND_ENABLE_CUDA)
   set(awtargets "${amr_wind_lib_name};${amr_wind_exe_name}")
   foreach(tgt IN LISTS awtargets)
     set_cuda_build_properties(${tgt})
@@ -177,7 +179,7 @@ if (AMR_WIND_ENABLE_CUDA)
   endforeach()
 endif()
 
-if (AMR_WIND_ENABLE_UNIT_TESTS OR AMR_WIND_ENABLE_TESTS)
+if(AMR_WIND_ENABLE_UNIT_TESTS OR AMR_WIND_ENABLE_TESTS)
   add_executable(${amr_wind_unit_test_exe_name})
   if(CLANG_TIDY_EXE)
     set_target_properties(${amr_wind_unit_test_exe_name}
@@ -186,7 +188,7 @@ if (AMR_WIND_ENABLE_UNIT_TESTS OR AMR_WIND_ENABLE_TESTS)
   add_subdirectory("submods/googletest")
   add_subdirectory("unit_tests")
   set_cuda_build_properties(${amr_wind_unit_test_exe_name})
-  # if (AMR_WIND_ENABLE_CUDA)
+  # if(AMR_WIND_ENABLE_CUDA)
   #   get_target_property(UTEST_SOURCES ${amr_wind_unit_test_exe_name} SOURCES)
   #   set_source_files_properties(${UTEST_SOURCES} PROPERTIES LANGUAGE CUDA)
   #   set_target_properties(${amr_wind_unit_test_exe_name} PROPERTIES

amr-wind/utilities/IOManager.H

@@ -132,6 +132,11 @@ private:
 
     //! Flag indicating whether we should allow missing restart fields
     bool m_allow_missing_restart_fields{true};
+
+#ifdef AMR_WIND_USE_HDF5
+    //! Flag indicating whether or not to output HDF5 plot files
+    bool m_output_hdf5_plotfile{false};
+#endif
 };
 
 } // namespace amr_wind

amr-wind/utilities/IOManager.cpp

@@ -16,6 +16,10 @@
 #include "AMReX_PlotFileUtil.H"
 #include "AMReX_MultiFabUtil.H"
 
+#ifdef AMR_WIND_USE_HDF5
+#include "AMReX_PlotFileUtilHDF5.H"
+#endif
+
 namespace amr_wind {
 
 IOManager::IOManager(CFDSim& sim)
@@ -40,6 +44,9 @@ void IOManager::initialize_io()
     pp.query("check_file", m_chk_prefix);
     pp.query("restart_file", m_restart_file);
     pp.query("allow_missing_restart_fields", m_allow_missing_restart_fields);
+#ifdef AMR_WIND_USE_HDF5
+    pp.query("output_hdf5_plotfile", m_output_hdf5_plotfile);
+#endif
 
     // ParmParse requires us to read in a vector
     pp.queryarr("outputs", out_vars);
@@ -150,11 +157,20 @@ void IOManager::write_plot_file()
     const auto& mesh = m_sim.mesh();
     amrex::Print() << "Writing plot file       " << plt_filename << " at time "
                    << m_sim.time().new_time() << std::endl;
-    amrex::WriteMultiLevelPlotfile(
-        plt_filename, nlevels, outfield->vec_const_ptrs(), m_plt_var_names,
-        mesh.Geom(), m_sim.time().new_time(), istep, mesh.refRatio());
-
-    write_info_file(plt_filename);
+#ifdef AMR_WIND_USE_HDF5
+    if (m_output_hdf5_plotfile) {
+        amrex::WriteMultiLevelPlotfileHDF5(
+            plt_filename, nlevels, outfield->vec_const_ptrs(), m_plt_var_names,
+            mesh.Geom(), m_sim.time().new_time(), istep, mesh.refRatio());
+    } else {
+#endif
+        amrex::WriteMultiLevelPlotfile(
+            plt_filename, nlevels, outfield->vec_const_ptrs(), m_plt_var_names,
+            mesh.Geom(), m_sim.time().new_time(), istep, mesh.refRatio());
+        write_info_file(plt_filename);
+#ifdef AMR_WIND_USE_HDF5
+    }
+#endif
 }
 
 void IOManager::write_checkpoint_file(const int start_level)

cmake/set_amrex_options.cmake

@@ -27,6 +27,7 @@ set(AMReX_SENSEI OFF)
 set(AMReX_SUNDIALS OFF)
 set(AMReX_TINY_PROFILE ${AMR_WIND_ENABLE_TINY_PROFILE})
 set(AMReX_TRACE_PROFILE OFF)
+set(AMReX_HDF5 ${AMR_WIND_ENABLE_HDF5})
 
 if (AMR_WIND_ENABLE_CUDA)
   set(AMReX_GPU_BACKEND CUDA CACHE STRING "AMReX GPU type" FORCE)

test/CMakeLists.txt

@@ -70,7 +70,7 @@ macro(setup_test)
       file(MAKE_DIRECTORY ${SAVED_GOLDS_DIRECTORY}/${TEST_NAME})
       set(SAVE_GOLDS_COMMAND "&& cp -R ${PLOT_TEST} ${SAVED_GOLDS_DIRECTORY}/${TEST_NAME}/")
     endif()
-    if(AMR_WIND_TEST_WITH_FCOMPARE)
+    if(AMR_WIND_TEST_WITH_FCOMPARE AND (NOT "${TEST_NAME}" MATCHES "hdf5$"))
       set(FCOMPARE_COMMAND "&& ${MPI_COMMANDS} ${FCOMPARE_EXE} ${FCOMPARE_TOLERANCE} ${PLOT_GOLD} ${PLOT_TEST}")
     endif()
 endmacro(setup_test)
@@ -199,13 +199,13 @@ add_test_re(vortex_ring_collision)
 add_test_re(fat_cored_vortex_ring)
 add_test_re(abl_bndry_output_native)
 
-if (NOT AMR_WIND_ENABLE_CUDA)
+if(NOT AMR_WIND_ENABLE_CUDA)
   add_test_re(ctv_godunov_plm)
   add_test_re(ctv_mol_mesh_map)
   add_test_re(ctv_mol_mesh_map_explicit)
 endif()
 
-if (AMR_WIND_ENABLE_NETCDF)
+if(AMR_WIND_ENABLE_NETCDF)
   add_test_re(abl_bndry_output)
   add_test_re(abl_virtual_lidar)
 endif()
@@ -217,22 +217,26 @@ if(AMR_WIND_ENABLE_MASA)
 endif()
 
 # TODO: Enable hypre capability on GPUs
-if (AMR_WIND_ENABLE_HYPRE)
+if(AMR_WIND_ENABLE_HYPRE)
   add_test_re(abl_godunov_hypre)
   add_test_re(channel_kwsst_hypre)
   add_test_re(channel_mol_mesh_map_x)
   add_test_re(channel_mol_mesh_map_y)
   add_test_re(channel_mol_mesh_map_z)
 endif()
 
+if(AMR_WIND_ENABLE_HDF5)
+  add_test_re(abl_stable_hdf5)
+endif()
+
 #=============================================================================
 # Regression tests excluded from CI with a test dependency
 #=============================================================================
 add_test_red(abl_bndry_input_native abl_bndry_output_native)
 add_test_red(abl_godunov_restart abl_godunov)
 add_test_red(abl_bndry_input_amr_native abl_bndry_output_native)
 
-if (AMR_WIND_ENABLE_NETCDF)
+if(AMR_WIND_ENABLE_NETCDF)
   add_test_red(abl_bndry_input abl_bndry_output)
   add_test_red(abl_bndry_input_amr abl_bndry_output)
 endif()

test/test_files/abl_stable_hdf5/abl_stable_hdf5.i

@@ -0,0 +1,77 @@
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#            SIMULATION STOP            #
+#.......................................#
+time.stop_time               = 28800.0     # Max (simulated) time to evolve
+time.max_step                =  10          # Max number of time steps
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#         TIME STEP COMPUTATION         #
+#.......................................#
+time.fixed_dt         =   0.25        # Use this constant dt if > 0
+time.cfl              =   0.95       # CFL factor
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#            INPUT AND OUTPUT           #
+#.......................................#
+time.plot_interval            =  10       # Steps between plot files
+time.checkpoint_interval      =  -1         # Steps between checkpoint files
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#               PHYSICS                 #
+#.......................................#
+incflo.gravity        =  0.0  0.0 -9.81  # Gravitational force (3D)
+incflo.density        =  1.3223            # Reference density
+incflo.use_godunov = 1
+incflo.godunov_type = "ppm_nolim"
+transport.viscosity = 0.0
+transport.laminar_prandtl = 0.7
+transport.turbulent_prandtl = 0.3333
+turbulence.model = OneEqKsgsM84
+incflo.physics = ABL
+ICNS.source_terms = BoussinesqBuoyancy CoriolisForcing GeostrophicForcing
+TKE.source_terms = KsgsM84Src
+BoussinesqBuoyancy.reference_temperature = 263.5
+CoriolisForcing.east_vector = 1.0 0.0 0.0
+CoriolisForcing.north_vector = 0.0 1.0 0.0
+CoriolisForcing.latitude = 90.0
+CoriolisForcing.rotational_time_period = 90405.5439881955
+GeostrophicForcing.geostrophic_wind = 8.0 0.0 0.0
+incflo.velocity = 8.0 0.0 0.0
+ABL.reference_temperature = 263.5
+ABL.temperature_heights = 0.0 100 400.0
+ABL.temperature_values = 265.0 265.0 268.0
+ABL.perturb_temperature = false
+ABL.cutoff_height = 50.0
+ABL.perturb_velocity = true
+ABL.perturb_ref_height = 50.0
+ABL.Uperiods = 4.0
+ABL.Vperiods = 4.0
+ABL.deltaU = 1e-5
+ABL.deltaV = 1e-5
+ABL.kappa = .40
+ABL.mo_gamma_m = 4.8
+ABL.mo_gamma_h = 7.8
+ABL.surface_roughness_z0 = 0.1
+ABL.surface_temp_rate = -0.25
+ABL.surface_temp_init = 265.0
+ABL.normal_direction = 2
+ABL.stats_output_frequency = 100
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#        ADAPTIVE MESH REFINEMENT       #
+#.......................................#
+amr.n_cell              = 48 48 48 # Grid cells at coarsest AMRlevel
+amr.max_level           = 0           # Max AMR level in hierarchy
+#¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨¨#
+#              GEOMETRY                 #
+#.......................................#
+geometry.prob_lo        =   0.       0.     0.  # Lo corner coordinates
+geometry.prob_hi        =   400.   400.   400.  # Hi corner coordinates
+geometry.is_periodic    =   1   1   0   # Periodicity x y z (0/1)
+# Boundary conditions
+zlo.type =   "wall_model"
+zlo.temperature_type = "wall_model"
+zhi.type =   "slip_wall"
+zhi.temperature_type = "fixed_gradient"
+zhi.temperature = 0.01
+zlo.tke_type = "zero_gradient"
+#zhi.tke_type = "zero_gradient"
+incflo.verbose          =   0
+
+io.output_hdf5_plotfile = true