Framework for LSM (#1389)

* This ran 1 step in debug. * This runs but has odd behavior in relaxation zone. The smag viscosity becomes VERY large; remains stable with setting k_turb=10. * Fix copy error. * Fix issue with nudge zone and most aware SmnSmn. Issue with PBL now. * Runs 100 steps. Will not restart. * This will restart. --------- Co-authored-by: Ann Almgren <[email protected]>
erf-model · Feb 1, 2024 · fbe71ac · fbe71ac
1 parent 709d24e
commit fbe71ac
Show file tree

Hide file tree

Showing 33 changed files with 1,233 additions and 62 deletions.
diff --git a/CMake/BuildERFExe.cmake b/CMake/BuildERFExe.cmake
@@ -136,6 +136,7 @@ function(build_erf_lib erf_lib_name)
  ${SRC_DIR}/TimeIntegration/ERF_TimeStep.cpp
  ${SRC_DIR}/TimeIntegration/ERF_advance_dycore.cpp
  ${SRC_DIR}/TimeIntegration/ERF_advance_microphysics.cpp
+ ${SRC_DIR}/TimeIntegration/ERF_advance_lsm.cpp
  ${SRC_DIR}/TimeIntegration/ERF_advance_radiation.cpp
  ${SRC_DIR}/TimeIntegration/ERF_make_buoyancy.cpp
  ${SRC_DIR}/TimeIntegration/ERF_make_fast_coeffs.cpp
@@ -164,6 +165,8 @@ function(build_erf_lib erf_lib_name)
  ${SRC_DIR}/Microphysics/FastEddy/FastEddy.cpp
  ${SRC_DIR}/Microphysics/FastEddy/Diagnose_FE.cpp
  ${SRC_DIR}/Microphysics/FastEddy/Update_FE.cpp
+ ${SRC_DIR}/LandSurfaceModel/SLM/SLM.cpp
+ ${SRC_DIR}/LandSurfaceModel/MM5/MM5.cpp
  )
 
  if(NOT "${erf_exe_name}" STREQUAL "erf_unit_tests")
@@ -208,8 +211,12 @@ function(build_erf_lib erf_lib_name)
  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Null)
  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/SAM)
  target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/Kessler)
- target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/FastEddy)
-
+ target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/Microphysics/FastEddy) 
+ target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel)
+ target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel/Null)
+ target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel/SLM)
+ target_include_directories(${erf_lib_name} PUBLIC ${SRC_DIR}/LandSurfaceModel/MM5)
+
  if(ERF_ENABLE_RRTMGP)
  target_link_libraries(${erf_lib_name} PUBLIC yakl)
  target_link_libraries(${erf_lib_name} PUBLIC rrtmgp)

diff --git a/Exec/CMakeLists.txt b/Exec/CMakeLists.txt
@@ -27,5 +27,6 @@ else ()
  add_subdirectory(DevTests/ParticlesOverWoA)
  add_subdirectory(DevTests/MiguelDev)
  add_subdirectory(DevTests/MetGrid)
+ add_subdirectory(DevTests/LandSurfaceModel)
  add_subdirectory(DevTests/TemperatureSource)
 endif()
diff --git a/Exec/DevTests/LandSurfaceModel/CMakeLists.txt b/Exec/DevTests/LandSurfaceModel/CMakeLists.txt
@@ -0,0 +1,15 @@
+set(erf_exe_name LandSurfaceModel)
+
+add_executable(${erf_exe_name} "")
+target_sources(${erf_exe_name}
+ PRIVATE
+ prob.cpp
+)
+
+target_include_directories(${erf_exe_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+
+include(${CMAKE_SOURCE_DIR}/CMake/BuildERFExe.cmake)
+build_erf_exe(${erf_exe_name})
+
+#find_package(AMReX)
+#target_link_libraries( ${_target} AMReX::amrex)
diff --git a/Exec/DevTests/LandSurfaceModel/GNUmakefile b/Exec/DevTests/LandSurfaceModel/GNUmakefile
@@ -0,0 +1,35 @@
+# AMReX
+COMP = gnu
+PRECISION = DOUBLE
+
+# Profiling
+PROFILE = FALSE
+TINY_PROFILE = FALSE
+COMM_PROFILE = FALSE
+TRACE_PROFILE = FALSE
+MEM_PROFILE = FALSE
+USE_GPROF = FALSE
+
+# Performance
+USE_MPI = TRUE
+USE_OMP = FALSE
+
+USE_CUDA = FALSE
+USE_HIP = FALSE
+USE_SYCL = FALSE
+
+# Debugging
+DEBUG = FALSE
+
+TEST = TRUE
+USE_ASSERTION = TRUE
+
+USE_NETCDF = TRUE
+
+# GNU Make
+Bpack := ./Make.package
+Blocs := .
+
+ERF_HOME := ../../..
+ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/DevTests/LandSurfaceModel
+include $(ERF_HOME)/Exec/Make.ERF
diff --git a/Exec/DevTests/LandSurfaceModel/Make.package b/Exec/DevTests/LandSurfaceModel/Make.package
@@ -0,0 +1,2 @@
+CEXE_headers += prob.H
+CEXE_sources += prob.cpp
diff --git a/Exec/DevTests/LandSurfaceModel/README b/Exec/DevTests/LandSurfaceModel/README
@@ -0,0 +1 @@
+This problem setup is to test the ability of ERF to handle a land surface model.
diff --git a/Exec/DevTests/LandSurfaceModel/inputs b/Exec/DevTests/LandSurfaceModel/inputs
@@ -0,0 +1,74 @@
+# ------------------ INPUTS TO MAIN PROGRAM -------------------
+max_step = 100
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1
+
+# PROBLEM SIZE & GEOMETRY
+geometry.prob_extent = 2430000 2673000 12000
+amr.n_cell = 90 99 50
+
+geometry.is_periodic = 0 0 0
+
+xlo.type = "Outflow"
+xhi.type = "Outflow"
+
+ylo.type = "Outflow"
+yhi.type = "Outflow"
+
+zlo.type = "Most"
+erf.most.z0 = 0.1
+erf.most.zref = 120.0
+zhi.type = "SlipWall"
+
+# TIME STEP CONTROL
+erf.fixed_dt = 0.01
+erf.fixed_mri_dt_ratio = 4
+
+# DIAGNOSTICS & VERBOSITY
+erf.sum_interval = -1 # timesteps between computing mass
+#erf.data_log = my_data my_1d_data
+erf.v = 1 # verbosity in ERF.cpp
+amr.v = 1 # verbosity in Amr.cpp
+
+# REFINEMENT / REGRIDDING
+amr.max_level = 0 # maximum level number allowed
+
+# CHECKPOINT FILES
+erf.check_file = chk # root name of checkpoint file
+erf.check_int = 100 # number of timesteps between checkpoints
+erf.restart_type = "native"
+# PLOTFILES
+erf.plot_file_1 = plt # prefix of plotfile name
+erf.plot_int_1 = 1 # number of timesteps between plotfiles
+erf.plot_vars_1 = density rhotheta rhoadv_0 x_velocity y_velocity z_velocity pressure theta scalar temp pres_hse dens_hse pert_pres pert_dens
+
+erf.plot_lsm = true
+
+# SOLVER CHOICE
+erf.alpha_T = 1.0
+erf.alpha_C = 1.0
+erf.use_gravity = true
+
+erf.molec_diff_type = "None"
+erf.les_type = "Smagorinsky"
+erf.Cs = 0.1
+#erf.pbl_type = "MYNN2.5"
+#erf.QKE_0 = 0.5
+
+erf.use_terrain = true
+erf.terrain_smoothing = 2
+
+erf.moisture_model = "NullMoist"
+
+erf.land_surface_model = "MM5"
+
+# INITIALIZATION WITH METGRID DATA
+erf.metgrid_bdy_width = 5
+erf.metgrid_bdy_set_width = 1
+erf.init_type = "metgrid"
+erf.nc_init_file_0 = "met_em.d01.2016-10-06_00_00_00.nc" "met_em.d01.2016-10-06_06_00_00.nc"
+
+# NO CACHE TILING
+fabarray.mfiter_tile_size = 1024 1024 1024
diff --git a/Exec/DevTests/LandSurfaceModel/prob.H b/Exec/DevTests/LandSurfaceModel/prob.H
@@ -0,0 +1,24 @@
+#ifndef _PROB_H_
+#define _PROB_H_
+
+#include <string>
+
+#include "AMReX_REAL.H"
+
+#include "prob_common.H"
+
+struct ProbParm : ProbParmDefaults {
+}; // namespace ProbParm
+class Problem : public ProblemBase
+{
+public:
+ Problem(const amrex::Real* /*problo*/, const amrex::Real* /*probhi*/);
+
+protected:
+ std::string name() override { return "LandSurfaceModel"; }
+
+private:
+ ProbParm parms;
+};
+
+#endif
diff --git a/Exec/DevTests/LandSurfaceModel/prob.cpp b/Exec/DevTests/LandSurfaceModel/prob.cpp
@@ -0,0 +1,12 @@
+#include "prob.H"
+
+using namespace amrex;
+
+std::unique_ptr<ProblemBase>
+amrex_probinit(const amrex_real* problo, const amrex_real* probhi)
+{
+ return std::make_unique<Problem>(problo, probhi);
+}
+
+Problem::Problem(const amrex::Real* /*problo*/, const amrex::Real* /*probhi*/)
+{}
diff --git a/Exec/Make.ERF b/Exec/Make.ERF
@@ -127,6 +127,26 @@ include $(ERF_MOISTURE_KESSLER_DIR)/Make.package
 VPATH_LOCATIONS += $(ERF_MOISTURE_KESSLER_DIR)
 INCLUDE_LOCATIONS += $(ERF_MOISTURE_KESSLER_DIR)
 
+ERF_LSM_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel
+include $(ERF_LSM_DIR)/Make.package
+VPATH_LOCATIONS += $(ERF_LSM_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_DIR)
+
+ERF_LSM_NULL_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/Null
+include $(ERF_LSM_NULL_DIR)/Make.package
+VPATH_LOCATIONS += $(ERF_LSM_NULL_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_NULL_DIR)
+
+ERF_LSM_SLM_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/SLM
+include $(ERF_LSM_SLM_DIR)/Make.package
+VPATH_LOCATIONS += $(ERF_LSM_SLM_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_SLM_DIR)
+
+ERF_LSM_MM5_DIR = $(ERF_SOURCE_DIR)/LandSurfaceModel/MM5
+include $(ERF_LSM_MM5_DIR)/Make.package
+VPATH_LOCATIONS += $(ERF_LSM_MM5_DIR)
+INCLUDE_LOCATIONS += $(ERF_LSM_MM5_DIR)
+
 ifeq ($(COMPUTE_ERROR), TRUE)
  DEFINES += -DERF_COMPUTE_ERROR
 endif

diff --git a/Source/BoundaryConditions/ABLMost.H b/Source/BoundaryConditions/ABLMost.H
@@ -35,6 +35,8 @@ public:
  amrex::Vector<std::unique_ptr<amrex::MultiFab>>& z_phys_nd,
  amrex::Vector<amrex::Vector<std::unique_ptr<amrex::MultiFab>>>& sst_lev,
  amrex::Vector<amrex::Vector<std::unique_ptr<amrex::iMultiFab>>>& lmask_lev,
+ amrex::Vector<amrex::Vector<amrex::MultiFab*>> lsm_data,
+ amrex::Vector<amrex::Vector<amrex::MultiFab*>> lsm_flux,
  amrex::Real start_bdy_time = 0.0,
  amrex::Real bdy_time_interval = 0.0)
  : m_start_bdy_time(start_bdy_time),
@@ -113,6 +115,19 @@ public:
  }
  }
 
+ // Get pointers to LSM data and Fluxes
+ m_lsm_data_lev.resize(nlevs);
+ m_lsm_flux_lev.resize(nlevs);
+ for (int lev(0); lev<nlevs; ++lev) {
+ int nvar = lsm_data[lev].size();
+ m_lsm_data_lev[lev].resize(nvar);
+ m_lsm_flux_lev[lev].resize(nvar);
+ for (int n(0); n<nvar; ++n) {
+ m_lsm_data_lev[lev][n] = lsm_data[lev][n];
+ m_lsm_flux_lev[lev][n] = lsm_flux[lev][n];
+ }
+ }
+
  for (int lev = 0; lev < nlevs; lev++) {
  // Attributes for MFs and FABs
  //--------------------------------------------------------
@@ -149,10 +164,12 @@ public:
 
  t_surf[lev] = std::make_unique<amrex::MultiFab>(ba2d,dm,ncomp,ng);
 
- if (m_sst_lev[lev][0]) { // Valid SST data at t==0
+ // TODO: Do we want lsm_data to have theta at 0 index always?
+ // Do we want an external enum struct for indexing?
+ if (m_sst_lev[lev][0] || m_lsm_data_lev[lev][0]) {
+ // Valid SST or LSM data; t_surf set before computing fluxes (avoids extended lambda capture)
  theta_type = ThetaCalcType::SURFACE_TEMPERATURE;
- amrex::MultiFab::Copy(*(t_surf[lev]), *(m_sst_lev[lev][0]), 0, 0, 1, ng);
- } else if (erf_st) { // Constant temp
+ } else if (erf_st) {
  t_surf[lev]->setVal(surf_temp);
  } else {
  t_surf[lev]->setVal(0.0);
@@ -187,8 +204,11 @@ public:
  const FluxCalc& flux_comp);
 
  void
- time_interp_tsurf(const int& lev,
- const amrex::Real& time);
+ time_interp_sst (const int& lev,
+ const amrex::Real& time);
+
+ void
+ get_lsm_tsurf (const int& lev);
 
  void
  update_surf_temp (const amrex::Real& time)
@@ -265,6 +285,8 @@ private:
 
  amrex::Vector<amrex::Vector<amrex::MultiFab*>> m_sst_lev;
  amrex::Vector<amrex::Vector<amrex::iMultiFab*>> m_lmask_lev;
+ amrex::Vector<amrex::Vector<amrex::MultiFab*>> m_lsm_data_lev;
+ amrex::Vector<amrex::Vector<amrex::MultiFab*>> m_lsm_flux_lev;
 };
 
 #endif /* ABLMOST_H */