From 68de864bcc48e19291db4321c300d8cddf9f5a26 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 2 Jun 2023 19:05:54 +0200 Subject: [PATCH 01/48] Move cluster pgen to MeshPgen to allow for reductions --- src/main.cpp | 2 +- src/pgen/cluster.cpp | 335 ++++++++++++++++++++++--------------------- src/pgen/pgen.hpp | 2 +- 3 files changed, 174 insertions(+), 165 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 1d9ef467..b01ddad2 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -85,7 +85,7 @@ int main(int argc, char *argv[]) { Hydro::ProblemInitPackageData = rand_blast::ProblemInitPackageData; Hydro::ProblemSourceFirstOrder = rand_blast::RandomBlasts; } else if (problem == "cluster") { - pman.app_input->ProblemGenerator = cluster::ProblemGenerator; + pman.app_input->MeshProblemGenerator = cluster::ProblemGenerator; pman.app_input->MeshBlockUserWorkBeforeOutput = cluster::UserWorkBeforeOutput; Hydro::ProblemInitPackageData = cluster::ProblemInitPackageData; Hydro::ProblemSourceUnsplit = cluster::ClusterSrcTerm; diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 69385ea0..b85e3eb1 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -247,192 +247,201 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd } //======================================================================================== -//! \fn void MeshBlock::ProblemGenerator(ParameterInput *pin) -//! \brief Generate problem data on each meshblock +//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) +//! \brief Generate problem data for all blocks on rank +// +// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that +// reductions work //======================================================================================== -void ProblemGenerator(MeshBlock *pmb, parthenon::ParameterInput *pin) { - auto hydro_pkg = pmb->packages.Get("Hydro"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + // This could be more optimized, but require a refactor of init routines being called. + // However, given that it's just called during initial setup, this should not be a + // performance concern. + for (int b; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - auto &coords = pmb->coords; + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); + auto &coords = pmb->coords; - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = - sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; + // Get Adiabatic Index + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); /************************************************************ - * Initialize an initial magnetic tower + * Initialize the initial hydro state ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb, a_kb, a_jb, a_ib, A); + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + if (init_uniform_gas) { + const Real rho = hydro_pkg->Param("uniform_gas_rho"); + const Real ux = hydro_pkg->Param("uniform_gas_ux"); + const Real uy = hydro_pkg->Param("uniform_gas_uy"); + const Real uz = hydro_pkg->Param("uniform_gas_uz"); + const Real pres = hydro_pkg->Param("uniform_gas_pres"); + + const Real Mx = rho * ux; + const Real My = rho * uy; + const Real Mz = rho * uz; + const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); + u(IDN, k, j, i) = rho; + u(IM1, k, j, i) = Mx; + u(IM2, k, j, i) = My; + u(IM3, k, j, i) = Mz; + u(IEN, k, j, i) = E; + }); - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + // end if(init_uniform_gas) + } else { + /************************************************************ + * Initialize a HydrostaticEquilibriumSphere + ************************************************************/ + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + // Get pressure and density from generated profile + const Real P_r = P_rho_profile.P_from_r(r); + const Real rho_r = P_rho_profile.rho_from_r(r); + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; }); } - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + /************************************************************ + * Initialize the initial magnetic field state via a vector potential + ************************************************************/ + parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + /************************************************************ + * Initialize an initial magnetic tower + ************************************************************/ + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + + /************************************************************ + * Add dipole magnetic field to the magnetic potential + ************************************************************/ + const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); + if (init_dipole_b_field) { + const Real mx = hydro_pkg->Param("dipole_b_field_mx"); + const Real my = hydro_pkg->Param("dipole_b_field_my"); + const Real mz = hydro_pkg->Param("dipole_b_field_mz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + + const Real m_cross_r_x = my * z - mz * y; + const Real m_cross_r_y = mz * x - mx * z; + const Real m_cross_r_z = mx * y - mx * y; + + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); + A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); + A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + }); + } + + /************************************************************ + * Apply the potential to the conserved variables + ************************************************************/ parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + u(IB1, k, j, i) = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + + SQR(u(IB3, k, j, i))); }); - // end if(init_uniform_b_field) - } - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) + /************************************************************ + * Add uniform magnetic field to the conserved variables + ************************************************************/ + const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); + if (init_uniform_b_field) { + const Real bx = hydro_pkg->Param("uniform_b_field_bx"); + const Real by = hydro_pkg->Param("uniform_b_field_by"); + const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + const Real bx_i = u(IB1, k, j, i); + const Real by_i = u(IB2, k, j, i); + const Real bz_i = u(IB3, k, j, i); + + u(IB1, k, j, i) += bx; + u(IB2, k, j, i) += by; + u(IB3, k, j, i) += bz; + + // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + + // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that + u(IEN, k, j, i) += + bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + }); + // end if(init_uniform_b_field) + } + + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) + } } void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { diff --git a/src/pgen/pgen.hpp b/src/pgen/pgen.hpp index 84e36b00..3050cbb5 100644 --- a/src/pgen/pgen.hpp +++ b/src/pgen/pgen.hpp @@ -98,7 +98,7 @@ using namespace parthenon::driver::prelude; void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg); void InitUserMeshData(ParameterInput *pin); -void ProblemGenerator(MeshBlock *pmb, parthenon::ParameterInput *pin); +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md); void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin); void ClusterSrcTerm(MeshData *md, const parthenon::SimTime &tm, const Real beta_dt); parthenon::Real ClusterEstimateTimestep(MeshData *md); From abc082224a6f01c67f1abc7add4299eec17d375d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 2 Jun 2023 20:23:28 +0200 Subject: [PATCH 02/48] Prepare hse test case for init pert --- tst/regression/CMakeLists.txt | 2 +- .../test_suites/cluster_hse/cluster_hse.py | 30 ++++++++----------- 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 31d2876f..3461a817 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -35,7 +35,7 @@ setup_test_serial("performance" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ --driver_input ${PROJECT_SOURCE_DIR}/inputs/linear_wave3d.in --num_steps 21" "performance") setup_test_both("cluster_hse" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ - --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/hse.in --num_steps 1" "convergence") + --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/hse.in --num_steps 2" "convergence") setup_test_serial("cluster_tabular_cooling" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/cooling.in --num_steps 11" "convergence") diff --git a/tst/regression/test_suites/cluster_hse/cluster_hse.py b/tst/regression/test_suites/cluster_hse/cluster_hse.py index 8cd1d1cb..044592fd 100644 --- a/tst/regression/test_suites/cluster_hse/cluster_hse.py +++ b/tst/regression/test_suites/cluster_hse/cluster_hse.py @@ -33,7 +33,6 @@ class TestCase(utils.test_case.TestCaseAbs): def __init__(self): - # Define cluster parameters # Setup units unyt.define_unit("code_length", (1, "Mpc")) @@ -92,6 +91,9 @@ def __init__(self): self.norm_tol = 1e-3 + self.sigma_v = unyt.unyt_quantity(75.0, "km/s") + self.sigma_B = unyt.unyt_quantity(1e-8, "G") + def Prepare(self, parameters, step): """ Any preprocessing that is needed before the drive is run can be done in @@ -140,29 +142,23 @@ def Prepare(self, parameters, step): f"problem/cluster/hydrostatic_equilibrium/r_fix={self.R_fix.in_units('code_length').v}", f"problem/cluster/hydrostatic_equilibrium/rho_fix={self.rho_fix.in_units('code_mass/code_length**3').v}", f"problem/cluster/hydrostatic_equilibrium/r_sampling={self.R_sampling}", + f"problem/cluster/init_perturb/sigma_v={0.0 if step == 2 else self.sigma_v.in_units('code_length/code_time').v}", + f"problem/cluster/init_perturb/sigma_B={0.0 if step == 2 else self.sigma_B.in_units('(code_mass/code_length)**0.5/code_time').v}", + f"parthenon/output2/id={'prim' if step == 2 else 'prim_perturb'}", + f"parthenon/time/nlim={-1 if step == 2 else 1}", ] return parameters def Analyse(self, parameters): - """ - Analyze the output and determine if the test passes. - - This function is called after the driver has been executed. It is - responsible for reading whatever data it needs and making a judgment - about whether or not the test passes. It takes no inputs. Output should - be True (test passes) or False (test fails). - - The parameters that are passed in provide the paths to relevant - locations and commands. Of particular importance is the path to the - output folder. All files from a drivers run should appear in and output - folder located in - parthenon/tst/regression/test_suites/test_name/output. + analyze_status = self.AnalyseHSE(parameters) + analyze_status &= self.AnalyseInitPert(parameters) + return analyze_status - It is possible in this function to read any of the output files such as - hdf5 output and compare them to expected quantities. + def AnalyseInitPert(self, parameters): + return True - """ + def AnalyseHSE(self, parameters): analyze_status = True self.Yp = self.He_mass_fraction From 3905cc0b9b51984d1b69096c9d9037fba91efb78 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 3 Jun 2023 12:27:05 +0200 Subject: [PATCH 03/48] Git add sigma_v plus test for cluster pgen --- src/pgen/cluster.cpp | 58 +++++++++++++++ .../test_suites/cluster_hse/cluster_hse.py | 70 ++++++++++++++++++- 2 files changed, 127 insertions(+), 1 deletion(-) diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index b85e3eb1..702edbd3 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -442,6 +442,64 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) } + + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto fluid = hydro_pkg->Param("fluid"); + auto const &cons = md->PackVariables(std::vector{"cons"}); + const auto num_blocks = md->NumBlocks(); + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + const auto sigma_B = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_B", 0.0); + + if (sigma_v != 0.0) { + + Real v2_sum = 0.0; // used for normalization + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + auto rho = u(IDN, k, j, i); + + u(IM1, k, j, i) = rho * (i + b); + u(IM2, k, j, i) = rho * (j + b); + u(IM3, k, j, i) = rho * (k + b); + // No need to touch the energy yet as we'll normalize later + + lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * + coords.CellVolume(k, j, i) / SQR(rho); + }, + v2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); + + pmb->par_for( + "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IM1, k, j, i) /= v_norm; + u(IM2, k, j, i) /= v_norm; + u(IM3, k, j, i) /= v_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / + u(IDN, k, j, i); + }); + } } void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { diff --git a/tst/regression/test_suites/cluster_hse/cluster_hse.py b/tst/regression/test_suites/cluster_hse/cluster_hse.py index 044592fd..0a52bb6a 100644 --- a/tst/regression/test_suites/cluster_hse/cluster_hse.py +++ b/tst/regression/test_suites/cluster_hse/cluster_hse.py @@ -38,6 +38,7 @@ def __init__(self): unyt.define_unit("code_length", (1, "Mpc")) unyt.define_unit("code_mass", (1e14, "Msun")) unyt.define_unit("code_time", (1, "Gyr")) + unyt.define_unit("code_velocity", (1, "code_length/code_time")) self.code_length = unyt.unyt_quantity(1, "code_length") self.code_mass = unyt.unyt_quantity(1, "code_mass") self.code_time = unyt.unyt_quantity(1, "code_time") @@ -156,7 +157,74 @@ def Analyse(self, parameters): return analyze_status def AnalyseInitPert(self, parameters): - return True + analyze_status = True + sys.path.insert( + 1, + parameters.parthenon_path + + "/scripts/python/packages/parthenon_tools/parthenon_tools", + ) + + try: + import phdf + except ModuleNotFoundError: + print("Couldn't find module to load Parthenon hdf5 files.") + return False + + data_file = phdf.phdf( + f"{parameters.output_path}/parthenon.prim_perturb.00000.phdf" + ) + dx = data_file.xf[:, 1:] - data_file.xf[:, :-1] + dy = data_file.yf[:, 1:] - data_file.yf[:, :-1] + dz = data_file.zf[:, 1:] - data_file.zf[:, :-1] + + # create array of volume with (block, k, j, i) indices + cell_vol = np.empty( + ( + data_file.x.shape[0], + data_file.z.shape[1], + data_file.y.shape[1], + data_file.x.shape[1], + ) + ) + for block in range(dx.shape[0]): + dz3d, dy3d, dx3d = np.meshgrid( + dz[block], dy[block], dx[block], indexing="ij" + ) + cell_vol[block, :, :, :] = dx3d * dy3d * dz3d + + # flatten array as prim var are also flattended + cell_vol = cell_vol.ravel() + + prim = data_file.Get("prim") + + # FIXME: For now this is hard coded - a component mapping should be done by phdf + prim_col_dict = { + "velocity_1": 1, + "velocity_2": 2, + "velocity_3": 3, + } + + vx = prim[prim_col_dict["velocity_1"]] + vy = prim[prim_col_dict["velocity_2"]] + vz = prim[prim_col_dict["velocity_3"]] + + # volume weighted rms velocity + rms_v = np.sqrt( + np.sum((vx**2 + vy**2 + vz**2) * cell_vol) / np.sum(cell_vol) + ) + + sigma_v_match = np.isclose( + rms_v, self.sigma_v.in_units("code_velocity").v, rtol=1e-14, atol=1e-14 + ) + + if not sigma_v_match: + analyze_status = False + print( + f"ERROR: velocity perturbation too large\n" + f"Expected {self.sigma_v.in_units('code_velocity')} but got {rms_v}\n" + ) + + return analyze_status def AnalyseHSE(self, parameters): analyze_status = True From 196afe969e2052078f191010af93acf2000e5bfd Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 3 Jun 2023 12:27:27 +0200 Subject: [PATCH 04/48] Fix index error in cooling test --- .../cluster_tabular_cooling/cluster_tabular_cooling.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py b/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py index c5b6d9ed..e04cdc60 100644 --- a/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py +++ b/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py @@ -36,7 +36,6 @@ class TestCase(utils.test_case.TestCaseAbs): def __init__(self): - # Define cluster parameters # Setup units unyt.define_unit("code_length", (1, "Mpc")) @@ -322,10 +321,10 @@ def zero_corrected_linf_err(gold, test): } rho = unyt.unyt_array( - prim[:, prim_col_dict["density"]], "code_mass*code_length**-3" + prim[prim_col_dict["density"], :], "code_mass*code_length**-3" ) pres = unyt.unyt_array( - prim[:, prim_col_dict["pressure"]], + prim[prim_col_dict["pressure"], :], "code_mass*code_length**-1*code_time**-2", ) From 8d058080937ffec2ca893aeac97e3d3975b2f7b8 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Sat, 3 Jun 2023 16:04:34 +0200 Subject: [PATCH 05/48] Move iFT to utils from turb driver --- src/pgen/turbulence.cpp | 27 +++++--------------- src/utils/few_modes_ft.hpp | 50 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+), 21 deletions(-) create mode 100644 src/utils/few_modes_ft.hpp diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 91bb935a..93d1490a 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -1,6 +1,6 @@ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 20212-2022, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file turbulence.cpp @@ -27,13 +27,13 @@ // AthenaPK headers #include "../main.hpp" #include "../units.hpp" +#include "../utils/few_modes_ft.hpp" namespace turbulence { using namespace parthenon::package::prelude; - -typedef Kokkos::complex Complex; using parthenon::DevMemSpace; using parthenon::ParArray2D; +using utils::few_modes_ft::Complex; // Defining these "globally" as they are fixed across all blocks ParArray2D accel_hat_, accel_hat_new_; @@ -600,25 +600,10 @@ void Generate(MeshData *md, Real dt) { auto phases_i = md->PackVariables(std::vector{"phases_i"}); auto phases_j = md->PackVariables(std::vector{"phases_j"}); auto phases_k = md->PackVariables(std::vector{"phases_k"}); - // implictly assuming cubic box of size L=1 - pmb->par_for( - "Inverse FT", 0, acc_pack.GetDim(5) - 1, 0, 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { - Complex phase, phase_i, phase_j, phase_k; - acc_pack(b, n, k, j, i) = 0.0; - for (int m = 0; m < num_modes; m++) { - phase_i = - Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); - phase_j = - Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); - phase_k = - Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); - phase = phase_i * phase_j * phase_k; - acc_pack(b, n, k, j, i) += 2. * (accel_hat(n, m).real() * phase.real() - - accel_hat(n, m).imag() * phase.imag()); - } - }); + utils::few_modes_ft::InverseFT>( + acc_pack, phases_i, phases_j, phases_k, accel_hat, ib, jb, kb, acc_pack.GetDim(5), + num_modes); } //---------------------------------------------------------------------------------------- diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp new file mode 100644 index 00000000..95000856 --- /dev/null +++ b/src/utils/few_modes_ft.hpp @@ -0,0 +1,50 @@ + +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//======================================================================================== +//! \file few_modes_ft.hpp +// \brief Helper functions for an inverse (explicit complex to real) FT + +// Parthenon headers +#include "basic_types.hpp" +#include "config.hpp" +#include "kokkos_abstraction.hpp" + +// AthenaPK headers +#include "../main.hpp" +#include "mesh/domain.hpp" + +namespace utils::few_modes_ft { +using Complex = Kokkos::complex; +using parthenon::IndexRange; + +template +void InverseFT(const TPack &out_pack, const TPack &phases_i, const TPack &phases_j, + const TPack &phases_k, const THat &in_hat, const IndexRange &ib, + const IndexRange &jb, const IndexRange &kb, const int num_blocks, + const int num_modes) { + // implictly assuming cubic box of size L=1 + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "Inverse FT", parthenon::DevExecSpace(), 0, num_blocks - 1, 0, + 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { + Complex phase, phase_i, phase_j, phase_k; + out_pack(b, n, k, j, i) = 0.0; + + for (int m = 0; m < num_modes; m++) { + phase_i = + Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); + phase_j = + Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); + phase_k = + Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); + phase = phase_i * phase_j * phase_k; + out_pack(b, n, k, j, i) += 2. * (in_hat(n, m).real() * phase.real() - + in_hat(n, m).imag() * phase.imag()); + } + }); +} +} // namespace utils::few_modes_ft \ No newline at end of file From e046a6643b03d6b21b177f684cc4deb597a5eb18 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 5 Jun 2023 15:24:11 +0200 Subject: [PATCH 06/48] Isolate fmft construtor --- src/CMakeLists.txt | 1 + src/pgen/turbulence.cpp | 41 ++++++++++---------------------------- src/utils/few_modes_ft.hpp | 21 +++++++++++++++++-- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 2066812d..076fbb9f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -17,6 +17,7 @@ add_executable( hydro/srcterms/tabular_cooling.cpp refinement/gradient.cpp refinement/other.cpp + utils/few_modes_ft.cpp ) add_subdirectory(pgen) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 93d1490a..c703f8f5 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -36,8 +36,6 @@ using parthenon::ParArray2D; using utils::few_modes_ft::Complex; // Defining these "globally" as they are fixed across all blocks -ParArray2D accel_hat_, accel_hat_new_; -ParArray2D k_vec_; Kokkos::View random_num_; Kokkos::View random_num_host; std::mt19937 rng; @@ -131,26 +129,6 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg auto num_modes = pin->GetInteger("problem/turbulence", "num_modes"); // number of wavemodes - if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { - std::cout << "### WARNING using more than 100 explicit modes will significantly " - << "increase the runtime." << std::endl - << "If many modes are required in the acceleration field consider using " - << "the driving mechanism based on full FFTs." << std::endl; - } - pkg->AddParam<>("turbulence/num_modes", num_modes); - - const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); - const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); - const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx1}), "phases_i"); - pkg->AddField("phases_i", m); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx2}), "phases_j"); - pkg->AddField("phases_j", m); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx3}), "phases_k"); - pkg->AddField("phases_k", m); uint32_t rseed = pin->GetOrAddInteger("problem/turbulence", "rseed", -1); // seed for random number. @@ -171,20 +149,20 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg Real sol_weight = pin->GetReal("problem/turbulence", "sol_weight"); // solenoidal weight pkg->AddParam<>("turbulence/sol_weight", sol_weight); - // Acceleration field in Fourier space using complex to real transform. - accel_hat_ = ParArray2D("accel_hat", 3, num_modes); - accel_hat_new_ = ParArray2D("accel_hat_new", 3, num_modes); - // list of wavenumber vectors - k_vec_ = ParArray2D("k_vec", 3, num_modes); - auto k_vec_host = Kokkos::create_mirror_view(k_vec_); + auto k_vec = ParArray2D("k_vec", 3, num_modes); + auto k_vec_host = Kokkos::create_mirror_view(k_vec); for (int j = 0; j < 3; j++) { for (int i = 1; i <= num_modes; i++) { k_vec_host(j, i - 1) = pin->GetInteger("modes", "k_" + std::to_string(i) + "_" + std::to_string(j)); } } - Kokkos::deep_copy(k_vec_, k_vec_host); + Kokkos::deep_copy(k_vec, k_vec_host); + + auto few_modes_ft = + utils::few_modes_ft::FewModesFT(pin, pkg, "turbulence", num_modes, k_vec); + pkg->AddParam<>("turbulence/few_modes_ft", few_modes_ft); random_num_ = Kokkos::View("random_num", 3, num_modes, 2); @@ -193,7 +171,8 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg // Check if this is is a restart and restore previous state if (pin->DoesParameterExist("problem/turbulence", "accel_hat_0_0_r")) { // Restore (common) acceleration field in spectral space - auto accel_hat_host = Kokkos::create_mirror_view(accel_hat_); + auto accel_hat = few_modes_ft.GetVarHat(); + auto accel_hat_host = Kokkos::create_mirror_view(accel_hat); for (int i = 0; i < 3; i++) { for (int m = 0; m < num_modes; m++) { auto real = @@ -205,7 +184,7 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg accel_hat_host(i, m) = Complex(real, imag); } } - Kokkos::deep_copy(accel_hat_, accel_hat_host); + Kokkos::deep_copy(accel_hat, accel_hat_host); // Restore state of random number gen { diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp index 95000856..af0f67d7 100644 --- a/src/utils/few_modes_ft.hpp +++ b/src/utils/few_modes_ft.hpp @@ -11,15 +11,32 @@ // Parthenon headers #include "basic_types.hpp" #include "config.hpp" -#include "kokkos_abstraction.hpp" +#include // AthenaPK headers #include "../main.hpp" #include "mesh/domain.hpp" namespace utils::few_modes_ft { -using Complex = Kokkos::complex; +using parthenon::Real; +using Complex = Kokkos::complex; using parthenon::IndexRange; +using parthenon::ParArray2D; + +class FewModesFT { + private: + int num_modes_; + std::string prefix_; + ParArray2D var_hat_, var_hat_new_; + ParArray2D k_vec_; + + public: + FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, + std::string prefix, int num_modes, ParArray2D k_vec); + + ParArray2D GetVarHat() { return var_hat_ } + void SetVarHat(ParArray2D var_hat) { var_hat_ = var_hat } +}; template void InverseFT(const TPack &out_pack, const TPack &phases_i, const TPack &phases_j, From fbbe2dcdcbbb3d3cac763c8d95d7f6265c333489 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 15 Jun 2023 10:59:50 +0200 Subject: [PATCH 07/48] Separate OU and FT parts from turbulence pgen --- src/main.cpp | 4 - src/pgen/cluster.cpp | 2 +- src/pgen/turbulence.cpp | 296 +++------------------------------ src/utils/few_modes_ft.cpp | 332 +++++++++++++++++++++++++++++++++++++ src/utils/few_modes_ft.hpp | 59 +++---- 5 files changed, 385 insertions(+), 308 deletions(-) create mode 100644 src/utils/few_modes_ft.cpp diff --git a/src/main.cpp b/src/main.cpp index b01ddad2..8b796610 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -119,10 +119,6 @@ int main(int argc, char *argv[]) { // This line actually runs the simulation driver.Execute(); } - // very ugly cleanup... - if (problem == "turbulence") { - turbulence::Cleanup(); - } // call MPI_Finalize and Kokkos::finalize if necessary pman.ParthenonFinalize(); diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 702edbd3..7012fc41 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -258,7 +258,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { // This could be more optimized, but require a refactor of init routines being called. // However, given that it's just called during initial setup, this should not be a // performance concern. - for (int b; b < md->NumBlocks(); b++) { + for (int b = 0; b < md->NumBlocks(); b++) { auto pmb = md->GetBlockData(b)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index c703f8f5..b47cda6a 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -34,12 +34,7 @@ using namespace parthenon::package::prelude; using parthenon::DevMemSpace; using parthenon::ParArray2D; using utils::few_modes_ft::Complex; - -// Defining these "globally" as they are fixed across all blocks -Kokkos::View random_num_; -Kokkos::View random_num_host; -std::mt19937 rng; -std::uniform_real_distribution<> dist(-1.0, 1.0); +using utils::few_modes_ft::FewModesFT; // TODO(?) until we are able to process multiple variables in a single hst function call // we'll use this enum to identify the various vars. @@ -134,17 +129,17 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg pin->GetOrAddInteger("problem/turbulence", "rseed", -1); // seed for random number. pkg->AddParam<>("turbulence/rseed", rseed); - auto kpeak = + auto k_peak = pin->GetOrAddReal("problem/turbulence", "kpeak", 0.0); // peak of the forcing spec - pkg->AddParam<>("turbulence/kpeak", kpeak); + pkg->AddParam<>("turbulence/kpeak", k_peak); auto accel_rms = pin->GetReal("problem/turbulence", "accel_rms"); // turbulence amplitude pkg->AddParam<>("turbulence/accel_rms", accel_rms); - auto tcorr = + auto t_corr = pin->GetReal("problem/turbulence", "corr_time"); // forcing autocorrelation time - pkg->AddParam<>("turbulence/tcorr", tcorr); + pkg->AddParam<>("turbulence/t_corr", t_corr); Real sol_weight = pin->GetReal("problem/turbulence", "sol_weight"); // solenoidal weight pkg->AddParam<>("turbulence/sol_weight", sol_weight); @@ -161,13 +156,9 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg Kokkos::deep_copy(k_vec, k_vec_host); auto few_modes_ft = - utils::few_modes_ft::FewModesFT(pin, pkg, "turbulence", num_modes, k_vec); + FewModesFT(pin, pkg, "turbulence", num_modes, k_vec, k_peak, sol_weight, t_corr); pkg->AddParam<>("turbulence/few_modes_ft", few_modes_ft); - random_num_ = Kokkos::View("random_num", 3, - num_modes, 2); - random_num_host = Kokkos::create_mirror_view(random_num_); - // Check if this is is a restart and restore previous state if (pin->DoesParameterExist("problem/turbulence", "accel_hat_0_0_r")) { // Restore (common) acceleration field in spectral space @@ -189,128 +180,25 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg // Restore state of random number gen { std::istringstream iss(pin->GetString("problem/turbulence", "state_rng")); - iss >> rng; + few_modes_ft.RestoreRNG(iss); } // Restore state of dist { std::istringstream iss(pin->GetString("problem/turbulence", "state_dist")); - iss >> dist; + few_modes_ft.RestoreDist(iss); } } else { // init RNG - rng.seed(rseed); + few_modes_ft.SeedRNG(rseed); } } +// SetPhases is used as InitMeshBlockUserData because phases need to be reset on remeshing void SetPhases(MeshBlock *pmb, ParameterInput *pin) { - auto pm = pmb->pmy_mesh; auto hydro_pkg = pmb->packages.Get("Hydro"); - - // The following restriction could technically be lifted if the turbulence driver is - // directly embedded in the hydro driver rather than a user defined source as well as - // fixing the pack_size=-1 when using the Mesh- (not MeshBlock-)based problem generator. - // The restriction stems from requiring a collective MPI comm to normalize the - // acceleration and magnetic field, respectively. Note, that the restriction does not - // apply here, but for the ProblemGenerator() and Driving() function below. The check is - // just added here for convenience as this function is called during problem - // initializtion. From my (pgrete) point of view, it's currently cleaner to keep things - // separate and not touch the main driver at the expense of using one pack per rank -- - // which is typically fastest on devices anyway. - const auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size"); - PARTHENON_REQUIRE_THROWS(pack_size == -1, - "Turbulence pgen currently needs parthenon/mesh/pack_size=-1 " - "to work because of global reductions.") - - auto Lx = pm->mesh_size.x1max - pm->mesh_size.x1min; - auto Ly = pm->mesh_size.x2max - pm->mesh_size.x2min; - auto Lz = pm->mesh_size.x3max - pm->mesh_size.x3min; - // should also be easily fixed, just need to double check transforms and volume - // weighting everywhere - if ((Lx != 1.0) || (Ly != 1.0) || (Lz != 1.0)) { - std::stringstream msg; - msg << "### FATAL ERROR in turbulence driver" << std::endl - << "Only domain sizes with edge lengths of 1 are supported." << std::endl; - throw std::runtime_error(msg.str().c_str()); - } - - auto gnx1 = pm->mesh_size.nx1; - auto gnx2 = pm->mesh_size.nx2; - auto gnx3 = pm->mesh_size.nx3; - // as above, this restriction should/could be easily lifted - if ((gnx1 != gnx2) || (gnx2 != gnx3)) { - std::stringstream msg; - msg << "### FATAL ERROR in turbulence driver" << std::endl - << "Only cubic mesh sizes are supported." << std::endl; - throw std::runtime_error(msg.str().c_str()); - } - - const auto nx1 = pmb->block_size.nx1; - const auto nx2 = pmb->block_size.nx2; - const auto nx3 = pmb->block_size.nx3; - - const auto gis = pmb->loc.lx1 * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2 * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3 * pmb->block_size.nx3; - - const auto num_modes = hydro_pkg->Param("turbulence/num_modes"); - - // make local ref to capure in lambda - auto &k_vec = k_vec_; - - Complex I(0.0, 1.0); - - auto &base = pmb->meshblock_data.Get(); - auto &phases_i = base->Get("phases_i").data; - auto &phases_j = base->Get("phases_j").data; - auto &phases_k = base->Get("phases_k").data; - - pmb->par_for( - "forcing: calc phases_i", 0, nx1 - 1, KOKKOS_LAMBDA(int i) { - Real gi = static_cast(i + gis); - Real w_kx; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_kx = k_vec(0, m) * 2. * M_PI / static_cast(gnx1); - // adjust phase factor to Complex->Real IFT: u_hat*(k) = u_hat(-k) - if (k_vec(0, m) == 0.0) { - phase = 0.5 * Kokkos::exp(I * w_kx * gi); - } else { - phase = Kokkos::exp(I * w_kx * gi); - } - phases_i(i, m, 0) = phase.real(); - phases_i(i, m, 1) = phase.imag(); - } - }); - - pmb->par_for( - "forcing: calc phases_j", 0, nx2 - 1, KOKKOS_LAMBDA(int j) { - Real gj = static_cast(j + gjs); - Real w_ky; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_ky = k_vec(1, m) * 2. * M_PI / static_cast(gnx2); - phase = Kokkos::exp(I * w_ky * gj); - phases_j(j, m, 0) = phase.real(); - phases_j(j, m, 1) = phase.imag(); - } - }); - - pmb->par_for( - "forcing: calc phases_k", 0, nx3 - 1, KOKKOS_LAMBDA(int k) { - Real gk = static_cast(k + gks); - Real w_kz; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_kz = k_vec(2, m) * 2. * M_PI / static_cast(gnx3); - phase = Kokkos::exp(I * w_kz * gk); - phases_k(k, m, 0) = phase.real(); - phases_k(k, m, 1) = phase.imag(); - } - }); + auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); + few_modes_ft.SetPhases(pmb, pin); } //======================================================================================== @@ -454,135 +342,8 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { void Generate(MeshData *md, Real dt) { auto pmb = md->GetBlockData(0)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); - - const auto num_modes = hydro_pkg->Param("turbulence/num_modes"); - - Complex I(0.0, 1.0); - auto &random_num = random_num_; - - // get a set of random numbers from the CPU so that they are deterministic - // when run on GPUs - Real v1, v2, v_sqr; - for (int n = 0; n < 3; n++) - for (int m = 0; m < num_modes; m++) { - do { - v1 = dist(rng); - v2 = dist(rng); - v_sqr = v1 * v1 + v2 * v2; - } while (v_sqr >= 1.0 || v_sqr == 0.0); - - random_num_host(n, m, 0) = v1; - random_num_host(n, m, 1) = v2; - } - Kokkos::deep_copy(random_num, random_num_host); - - // make local ref to capure in lambda - auto &k_vec = k_vec_; - auto &accel_hat = accel_hat_; - auto &accel_hat_new = accel_hat_new_; - - const auto kpeak = hydro_pkg->Param("turbulence/kpeak"); - // generate new power spectrum (injection) - pmb->par_for( - "forcing: new power spec", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - Real kmag, tmp, norm, v_sqr; - - Real kx = k_vec(0, m); - Real ky = k_vec(1, m); - Real kz = k_vec(2, m); - - kmag = std::sqrt(kx * kx + ky * ky + kz * kz); - - accel_hat_new(n, m) = Complex(0., 0.); - - tmp = std::pow(kmag / kpeak, 2.) * (2. - std::pow(kmag / kpeak, 2.)); - if (tmp < 0.) tmp = 0.; - v_sqr = SQR(random_num(n, m, 0)) + SQR(random_num(n, m, 1)); - norm = std::sqrt(-2.0 * std::log(v_sqr) / v_sqr); - - accel_hat_new(n, m) = - Complex(tmp * norm * random_num(n, m, 0), tmp * norm * random_num(n, m, 1)); - }); - - // enforce symmetry of complex to real transform - pmb->par_for( - "forcing: enforce symmetry", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - if (k_vec(0, m) == 0.) { - for (int m2 = 0; m2 < m; m2++) { - if (k_vec(1, m) == -k_vec(1, m2) && k_vec(2, m) == -k_vec(2, m2)) - accel_hat_new(n, m) = - Complex(accel_hat_new(n, m2).real(), -accel_hat_new(n, m2).imag()); - } - } - }); - - const auto sol_weight = hydro_pkg->Param("turbulence/sol_weight"); - // project - pmb->par_for( - "forcing: projection", 0, num_modes - 1, KOKKOS_LAMBDA(const int m) { - Real kmag; - - Real kx = k_vec(0, m); - Real ky = k_vec(1, m); - Real kz = k_vec(2, m); - - kmag = std::sqrt(kx * kx + ky * ky + kz * kz); - - // setting kmag to 1 as a "continue" doesn't work within the parallel_for - // construct and it doesn't affect anything (there should never be power in the - // k=0 mode) - if (kmag == 0.) kmag = 1.; - - // make it a unit vector - kx /= kmag; - ky /= kmag; - kz /= kmag; - - Complex dot(accel_hat_new(0, m).real() * kx + accel_hat_new(1, m).real() * ky + - accel_hat_new(2, m).real() * kz, - accel_hat_new(0, m).imag() * kx + accel_hat_new(1, m).imag() * ky + - accel_hat_new(2, m).imag() * kz); - - accel_hat_new(0, m) = Complex(accel_hat_new(0, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * kx, - accel_hat_new(0, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * kx); - accel_hat_new(1, m) = Complex(accel_hat_new(1, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * ky, - accel_hat_new(1, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * ky); - accel_hat_new(2, m) = Complex(accel_hat_new(2, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * kz, - accel_hat_new(2, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * kz); - }); - - // evolve - const auto tcorr = hydro_pkg->Param("turbulence/tcorr"); - Real c_drift = std::exp(-dt / tcorr); - Real c_diff = std::sqrt(1.0 - c_drift * c_drift); - - pmb->par_for( - "forcing: evolve spec", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - accel_hat(n, m) = Complex( - accel_hat(n, m).real() * c_drift + accel_hat_new(n, m).real() * c_diff, - accel_hat(n, m).imag() * c_drift + accel_hat_new(n, m).imag() * c_diff); - }); - - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - auto acc_pack = md->PackVariables(std::vector{"acc"}); - auto phases_i = md->PackVariables(std::vector{"phases_i"}); - auto phases_j = md->PackVariables(std::vector{"phases_j"}); - auto phases_k = md->PackVariables(std::vector{"phases_k"}); - - utils::few_modes_ft::InverseFT>( - acc_pack, phases_i, phases_j, phases_k, accel_hat, ib, jb, kb, acc_pack.GetDim(5), - num_modes); + auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); + few_modes_ft.Generate(md, dt, "acc"); } //---------------------------------------------------------------------------------------- @@ -685,24 +446,17 @@ void Driving(MeshData *md, const parthenon::SimTime &tm, const Real dt) { Perturb(md, dt); } -void Cleanup() { - // Ensure the Kokkos views are gargabe collected before finalized is called - k_vec_ = {}; - accel_hat_ = {}; - accel_hat_new_ = {}; - random_num_ = {}; - random_num_host = {}; -} - void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto num_modes = hydro_pkg->Param("turbulence/num_modes"); // Store (common) acceleration field in spectral space + auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); + auto var_hat = few_modes_ft.GetVarHat(); auto accel_hat_host = - Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), accel_hat_); + Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), var_hat); + const auto num_modes = few_modes_ft.GetNumModes(); for (int i = 0; i < 3; i++) { for (int m = 0; m < num_modes; m++) { pin->SetReal("problem/turbulence", @@ -714,17 +468,11 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { } } // store state of random number gen - { - std::ostringstream oss; - oss << rng; - pin->SetString("problem/turbulence", "state_rng", oss.str()); - } + auto state_rng = few_modes_ft.GetRNGState(); + pin->SetString("problem/turbulence", "state_rng", state_rng); // store state of distribution - { - std::ostringstream oss; - oss << dist; - pin->SetString("problem/turbulence", "state_dist", oss.str()); - } + auto state_dist = few_modes_ft.GetDistState(); + pin->SetString("problem/turbulence", "state_dist", state_dist); } } // namespace turbulence diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp new file mode 100644 index 00000000..8d1f3b3d --- /dev/null +++ b/src/utils/few_modes_ft.cpp @@ -0,0 +1,332 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//======================================================================================== +//! \file few_modes_ft.cpp +// \brief Helper functions for an inverse (explicit complex to real) FT + +// C++ headers +#include + +// Parthenon headers +#include "basic_types.hpp" +#include "config.hpp" +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/meshblock_pack.hpp" + +// AthenaPK headers +#include "../main.hpp" +#include "few_modes_ft.hpp" +#include "utils/error_checking.hpp" + +namespace utils::few_modes_ft { +using Complex = Kokkos::complex; +using parthenon::IndexRange; +using parthenon::Metadata; + +FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, + std::string prefix, int num_modes, ParArray2D k_vec, + Real k_peak, Real sol_weight, Real t_corr) + : prefix_(prefix), num_modes_(num_modes), k_vec_(k_vec), k_peak_(k_peak), + t_corr_(t_corr), dist_(-1.0, 1.0) { + + if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { + std::cout << "### WARNING using more than 100 explicit modes will significantly " + << "increase the runtime." << std::endl + << "If many modes are required in the transform field consider using " + << "the driving mechanism based on full FFTs." << std::endl; + } + const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); + const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); + const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); + auto m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, + std::vector({2, num_modes, nx1}), prefix + "_phases_i"); + pkg->AddField(prefix + "_phases_i", m); + m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, + std::vector({2, num_modes, nx2}), prefix + "_phases_j"); + pkg->AddField(prefix + "_phases_j", m); + m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, + std::vector({2, num_modes, nx3}), prefix + "_phases_k"); + pkg->AddField(prefix + "_phases_k", m); + + // Variable (e.g., acceleration field for turbulence driver) in Fourier space using + // complex to real transform. + var_hat_ = ParArray2D(prefix + "_var_hat", 3, num_modes); + var_hat_new_ = ParArray2D(prefix + "_var_hat_new", 3, num_modes); + + PARTHENON_REQUIRE((sol_weight == -1.0) || (sol_weight >= 0.0 && sol_weight <= 1.0), + "sol_weight for projection in few modes fft module needs to be " + "between 0.0 and 1.0 or set to -1.0 (to disable projection).") + sol_weight_ = sol_weight; + + random_num_ = Kokkos::View( + "random_num", 3, num_modes, 2); + random_num_host_ = Kokkos::create_mirror_view(random_num_); +} + +void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { + auto pm = pmb->pmy_mesh; + auto hydro_pkg = pmb->packages.Get("Hydro"); + + // The following restriction could technically be lifted if the turbulence driver is + // directly embedded in the hydro driver rather than a user defined source as well as + // fixing the pack_size=-1 when using the Mesh- (not MeshBlock-)based problem generator. + // The restriction stems from requiring a collective MPI comm to normalize the + // acceleration and magnetic field, respectively. Note, that the restriction does not + // apply here, but for the ProblemGenerator() and Driving() function below. The check is + // just added here for convenience as this function is called during problem + // initializtion. From my (pgrete) point of view, it's currently cleaner to keep things + // separate and not touch the main driver at the expense of using one pack per rank -- + // which is typically fastest on devices anyway. + const auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size"); + PARTHENON_REQUIRE_THROWS(pack_size == -1, + "Few modes FT currently needs parthenon/mesh/pack_size=-1 " + "to work because of global reductions.") + + auto Lx = pm->mesh_size.x1max - pm->mesh_size.x1min; + auto Ly = pm->mesh_size.x2max - pm->mesh_size.x2min; + auto Lz = pm->mesh_size.x3max - pm->mesh_size.x3min; + // should also be easily fixed, just need to double check transforms and volume + // weighting everywhere + if ((Lx != 1.0) || (Ly != 1.0) || (Lz != 1.0)) { + std::stringstream msg; + msg << "### FATAL ERROR in turbulence driver" << std::endl + << "Only domain sizes with edge lengths of 1 are supported." << std::endl; + throw std::runtime_error(msg.str().c_str()); + } + + auto gnx1 = pm->mesh_size.nx1; + auto gnx2 = pm->mesh_size.nx2; + auto gnx3 = pm->mesh_size.nx3; + // as above, this restriction should/could be easily lifted + if ((gnx1 != gnx2) || (gnx2 != gnx3)) { + std::stringstream msg; + msg << "### FATAL ERROR in turbulence driver" << std::endl + << "Only cubic mesh sizes are supported." << std::endl; + throw std::runtime_error(msg.str().c_str()); + } + + const auto nx1 = pmb->block_size.nx1; + const auto nx2 = pmb->block_size.nx2; + const auto nx3 = pmb->block_size.nx3; + + const auto gis = pmb->loc.lx1 * pmb->block_size.nx1; + const auto gjs = pmb->loc.lx2 * pmb->block_size.nx2; + const auto gks = pmb->loc.lx3 * pmb->block_size.nx3; + + // make local ref to capure in lambda + const auto num_modes = num_modes_; + auto &k_vec = k_vec_; + + Complex I(0.0, 1.0); + + auto &base = pmb->meshblock_data.Get(); + auto &phases_i = base->Get(prefix_ + "_phases_i").data; + auto &phases_j = base->Get(prefix_ + "_phases_j").data; + auto &phases_k = base->Get(prefix_ + "_phases_k").data; + + pmb->par_for( + "FMFT: calc phases_i", 0, nx1 - 1, KOKKOS_LAMBDA(int i) { + Real gi = static_cast(i + gis); + Real w_kx; + Complex phase; + + for (int m = 0; m < num_modes; m++) { + w_kx = k_vec(0, m) * 2. * M_PI / static_cast(gnx1); + // adjust phase factor to Complex->Real IFT: u_hat*(k) = u_hat(-k) + if (k_vec(0, m) == 0.0) { + phase = 0.5 * Kokkos::exp(I * w_kx * gi); + } else { + phase = Kokkos::exp(I * w_kx * gi); + } + phases_i(i, m, 0) = phase.real(); + phases_i(i, m, 1) = phase.imag(); + } + }); + + pmb->par_for( + "FMFT: calc phases_j", 0, nx2 - 1, KOKKOS_LAMBDA(int j) { + Real gj = static_cast(j + gjs); + Real w_ky; + Complex phase; + + for (int m = 0; m < num_modes; m++) { + w_ky = k_vec(1, m) * 2. * M_PI / static_cast(gnx2); + phase = Kokkos::exp(I * w_ky * gj); + phases_j(j, m, 0) = phase.real(); + phases_j(j, m, 1) = phase.imag(); + } + }); + + pmb->par_for( + "FMFT: calc phases_k", 0, nx3 - 1, KOKKOS_LAMBDA(int k) { + Real gk = static_cast(k + gks); + Real w_kz; + Complex phase; + + for (int m = 0; m < num_modes; m++) { + w_kz = k_vec(2, m) * 2. * M_PI / static_cast(gnx3); + phase = Kokkos::exp(I * w_kz * gk); + phases_k(k, m, 0) = phase.real(); + phases_k(k, m, 1) = phase.imag(); + } + }); +} + +void FewModesFT::Generate(MeshData *md, const Real dt, + const std::string &var_name) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + + const auto num_modes = num_modes_; + + Complex I(0.0, 1.0); + auto &random_num = random_num_; + + // get a set of random numbers from the CPU so that they are deterministic + // when run on GPUs + Real v1, v2, v_sqr; + for (int n = 0; n < 3; n++) + for (int m = 0; m < num_modes; m++) { + do { + v1 = dist_(rng_); + v2 = dist_(rng_); + v_sqr = v1 * v1 + v2 * v2; + } while (v_sqr >= 1.0 || v_sqr == 0.0); + + random_num_host_(n, m, 0) = v1; + random_num_host_(n, m, 1) = v2; + } + Kokkos::deep_copy(random_num, random_num_host_); + + // make local ref to capure in lambda + auto &k_vec = k_vec_; + auto &var_hat = var_hat_; + auto &var_hat_new = var_hat_new_; + + const auto kpeak = k_peak_; + + // generate new power spectrum (injection) + pmb->par_for( + "FMFT: new power spec", 0, 2, 0, num_modes - 1, + KOKKOS_LAMBDA(const int n, const int m) { + Real kmag, tmp, norm, v_sqr; + + Real kx = k_vec(0, m); + Real ky = k_vec(1, m); + Real kz = k_vec(2, m); + + kmag = std::sqrt(kx * kx + ky * ky + kz * kz); + + var_hat_new(n, m) = Complex(0., 0.); + + tmp = std::pow(kmag / kpeak, 2.) * (2. - std::pow(kmag / kpeak, 2.)); + if (tmp < 0.) tmp = 0.; + v_sqr = SQR(random_num(n, m, 0)) + SQR(random_num(n, m, 1)); + norm = std::sqrt(-2.0 * std::log(v_sqr) / v_sqr); + + var_hat_new(n, m) = + Complex(tmp * norm * random_num(n, m, 0), tmp * norm * random_num(n, m, 1)); + }); + + // enforce symmetry of complex to real transform + pmb->par_for( + "forcing: enforce symmetry", 0, 2, 0, num_modes - 1, + KOKKOS_LAMBDA(const int n, const int m) { + if (k_vec(0, m) == 0.) { + for (int m2 = 0; m2 < m; m2++) { + if (k_vec(1, m) == -k_vec(1, m2) && k_vec(2, m) == -k_vec(2, m2)) + var_hat_new(n, m) = + Complex(var_hat_new(n, m2).real(), -var_hat_new(n, m2).imag()); + } + } + }); + + const auto sol_weight = sol_weight_; + if (sol_weight_ >= 0.0) { + // project + pmb->par_for( + "forcing: projection", 0, num_modes - 1, KOKKOS_LAMBDA(const int m) { + Real kmag; + + Real kx = k_vec(0, m); + Real ky = k_vec(1, m); + Real kz = k_vec(2, m); + + kmag = std::sqrt(kx * kx + ky * ky + kz * kz); + + // setting kmag to 1 as a "continue" doesn't work within the parallel_for + // construct and it doesn't affect anything (there should never be power in the + // k=0 mode) + if (kmag == 0.) kmag = 1.; + + // make it a unit vector + kx /= kmag; + ky /= kmag; + kz /= kmag; + + Complex dot(var_hat_new(0, m).real() * kx + var_hat_new(1, m).real() * ky + + var_hat_new(2, m).real() * kz, + var_hat_new(0, m).imag() * kx + var_hat_new(1, m).imag() * ky + + var_hat_new(2, m).imag() * kz); + + var_hat_new(0, m) = Complex(var_hat_new(0, m).real() * sol_weight + + (1. - 2. * sol_weight) * dot.real() * kx, + var_hat_new(0, m).imag() * sol_weight + + (1. - 2. * sol_weight) * dot.imag() * kx); + var_hat_new(1, m) = Complex(var_hat_new(1, m).real() * sol_weight + + (1. - 2. * sol_weight) * dot.real() * ky, + var_hat_new(1, m).imag() * sol_weight + + (1. - 2. * sol_weight) * dot.imag() * ky); + var_hat_new(2, m) = Complex(var_hat_new(2, m).real() * sol_weight + + (1. - 2. * sol_weight) * dot.real() * kz, + var_hat_new(2, m).imag() * sol_weight + + (1. - 2. * sol_weight) * dot.imag() * kz); + }); + } + + // evolve + const auto t_corr = t_corr_; + Real c_drift = std::exp(-dt / t_corr); + Real c_diff = std::sqrt(1.0 - c_drift * c_drift); + + pmb->par_for( + "FMFT: evolve spec", 0, 2, 0, num_modes - 1, + KOKKOS_LAMBDA(const int n, const int m) { + var_hat(n, m) = + Complex(var_hat(n, m).real() * c_drift + var_hat_new(n, m).real() * c_diff, + var_hat(n, m).imag() * c_drift + var_hat_new(n, m).imag() * c_diff); + }); + + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + auto var_pack = md->PackVariables(std::vector{var_name}); + auto phases_i = md->PackVariables(std::vector{prefix_ + "_phases_i"}); + auto phases_j = md->PackVariables(std::vector{prefix_ + "_phases_j"}); + auto phases_k = md->PackVariables(std::vector{prefix_ + "_phases_k"}); + + // implictly assuming cubic box of size L=1 + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "FMFT: Inverse FT", parthenon::DevExecSpace(), 0, + md->NumBlocks() - 1, 0, 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { + Complex phase, phase_i, phase_j, phase_k; + var_pack(b, n, k, j, i) = 0.0; + + for (int m = 0; m < num_modes; m++) { + phase_i = + Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); + phase_j = + Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); + phase_k = + Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); + phase = phase_i * phase_j * phase_k; + var_pack(b, n, k, j, i) += 2. * (var_hat(n, m).real() * phase.real() - + var_hat(n, m).imag() * phase.imag()); + } + }); +} +} // namespace utils::few_modes_ft \ No newline at end of file diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp index af0f67d7..3e1ba9f6 100644 --- a/src/utils/few_modes_ft.hpp +++ b/src/utils/few_modes_ft.hpp @@ -12,12 +12,15 @@ #include "basic_types.hpp" #include "config.hpp" #include +#include +#include // AthenaPK headers #include "../main.hpp" #include "mesh/domain.hpp" namespace utils::few_modes_ft { +using namespace parthenon::package::prelude; using parthenon::Real; using Complex = Kokkos::complex; using parthenon::IndexRange; @@ -29,39 +32,37 @@ class FewModesFT { std::string prefix_; ParArray2D var_hat_, var_hat_new_; ParArray2D k_vec_; + Real k_peak_; // peak of the power spectrum + Kokkos::View random_num_; + Kokkos::View random_num_host_; + std::mt19937 rng_; + std::uniform_real_distribution<> dist_; + Real sol_weight_; // power in solenoidal modes for projection. Set to negative to + // disable projection + Real t_corr_; // correlation time for evolution of Ornstein-Uhlenbeck process public: FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, - std::string prefix, int num_modes, ParArray2D k_vec); + std::string prefix, int num_modes, ParArray2D k_vec, Real k_peak, + Real sol_weight, Real t_corr); - ParArray2D GetVarHat() { return var_hat_ } - void SetVarHat(ParArray2D var_hat) { var_hat_ = var_hat } + ParArray2D GetVarHat() { return var_hat_; } + int GetNumModes() { return num_modes_; } + void SetPhases(MeshBlock *pmb, ParameterInput *pin); + void Generate(MeshData *md, const Real dt, const std::string &var_name); + void RestoreRNG(std::istringstream &iss) { iss >> rng_; } + void RestoreDist(std::istringstream &iss) { iss >> dist_; } + void SeedRNG(uint32_t rseed) { rng_.seed(rseed); } + std::string GetRNGState() { + std::ostringstream oss; + oss << rng_; + return oss.str(); + } + std::string GetDistState() { + std::ostringstream oss; + oss << dist_; + return oss.str(); + } }; -template -void InverseFT(const TPack &out_pack, const TPack &phases_i, const TPack &phases_j, - const TPack &phases_k, const THat &in_hat, const IndexRange &ib, - const IndexRange &jb, const IndexRange &kb, const int num_blocks, - const int num_modes) { - // implictly assuming cubic box of size L=1 - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Inverse FT", parthenon::DevExecSpace(), 0, num_blocks - 1, 0, - 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { - Complex phase, phase_i, phase_j, phase_k; - out_pack(b, n, k, j, i) = 0.0; - - for (int m = 0; m < num_modes; m++) { - phase_i = - Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); - phase_j = - Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); - phase_k = - Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); - phase = phase_i * phase_j * phase_k; - out_pack(b, n, k, j, i) += 2. * (in_hat(n, m).real() * phase.real() - - in_hat(n, m).imag() * phase.imag()); - } - }); -} } // namespace utils::few_modes_ft \ No newline at end of file From cd2420c6c427900ec9d37baacaeaf00446520d74 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 15 Jun 2023 13:22:14 +0200 Subject: [PATCH 08/48] Make FMFT smooth and consistent for mesh refinement runs --- src/pgen/turbulence.cpp | 1 - src/utils/few_modes_ft.cpp | 12 +++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index b47cda6a..7d728fc4 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -449,7 +449,6 @@ void Driving(MeshData *md, const parthenon::SimTime &tm, const Real dt) { void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { auto hydro_pkg = pmb->packages.Get("Hydro"); - // Store (common) acceleration field in spectral space auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); auto var_hat = few_modes_ft.GetVarHat(); diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index 8d1f3b3d..a96d5cff 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -89,6 +89,7 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { auto Lx = pm->mesh_size.x1max - pm->mesh_size.x1min; auto Ly = pm->mesh_size.x2max - pm->mesh_size.x2min; auto Lz = pm->mesh_size.x3max - pm->mesh_size.x3min; + // should also be easily fixed, just need to double check transforms and volume // weighting everywhere if ((Lx != 1.0) || (Ly != 1.0) || (Lz != 1.0)) { @@ -98,9 +99,14 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { throw std::runtime_error(msg.str().c_str()); } - auto gnx1 = pm->mesh_size.nx1; - auto gnx2 = pm->mesh_size.nx2; - auto gnx3 = pm->mesh_size.nx3; + // Adjust (logical) grid size at levels other than the root level. + // This is required for simulation with mesh refinement so that the phases calculated + // below take the logical grid size into account. For example, the local phases at level + // 1 should be calculated assuming a grid that is twice as large as the root grid. + const auto root_level = pm->GetRootLevel(); + auto gnx1 = pm->mesh_size.nx1 * std::pow(2, pmb->loc.level - root_level); + auto gnx2 = pm->mesh_size.nx2 * std::pow(2, pmb->loc.level - root_level); + auto gnx3 = pm->mesh_size.nx3 * std::pow(2, pmb->loc.level - root_level); // as above, this restriction should/could be easily lifted if ((gnx1 != gnx2) || (gnx2 != gnx3)) { std::stringstream msg; From 83de45464a9d7455b5f21e889b32acea7f1ae8f2 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 15 Jun 2023 22:10:22 +0200 Subject: [PATCH 09/48] Fix update of internal RNG state --- src/pgen/turbulence.cpp | 25 +++++++++++++------------ src/utils/few_modes_ft.cpp | 13 ++++++++----- src/utils/few_modes_ft.hpp | 3 +-- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 7d728fc4..58882f52 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -155,14 +155,18 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg } Kokkos::deep_copy(k_vec, k_vec_host); - auto few_modes_ft = - FewModesFT(pin, pkg, "turbulence", num_modes, k_vec, k_peak, sol_weight, t_corr); - pkg->AddParam<>("turbulence/few_modes_ft", few_modes_ft); + auto few_modes_ft = FewModesFT(pin, pkg, "turbulence", num_modes, k_vec, k_peak, + sol_weight, t_corr, rseed); + // object must be mutable to update the internal state of the RNG + pkg->AddParam<>("turbulence/few_modes_ft", few_modes_ft, true); // Check if this is is a restart and restore previous state if (pin->DoesParameterExist("problem/turbulence", "accel_hat_0_0_r")) { + // Need to extract mutable object from Params here as the original few_modes_ft above + // and the one in Params are different instances + auto *pfew_modes_ft = pkg->MutableParam("turbulence/few_modes_ft"); // Restore (common) acceleration field in spectral space - auto accel_hat = few_modes_ft.GetVarHat(); + auto accel_hat = pfew_modes_ft->GetVarHat(); auto accel_hat_host = Kokkos::create_mirror_view(accel_hat); for (int i = 0; i < 3; i++) { for (int m = 0; m < num_modes; m++) { @@ -180,17 +184,13 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg // Restore state of random number gen { std::istringstream iss(pin->GetString("problem/turbulence", "state_rng")); - few_modes_ft.RestoreRNG(iss); + pfew_modes_ft->RestoreRNG(iss); } // Restore state of dist { std::istringstream iss(pin->GetString("problem/turbulence", "state_dist")); - few_modes_ft.RestoreDist(iss); + pfew_modes_ft->RestoreDist(iss); } - - } else { - // init RNG - few_modes_ft.SeedRNG(rseed); } } @@ -342,8 +342,9 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { void Generate(MeshData *md, Real dt) { auto pmb = md->GetBlockData(0)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); - auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); - few_modes_ft.Generate(md, dt, "acc"); + // Must be mutable so the internal RNG state is updated + auto *few_modes_ft = hydro_pkg->MutableParam("turbulence/few_modes_ft"); + few_modes_ft->Generate(md, dt, "acc"); } //---------------------------------------------------------------------------------------- diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index a96d5cff..4cfba1c3 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -8,6 +8,7 @@ // \brief Helper functions for an inverse (explicit complex to real) FT // C++ headers +#include #include // Parthenon headers @@ -29,9 +30,9 @@ using parthenon::Metadata; FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, std::string prefix, int num_modes, ParArray2D k_vec, - Real k_peak, Real sol_weight, Real t_corr) + Real k_peak, Real sol_weight, Real t_corr, uint32_t rseed) : prefix_(prefix), num_modes_(num_modes), k_vec_(k_vec), k_peak_(k_peak), - t_corr_(t_corr), dist_(-1.0, 1.0) { + t_corr_(t_corr) { if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { std::cout << "### WARNING using more than 100 explicit modes will significantly " @@ -65,6 +66,9 @@ FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescripto random_num_ = Kokkos::View( "random_num", 3, num_modes, 2); random_num_host_ = Kokkos::create_mirror_view(random_num_); + + rng_.seed(rseed); + dist_ = std::uniform_real_distribution<>(-1.0, 1.0); } void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { @@ -294,9 +298,8 @@ void FewModesFT::Generate(MeshData *md, const Real dt, } // evolve - const auto t_corr = t_corr_; - Real c_drift = std::exp(-dt / t_corr); - Real c_diff = std::sqrt(1.0 - c_drift * c_drift); + const auto c_drift = std::exp(-dt / t_corr_); + const auto c_diff = std::sqrt(1.0 - c_drift * c_drift); pmb->par_for( "FMFT: evolve spec", 0, 2, 0, num_modes - 1, diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp index 3e1ba9f6..572f163e 100644 --- a/src/utils/few_modes_ft.hpp +++ b/src/utils/few_modes_ft.hpp @@ -44,7 +44,7 @@ class FewModesFT { public: FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, std::string prefix, int num_modes, ParArray2D k_vec, Real k_peak, - Real sol_weight, Real t_corr); + Real sol_weight, Real t_corr, uint32_t rseed); ParArray2D GetVarHat() { return var_hat_; } int GetNumModes() { return num_modes_; } @@ -52,7 +52,6 @@ class FewModesFT { void Generate(MeshData *md, const Real dt, const std::string &var_name); void RestoreRNG(std::istringstream &iss) { iss >> rng_; } void RestoreDist(std::istringstream &iss) { iss >> dist_; } - void SeedRNG(uint32_t rseed) { rng_.seed(rseed); } std::string GetRNGState() { std::ostringstream oss; oss << rng_; From 8c74b1dde89ec3690398440a13178c24badd3fff Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 15 Jun 2023 23:02:09 +0200 Subject: [PATCH 10/48] Remove restriction on unit domain dims --- src/utils/few_modes_ft.cpp | 40 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index 4cfba1c3..e7a329d5 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -40,6 +40,17 @@ FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescripto << "If many modes are required in the transform field consider using " << "the driving mechanism based on full FFTs." << std::endl; } + // Ensure that all all wavevectors can be represented on the root grid + const auto gnx1 = pin->GetInteger("parthenon/mesh", "nx1"); + const auto gnx2 = pin->GetInteger("parthenon/mesh", "nx2"); + const auto gnx3 = pin->GetInteger("parthenon/mesh", "nx3"); + Kokkos::parallel_for( + "FMFT: Check k_vec", num_modes, KOKKOS_LAMBDA(const int i) { + PARTHENON_REQUIRE(std::abs(k_vec_(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); + }); + const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); @@ -90,18 +101,9 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { "Few modes FT currently needs parthenon/mesh/pack_size=-1 " "to work because of global reductions.") - auto Lx = pm->mesh_size.x1max - pm->mesh_size.x1min; - auto Ly = pm->mesh_size.x2max - pm->mesh_size.x2min; - auto Lz = pm->mesh_size.x3max - pm->mesh_size.x3min; - - // should also be easily fixed, just need to double check transforms and volume - // weighting everywhere - if ((Lx != 1.0) || (Ly != 1.0) || (Lz != 1.0)) { - std::stringstream msg; - msg << "### FATAL ERROR in turbulence driver" << std::endl - << "Only domain sizes with edge lengths of 1 are supported." << std::endl; - throw std::runtime_error(msg.str().c_str()); - } + auto Lx1 = pm->mesh_size.x1max - pm->mesh_size.x1min; + auto Lx2 = pm->mesh_size.x2max - pm->mesh_size.x2min; + auto Lx3 = pm->mesh_size.x3max - pm->mesh_size.x3min; // Adjust (logical) grid size at levels other than the root level. // This is required for simulation with mesh refinement so that the phases calculated @@ -111,13 +113,13 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { auto gnx1 = pm->mesh_size.nx1 * std::pow(2, pmb->loc.level - root_level); auto gnx2 = pm->mesh_size.nx2 * std::pow(2, pmb->loc.level - root_level); auto gnx3 = pm->mesh_size.nx3 * std::pow(2, pmb->loc.level - root_level); - // as above, this restriction should/could be easily lifted - if ((gnx1 != gnx2) || (gnx2 != gnx3)) { - std::stringstream msg; - msg << "### FATAL ERROR in turbulence driver" << std::endl - << "Only cubic mesh sizes are supported." << std::endl; - throw std::runtime_error(msg.str().c_str()); - } + + // Restriction should also be easily fixed, just need to double check transforms and + // volume weighting everywhere + PARTHENON_REQUIRE_THROWS(((gnx1 == gnx2) && (gnx2 == gnx3)) && + ((Lx1 == Lx2) && (Lx2 == Lx3)), + "FMFT has only been tested with cubic meshes and constant " + "dx/dy/dz. Remove this warning at your own risk.") const auto nx1 = pmb->block_size.nx1; const auto nx2 = pmb->block_size.nx2; From 6ed81701d36562a5249931e146895c3882224bb2 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 09:06:18 +0200 Subject: [PATCH 11/48] Fix k_vec check on device --- src/utils/few_modes_ft.cpp | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index e7a329d5..01f8d2d6 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -44,12 +44,14 @@ FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescripto const auto gnx1 = pin->GetInteger("parthenon/mesh", "nx1"); const auto gnx2 = pin->GetInteger("parthenon/mesh", "nx2"); const auto gnx3 = pin->GetInteger("parthenon/mesh", "nx3"); - Kokkos::parallel_for( - "FMFT: Check k_vec", num_modes, KOKKOS_LAMBDA(const int i) { - PARTHENON_REQUIRE(std::abs(k_vec_(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); - PARTHENON_REQUIRE(std::abs(k_vec_(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); - PARTHENON_REQUIRE(std::abs(k_vec_(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); - }); + // Need to make this comparison on the host as (for some reason) an extended cuda device + // lambda cannot live in the constructor of an object. + auto k_vec_host = k_vec.GetHostMirrorAndCopy(); + for (int i = 0; i < num_modes; i++) { + PARTHENON_REQUIRE(std::abs(k_vec(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); + } const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); From 2571e1b6ae845d6279be6ea143031bc761b5b63a Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 13:38:52 +0200 Subject: [PATCH 12/48] Add init perturb for velocity field --- src/pgen/cluster.cpp | 105 +++++++++++++++++- .../test_suites/cluster_hse/cluster_hse.py | 7 ++ 2 files changed, 109 insertions(+), 3 deletions(-) diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 7012fc41..992b81fc 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -25,6 +25,7 @@ #include // c_str() // Parthenon headers +#include "kokkos_abstraction.hpp" #include "mesh/domain.hpp" #include "mesh/mesh.hpp" #include @@ -35,6 +36,7 @@ #include "../hydro/srcterms/gravitational_field.hpp" #include "../hydro/srcterms/tabular_cooling.hpp" #include "../main.hpp" +#include "../utils/few_modes_ft.hpp" // Cluster headers #include "cluster/agn_feedback.hpp" @@ -44,10 +46,13 @@ #include "cluster/hydrostatic_equilibrium_sphere.hpp" #include "cluster/magnetic_tower.hpp" #include "cluster/snia_feedback.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" namespace cluster { using namespace parthenon::driver::prelude; using namespace parthenon::package::prelude; +using utils::few_modes_ft::FewModesFT; void ClusterSrcTerm(MeshData *md, const parthenon::SimTime &tm, const Real beta_dt) { @@ -244,6 +249,87 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // plasma beta hydro_pkg->AddField("plasma_beta", m); } + + /************************************************************ + * Add infrastructure for initial pertubations + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + if (sigma_v != 0.0) { + // peak of init vel perturb + auto k_peak_v = pin->GetReal("problem/cluster/init_perturb", "k_peak_v"); + auto num_modes_v = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); + auto sol_weight_v = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); + uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); + // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const auto t_corr = 1e-10; + + // Construct list of wavenumers for inital perturbation.We pick randomly between + // kpeak/2 and 2kpeak. + auto k_vec_v = parthenon::ParArray2D("k_vec", 3, num_modes_v); + auto k_vec_v_h = + Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), k_vec_v); + + const int k_low = std::floor(k_peak_v / 2); + const int k_high = std::ceil(2 * k_peak_v); + + std::mt19937 rng; + rng.seed(rseed_v); + std::uniform_int_distribution<> dist(-k_high, k_high); + + int n_mode = 0; + int n_attempt = 0; + constexpr int max_attempts = 1000000; + Real kx1, kx2, kx3, k_mag, ampl; + bool mode_exists = false; + while (n_mode < num_modes_v && n_attempt < max_attempts) { + n_attempt += 1; + + kx1 = dist(rng); + kx2 = dist(rng); + kx3 = dist(rng); + k_mag = std::sqrt(SQR(kx1) + SQR(kx2) + SQR(kx3)); + + // Expected amplitude of the spectral function. If this is changed, it also needs to + // be changed in the FMFT class (or abstracted). + ampl = SQR(k_mag / k_peak_v) * (2.0 - SQR(k_mag / k_peak_v)); + + // Check is mode was already picked by chance + mode_exists = false; + for (int n_mode_exsist = 0; n_mode_exsist < n_mode; n_mode_exsist++) { + if (k_vec_v_h(0, n_mode_exsist) == kx1 && k_vec_v_h(1, n_mode_exsist) == kx2 && + k_vec_v_h(2, n_mode_exsist) == kx3) { + mode_exists = true; + } + } + + // kx1 < 0.0 because we use a explicit symmetric Complex to Real transform + if (ampl < 0 || k_mag < k_low || k_mag > k_high || mode_exists || kx1 < 0.0) { + continue; + } + k_vec_v_h(0, n_mode) = kx1; + k_vec_v_h(1, n_mode) = kx2; + k_vec_v_h(2, n_mode) = kx3; + n_mode++; + } + PARTHENON_REQUIRE_THROWS( + n_attempt < max_attempts, + "Cluster init did not succeed in calculating perturbation modes.") + Kokkos::deep_copy(k_vec_v, k_vec_v_h); + + auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, + k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); + hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } } //======================================================================================== @@ -456,9 +542,22 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const auto sigma_B = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_B", 0.0); if (sigma_v != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); Real v2_sum = 0.0; // used for normalization + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + pmb->par_reduce( "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { @@ -466,9 +565,9 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const auto &u = cons(b); auto rho = u(IDN, k, j, i); - u(IM1, k, j, i) = rho * (i + b); - u(IM2, k, j, i) = rho * (j + b); - u(IM3, k, j, i) = rho * (k + b); + u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); + u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); + u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); // No need to touch the energy yet as we'll normalize later lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * diff --git a/tst/regression/test_suites/cluster_hse/cluster_hse.py b/tst/regression/test_suites/cluster_hse/cluster_hse.py index 0a52bb6a..9601b4f8 100644 --- a/tst/regression/test_suites/cluster_hse/cluster_hse.py +++ b/tst/regression/test_suites/cluster_hse/cluster_hse.py @@ -93,6 +93,12 @@ def __init__(self): self.norm_tol = 1e-3 self.sigma_v = unyt.unyt_quantity(75.0, "km/s") + char_v_lengthscale = unyt.unyt_quantity(100.0, "kpc") + # Note that the box scale is set in the input file directly (-0.1 to 0.1), + # so if the input file changes, the following line should change, too. + Lbox = 0.2 * self.code_length + self.k_peak_v = Lbox / char_v_lengthscale + self.sigma_B = unyt.unyt_quantity(1e-8, "G") def Prepare(self, parameters, step): @@ -144,6 +150,7 @@ def Prepare(self, parameters, step): f"problem/cluster/hydrostatic_equilibrium/rho_fix={self.rho_fix.in_units('code_mass/code_length**3').v}", f"problem/cluster/hydrostatic_equilibrium/r_sampling={self.R_sampling}", f"problem/cluster/init_perturb/sigma_v={0.0 if step == 2 else self.sigma_v.in_units('code_length/code_time').v}", + f"problem/cluster/init_perturb/k_peak_v={0.0 if step == 2 else self.k_peak_v.v}", f"problem/cluster/init_perturb/sigma_B={0.0 if step == 2 else self.sigma_B.in_units('(code_mass/code_length)**0.5/code_time').v}", f"parthenon/output2/id={'prim' if step == 2 else 'prim_perturb'}", f"parthenon/time/nlim={-1 if step == 2 else 1}", From 429d1b1c19db8fbe69b378403f529c2a9d40acec Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 15:17:35 +0200 Subject: [PATCH 13/48] Move construction of modes to shared util --- src/pgen/cluster.cpp | 53 +----------------------------------- src/utils/few_modes_ft.cpp | 56 ++++++++++++++++++++++++++++++++++++++ src/utils/few_modes_ft.hpp | 3 ++ 3 files changed, 60 insertions(+), 52 deletions(-) diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 992b81fc..c803fc2c 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -267,58 +267,7 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // the perturbation (and is normalized in the following to get the desired sigma_v) const auto t_corr = 1e-10; - // Construct list of wavenumers for inital perturbation.We pick randomly between - // kpeak/2 and 2kpeak. - auto k_vec_v = parthenon::ParArray2D("k_vec", 3, num_modes_v); - auto k_vec_v_h = - Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), k_vec_v); - - const int k_low = std::floor(k_peak_v / 2); - const int k_high = std::ceil(2 * k_peak_v); - - std::mt19937 rng; - rng.seed(rseed_v); - std::uniform_int_distribution<> dist(-k_high, k_high); - - int n_mode = 0; - int n_attempt = 0; - constexpr int max_attempts = 1000000; - Real kx1, kx2, kx3, k_mag, ampl; - bool mode_exists = false; - while (n_mode < num_modes_v && n_attempt < max_attempts) { - n_attempt += 1; - - kx1 = dist(rng); - kx2 = dist(rng); - kx3 = dist(rng); - k_mag = std::sqrt(SQR(kx1) + SQR(kx2) + SQR(kx3)); - - // Expected amplitude of the spectral function. If this is changed, it also needs to - // be changed in the FMFT class (or abstracted). - ampl = SQR(k_mag / k_peak_v) * (2.0 - SQR(k_mag / k_peak_v)); - - // Check is mode was already picked by chance - mode_exists = false; - for (int n_mode_exsist = 0; n_mode_exsist < n_mode; n_mode_exsist++) { - if (k_vec_v_h(0, n_mode_exsist) == kx1 && k_vec_v_h(1, n_mode_exsist) == kx2 && - k_vec_v_h(2, n_mode_exsist) == kx3) { - mode_exists = true; - } - } - - // kx1 < 0.0 because we use a explicit symmetric Complex to Real transform - if (ampl < 0 || k_mag < k_low || k_mag > k_high || mode_exists || kx1 < 0.0) { - continue; - } - k_vec_v_h(0, n_mode) = kx1; - k_vec_v_h(1, n_mode) = kx2; - k_vec_v_h(2, n_mode) = kx3; - n_mode++; - } - PARTHENON_REQUIRE_THROWS( - n_attempt < max_attempts, - "Cluster init did not succeed in calculating perturbation modes.") - Kokkos::deep_copy(k_vec_v, k_vec_v_h); + auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index 01f8d2d6..4e57ca64 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -342,4 +342,60 @@ void FewModesFT::Generate(MeshData *md, const Real dt, } }); } + +// Creates a random set of wave vectors with k_mag within k_peak/2 and 2*k_peak +ParArray2D MakeRandomModes(const int num_modes, const Real k_peak, + uint32_t rseed = 31224) { + auto k_vec = parthenon::ParArray2D("k_vec", 3, num_modes); + auto k_vec_h = Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), k_vec); + + const int k_low = std::floor(k_peak / 2); + const int k_high = std::ceil(2 * k_peak); + + std::mt19937 rng; + rng.seed(rseed); + std::uniform_int_distribution<> dist(-k_high, k_high); + + int n_mode = 0; + int n_attempt = 0; + constexpr int max_attempts = 1000000; + Real kx1, kx2, kx3, k_mag, ampl; + bool mode_exists = false; + while (n_mode < num_modes && n_attempt < max_attempts) { + n_attempt += 1; + + kx1 = dist(rng); + kx2 = dist(rng); + kx3 = dist(rng); + k_mag = std::sqrt(SQR(kx1) + SQR(kx2) + SQR(kx3)); + + // Expected amplitude of the spectral function. If this is changed, it also needs to + // be changed in the FMFT class (or abstracted). + ampl = SQR(k_mag / k_peak) * (2.0 - SQR(k_mag / k_peak)); + + // Check is mode was already picked by chance + mode_exists = false; + for (int n_mode_exsist = 0; n_mode_exsist < n_mode; n_mode_exsist++) { + if (k_vec_h(0, n_mode_exsist) == kx1 && k_vec_h(1, n_mode_exsist) == kx2 && + k_vec_h(2, n_mode_exsist) == kx3) { + mode_exists = true; + } + } + + // kx1 < 0.0 because we use a explicit symmetric Complex to Real transform + if (ampl < 0 || k_mag < k_low || k_mag > k_high || mode_exists || kx1 < 0.0) { + continue; + } + k_vec_h(0, n_mode) = kx1; + k_vec_h(1, n_mode) = kx2; + k_vec_h(2, n_mode) = kx3; + n_mode++; + } + PARTHENON_REQUIRE_THROWS( + n_attempt < max_attempts, + "Cluster init did not succeed in calculating perturbation modes.") + Kokkos::deep_copy(k_vec, k_vec_h); + + return k_vec; +} } // namespace utils::few_modes_ft \ No newline at end of file diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp index 572f163e..f544db97 100644 --- a/src/utils/few_modes_ft.hpp +++ b/src/utils/few_modes_ft.hpp @@ -64,4 +64,7 @@ class FewModesFT { } }; +// Creates a random set of wave vectors with k_mag within k_peak/2 and 2*k_peak +ParArray2D MakeRandomModes(const int num_modes, const Real k_peak, uint32_t rseed); + } // namespace utils::few_modes_ft \ No newline at end of file From a616d93b64e7607f87bdcb498485244cf41b25c8 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 15:39:14 +0200 Subject: [PATCH 14/48] Allow ghost zones filling in FMFT --- src/utils/few_modes_ft.cpp | 33 +++++++++++++++++++-------------- src/utils/few_modes_ft.hpp | 9 +++++---- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index 4e57ca64..684a6924 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -14,6 +14,7 @@ // Parthenon headers #include "basic_types.hpp" #include "config.hpp" +#include "globals.hpp" #include "kokkos_abstraction.hpp" #include "mesh/domain.hpp" #include "mesh/meshblock_pack.hpp" @@ -30,9 +31,10 @@ using parthenon::Metadata; FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, std::string prefix, int num_modes, ParArray2D k_vec, - Real k_peak, Real sol_weight, Real t_corr, uint32_t rseed) + Real k_peak, Real sol_weight, Real t_corr, uint32_t rseed, + bool fill_ghosts) : prefix_(prefix), num_modes_(num_modes), k_vec_(k_vec), k_peak_(k_peak), - t_corr_(t_corr) { + t_corr_(t_corr), fill_ghosts_(fill_ghosts) { if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { std::cout << "### WARNING using more than 100 explicit modes will significantly " @@ -56,14 +58,15 @@ FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescripto const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); + const auto ng_tot = fill_ghosts_ ? 2 * parthenon::Globals::nghost : 0; auto m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx1}), prefix + "_phases_i"); + std::vector({2, num_modes, nx1 + ng_tot}), prefix + "_phases_i"); pkg->AddField(prefix + "_phases_i", m); m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx2}), prefix + "_phases_j"); + std::vector({2, num_modes, nx2 + ng_tot}), prefix + "_phases_j"); pkg->AddField(prefix + "_phases_j", m); m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx3}), prefix + "_phases_k"); + std::vector({2, num_modes, nx3 + ng_tot}), prefix + "_phases_k"); pkg->AddField(prefix + "_phases_k", m); // Variable (e.g., acceleration field for turbulence driver) in Fourier space using @@ -142,9 +145,10 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { auto &phases_j = base->Get(prefix_ + "_phases_j").data; auto &phases_k = base->Get(prefix_ + "_phases_k").data; + const auto ng = fill_ghosts_ ? parthenon::Globals::nghost : 0; pmb->par_for( - "FMFT: calc phases_i", 0, nx1 - 1, KOKKOS_LAMBDA(int i) { - Real gi = static_cast(i + gis); + "FMFT: calc phases_i", 0, nx1 - 1 + 2 * ng, KOKKOS_LAMBDA(int i) { + Real gi = static_cast((i + gis - ng) % static_cast(gnx1)); Real w_kx; Complex phase; @@ -162,8 +166,8 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { }); pmb->par_for( - "FMFT: calc phases_j", 0, nx2 - 1, KOKKOS_LAMBDA(int j) { - Real gj = static_cast(j + gjs); + "FMFT: calc phases_j", 0, nx2 - 1 + 2 * ng, KOKKOS_LAMBDA(int j) { + Real gj = static_cast((j + gjs - ng) % static_cast(gnx2)); Real w_ky; Complex phase; @@ -176,8 +180,8 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { }); pmb->par_for( - "FMFT: calc phases_k", 0, nx3 - 1, KOKKOS_LAMBDA(int k) { - Real gk = static_cast(k + gks); + "FMFT: calc phases_k", 0, nx3 - 1 + 2 * ng, KOKKOS_LAMBDA(int k) { + Real gk = static_cast((k + gks - ng) % static_cast(gnx3)); Real w_kz; Complex phase; @@ -313,9 +317,10 @@ void FewModesFT::Generate(MeshData *md, const Real dt, var_hat(n, m).imag() * c_drift + var_hat_new(n, m).imag() * c_diff); }); - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + auto domain = fill_ghosts_ ? IndexDomain::entire : IndexDomain::interior; + IndexRange ib = md->GetBlockData(0)->GetBoundsI(domain); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(domain); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(domain); auto var_pack = md->PackVariables(std::vector{var_name}); auto phases_i = md->PackVariables(std::vector{prefix_ + "_phases_i"}); auto phases_j = md->PackVariables(std::vector{prefix_ + "_phases_j"}); diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp index f544db97..ce17401c 100644 --- a/src/utils/few_modes_ft.hpp +++ b/src/utils/few_modes_ft.hpp @@ -37,14 +37,15 @@ class FewModesFT { Kokkos::View random_num_host_; std::mt19937 rng_; std::uniform_real_distribution<> dist_; - Real sol_weight_; // power in solenoidal modes for projection. Set to negative to - // disable projection - Real t_corr_; // correlation time for evolution of Ornstein-Uhlenbeck process + Real sol_weight_; // power in solenoidal modes for projection. Set to negative to + // disable projection + Real t_corr_; // correlation time for evolution of Ornstein-Uhlenbeck process + bool fill_ghosts_; // if the inverse transform should also fill ghost zones public: FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, std::string prefix, int num_modes, ParArray2D k_vec, Real k_peak, - Real sol_weight, Real t_corr, uint32_t rseed); + Real sol_weight, Real t_corr, uint32_t rseed, bool fill_ghosts = false); ParArray2D GetVarHat() { return var_hat_; } int GetNumModes() { return num_modes_; } From 639fcc1c947f6ef923266cbb2f957c31693b7da9 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 17:09:21 +0200 Subject: [PATCH 15/48] Add initial magnetic field perturbations --- src/pgen/cluster.cpp | 119 +++++++++++++++++- .../test_suites/cluster_hse/cluster_hse.py | 41 +++++- 2 files changed, 153 insertions(+), 7 deletions(-) diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index c803fc2c..e4791a9e 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -279,6 +279,38 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd std::vector({3})); hydro_pkg->AddField("tmp_perturb", m); } + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, + "Requested initial magnetic field perturbations but not " + "solving the MHD equations.") + // peak of init vel perturb + auto k_peak_b = pin->GetReal("problem/cluster/init_perturb", "k_peak_b"); + auto num_modes_b = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); + uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); + // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const auto t_corr = 1e-10; + // This field should by construction have no compressive modes, so we fix the number. + const auto sol_weight_b = 1.0; + + auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); + + const bool fill_ghosts = true; // as we fill a vector potential to calc B + auto few_modes_ft = + FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, + sol_weight_b, t_corr, rseed_b, fill_ghosts); + hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now). Only add if not already done for the velocity. + if (sigma_v == 0.0) { + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + } } //======================================================================================== @@ -478,6 +510,10 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) } + /************************************************************ + * Set initial velocity perturbations (requires no other velocities for now) + ************************************************************/ + auto pmb = md->GetBlockData(0)->GetBlockPointer(); IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); @@ -488,7 +524,6 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const auto num_blocks = md->NumBlocks(); const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - const auto sigma_B = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_B", 0.0); if (sigma_v != 0.0) { auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); @@ -513,6 +548,11 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const auto &coords = cons.GetCoords(b); const auto &u = cons(b); auto rho = u(IDN, k, j, i); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, + "Found existing non-zero velocity when setting velocity perturbations."); u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); @@ -548,6 +588,83 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { u(IDN, k, j, i); }); } + + /************************************************************ + * Set initial magnetic field perturbations (resets magnetic field field) + ************************************************************/ + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real b2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0; + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * + coords.CellVolume(k, j, i); + }, + b2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); + }); + } } void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { diff --git a/tst/regression/test_suites/cluster_hse/cluster_hse.py b/tst/regression/test_suites/cluster_hse/cluster_hse.py index 9601b4f8..0b3d999a 100644 --- a/tst/regression/test_suites/cluster_hse/cluster_hse.py +++ b/tst/regression/test_suites/cluster_hse/cluster_hse.py @@ -39,6 +39,7 @@ def __init__(self): unyt.define_unit("code_mass", (1e14, "Msun")) unyt.define_unit("code_time", (1, "Gyr")) unyt.define_unit("code_velocity", (1, "code_length/code_time")) + unyt.define_unit("code_magnetic", (np.sqrt(4*np.pi), "(code_mass/code_length)**0.5/code_time")) self.code_length = unyt.unyt_quantity(1, "code_length") self.code_mass = unyt.unyt_quantity(1, "code_mass") self.code_time = unyt.unyt_quantity(1, "code_time") @@ -96,10 +97,13 @@ def __init__(self): char_v_lengthscale = unyt.unyt_quantity(100.0, "kpc") # Note that the box scale is set in the input file directly (-0.1 to 0.1), # so if the input file changes, the following line should change, too. - Lbox = 0.2 * self.code_length - self.k_peak_v = Lbox / char_v_lengthscale + l_box = 0.2 * self.code_length + self.k_peak_v = l_box / char_v_lengthscale - self.sigma_B = unyt.unyt_quantity(1e-8, "G") + self.sigma_b = unyt.unyt_quantity(1e-8, "G") + # using a different one than for the velocity + char_b_lengthscale = unyt.unyt_quantity(50.0, "kpc") + self.k_peak_b = l_box / char_b_lengthscale def Prepare(self, parameters, step): """ @@ -149,9 +153,11 @@ def Prepare(self, parameters, step): f"problem/cluster/hydrostatic_equilibrium/r_fix={self.R_fix.in_units('code_length').v}", f"problem/cluster/hydrostatic_equilibrium/rho_fix={self.rho_fix.in_units('code_mass/code_length**3').v}", f"problem/cluster/hydrostatic_equilibrium/r_sampling={self.R_sampling}", - f"problem/cluster/init_perturb/sigma_v={0.0 if step == 2 else self.sigma_v.in_units('code_length/code_time').v}", + f"hydro/fluid={'euler' if step == 2 else 'glmmhd'}", + f"problem/cluster/init_perturb/sigma_v={0.0 if step == 2 else self.sigma_v.in_units('code_velocity').v}", f"problem/cluster/init_perturb/k_peak_v={0.0 if step == 2 else self.k_peak_v.v}", - f"problem/cluster/init_perturb/sigma_B={0.0 if step == 2 else self.sigma_B.in_units('(code_mass/code_length)**0.5/code_time').v}", + f"problem/cluster/init_perturb/sigma_b={0.0 if step == 2 else self.sigma_b.in_units('code_magnetic').v}", + f"problem/cluster/init_perturb/k_peak_b={0.0 if step == 2 else self.k_peak_b.v}", f"parthenon/output2/id={'prim' if step == 2 else 'prim_perturb'}", f"parthenon/time/nlim={-1 if step == 2 else 1}", ] @@ -209,6 +215,9 @@ def AnalyseInitPert(self, parameters): "velocity_1": 1, "velocity_2": 2, "velocity_3": 3, + "magnetic_field_1": 5, + "magnetic_field_2": 6, + "magnetic_field_3": 7, } vx = prim[prim_col_dict["velocity_1"]] @@ -227,10 +236,30 @@ def AnalyseInitPert(self, parameters): if not sigma_v_match: analyze_status = False print( - f"ERROR: velocity perturbation too large\n" + f"ERROR: velocity perturbations don't match\n" f"Expected {self.sigma_v.in_units('code_velocity')} but got {rms_v}\n" ) + bx = prim[prim_col_dict["magnetic_field_1"]] + by = prim[prim_col_dict["magnetic_field_2"]] + bz = prim[prim_col_dict["magnetic_field_3"]] + + # volume weighted rms magnetic field + rms_b = np.sqrt( + np.sum((bx**2 + by**2 + bz**2) * cell_vol) / np.sum(cell_vol) + ) + + sigma_b_match = np.isclose( + rms_b, self.sigma_b.in_units("code_magnetic").v, rtol=1e-14, atol=1e-14 + ) + + if not sigma_b_match: + analyze_status = False + print( + f"ERROR: magnetic field perturbations don't match\n" + f"Expected {self.sigma_b.in_units('code_magnetic')} but got {rms_b}\n" + ) + return analyze_status def AnalyseHSE(self, parameters): From b31f88ead2590a555283dbca317922b38ff50601 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 17:23:38 +0200 Subject: [PATCH 16/48] Add doc for cluster init perturb --- docs/cluster.md | 33 +++++++++++++++++++ .../test_suites/cluster_hse/cluster_hse.py | 5 ++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/docs/cluster.md b/docs/cluster.md index 9fb6f20a..134f9f22 100644 --- a/docs/cluster.md +++ b/docs/cluster.md @@ -162,6 +162,39 @@ r_sampling = 4.0 Specifically, the resolution of the 1D profile for each meshblock is either `min(dx,dy,dz)/r_sampling` or `r_k/r_sampling`, whichever is smaller. +## Initial perturbations + +Initial perturbations for both the velocity field and the magnetic field are +supported. + +*Note* that these intial perturbation are currently incompatible with +other initial conditions that modify the velocity or magnetic field, e.g., +an initial magnetic dipole or a uniform field. +This restriction could be lifted if required/desired but the normalization +of the fields would need to be adjusted. + +In general, the perturbations will be seeded by an inverse parabolic +spectral profile centered on a peak wavenumber (in normalized units, i.e., +`k_peak = 2` mean half the box size) with a finite number of modes (default 40) +randomly chosen between `k_peak/2` and `2*k_peak`. + +Pertubations are controlled by the following parameters: +``` + +# for the velocity field +sigma_v = 0.0 # in code velocity; default: 0.0 (disabled) +k_peak_v = ??? # wavenumber in normalized units where the velocity spectrum peaks. No default value. +num_modes_v = 40 # (optional) number of wavemodes in spectral space; default: 40 +sol_weight_v = 1.0 # (optional) power in solenoidal (rotational) modes of the perturbation. Range between 0 (fully compressive) and 1.0 (default, fully solenoidal). +rseed_v = 1 # (optional) integer seed for RNG for wavenumbers and amplitudes + +# for the magnetic field +sigma_b = 0.0 # in code magnetic; default: 0.0 (disabled) +k_peak_b = ??? # wavenumber in normalized units where the magnetic field spectrum peaks. No default value. +num_modes_b = 40 # (optional) number of wavemodes in spectral space; default: 40 +rseed_b = 2 # (optional) integer seed for RNG for wavenumbers and amplitudes +``` + ## AGN Triggering If AGN triggering is enabled, at the end of each time step, a mass accretion diff --git a/tst/regression/test_suites/cluster_hse/cluster_hse.py b/tst/regression/test_suites/cluster_hse/cluster_hse.py index 0b3d999a..0f567e4f 100644 --- a/tst/regression/test_suites/cluster_hse/cluster_hse.py +++ b/tst/regression/test_suites/cluster_hse/cluster_hse.py @@ -39,7 +39,10 @@ def __init__(self): unyt.define_unit("code_mass", (1e14, "Msun")) unyt.define_unit("code_time", (1, "Gyr")) unyt.define_unit("code_velocity", (1, "code_length/code_time")) - unyt.define_unit("code_magnetic", (np.sqrt(4*np.pi), "(code_mass/code_length)**0.5/code_time")) + unyt.define_unit( + "code_magnetic", + (np.sqrt(4 * np.pi), "(code_mass/code_length)**0.5/code_time"), + ) self.code_length = unyt.unyt_quantity(1, "code_length") self.code_mass = unyt.unyt_quantity(1, "code_mass") self.code_time = unyt.unyt_quantity(1, "code_time") From 3ebd164441e3f2ecdfec05270d3924d2cf10fabe Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 16 Jun 2023 17:57:03 +0200 Subject: [PATCH 17/48] Fix typo for using host view --- src/utils/few_modes_ft.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index 684a6924..f80fd103 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -50,9 +50,9 @@ FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescripto // lambda cannot live in the constructor of an object. auto k_vec_host = k_vec.GetHostMirrorAndCopy(); for (int i = 0; i < num_modes; i++) { - PARTHENON_REQUIRE(std::abs(k_vec(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); - PARTHENON_REQUIRE(std::abs(k_vec(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); - PARTHENON_REQUIRE(std::abs(k_vec(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_host(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_host(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_host(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); } const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); From ec12f612620f7bde53cc68884a5be294446d974d Mon Sep 17 00:00:00 2001 From: Forrest Glines Date: Wed, 21 Jun 2023 10:55:44 -0600 Subject: [PATCH 18/48] Update Parthenon to PR887 --- external/parthenon | 2 +- src/hydro/hydro_driver.cpp | 18 +++++++++--------- src/pgen/cloud.cpp | 3 ++- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/external/parthenon b/external/parthenon index 7da5c6a2..462f9282 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 7da5c6a29792b47cff2411390d91ef2c8f386304 +Subproject commit 462f928231baadd762ac9ed8481183074245d536 diff --git a/src/hydro/hydro_driver.cpp b/src/hydro/hydro_driver.cpp index f0418ba6..3bbd65d8 100644 --- a/src/hydro/hydro_driver.cpp +++ b/src/hydro/hydro_driver.cpp @@ -12,7 +12,7 @@ // Parthenon headers #include "amr_criteria/refinement_package.hpp" -#include "bvals/cc/bvals_cc_in_one.hpp" +#include "bvals/comms/bvals_in_one.hpp" #include "prolong_restrict/prolong_restrict.hpp" #include // AthenaPK headers @@ -277,7 +277,7 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { auto &tl = single_tasklist_per_pack_region[i]; auto &mu0 = pmesh->mesh_data.GetOrAdd("base", i); auto &mu1 = pmesh->mesh_data.GetOrAdd("u1", i); - tl.AddTask(none, parthenon::cell_centered_bvars::StartReceiveFluxCorrections, mu0); + tl.AddTask(none, parthenon::StartReceiveFluxCorrections, mu0); const auto flux_str = (stage == 1) ? "flux_first_stage" : "flux_other_stage"; FluxFun_t *calc_flux_fun = hydro_pkg->Param(flux_str); @@ -298,12 +298,12 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { auto send_flx = tl.AddTask(first_order_flux_correct, - parthenon::cell_centered_bvars::LoadAndSendFluxCorrections, mu0); + parthenon::LoadAndSendFluxCorrections, mu0); auto recv_flx = tl.AddTask(first_order_flux_correct, - parthenon::cell_centered_bvars::ReceiveFluxCorrections, mu0); + parthenon::ReceiveFluxCorrections, mu0); auto set_flx = - tl.AddTask(recv_flx, parthenon::cell_centered_bvars::SetFluxCorrections, mu0); + tl.AddTask(recv_flx, parthenon::SetFluxCorrections, mu0); // compute the divergence of fluxes of conserved variables auto update = tl.AddTask( @@ -339,7 +339,7 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { // TODO(someone) experiment with split (local/nonlocal) comms with respect to // performance for various tests (static, amr, block sizes) and then decide on the // best impl. Go with default call (split local/nonlocal) for now. - parthenon::cell_centered_bvars::AddBoundaryExchangeTasks(source_split_first_order, tl, + parthenon::AddBoundaryExchangeTasks(source_split_first_order, tl, mu0, pmesh->multilevel); } @@ -348,9 +348,9 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { auto &tl = async_region_3[i]; auto &u0 = blocks[i]->meshblock_data.Get("base"); auto prolongBound = none; - if (pmesh->multilevel) { - prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, u0); - } + //if (pmesh->multilevel) { + // prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, u0); + //} // set physical boundaries auto set_bc = tl.AddTask(prolongBound, parthenon::ApplyBoundaryConditions, u0); diff --git a/src/pgen/cloud.cpp b/src/pgen/cloud.cpp index ed39b2c2..1a86e52e 100644 --- a/src/pgen/cloud.cpp +++ b/src/pgen/cloud.cpp @@ -16,6 +16,7 @@ #include "mesh/mesh.hpp" #include #include +#include #include #include #include @@ -222,7 +223,7 @@ void InflowWindX2(std::shared_ptr> &mbd, bool coarse) { const auto Bx_ = Bx; const auto By_ = By; pmb->par_for_bndry( - "InflowWindX2", nb, IndexDomain::inner_x2, coarse, + "InflowWindX2", nb, IndexDomain::inner_x2, parthenon::TopologicalElement::CC, coarse, KOKKOS_LAMBDA(const int, const int &k, const int &j, const int &i) { cons(IDN, k, j, i) = rho_wind_; cons(IM2, k, j, i) = mom_wind_; From 44b94aa040fbc9a2b8a7d8e6d467e5ec01308190 Mon Sep 17 00:00:00 2001 From: Forrest Glines Date: Wed, 21 Jun 2023 12:42:45 -0600 Subject: [PATCH 19/48] Bug fixes --- external/parthenon | 2 +- src/pgen/cluster/agn_triggering.cpp | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/external/parthenon b/external/parthenon index 462f9282..d431cec6 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 462f928231baadd762ac9ed8481183074245d536 +Subproject commit d431cec6bd25f1c36a74670cfd7f8771cc72c80a diff --git a/src/pgen/cluster/agn_triggering.cpp b/src/pgen/cluster/agn_triggering.cpp index 0efb97ae..f0374df6 100644 --- a/src/pgen/cluster/agn_triggering.cpp +++ b/src/pgen/cluster/agn_triggering.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -82,15 +83,15 @@ AGNTriggering::AGNTriggering(parthenon::ParameterInput *pin, } switch (triggering_mode_) { case AGNTriggeringMode::COLD_GAS: { - hydro_pkg->AddParam("agn_triggering_cold_mass", 0, true); + hydro_pkg->AddParam("agn_triggering_cold_mass", 0, Params::Mutability::Restart); break; } case AGNTriggeringMode::BOOSTED_BONDI: case AGNTriggeringMode::BOOTH_SCHAYE: { - hydro_pkg->AddParam("agn_triggering_total_mass", 0, true); - hydro_pkg->AddParam("agn_triggering_mass_weighted_density", 0, true); - hydro_pkg->AddParam("agn_triggering_mass_weighted_velocity", 0, true); - hydro_pkg->AddParam("agn_triggering_mass_weighted_cs", 0, true); + hydro_pkg->AddParam("agn_triggering_total_mass", 0, Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_density", 0, Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_velocity", 0, Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_cs", 0, Params::Mutability::Restart); break; } case AGNTriggeringMode::NONE: { From 3a4a572fd3f47fb69f07aa6cd89548d81ba65dd1 Mon Sep 17 00:00:00 2001 From: par-hermes Date: Wed, 21 Jun 2023 18:46:22 +0000 Subject: [PATCH 20/48] cpp-py-formatter --- src/hydro/hydro_driver.cpp | 15 ++++++--------- src/pgen/cloud.cpp | 6 +++--- src/pgen/cluster/agn_triggering.cpp | 14 +++++++++----- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/hydro/hydro_driver.cpp b/src/hydro/hydro_driver.cpp index 3bbd65d8..b90b3303 100644 --- a/src/hydro/hydro_driver.cpp +++ b/src/hydro/hydro_driver.cpp @@ -297,13 +297,10 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { } auto send_flx = - tl.AddTask(first_order_flux_correct, - parthenon::LoadAndSendFluxCorrections, mu0); + tl.AddTask(first_order_flux_correct, parthenon::LoadAndSendFluxCorrections, mu0); auto recv_flx = - tl.AddTask(first_order_flux_correct, - parthenon::ReceiveFluxCorrections, mu0); - auto set_flx = - tl.AddTask(recv_flx, parthenon::SetFluxCorrections, mu0); + tl.AddTask(first_order_flux_correct, parthenon::ReceiveFluxCorrections, mu0); + auto set_flx = tl.AddTask(recv_flx, parthenon::SetFluxCorrections, mu0); // compute the divergence of fluxes of conserved variables auto update = tl.AddTask( @@ -339,8 +336,8 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { // TODO(someone) experiment with split (local/nonlocal) comms with respect to // performance for various tests (static, amr, block sizes) and then decide on the // best impl. Go with default call (split local/nonlocal) for now. - parthenon::AddBoundaryExchangeTasks(source_split_first_order, tl, - mu0, pmesh->multilevel); + parthenon::AddBoundaryExchangeTasks(source_split_first_order, tl, mu0, + pmesh->multilevel); } TaskRegion &async_region_3 = tc.AddRegion(num_task_lists_executed_independently); @@ -348,7 +345,7 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { auto &tl = async_region_3[i]; auto &u0 = blocks[i]->meshblock_data.Get("base"); auto prolongBound = none; - //if (pmesh->multilevel) { + // if (pmesh->multilevel) { // prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, u0); //} diff --git a/src/pgen/cloud.cpp b/src/pgen/cloud.cpp index 1a86e52e..bc9fed25 100644 --- a/src/pgen/cloud.cpp +++ b/src/pgen/cloud.cpp @@ -14,9 +14,9 @@ // Parthenon headers #include "mesh/mesh.hpp" +#include #include #include -#include #include #include #include @@ -223,8 +223,8 @@ void InflowWindX2(std::shared_ptr> &mbd, bool coarse) { const auto Bx_ = Bx; const auto By_ = By; pmb->par_for_bndry( - "InflowWindX2", nb, IndexDomain::inner_x2, parthenon::TopologicalElement::CC, coarse, - KOKKOS_LAMBDA(const int, const int &k, const int &j, const int &i) { + "InflowWindX2", nb, IndexDomain::inner_x2, parthenon::TopologicalElement::CC, + coarse, KOKKOS_LAMBDA(const int, const int &k, const int &j, const int &i) { cons(IDN, k, j, i) = rho_wind_; cons(IM2, k, j, i) = mom_wind_; cons(IEN, k, j, i) = rhoe_wind_ + 0.5 * mom_wind_ * mom_wind_ / rho_wind_; diff --git a/src/pgen/cluster/agn_triggering.cpp b/src/pgen/cluster/agn_triggering.cpp index f0374df6..1f241d88 100644 --- a/src/pgen/cluster/agn_triggering.cpp +++ b/src/pgen/cluster/agn_triggering.cpp @@ -12,8 +12,8 @@ // Parthenon headers #include #include -#include #include +#include #include #include #include @@ -88,10 +88,14 @@ AGNTriggering::AGNTriggering(parthenon::ParameterInput *pin, } case AGNTriggeringMode::BOOSTED_BONDI: case AGNTriggeringMode::BOOTH_SCHAYE: { - hydro_pkg->AddParam("agn_triggering_total_mass", 0, Params::Mutability::Restart); - hydro_pkg->AddParam("agn_triggering_mass_weighted_density", 0, Params::Mutability::Restart); - hydro_pkg->AddParam("agn_triggering_mass_weighted_velocity", 0, Params::Mutability::Restart); - hydro_pkg->AddParam("agn_triggering_mass_weighted_cs", 0, Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_total_mass", 0, + Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_density", 0, + Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_velocity", 0, + Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_cs", 0, + Params::Mutability::Restart); break; } case AGNTriggeringMode::NONE: { From de70cf8b5c6e9ec401ab1b33672bd0e1ec2f0b83 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 22 Jun 2023 08:33:05 -0400 Subject: [PATCH 21/48] Bump Kokkos and Parthenon --- external/Kokkos | 2 +- external/parthenon | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/external/Kokkos b/external/Kokkos index af5caae7..62d2b6c8 160000 --- a/external/Kokkos +++ b/external/Kokkos @@ -1 +1 @@ -Subproject commit af5caae75ef07e93177dcc23b180ec6983942696 +Subproject commit 62d2b6c879b74b6ae7bd06eb3e5e80139c4708e6 diff --git a/external/parthenon b/external/parthenon index d431cec6..26c5097c 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit d431cec6bd25f1c36a74670cfd7f8771cc72c80a +Subproject commit 26c5097cd0a709fab2b875cc4076f4c2d867704c From 7e70e9540589f15f556799c207bae988275bc962 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 22 Jun 2023 15:19:24 +0200 Subject: [PATCH 22/48] Add forced refinement --- src/hydro/hydro.cpp | 2 ++ src/refinement/other.cpp | 5 +++++ src/refinement/refinement.hpp | 1 + 3 files changed, 8 insertions(+) diff --git a/src/hydro/hydro.cpp b/src/hydro/hydro.cpp index 40cdd27b..c6408c1e 100644 --- a/src/hydro/hydro.cpp +++ b/src/hydro/hydro.cpp @@ -582,6 +582,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { "refinement/maxdensity_refine_above"); pkg->AddParam("refinement/maxdensity_deref_below", deref_below); pkg->AddParam("refinement/maxdensity_refine_above", refine_above); + } else if (refine_str == "always") { + pkg->CheckRefinementBlock = refinement::other::Always; } else if (refine_str == "user") { pkg->CheckRefinementBlock = Hydro::ProblemCheckRefinementBlock; } diff --git a/src/refinement/other.cpp b/src/refinement/other.cpp index 586521f9..125292e4 100644 --- a/src/refinement/other.cpp +++ b/src/refinement/other.cpp @@ -14,6 +14,11 @@ namespace other { using parthenon::IndexDomain; using parthenon::IndexRange; +// refinement condition: check max density +parthenon::AmrTag Always(MeshBlockData *rc) { + return parthenon::AmrTag::refine; +} + // refinement condition: check max density parthenon::AmrTag MaxDensity(MeshBlockData *rc) { auto pmb = rc->GetBlockPointer(); diff --git a/src/refinement/refinement.hpp b/src/refinement/refinement.hpp index 436bc327..0373ad52 100644 --- a/src/refinement/refinement.hpp +++ b/src/refinement/refinement.hpp @@ -19,6 +19,7 @@ AmrTag PressureGradient(MeshBlockData *rc); AmrTag VelocityGradient(MeshBlockData *rc); } // namespace gradient namespace other { +parthenon::AmrTag Always(MeshBlockData *rc); parthenon::AmrTag MaxDensity(MeshBlockData *rc); } From 8910136ecf46e2b122436ccd45186712726854a2 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 21 Aug 2023 17:21:25 +0200 Subject: [PATCH 23/48] Add rescaling for turb sims, need to fix norm --- src/pgen/turbulence.cpp | 126 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 58882f52..c2705ddc 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -14,6 +14,7 @@ // Parthenon headers #include "basic_types.hpp" +#include "globals.hpp" #include "kokkos_abstraction.hpp" #include "mesh/mesh.hpp" #include @@ -28,6 +29,7 @@ #include "../main.hpp" #include "../units.hpp" #include "../utils/few_modes_ft.hpp" +#include "utils/error_checking.hpp" namespace turbulence { using namespace parthenon::package::prelude; @@ -192,6 +194,31 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg pfew_modes_ft->RestoreDist(iss); } } + + // Parameters to rescale the simulation to a target Mach number at a given cycle, time, + // or restart + auto rescale_once_at_time = + pin->GetOrAddReal("problem/turbulence", "rescale_once_at_time", -1.0); + auto rescale_once_at_cycle = + pin->GetOrAddInteger("problem/turbulence", "rescale_once_at_cycle", -1); + auto rescale_once_on_restart = + pin->GetOrAddBoolean("problem/turbulence", "rescale_once_on_restart", false); + + PARTHENON_REQUIRE_THROWS( + (rescale_once_at_time < 0.0 && rescale_once_at_cycle < 0 && + !rescale_once_on_restart) || + (rescale_once_at_cycle * rescale_once_at_time < 0.0 && + !rescale_once_on_restart) || + (rescale_once_at_cycle * rescale_once_at_time > 0.0 && rescale_once_on_restart), + "Rescaling should only be set for one option (or none at all)."); + // Make Params mutable as they're reset after rescale + pkg->AddParam<>("turbulence/rescale_once_at_time", rescale_once_at_time, true); + pkg->AddParam<>("turbulence/rescale_once_at_cycle", rescale_once_at_cycle, true); + pkg->AddParam<>("turbulence/rescale_once_on_restart", rescale_once_on_restart, true); + + auto rescale_to_rms_Ms = + pin->GetOrAddReal("problem/turbulence", "rescale_to_rms_Ms", -1.0); + pkg->AddParam<>("turbulence/rescale_to_rms_Ms", rescale_to_rms_Ms); } // SetPhases is used as InitMeshBlockUserData because phases need to be reset on remeshing @@ -212,6 +239,12 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + const auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size"); + PARTHENON_REQUIRE_THROWS( + pack_size == -1, "Turbulence problem generator currently relies on synchronous MPI " + "Allreduce. Therefore, only a `parthenon/mesh/pack_size=-1` is " + "supported. Please get in contact if this is an issue."); + auto hydro_pkg = pmb->packages.Get("Hydro"); const auto fluid = hydro_pkg->Param("fluid"); const auto gm1 = pin->GetReal("hydro", "gamma") - 1.0; @@ -435,6 +468,96 @@ void Perturb(MeshData *md, const Real dt) { }); } +void Rescale(MeshData *md, const parthenon::SimTime &tm, const Real dt) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto pkg = pmb->packages.Get("Hydro"); + + const auto rescale_once_at_time = pkg->Param("turbulence/rescale_once_at_time"); + const auto rescale_once_at_cycle = pkg->Param("turbulence/rescale_once_at_cycle"); + const auto rescale_once_on_restart = + pkg->Param("turbulence/rescale_once_on_restart"); + + // Check if any condition is met for rescaling + if (!((rescale_once_at_time >= tm.time && rescale_once_at_time < tm.time + dt) || + (rescale_once_at_cycle == tm.ncycle) || rescale_once_on_restart)) { + return; + } + + // Always disable rescaling as the original value doesn't matter + pkg->UpdateParam("turbulence/rescale_once_at_time", -1.0); + pkg->UpdateParam("turbulence/rescale_once_at_cycle", -1); + pkg->UpdateParam("turbulence/rescale_once_on_restart", false); + + const auto rescale_to_rms_Ms = pkg->Param("turbulence/rescale_to_rms_Ms"); + PARTHENON_REQUIRE_THROWS(rescale_to_rms_Ms > 0.0, "What's a negative Mach number?"); + + if (parthenon::Globals::my_rank == 0) { + std::stringstream msg; + msg << std::setprecision(2); + msg << "\n# Turbulence driver: rescaling to an RMS Ms of " << rescale_to_rms_Ms; + msg << " by resetting the temperature.\n\n"; + std::cout << msg.str(); + } + + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + + auto cons_pack = md->PackVariables(std::vector{"cons"}); + + const auto fluid = pkg->Param("fluid"); + // To fix this, we'd just have to account for the magnetic energy in the reduction + PARTHENON_REQUIRE(fluid == Fluid::euler, + "Rescaling only supported for hydro sims at the moment."); + + const auto gamma = pkg->Param("AdiabaticIndex"); + + Real Ms2_sum; + Kokkos::parallel_reduce( + "turbulence: calc RMS Ms", + Kokkos::MDRangePolicy>( + {0, kb.s, jb.s, ib.s}, {cons_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lMs2_sum) { + const auto &coords = cons_pack.GetCoords(b); + auto &cons = cons_pack(b); + + const auto kin_en_density = (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + + SQR(cons(IM3, k, j, i))) / + cons(IDN, k, j, i); + auto pres = (gamma - 1.0) * (cons(IEN, k, j, i) - kin_en_density); + lMs2_sum += kin_en_density / (gamma * pres) * coords.CellVolume(k, j, i); + }, + Ms2_sum); + +#ifdef MPI_PARALLEL + // Sum the perturbations over all processors + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &Ms2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmb->pmy_mesh->mesh_size.x1max - pmb->pmy_mesh->mesh_size.x1min; + const auto Ly = pmb->pmy_mesh->mesh_size.x2max - pmb->pmy_mesh->mesh_size.x2min; + const auto Lz = pmb->pmy_mesh->mesh_size.x3max - pmb->pmy_mesh->mesh_size.x3min; + auto norm = rescale_to_rms_Ms / std::sqrt(Ms2_sum / (Lx * Ly * Lz)); + + pmb->par_for( + "Rescale temperature to target rms Ms", 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, + jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &coords = cons_pack.GetCoords(b); + auto &cons = cons_pack(b); + + const auto kin_en_density = (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + + SQR(cons(IM3, k, j, i))) / + cons(IDN, k, j, i); + + auto e = (cons(IEN, k, j, i) - kin_en_density) / cons(IDN, k, j, i); + + cons(IEN, k, j, i) = kin_en_density + e * norm * cons(IDN, k, j, i); + }); +} + //---------------------------------------------------------------------------------------- //! \fn void FewModesTurbulenceDriver::Driving(void) // \brief Generate and Perturb the velocity field @@ -445,6 +568,9 @@ void Driving(MeshData *md, const parthenon::SimTime &tm, const Real dt) { // actually drive turbulence Perturb(md, dt); + + // Magic rescaling of simulation to target regime + Rescale(md, tm, dt); } void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { From f1195ca7bf7f822c5b008a7ec46097a93a414471 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 21 Aug 2023 22:44:13 +0200 Subject: [PATCH 24/48] Fix rescale norm --- src/pgen/turbulence.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index c2705ddc..01e6471c 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -522,11 +522,12 @@ void Rescale(MeshData *md, const parthenon::SimTime &tm, const Real dt) { const auto &coords = cons_pack.GetCoords(b); auto &cons = cons_pack(b); - const auto kin_en_density = (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + + const auto kin_en_density = 0.5 * + (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + SQR(cons(IM3, k, j, i))) / cons(IDN, k, j, i); auto pres = (gamma - 1.0) * (cons(IEN, k, j, i) - kin_en_density); - lMs2_sum += kin_en_density / (gamma * pres) * coords.CellVolume(k, j, i); + lMs2_sum += 2.0 * kin_en_density / (gamma * pres) * coords.CellVolume(k, j, i); }, Ms2_sum); @@ -539,7 +540,7 @@ void Rescale(MeshData *md, const parthenon::SimTime &tm, const Real dt) { const auto Lx = pmb->pmy_mesh->mesh_size.x1max - pmb->pmy_mesh->mesh_size.x1min; const auto Ly = pmb->pmy_mesh->mesh_size.x2max - pmb->pmy_mesh->mesh_size.x2min; const auto Lz = pmb->pmy_mesh->mesh_size.x3max - pmb->pmy_mesh->mesh_size.x3min; - auto norm = rescale_to_rms_Ms / std::sqrt(Ms2_sum / (Lx * Ly * Lz)); + auto norm = SQR(rescale_to_rms_Ms) / (Ms2_sum / (Lx * Ly * Lz)); pmb->par_for( "Rescale temperature to target rms Ms", 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, @@ -548,13 +549,14 @@ void Rescale(MeshData *md, const parthenon::SimTime &tm, const Real dt) { const auto &coords = cons_pack.GetCoords(b); auto &cons = cons_pack(b); - const auto kin_en_density = (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + + const auto kin_en_density = 0.5 * + (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + SQR(cons(IM3, k, j, i))) / cons(IDN, k, j, i); auto e = (cons(IEN, k, j, i) - kin_en_density) / cons(IDN, k, j, i); - cons(IEN, k, j, i) = kin_en_density + e * norm * cons(IDN, k, j, i); + cons(IEN, k, j, i) = kin_en_density + e / norm * cons(IDN, k, j, i); }); } From 882745deb75fe727e7c51c82c34667f842fe0f4e Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 21 Aug 2023 23:28:47 +0200 Subject: [PATCH 25/48] Add blob injection --- src/pgen/turbulence.cpp | 145 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 143 insertions(+), 2 deletions(-) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 01e6471c..6c8861de 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -24,6 +24,7 @@ #include #include #include +#include // AthenaPK headers #include "../main.hpp" @@ -195,8 +196,8 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg } } - // Parameters to rescale the simulation to a target Mach number at a given cycle, time, - // or restart + // Parameters to rescale the simulation to a target Mach number at a given cycle, + // time, or restart auto rescale_once_at_time = pin->GetOrAddReal("problem/turbulence", "rescale_once_at_time", -1.0); auto rescale_once_at_cycle = @@ -219,6 +220,44 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg auto rescale_to_rms_Ms = pin->GetOrAddReal("problem/turbulence", "rescale_to_rms_Ms", -1.0); pkg->AddParam<>("turbulence/rescale_to_rms_Ms", rescale_to_rms_Ms); + + // Parameters to inject overdense blobs into the simulation with a target overdensity + // and radius at a given cycle, time, or restart + auto inject_once_at_time = + pin->GetOrAddReal("problem/turbulence", "inject_once_at_time", -1.0); + auto inject_once_at_cycle = + pin->GetOrAddInteger("problem/turbulence", "inject_once_at_cycle", -1); + auto inject_once_on_restart = + pin->GetOrAddBoolean("problem/turbulence", "inject_once_on_restart", false); + + PARTHENON_REQUIRE_THROWS( + (inject_once_at_time < 0.0 && inject_once_at_cycle < 0 && + !inject_once_on_restart) || + (inject_once_at_cycle * inject_once_at_time < 0.0 && !inject_once_on_restart) || + (inject_once_at_cycle * inject_once_at_time > 0.0 && inject_once_on_restart), + "injectng should only be set for one option (or none at all)."); + // Make Params mutable as they're reset after inject + pkg->AddParam<>("turbulence/inject_once_at_time", inject_once_at_time, true); + pkg->AddParam<>("turbulence/inject_once_at_cycle", inject_once_at_cycle, true); + pkg->AddParam<>("turbulence/inject_once_on_restart", inject_once_on_restart, true); + + auto inject_n_blobs = pin->GetOrAddInteger("problem/turbulence", "inject_n_blobs", -1); + pkg->AddParam<>("turbulence/inject_n_blobs", inject_n_blobs); + + for (int i = 0; i < inject_n_blobs; i++) { + auto inject_blob_radius = + pin->GetReal("problem/turbulence", "inject_blob_radius_" + std::to_string(i)); + pkg->AddParam<>("turbulence/inject_blob_radius_" + std::to_string(i), + inject_blob_radius); + + auto inject_blob_loc = pin->GetVector("problem/turbulence", + "inject_blob_loc_" + std::to_string(i)); + pkg->AddParam<>("turbulence/inject_blob_loc_" + std::to_string(i), inject_blob_loc); + + auto inject_blob_chi = + pin->GetReal("problem/turbulence", "inject_blob_chi_" + std::to_string(i)); + pkg->AddParam<>("turbulence/inject_blob_chi_" + std::to_string(i), inject_blob_chi); + } } // SetPhases is used as InitMeshBlockUserData because phases need to be reset on remeshing @@ -560,6 +599,105 @@ void Rescale(MeshData *md, const parthenon::SimTime &tm, const Real dt) { }); } +void InjectBlob(MeshData *md, const parthenon::SimTime &tm, const Real dt) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto pkg = pmb->packages.Get("Hydro"); + + const auto inject_once_at_time = pkg->Param("turbulence/inject_once_at_time"); + const auto inject_once_at_cycle = pkg->Param("turbulence/inject_once_at_cycle"); + const auto inject_once_on_restart = + pkg->Param("turbulence/inject_once_on_restart"); + + // Check if any condition is met for injecting + if (!((inject_once_at_time >= tm.time && inject_once_at_time < tm.time + dt) || + (inject_once_at_cycle == tm.ncycle) || inject_once_on_restart)) { + return; + } + + // Always disable injecting as the original value doesn't matter + pkg->UpdateParam("turbulence/inject_once_at_time", -1.0); + pkg->UpdateParam("turbulence/inject_once_at_cycle", -1); + pkg->UpdateParam("turbulence/inject_once_on_restart", false); + + const auto inject_n_blobs = pkg->Param("turbulence/inject_n_blobs"); + PARTHENON_REQUIRE_THROWS(inject_n_blobs > 0, "Need to inject at least one blob"); + + for (int n_blob = 0; n_blob < inject_n_blobs; n_blob++) { + const auto radius = + pkg->Param("turbulence/inject_blob_radius_" + std::to_string(n_blob)); + const auto chi = + pkg->Param("turbulence/inject_blob_chi_" + std::to_string(n_blob)); + const auto loc = pkg->Param>("turbulence/inject_blob_loc_" + + std::to_string(n_blob)); + + // redef vars for easier capture (std::vector does not work) + const auto loc_x = loc[0]; + const auto loc_y = loc[1]; + const auto loc_z = loc[2]; + if (parthenon::Globals::my_rank == 0) { + std::stringstream msg; + msg << std::setprecision(2); + msg << "\n# Turbulence driver: injecting blob number " << n_blob; + msg << " at location " << loc_x << " " << loc_y << " " << loc_z + << " with overdensity " << chi << ".\n\n "; + std::cout << msg.str(); + } + + const auto *const error_msg = + "Blob bounds crossing domain bounds currently not supported."; + PARTHENON_REQUIRE_THROWS(loc_x + radius < pmb->pmy_mesh->mesh_size.x1max, error_msg) + PARTHENON_REQUIRE_THROWS(loc_x - radius > pmb->pmy_mesh->mesh_size.x1min, error_msg) + PARTHENON_REQUIRE_THROWS(loc_y + radius < pmb->pmy_mesh->mesh_size.x2max, error_msg) + PARTHENON_REQUIRE_THROWS(loc_y - radius > pmb->pmy_mesh->mesh_size.x2min, error_msg) + PARTHENON_REQUIRE_THROWS(loc_z + radius < pmb->pmy_mesh->mesh_size.x3max, error_msg) + PARTHENON_REQUIRE_THROWS(loc_z - radius > pmb->pmy_mesh->mesh_size.x3min, error_msg) + + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + + auto cons_pack = md->PackVariables(std::vector{"cons"}); + + const auto fluid = pkg->Param("fluid"); + // To fix this, we'd just have to account for the magnetic energy in the reduction + PARTHENON_REQUIRE(fluid == Fluid::euler, + "Injecting only supported for hydro sims at the moment."); + + const auto gamma = pkg->Param("AdiabaticIndex"); + + pmb->par_for( + "turbulence: inject blob", 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, + ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &coords = cons_pack.GetCoords(b); + auto &cons = cons_pack(b); + + const auto x = coords.Xc<1>(i) - loc_x; + const auto y = coords.Xc<2>(j) - loc_y; + const auto z = coords.Xc<3>(k) - loc_z; + const auto r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); + + if (r < radius) { + const auto kin_en_density = + 0.5 * + (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + + SQR(cons(IM3, k, j, i))) / + cons(IDN, k, j, i); + auto rho_e = cons(IEN, k, j, i) - kin_en_density; + + // increase density according to overdensity + cons(IDN, k, j, i) *= chi; + // adjust momentum (so that the velocity remains constant) + cons(IM1, k, j, i) *= chi; + cons(IM2, k, j, i) *= chi; + cons(IM2, k, j, i) *= chi; + // adjust total energy density (using original rho_e translates to an increase + // of 1/chi in temperature) + cons(IEN, k, j, i) = kin_en_density * chi + rho_e; + } + }); + } +} + //---------------------------------------------------------------------------------------- //! \fn void FewModesTurbulenceDriver::Driving(void) // \brief Generate and Perturb the velocity field @@ -573,6 +711,9 @@ void Driving(MeshData *md, const parthenon::SimTime &tm, const Real dt) { // Magic rescaling of simulation to target regime Rescale(md, tm, dt); + + // Magic injection of blobs into the simulation + InjectBlob(md, tm, dt); } void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { From a633576ac21c5a8263777fbcda5f82bba372be7d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 22 Aug 2023 14:00:29 +0200 Subject: [PATCH 26/48] Add rescale and blob inject doc --- docs/turbulence.md | 68 +++++++++++++++++++++++++++++++++++ src/refinement/other.cpp | 4 +-- src/refinement/refinement.hpp | 2 +- 3 files changed, 70 insertions(+), 4 deletions(-) create mode 100644 docs/turbulence.md diff --git a/docs/turbulence.md b/docs/turbulence.md new file mode 100644 index 00000000..e2f404e8 --- /dev/null +++ b/docs/turbulence.md @@ -0,0 +1,68 @@ +# Turbulence problem generator + +The problem generator has been refactored from its implementation in K-Athena. +Until the documentation is fully adapted, please consult https://gitlab.com/pgrete/kathena/-/wikis/turbulence for general information. + +A sample input file is provided in [turbulence.in](../inputs/turbulence.in) + +## Blob injection + +In order to study the evolution of (cold) clouds in a turbulent environment, +blobs can be injected to the simulation. +The injection is a one off mechanism controlled by the following parameters + +``` + +# only one of the following three conditions can be set at a given time +inject_once_at_time = -1.0 +inject_once_at_cycle = -1 +inject_once_on_restart = false # should not be set in the input file, but only via the command line upon restart + +inject_n_blobs = -1 # number of blob to inject + +# then for the given number of blobs follow parameters need to be given (starting to count with 0) +inject_blob_radius_0 = ... # float, in code length units, no default value +inject_blob_loc_0 = ...,...,... # location vector of three comma-separated floats, in code length units, no default value +inject_blob_chi_0 = ... # float, dimensionless, no default value, density ratio to existing value + +inject_blob_radius_1 = ... +... +``` + +In practice, this will result in blobs being seeded at a given time, cycle, or upon restart +by adjusting the density within the blob's radius by a factor of $\chi$ and +at the same time adjusting the temperature by a factor of $1/\chi$ so that the +blob remain in pressure equilibrium. + +While this is an action that is performed once, it can be repeated upon restart (or a later +time) by resetting the variables. + +A current restriction is that the blobs cannot be seeded across a domain boundary (i.e., +the periodicity of the box is not taken into account). + +## Rescaling **not recommended* + +*The rescaling described in the following is generally not recommended, as it result in a +state that is not naturally reached. +Moreover, given the artificial nature of a hard reset, some time after the rescaling is +required for the system to readjust. + +For non-isothermal simulations, the plasma will eventually heat up over time due to dissipation. +One possibility to remove that extra heat (or add heat), is to rescale the temperature in the simulation. +This can be done via the following parameters: + +``` + +# only one of the following three conditions can be set at a given time +rescale_once_at_time = -1.0 +rescale_once_at_cycle = -1 +rescale_once_on_restart = false # should not be set in the input file, but only via the command line upon restart + +rescale_to_rms_Ms = -1.0 +``` + +As the parameters suggest, rescaling is a one off action (though it can be repeated when +the parameters are set again for a subsequent restart). +The density and velocity field are not changed, only the specific internal energy is +adjusted so that the volume-weighted root mean squared sonic Mach number matches +the target value. diff --git a/src/refinement/other.cpp b/src/refinement/other.cpp index 125292e4..66e20478 100644 --- a/src/refinement/other.cpp +++ b/src/refinement/other.cpp @@ -15,9 +15,7 @@ using parthenon::IndexDomain; using parthenon::IndexRange; // refinement condition: check max density -parthenon::AmrTag Always(MeshBlockData *rc) { - return parthenon::AmrTag::refine; -} +parthenon::AmrTag Always(MeshBlockData *rc) { return parthenon::AmrTag::refine; } // refinement condition: check max density parthenon::AmrTag MaxDensity(MeshBlockData *rc) { diff --git a/src/refinement/refinement.hpp b/src/refinement/refinement.hpp index 0373ad52..7473fa0e 100644 --- a/src/refinement/refinement.hpp +++ b/src/refinement/refinement.hpp @@ -21,7 +21,7 @@ AmrTag VelocityGradient(MeshBlockData *rc); namespace other { parthenon::AmrTag Always(MeshBlockData *rc); parthenon::AmrTag MaxDensity(MeshBlockData *rc); -} +} // namespace other } // namespace refinement From 6853b2db1828839acc3d92b3ecdd0899e2b063e7 Mon Sep 17 00:00:00 2001 From: Forrest Wolfgang Glines Date: Tue, 29 Aug 2023 16:59:43 -0600 Subject: [PATCH 27/48] Updated Parthenon to PR930 --- external/parthenon | 2 +- src/hydro/hydro.cpp | 6 ++-- src/pgen/cluster.cpp | 64 ++++++++++++++++++++---------------- src/pgen/cpaw.cpp | 6 ++-- src/pgen/field_loop.cpp | 6 ++-- src/pgen/linear_wave.cpp | 11 ++++--- src/pgen/linear_wave_mhd.cpp | 11 ++++--- src/pgen/turbulence.cpp | 14 ++++---- src/utils/few_modes_ft.cpp | 24 +++++++------- 9 files changed, 78 insertions(+), 66 deletions(-) diff --git a/external/parthenon b/external/parthenon index 5b6bb619..60d54786 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 5b6bb61906f7c278f9724ee9f38e79dee8707098 +Subproject commit 60d5478693e26fa9b4836beb60ba7aeedfc608a4 diff --git a/src/hydro/hydro.cpp b/src/hydro/hydro.cpp index d22dddef..846adb9d 100644 --- a/src/hydro/hydro.cpp +++ b/src/hydro/hydro.cpp @@ -794,8 +794,8 @@ TaskStatus CalculateFluxes(std::shared_ptr> &md) { int il, iu, jl, ju, kl, ku; jl = jb.s, ju = jb.e, kl = kb.s, ku = kb.e; // TODO(pgrete): are these looop limits are likely too large for 2nd order - if (pmb->block_size.nx2 > 1) { - if (pmb->block_size.nx3 == 1) // 2D + if (pmb->block_size.nx(X2DIR) > 1) { + if (pmb->block_size.nx(X3DIR)== 1) // 2D jl = jb.s - 1, ju = jb.e + 1, kl = kb.s, ku = kb.e; else // 3D jl = jb.s - 1, ju = jb.e + 1, kl = kb.s - 1, ku = kb.e + 1; @@ -867,7 +867,7 @@ TaskStatus CalculateFluxes(std::shared_ptr> &md) { parthenon::ScratchPad2D::shmem_size(num_scratch_vars, nx1) * 3; // set the loop limits il = ib.s - 1, iu = ib.e + 1, kl = kb.s, ku = kb.e; - if (pmb->block_size.nx3 == 1) // 2D + if (pmb->block_size.nx(X3DIR) == 1) // 2D kl = kb.s, ku = kb.e; else // 3D kl = kb.s - 1, ku = kb.e + 1; diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 96a781ba..de489177 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -59,8 +59,9 @@ using namespace parthenon::driver::prelude; using namespace parthenon::package::prelude; using utils::few_modes_ft::FewModesFT; + void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { + const Real beta_dt) { auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); @@ -78,11 +79,17 @@ void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); + //const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + //snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); + + //ApplyClusterClips(md, tm, beta_dt); + + const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); + stellar_feedback.FeedbackSrcTerm(md, beta_dt, tm); + }; void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { + const Real dt) { auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); @@ -288,49 +295,50 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd made and a history output is not, then the mass/energy between the last history output and the restart dump is lost */ - std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", - "removed_eceil_energy", "removed_vceil_energy", - "added_vAceil_mass"}; + std::string reduction_strs[] = {"stellar_mass","added_dfloor_mass", "removed_eceil_energy", + "removed_vceil_energy", "added_vAceil_mass"}; - // Add a param for each reduction, then add it as a summation reduction for - // history outputs + //Add a param for each reduction, then add it as a summation reduction for + //history outputs auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - for (auto reduction_str : reduction_strs) { - hydro_pkg->AddParam(reduction_str, 0.0, true); + for( auto reduction_str : reduction_strs ) { + hydro_pkg->AddParam(reduction_str, 0.0, true); hst_vars.emplace_back(parthenon::HistoryOutputVar( parthenon::UserHistoryOperation::sum, [reduction_str](MeshData *md) { auto pmb = md->GetBlockData(0)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); const Real reduction = hydro_pkg->Param(reduction_str); - // Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str, 0.0); + //Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str,0.0); return reduction; }, reduction_str)); } - // Add history reduction for total cold gas using stellar mass threshold + //Add history reduction for total cold gas using stellar mass threshold const Real cold_thresh = pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if (cold_thresh > 0) { + if( cold_thresh > 0){ hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, + "cold_mass")); } const Real agn_tracer_thresh = pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if (agn_tracer_thresh >= 0) { - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - PARTHENON_REQUIRE( - pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); + if( agn_tracer_thresh >= 0){ + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + PARTHENON_REQUIRE(pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, + "agn_extent")); } + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); /************************************************************ @@ -706,9 +714,9 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + const auto Lx = pmesh->mesh_size.xmax(X1DIR) - pmesh->mesh_size.xmin(X1DIR); + const auto Ly = pmesh->mesh_size.xmax(X2DIR) - pmesh->mesh_size.xmin(X2DIR); + const auto Lz = pmesh->mesh_size.xmax(X3DIR) - pmesh->mesh_size.xmin(X3DIR); auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); pmb->par_for( @@ -784,9 +792,9 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + const auto Lx = pmesh->mesh_size.xmax(X1DIR) - pmesh->mesh_size.xmin(X1DIR); + const auto Ly = pmesh->mesh_size.xmax(X2DIR) - pmesh->mesh_size.xmin(X2DIR); + const auto Lz = pmesh->mesh_size.xmax(X3DIR) - pmesh->mesh_size.xmin(X3DIR); auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); pmb->par_for( diff --git a/src/pgen/cpaw.cpp b/src/pgen/cpaw.cpp index c269c378..0dcefacb 100644 --- a/src/pgen/cpaw.cpp +++ b/src/pgen/cpaw.cpp @@ -34,6 +34,8 @@ namespace cpaw { using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; + // Parameters which define initial solution -- made global so that they can be shared // with functions A1,2,3 which compute vector potentials Real den, pres, gm1, b_par, b_perp, v_perp, v_par; @@ -213,8 +215,8 @@ void UserWorkAfterLoop(Mesh *mesh, ParameterInput *pin, parthenon::SimTime &tm) } // write errors - std::fprintf(pfile, "%d %d", mesh->mesh_size.nx1, mesh->mesh_size.nx2); - std::fprintf(pfile, " %d %d %e", mesh->mesh_size.nx3, tm.ncycle, rms_err); + std::fprintf(pfile, "%d %d", mesh->mesh_size.nx(X1DIR), mesh->mesh_size.nx(X2DIR)); + std::fprintf(pfile, " %d %d %e", mesh->mesh_size.nx(X3DIR), tm.ncycle, rms_err); std::fprintf(pfile, " %e %e %e %e", err[IDN], err[IM1], err[IM2], err[IM3]); std::fprintf(pfile, " %e", err[IEN]); std::fprintf(pfile, " %e %e %e", err[IB1], err[IB2], err[IB3]); diff --git a/src/pgen/field_loop.cpp b/src/pgen/field_loop.cpp index 66940d56..0ccdb69a 100644 --- a/src/pgen/field_loop.cpp +++ b/src/pgen/field_loop.cpp @@ -135,13 +135,13 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { int iprob = pin->GetInteger("problem/field_loop", "iprob"); Real ang_2, cos_a2(0.0), sin_a2(0.0), lambda(0.0); - Real x1size = pmb->pmy_mesh->mesh_size.x1max - pmb->pmy_mesh->mesh_size.x1min; - Real x2size = pmb->pmy_mesh->mesh_size.x2max - pmb->pmy_mesh->mesh_size.x2min; + Real x1size = pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); + Real x2size = pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); const bool two_d = pmb->pmy_mesh->ndim < 3; // for 2D sim set x3size to zero so that v_z is 0 below Real x3size = - two_d ? 0 : pmb->pmy_mesh->mesh_size.x3max - pmb->pmy_mesh->mesh_size.x3min; + two_d ? 0 : pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); // For (iprob=4) -- rotated cylinder in 3D -- set up rotation angle and wavelength if (iprob == 4) { diff --git a/src/pgen/linear_wave.cpp b/src/pgen/linear_wave.cpp index bc504b06..5480875e 100644 --- a/src/pgen/linear_wave.cpp +++ b/src/pgen/linear_wave.cpp @@ -32,6 +32,7 @@ namespace linear_wave { using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; // TODO(pgrete) temp fix to address removal in Parthenon. Update when merging with MHD constexpr int NWAVE = 5; @@ -280,9 +281,9 @@ void UserWorkAfterLoop(Mesh *mesh, ParameterInput *pin, parthenon::SimTime &tm) if (parthenon::Globals::my_rank == 0) { // normalize errors by number of cells const auto mesh_size = mesh->mesh_size; - const auto vol = (mesh_size.x1max - mesh_size.x1min) * - (mesh_size.x2max - mesh_size.x2min) * - (mesh_size.x3max - mesh_size.x3min); + const auto vol = (mesh_size.xmax(X1DIR) - mesh_size.xmin(X1DIR)) * + (mesh_size.xmax(X2DIR) - mesh_size.xmin(X2DIR)) * + (mesh_size.xmax(X3DIR) - mesh_size.xmin(X3DIR)); for (int i = 0; i < (NHYDRO + NFIELD); ++i) l1_err[i] = l1_err[i] / vol; // compute rms error @@ -320,8 +321,8 @@ void UserWorkAfterLoop(Mesh *mesh, ParameterInput *pin, parthenon::SimTime &tm) } // write errors - std::fprintf(pfile, "%d %d", mesh_size.nx1, mesh_size.nx2); - std::fprintf(pfile, " %d %d", mesh_size.nx3, tm.ncycle); + std::fprintf(pfile, "%d %d", mesh_size.nx(X1DIR), mesh_size.nx(X2DIR)); + std::fprintf(pfile, " %d %d", mesh_size.nx(X2DIR), tm.ncycle); std::fprintf(pfile, " %e %e", rms_err, l1_err[IDN]); std::fprintf(pfile, " %e %e %e", l1_err[IM1], l1_err[IM2], l1_err[IM3]); std::fprintf(pfile, " %e", l1_err[IEN]); diff --git a/src/pgen/linear_wave_mhd.cpp b/src/pgen/linear_wave_mhd.cpp index 7f1b549b..67affbe7 100644 --- a/src/pgen/linear_wave_mhd.cpp +++ b/src/pgen/linear_wave_mhd.cpp @@ -32,6 +32,7 @@ namespace linear_wave_mhd { using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; constexpr int NMHDWAVE = 7; // Parameters which define initial solution -- made global so that they can be shared @@ -300,9 +301,9 @@ void UserWorkAfterLoop(Mesh *mesh, ParameterInput *pin, parthenon::SimTime &tm) if (parthenon::Globals::my_rank == 0) { // normalize errors by number of cells const auto mesh_size = mesh->mesh_size; - const auto vol = (mesh_size.x1max - mesh_size.x1min) * - (mesh_size.x2max - mesh_size.x2min) * - (mesh_size.x3max - mesh_size.x3min); + const auto vol = (mesh_size.xmax(X1DIR) - mesh_size.xmin(X1DIR)) * + (mesh_size.xmax(X2DIR) - mesh_size.xmin(X2DIR)) * + (mesh_size.xmax(X3DIR) - mesh_size.xmin(X3DIR)); for (int i = 0; i < (NGLMMHD); ++i) l1_err[i] = l1_err[i] / vol; // compute rms error @@ -342,8 +343,8 @@ void UserWorkAfterLoop(Mesh *mesh, ParameterInput *pin, parthenon::SimTime &tm) } // write errors - std::fprintf(pfile, "%d %d", mesh_size.nx1, mesh_size.nx2); - std::fprintf(pfile, " %d %d", mesh_size.nx3, tm.ncycle); + std::fprintf(pfile, "%d %d", mesh_size.nx(X1DIR), mesh_size.nx(X2DIR)); + std::fprintf(pfile, " %d %d", mesh_size.nx(X3DIR), tm.ncycle); std::fprintf(pfile, " %e %e", rms_err, l1_err[IDN]); std::fprintf(pfile, " %e %e %e", l1_err[IM1], l1_err[IM2], l1_err[IM3]); std::fprintf(pfile, " %e", l1_err[IEN]); diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 58882f52..9d0e96be 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -217,10 +217,10 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const auto gm1 = pin->GetReal("hydro", "gamma") - 1.0; const auto p0 = pin->GetReal("problem/turbulence", "p0"); const auto rho0 = pin->GetReal("problem/turbulence", "rho0"); - const auto x3min = pmesh->mesh_size.x3min; - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + const auto x3min = pmesh->mesh_size.xmin(X3DIR); + const auto Lx = pmesh->mesh_size.xmax(X1DIR) - pmesh->mesh_size.xmin(X1DIR); + const auto Ly = pmesh->mesh_size.xmax(X2DIR) - pmesh->mesh_size.xmin(X2DIR); + const auto Lz = pmesh->mesh_size.xmax(X3DIR) - pmesh->mesh_size.xmin(X3DIR); const auto kz = 2.0 * M_PI / Lz; // already pack data here to get easy access to coords in kernels @@ -402,9 +402,9 @@ void Perturb(MeshData *md, const Real dt) { MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - const auto Lx = pmb->pmy_mesh->mesh_size.x1max - pmb->pmy_mesh->mesh_size.x1min; - const auto Ly = pmb->pmy_mesh->mesh_size.x2max - pmb->pmy_mesh->mesh_size.x2min; - const auto Lz = pmb->pmy_mesh->mesh_size.x3max - pmb->pmy_mesh->mesh_size.x3min; + const auto Lx = pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); + const auto Ly = pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); + const auto Lz = pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); const auto accel_rms = hydro_pkg->Param("turbulence/accel_rms"); auto norm = accel_rms / std::sqrt(sums[0] / (Lx * Ly * Lz)); diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp index f477b211..e42edde2 100644 --- a/src/utils/few_modes_ft.cpp +++ b/src/utils/few_modes_ft.cpp @@ -106,18 +106,18 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { "Few modes FT currently needs parthenon/mesh/pack_size=-1 " "to work because of global reductions.") - auto Lx1 = pm->mesh_size.x1max - pm->mesh_size.x1min; - auto Lx2 = pm->mesh_size.x2max - pm->mesh_size.x2min; - auto Lx3 = pm->mesh_size.x3max - pm->mesh_size.x3min; + const auto Lx1 = pm->mesh_size.xmax(X1DIR) - pm->mesh_size.xmin(X1DIR); + const auto Lx2 = pm->mesh_size.xmax(X2DIR) - pm->mesh_size.xmin(X2DIR); + const auto Lx3 = pm->mesh_size.xmax(X3DIR) - pm->mesh_size.xmin(X3DIR); // Adjust (logical) grid size at levels other than the root level. // This is required for simulation with mesh refinement so that the phases calculated // below take the logical grid size into account. For example, the local phases at level // 1 should be calculated assuming a grid that is twice as large as the root grid. const auto root_level = pm->GetRootLevel(); - auto gnx1 = pm->mesh_size.nx1 * std::pow(2, pmb->loc.level() - root_level); - auto gnx2 = pm->mesh_size.nx2 * std::pow(2, pmb->loc.level() - root_level); - auto gnx3 = pm->mesh_size.nx3 * std::pow(2, pmb->loc.level() - root_level); + auto gnx1 = pm->mesh_size.nx(X1DIR) * std::pow(2, pmb->loc.level() - root_level); + auto gnx2 = pm->mesh_size.nx(X2DIR) * std::pow(2, pmb->loc.level() - root_level); + auto gnx3 = pm->mesh_size.nx(X3DIR) * std::pow(2, pmb->loc.level() - root_level); // Restriction should also be easily fixed, just need to double check transforms and // volume weighting everywhere @@ -126,13 +126,13 @@ void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { "FMFT has only been tested with cubic meshes and constant " "dx/dy/dz. Remove this warning at your own risk.") - const auto nx1 = pmb->block_size.nx1; - const auto nx2 = pmb->block_size.nx2; - const auto nx3 = pmb->block_size.nx3; + const auto nx1 = pmb->block_size.nx(X1DIR); + const auto nx2 = pmb->block_size.nx(X2DIR); + const auto nx3 = pmb->block_size.nx(X3DIR); - const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; + const auto gis = pmb->loc.lx1() * pmb->block_size.nx(X1DIR); + const auto gjs = pmb->loc.lx2() * pmb->block_size.nx(X2DIR); + const auto gks = pmb->loc.lx3() * pmb->block_size.nx(X3DIR); // make local ref to capure in lambda const auto num_modes = num_modes_; From 60551edf8a1ba07a137ccdb1f292fe8b33ea829b Mon Sep 17 00:00:00 2001 From: par-hermes Date: Tue, 29 Aug 2023 23:01:00 +0000 Subject: [PATCH 28/48] cpp-py-formatter --- src/hydro/hydro.cpp | 2 +- src/pgen/cluster.cpp | 49 +++++++++++++++++++---------------------- src/pgen/field_loop.cpp | 9 +++++--- src/pgen/turbulence.cpp | 9 +++++--- 4 files changed, 36 insertions(+), 33 deletions(-) diff --git a/src/hydro/hydro.cpp b/src/hydro/hydro.cpp index 846adb9d..b46db6b2 100644 --- a/src/hydro/hydro.cpp +++ b/src/hydro/hydro.cpp @@ -795,7 +795,7 @@ TaskStatus CalculateFluxes(std::shared_ptr> &md) { jl = jb.s, ju = jb.e, kl = kb.s, ku = kb.e; // TODO(pgrete): are these looop limits are likely too large for 2nd order if (pmb->block_size.nx(X2DIR) > 1) { - if (pmb->block_size.nx(X3DIR)== 1) // 2D + if (pmb->block_size.nx(X3DIR) == 1) // 2D jl = jb.s - 1, ju = jb.e + 1, kl = kb.s, ku = kb.e; else // 3D jl = jb.s - 1, ju = jb.e + 1, kl = kb.s - 1, ku = kb.e + 1; diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index de489177..d0f981d2 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -59,9 +59,8 @@ using namespace parthenon::driver::prelude; using namespace parthenon::package::prelude; using utils::few_modes_ft::FewModesFT; - void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { + const Real beta_dt) { auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); @@ -79,17 +78,16 @@ void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - //const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - //snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); + // const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + // snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); - //ApplyClusterClips(md, tm, beta_dt); + // ApplyClusterClips(md, tm, beta_dt); const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); stellar_feedback.FeedbackSrcTerm(md, beta_dt, tm); - }; void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { + const Real dt) { auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); @@ -295,50 +293,49 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd made and a history output is not, then the mass/energy between the last history output and the restart dump is lost */ - std::string reduction_strs[] = {"stellar_mass","added_dfloor_mass", "removed_eceil_energy", - "removed_vceil_energy", "added_vAceil_mass"}; + std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", + "removed_eceil_energy", "removed_vceil_energy", + "added_vAceil_mass"}; - //Add a param for each reduction, then add it as a summation reduction for - //history outputs + // Add a param for each reduction, then add it as a summation reduction for + // history outputs auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - for( auto reduction_str : reduction_strs ) { - hydro_pkg->AddParam(reduction_str, 0.0, true); + for (auto reduction_str : reduction_strs) { + hydro_pkg->AddParam(reduction_str, 0.0, true); hst_vars.emplace_back(parthenon::HistoryOutputVar( parthenon::UserHistoryOperation::sum, [reduction_str](MeshData *md) { auto pmb = md->GetBlockData(0)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); const Real reduction = hydro_pkg->Param(reduction_str); - //Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str,0.0); + // Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str, 0.0); return reduction; }, reduction_str)); } - //Add history reduction for total cold gas using stellar mass threshold + // Add history reduction for total cold gas using stellar mass threshold const Real cold_thresh = pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if( cold_thresh > 0){ + if (cold_thresh > 0) { hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, - "cold_mass")); + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); } const Real agn_tracer_thresh = pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if( agn_tracer_thresh >= 0){ - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - PARTHENON_REQUIRE(pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); + if (agn_tracer_thresh >= 0) { + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + PARTHENON_REQUIRE( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, - "agn_extent")); + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); } - hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); /************************************************************ diff --git a/src/pgen/field_loop.cpp b/src/pgen/field_loop.cpp index 0ccdb69a..45eeb60e 100644 --- a/src/pgen/field_loop.cpp +++ b/src/pgen/field_loop.cpp @@ -135,13 +135,16 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { int iprob = pin->GetInteger("problem/field_loop", "iprob"); Real ang_2, cos_a2(0.0), sin_a2(0.0), lambda(0.0); - Real x1size = pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); - Real x2size = pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); + Real x1size = + pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); + Real x2size = + pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); const bool two_d = pmb->pmy_mesh->ndim < 3; // for 2D sim set x3size to zero so that v_z is 0 below Real x3size = - two_d ? 0 : pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); + two_d ? 0 + : pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); // For (iprob=4) -- rotated cylinder in 3D -- set up rotation angle and wavelength if (iprob == 4) { diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 9d0e96be..2c9ff9a7 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -402,9 +402,12 @@ void Perturb(MeshData *md, const Real dt) { MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - const auto Lx = pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); - const auto Ly = pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); - const auto Lz = pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); + const auto Lx = + pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); + const auto Ly = + pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); + const auto Lz = + pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); const auto accel_rms = hydro_pkg->Param("turbulence/accel_rms"); auto norm = accel_rms / std::sqrt(sums[0] / (Lx * Ly * Lz)); From 6004203365cf1b719d451746c5f7a74a8186a2d1 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 26 Sep 2023 11:48:27 +0200 Subject: [PATCH 29/48] Bump parth to current dev --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 60d54786..39ad758a 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 60d5478693e26fa9b4836beb60ba7aeedfc608a4 +Subproject commit 39ad758a2c94ac5552200809aa39076a11bb8810 From fe532fd13e554c4044e90e5071174d40e3ec0821 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 17 Oct 2023 15:34:25 +0200 Subject: [PATCH 30/48] Bump parth to include analysis outputs --- external/parthenon | 2 +- src/pgen/cluster.cpp | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/external/parthenon b/external/parthenon index 39ad758a..8af38e42 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 39ad758a2c94ac5552200809aa39076a11bb8810 +Subproject commit 8af38e420c56baf9c86c4c68f0787f10e8824d9d diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index a26b1da6..e1017415 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -78,13 +78,9 @@ void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - // const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - // snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); + const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); - // ApplyClusterClips(md, tm, beta_dt); - - const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); - stellar_feedback.FeedbackSrcTerm(md, beta_dt, tm); }; void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, const Real dt) { From d83ef781d62cc49ba7de5c19648542d08a842acd Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 17 Oct 2023 15:37:16 +0200 Subject: [PATCH 31/48] Add bmag calc for cluster --- src/pgen/cluster.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index e1017415..3a0cfb9a 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -25,6 +25,7 @@ #include // c_str() // Parthenon headers +#include "Kokkos_MathematicalFunctions.hpp" #include "kokkos_abstraction.hpp" #include "mesh/domain.hpp" #include "mesh/mesh.hpp" @@ -80,7 +81,6 @@ void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); - }; void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, const Real dt) { @@ -360,6 +360,9 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // plasma beta hydro_pkg->AddField("plasma_beta", m); + + // plasma beta + hydro_pkg->AddField("B_mag", m); } /************************************************************ @@ -885,6 +888,7 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { if (pkg->Param("fluid") == Fluid::glmmhd) { auto &plasma_beta = data->Get("plasma_beta").data; auto &mach_alfven = data->Get("mach_alfven").data; + auto &b_mag = data->Get("B_mag").data; pmb->par_for( "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, @@ -897,6 +901,8 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { const Real Bz = prim(IB3, k, j, i); const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + b_mag(k, j, i) = Kokkos::sqrt(B2); + // compute Alfven mach number const Real v_A = std::sqrt(B2 / rho); const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS From 2eb01991893cc3a52c68a292678fa371c4f4b2c6 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 20 Oct 2023 09:31:25 +0200 Subject: [PATCH 32/48] Bump hist parthenon ver --- external/parthenon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/parthenon b/external/parthenon index 8af38e42..725deed2 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 8af38e420c56baf9c86c4c68f0787f10e8824d9d +Subproject commit 725deed20ca2dff61d59f3a18d41772d8c4a7644 From 090383b3534b64e4bc5b10d3598beafa4f4ef9c7 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 20 Oct 2023 18:19:27 +0200 Subject: [PATCH 33/48] Fix typo in blob injection and update interface --- src/pgen/turbulence.cpp | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 4fe9101e..08d66df1 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -14,6 +14,7 @@ // Parthenon headers #include "basic_types.hpp" +#include "defs.hpp" #include "globals.hpp" #include "kokkos_abstraction.hpp" #include "mesh/mesh.hpp" @@ -36,6 +37,9 @@ namespace turbulence { using namespace parthenon::package::prelude; using parthenon::DevMemSpace; using parthenon::ParArray2D; +using parthenon::X1DIR; +using parthenon::X2DIR; +using parthenon::X3DIR; using utils::few_modes_ft::Complex; using utils::few_modes_ft::FewModesFT; @@ -579,9 +583,12 @@ void Rescale(MeshData *md, const parthenon::SimTime &tm, const Real dt) { MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - const auto Lx = pmb->pmy_mesh->mesh_size.x1max - pmb->pmy_mesh->mesh_size.x1min; - const auto Ly = pmb->pmy_mesh->mesh_size.x2max - pmb->pmy_mesh->mesh_size.x2min; - const auto Lz = pmb->pmy_mesh->mesh_size.x3max - pmb->pmy_mesh->mesh_size.x3min; + const auto Lx = + pmb->pmy_mesh->mesh_size.xmax(X1DIR) - pmb->pmy_mesh->mesh_size.xmin(X1DIR); + const auto Ly = + pmb->pmy_mesh->mesh_size.xmax(X2DIR) - pmb->pmy_mesh->mesh_size.xmin(X2DIR); + const auto Lz = + pmb->pmy_mesh->mesh_size.xmax(X3DIR) - pmb->pmy_mesh->mesh_size.xmin(X3DIR); auto norm = SQR(rescale_to_rms_Ms) / (Ms2_sum / (Lx * Ly * Lz)); pmb->par_for( @@ -648,12 +655,18 @@ void InjectBlob(MeshData *md, const parthenon::SimTime &tm, const Real dt) const auto *const error_msg = "Blob bounds crossing domain bounds currently not supported."; - PARTHENON_REQUIRE_THROWS(loc_x + radius < pmb->pmy_mesh->mesh_size.x1max, error_msg) - PARTHENON_REQUIRE_THROWS(loc_x - radius > pmb->pmy_mesh->mesh_size.x1min, error_msg) - PARTHENON_REQUIRE_THROWS(loc_y + radius < pmb->pmy_mesh->mesh_size.x2max, error_msg) - PARTHENON_REQUIRE_THROWS(loc_y - radius > pmb->pmy_mesh->mesh_size.x2min, error_msg) - PARTHENON_REQUIRE_THROWS(loc_z + radius < pmb->pmy_mesh->mesh_size.x3max, error_msg) - PARTHENON_REQUIRE_THROWS(loc_z - radius > pmb->pmy_mesh->mesh_size.x3min, error_msg) + PARTHENON_REQUIRE_THROWS(loc_x + radius < pmb->pmy_mesh->mesh_size.xmax(X1DIR), + error_msg) + PARTHENON_REQUIRE_THROWS(loc_x - radius > pmb->pmy_mesh->mesh_size.xmin(X1DIR), + error_msg) + PARTHENON_REQUIRE_THROWS(loc_y + radius < pmb->pmy_mesh->mesh_size.xmax(X2DIR), + error_msg) + PARTHENON_REQUIRE_THROWS(loc_y - radius > pmb->pmy_mesh->mesh_size.xmin(X2DIR), + error_msg) + PARTHENON_REQUIRE_THROWS(loc_z + radius < pmb->pmy_mesh->mesh_size.xmax(X3DIR), + error_msg) + PARTHENON_REQUIRE_THROWS(loc_z - radius > pmb->pmy_mesh->mesh_size.xmin(X3DIR), + error_msg) IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); @@ -692,7 +705,7 @@ void InjectBlob(MeshData *md, const parthenon::SimTime &tm, const Real dt) // adjust momentum (so that the velocity remains constant) cons(IM1, k, j, i) *= chi; cons(IM2, k, j, i) *= chi; - cons(IM2, k, j, i) *= chi; + cons(IM3, k, j, i) *= chi; // adjust total energy density (using original rho_e translates to an increase // of 1/chi in temperature) cons(IEN, k, j, i) = kin_en_density * chi + rho_e; From d37d2d2ae729d65892d518190255e8c2d74a4707 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Mon, 23 Oct 2023 15:44:54 +0200 Subject: [PATCH 34/48] Add temperature field output for turb driver --- src/pgen/turbulence.cpp | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 08d66df1..ce97db87 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -16,6 +16,7 @@ #include "basic_types.hpp" #include "defs.hpp" #include "globals.hpp" +#include "interface/metadata.hpp" #include "kokkos_abstraction.hpp" #include "mesh/mesh.hpp" #include @@ -119,14 +120,22 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg } pkg->UpdateParam(parthenon::hist_param_key, hst_vars); + // Add a temperature field for easier access within Ascent and history files + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); + if (pin->GetOrAddBoolean("problem/turbulence", "calc_temperature", false)) { + PARTHENON_REQUIRE_THROWS(pkg->AllParams().hasKey("mbar_over_kb"), + "Using temperature fields requires units or mbar_over_kb."); + pkg->AddField("temperature", m); + } + // Step 2. Add appropriate fields required by this pgen // Using OneCopy here to save memory. We typically don't need to update/evolve the // acceleration field for various stages in a cycle as the "model" error of the // turbulence driver is larger than the numerical one any way. This may need to be // changed if an "as close as possible" comparison between methods/codes is the goal and // not turbulence from a physical point of view. - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); + m = Metadata({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); pkg->AddField("acc", m); auto num_modes = @@ -758,6 +767,28 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { // store state of distribution auto state_dist = few_modes_ft.GetDistState(); pin->SetString("problem/turbulence", "state_dist", state_dist); + + if (pin->GetOrAddBoolean("problem/turbulence", "calc_temperature", false)) { + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; + auto &temperature = data->Get("temperature").data; + + // for computing temperature from primitives + auto units = hydro_pkg->Param("units"); + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + pmb->par_for( + "Turbulence::UserWorkBeforeOutput calc temperature", kb.s, kb.e, jb.s, jb.e, ib.s, + ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + // compute temperature + temperature(k, j, i) = mbar_over_kb * P / rho; + }); + } } } // namespace turbulence From d397e22c2b3081e810623b78530d33ce7be4e1f4 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 15 Nov 2023 17:20:09 +0100 Subject: [PATCH 35/48] Add skeleton infrastructure for UserOutput --- external/parthenon | 2 +- src/CMakeLists.txt | 1 + src/outputs/per_block.cpp | 690 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 692 insertions(+), 1 deletion(-) create mode 100644 src/outputs/per_block.cpp diff --git a/external/parthenon b/external/parthenon index 725deed2..f84dd6a1 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 725deed20ca2dff61d59f3a18d41772d8c4a7644 +Subproject commit f84dd6a1bc08ad3144b29a78bf166f22c0d4437c diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5d8fd375..51a0e4b7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,6 +16,7 @@ add_executable( hydro/srcterms/gravitational_field.hpp hydro/srcterms/tabular_cooling.hpp hydro/srcterms/tabular_cooling.cpp + outputs/per_block.cpp refinement/gradient.cpp refinement/other.cpp utils/few_modes_ft.cpp diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp new file mode 100644 index 00000000..5b0ffcdc --- /dev/null +++ b/src/outputs/per_block.cpp @@ -0,0 +1,690 @@ +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2023 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2020-2023. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 for Los +// Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +// for the U.S. Department of Energy/National Nuclear Security Administration. All rights +// in the program are reserved by Triad National Security, LLC, and the U.S. Department +// of Energy/National Nuclear Security Administration. The Government is granted for +// itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +// license in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do so. +//======================================================================================== + +// options for building +#include "config.hpp" +#include "globals.hpp" +#include "utils/error_checking.hpp" + +// Only proceed if HDF5 output enabled +#ifdef ENABLE_HDF5 + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "driver/driver.hpp" +#include "interface/metadata.hpp" +#include "mesh/mesh.hpp" +#include "mesh/meshblock.hpp" +#include "outputs/output_utils.hpp" +#include "outputs/outputs.hpp" +#include "outputs/parthenon_hdf5.hpp" +#include "outputs/parthenon_xdmf.hpp" +#include "utils/string_utils.hpp" + +namespace parthenon { +namespace UserOutputHelper { +// TODO(JMM): Should this live in the base class or output_utils? +// TODO(pgete): yes, as they can be reused +void ComputeXminBlocks_(Mesh *pm, std::vector &data) { + int i = 0; + for (auto &pmb : pm->block_list) { + auto xmin = pmb->coords.GetXmin(); + data[i++] = xmin[0]; + if (pm->ndim > 1) { + data[i++] = xmin[1]; + } + if (pm->ndim > 2) { + data[i++] = xmin[2]; + } + } +} +// TODO(JMM): Should this live in the base class or output_utils? +// TODO(pgete): yes, as they can be reused +void ComputeLocs_(Mesh *pm, std::vector &locs) { + int i = 0; + for (auto &pmb : pm->block_list) { + locs[i++] = pmb->loc.lx1(); + locs[i++] = pmb->loc.lx2(); + locs[i++] = pmb->loc.lx3(); + } +} +// TODO(JMM): Should this live in the base class or output_utils? +// TODO(pgete): yes, as they can be reused +void ComputeIDsAndFlags_(Mesh *pm, std::vector &data) { + int i = 0; + for (auto &pmb : pm->block_list) { + data[i++] = pmb->loc.level(); + data[i++] = pmb->gid; + data[i++] = pmb->lid; + data[i++] = pmb->cnghost; + data[i++] = pmb->gflag; + } +} +// TODO(JMM): Should this live in the base class or output_utils? +// TODO(pgete): yes, as they can be reused +void ComputeCoords_(Mesh *pm, bool face, const IndexRange &ib, const IndexRange &jb, + const IndexRange &kb, std::vector &x, std::vector &y, + std::vector &z) { + std::size_t idx_x = 0, idx_y = 0, idx_z = 0; + + // note relies on casting of bool to int + for (auto &pmb : pm->block_list) { + for (int i = ib.s; i <= ib.e + face; ++i) { + x[idx_x++] = face ? pmb->coords.Xf<1>(i) : pmb->coords.Xc<1>(i); + } + for (int j = jb.s; j <= jb.e + face; ++j) { + y[idx_y++] = face ? pmb->coords.Xf<2>(j) : pmb->coords.Xc<2>(j); + } + for (int k = kb.s; k <= kb.e + face; ++k) { + z[idx_z++] = face ? pmb->coords.Xf<3>(k) : pmb->coords.Xc<3>(k); + } + } +} +} // namespace UserOutputHelper +//---------------------------------------------------------------------------------------- +//! \fn void PHDF5Output:::WriteOutputFileImpl(Mesh *pm, ParameterInput *pin, bool flag) +// \brief Cycles over all MeshBlocks and writes OutputData in the Parthenon HDF5 format, +// one file per output using parallel IO. +void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { + using namespace HDF5; + using namespace OutputUtils; + + Kokkos::Profiling::pushRegion("UserOutput::WriteOutputFile"); + + // writes all graphics variables to hdf file + // HDF5 structures + // Also writes companion xdmf file + + const int max_blocks_global = pm->nbtotal; + const int num_blocks_local = static_cast(pm->block_list.size()); + + const IndexDomain theDomain = + (output_params.include_ghost_zones ? IndexDomain::entire : IndexDomain::interior); + + auto const &first_block = *(pm->block_list.front()); + + const IndexRange out_ib = first_block.cellbounds.GetBoundsI(theDomain); + const IndexRange out_jb = first_block.cellbounds.GetBoundsJ(theDomain); + const IndexRange out_kb = first_block.cellbounds.GetBoundsK(theDomain); + + auto const nx1 = out_ib.e - out_ib.s + 1; + auto const nx2 = out_jb.e - out_jb.s + 1; + auto const nx3 = out_kb.e - out_kb.s + 1; + + const int rootLevel = pm->GetRootLevel(); + const int max_level = pm->GetCurrentLevel() - rootLevel; + const auto &nblist = pm->GetNbList(); + + // open HDF5 file + // Define output filename + std::string filename = "bla"; // GenerateFilename_(pin, tm, signal); + + // set file access property list + H5P const acc_file = H5P::FromHIDCheck(HDF5::GenerateFileAccessProps()); + + // now create the file + H5F file; + try { + file = H5F::FromHIDCheck( + H5Fcreate(filename.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, acc_file)); + } catch (std::exception &ex) { + std::stringstream err; + err << "### ERROR: Failed to create HDF5 output file '" << filename + << "' with the following error:" << std::endl + << ex.what() << std::endl; + PARTHENON_THROW(err) + } + + // -------------------------------------------------------------------------------- // + // WRITING ATTRIBUTES // + // -------------------------------------------------------------------------------- // + Kokkos::Profiling::pushRegion("write Attributes"); + { + Kokkos::Profiling::pushRegion("write input"); + // write input key-value pairs + std::ostringstream oss; + pin->ParameterDump(oss); + + // Mesh information + const H5G input_group = MakeGroup(file, "/Input"); + + HDF5WriteAttribute("File", oss.str().c_str(), input_group); + Kokkos::Profiling::popRegion(); // write input + } // Input section + + // we'll need this again at the end + const H5G info_group = MakeGroup(file, "/Info"); + { + Kokkos::Profiling::pushRegion("write Info"); + HDF5WriteAttribute("OutputFormatVersion", OUTPUT_VERSION_FORMAT, info_group); + + if (tm != nullptr) { + HDF5WriteAttribute("NCycle", tm->ncycle, info_group); + HDF5WriteAttribute("Time", tm->time, info_group); + HDF5WriteAttribute("dt", tm->dt, info_group); + } + + HDF5WriteAttribute("WallTime", Driver::elapsed_main(), info_group); + HDF5WriteAttribute("NumDims", pm->ndim, info_group); + HDF5WriteAttribute("NumMeshBlocks", pm->nbtotal, info_group); + HDF5WriteAttribute("MaxLevel", max_level, info_group); + // write whether we include ghost cells or not + HDF5WriteAttribute("IncludesGhost", output_params.include_ghost_zones ? 1 : 0, + info_group); + // write number of ghost cells in simulation + HDF5WriteAttribute("NGhost", Globals::nghost, info_group); + HDF5WriteAttribute("Coordinates", std::string(first_block.coords.Name()).c_str(), + info_group); + + // restart info, write always + HDF5WriteAttribute("NBNew", pm->nbnew, info_group); + HDF5WriteAttribute("NBDel", pm->nbdel, info_group); + HDF5WriteAttribute("RootLevel", rootLevel, info_group); + HDF5WriteAttribute("Refine", pm->adaptive ? 1 : 0, info_group); + HDF5WriteAttribute("Multilevel", pm->multilevel ? 1 : 0, info_group); + + HDF5WriteAttribute("BlocksPerPE", nblist, info_group); + + // Mesh block size + HDF5WriteAttribute("MeshBlockSize", std::vector{nx1, nx2, nx3}, info_group); + + // RootGridDomain - float[9] array with xyz mins, maxs, rats (dx(i)/dx(i-1)) + HDF5WriteAttribute( + "RootGridDomain", + std::vector{pm->mesh_size.xmin(X1DIR), pm->mesh_size.xmax(X1DIR), + pm->mesh_size.xrat(X1DIR), pm->mesh_size.xmin(X2DIR), + pm->mesh_size.xmax(X2DIR), pm->mesh_size.xrat(X2DIR), + pm->mesh_size.xmin(X3DIR), pm->mesh_size.xmax(X3DIR), + pm->mesh_size.xrat(X3DIR)}, + info_group); + + // Root grid size (number of cells at root level) + HDF5WriteAttribute("RootGridSize", + std::vector{pm->mesh_size.nx(X1DIR), pm->mesh_size.nx(X2DIR), + pm->mesh_size.nx(X3DIR)}, + info_group); + + // Boundary conditions + std::vector boundary_condition_str(BOUNDARY_NFACES); + for (size_t i = 0; i < boundary_condition_str.size(); i++) { + boundary_condition_str[i] = GetBoundaryString(pm->mesh_bcs[i]); + } + + HDF5WriteAttribute("BoundaryConditions", boundary_condition_str, info_group); + Kokkos::Profiling::popRegion(); // write Info + } // Info section + + // write Params + { + Kokkos::Profiling::pushRegion("behold: write Params"); + const H5G params_group = MakeGroup(file, "/Params"); + + for (const auto &package : pm->packages.AllPackages()) { + const auto state = package.second; + // Write all params that can be written as HDF5 attributes + state->AllParams().WriteAllToHDF5(state->label(), params_group); + } + Kokkos::Profiling::popRegion(); // behold: write Params + } // Params section + Kokkos::Profiling::popRegion(); // write Attributes + + // -------------------------------------------------------------------------------- // + // WRITING MESHBLOCK METADATA // + // -------------------------------------------------------------------------------- // + + // set local offset, always the same for all data sets + hsize_t my_offset = 0; + for (int i = 0; i < Globals::my_rank; i++) { + my_offset += nblist[i]; + } + + const std::array local_offset({my_offset, 0, 0, 0, 0, 0, 0}); + + // these can vary by data set, except index 0 is always the same + std::array local_count( + {static_cast(num_blocks_local), 1, 1, 1, 1, 1, 1}); + std::array global_count( + {static_cast(max_blocks_global), 1, 1, 1, 1, 1, 1}); + + // for convenience + const hsize_t *const p_loc_offset = local_offset.data(); + const hsize_t *const p_loc_cnt = local_count.data(); + const hsize_t *const p_glob_cnt = global_count.data(); + + H5P const pl_xfer = H5P::FromHIDCheck(H5Pcreate(H5P_DATASET_XFER)); + H5P const pl_dcreate = H5P::FromHIDCheck(H5Pcreate(H5P_DATASET_CREATE)); + + // Never write fill values to the dataset + PARTHENON_HDF5_CHECK(H5Pset_fill_time(pl_dcreate, H5D_FILL_TIME_NEVER)); + +#ifdef MPI_PARALLEL + PARTHENON_HDF5_CHECK(H5Pset_dxpl_mpio(pl_xfer, H5FD_MPIO_COLLECTIVE)); +#endif + + // write Blocks metadata + { + Kokkos::Profiling::pushRegion("write block metadata"); + const H5G gBlocks = MakeGroup(file, "/Blocks"); + + // write Xmin[ndim] for blocks + { + std::vector tmpData(num_blocks_local * 3); + UserOutputHelper::ComputeXminBlocks_(pm, tmpData); + local_count[1] = global_count[1] = pm->ndim; + HDF5Write2D(gBlocks, "xmin", tmpData.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, + pl_xfer); + } + + // write Block ID + { + // LOC.lx1,2,3 + hsize_t n = 3; + std::vector tmpLoc(num_blocks_local * n); + local_count[1] = global_count[1] = n; + UserOutputHelper::ComputeLocs_(pm, tmpLoc); + HDF5Write2D(gBlocks, "loc.lx123", tmpLoc.data(), p_loc_offset, p_loc_cnt, + p_glob_cnt, pl_xfer); + + // (LOC.)level, GID, LID, cnghost, gflag + n = 5; // this is NOT H5_NDIM + std::vector tmpID(num_blocks_local * n); + local_count[1] = global_count[1] = n; + UserOutputHelper::ComputeIDsAndFlags_(pm, tmpID); + HDF5Write2D(gBlocks, "loc.level-gid-lid-cnghost-gflag", tmpID.data(), p_loc_offset, + p_loc_cnt, p_glob_cnt, pl_xfer); + } + Kokkos::Profiling::popRegion(); // write block metadata + } // Block section + + // Write mesh coordinates to file + Kokkos::Profiling::pushRegion("write mesh coords"); + for (const bool face : {true, false}) { + const H5G gLocations = MakeGroup(file, face ? "/Locations" : "/VolumeLocations"); + + // write X coordinates + std::vector loc_x((nx1 + face) * num_blocks_local); + std::vector loc_y((nx2 + face) * num_blocks_local); + std::vector loc_z((nx3 + face) * num_blocks_local); + + UserOutputHelper::ComputeCoords_(pm, face, out_ib, out_jb, out_kb, loc_x, loc_y, + loc_z); + + local_count[1] = global_count[1] = nx1 + face; + HDF5Write2D(gLocations, "x", loc_x.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, + pl_xfer); + + local_count[1] = global_count[1] = nx2 + face; + HDF5Write2D(gLocations, "y", loc_y.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, + pl_xfer); + + local_count[1] = global_count[1] = nx3 + face; + HDF5Write2D(gLocations, "z", loc_z.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, + pl_xfer); + } + Kokkos::Profiling::popRegion(); // write mesh coords + + // Write Levels and Logical Locations with the level for each Meshblock loclist contains + // levels and logical locations for all meshblocks on all ranks + { + Kokkos::Profiling::pushRegion("write levels and locations"); + const auto &loclist = pm->GetLocList(); + + std::vector levels; + levels.reserve(pm->nbtotal); + + std::vector logicalLocations; + logicalLocations.reserve(pm->nbtotal * 3); + + for (const auto &loc : loclist) { + levels.push_back(loc.level() - pm->GetRootLevel()); + logicalLocations.push_back(loc.lx1()); + logicalLocations.push_back(loc.lx2()); + logicalLocations.push_back(loc.lx3()); + } + + // Only write levels on rank 0 since it has data for all ranks + local_count[0] = (Globals::my_rank == 0) ? pm->nbtotal : 0; + HDF5WriteND(file, "Levels", levels.data(), 1, local_offset.data(), local_count.data(), + global_count.data(), pl_xfer, H5P_DEFAULT); + + local_count[1] = global_count[1] = 3; + HDF5Write2D(file, "LogicalLocations", logicalLocations.data(), local_offset.data(), + local_count.data(), global_count.data(), pl_xfer); + + // reset for collective output + local_count[0] = num_blocks_local; + Kokkos::Profiling::popRegion(); // write levels and locations + } + + // -------------------------------------------------------------------------------- // + // WRITING VARIABLES DATA // + // -------------------------------------------------------------------------------- // + Kokkos::Profiling::pushRegion("write all variable data"); + + // All blocks have the same list of variable metadata that exist in the entire + // simulation, but not all variables may be allocated on all blocks + + auto get_vars = [=](const std::shared_ptr pmb) { + auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); + return GetAnyVariables(var_vec, output_params.variables); + }; + + // get list of all vars, just use the first block since the list is the same for all + // blocks + std::vector all_vars_info; + const auto vars = get_vars(pm->block_list.front()); + for (auto &v : vars) { + all_vars_info.emplace_back(v); + } + + // sort alphabetically + std::sort(all_vars_info.begin(), all_vars_info.end(), + [](const VarInfo &a, const VarInfo &b) { return a.label < b.label; }); + + // We need to add information about the sparse variables to the HDF5 file, namely: + // 1) Which variables are sparse + // 2) Is a sparse id of a particular sparse variable allocated on a given block + // + // This information is stored in the dataset called "SparseInfo". The data set + // contains an attribute "SparseFields" that is a vector of strings with the names + // of the sparse fields (field name with sparse id, i.e. "bar_28", "bar_7", foo_1", + // "foo_145"). The field names are in alphabetical order, which is the same order + // they show up in all_unique_vars (because it's a sorted set). + // + // The dataset SparseInfo itself is a 2D array of bools. The first index is the + // global block index and the second index is the sparse field (same order as the + // SparseFields attribute). SparseInfo[b][v] is true if the sparse field with index + // v is allocated on the block with index b, otherwise the value is false + + std::vector sparse_names; + std::unordered_map sparse_field_idx; + for (auto &vinfo : all_vars_info) { + if (vinfo.is_sparse) { + sparse_field_idx.insert({vinfo.label, sparse_names.size()}); + sparse_names.push_back(vinfo.label); + } + } + + hsize_t num_sparse = sparse_names.size(); + // can't use std::vector here because std::vector is the same as + // std::vector and it doesn't have .data() member + std::unique_ptr sparse_allocated(new hbool_t[num_blocks_local * num_sparse]); + + // allocate space for largest size variable + int varSize_max = 0; + for (auto &vinfo : all_vars_info) { + const int varSize = + vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * vinfo.nx3 * vinfo.nx2 * vinfo.nx1; + varSize_max = std::max(varSize_max, varSize); + } + + using OutT = Real; + std::vector tmpData(varSize_max * num_blocks_local); + + // create persistent spaces + local_count[0] = num_blocks_local; + global_count[0] = max_blocks_global; + local_count[4] = global_count[4] = nx3; + local_count[5] = global_count[5] = nx2; + local_count[6] = global_count[6] = nx1; + + // for each variable we write + for (auto &vinfo : all_vars_info) { + Kokkos::Profiling::pushRegion("write variable loop"); + // not really necessary, but doesn't hurt + memset(tmpData.data(), 0, tmpData.size() * sizeof(OutT)); + + const std::string var_name = vinfo.label; + const hsize_t nx6 = vinfo.nx6; + const hsize_t nx5 = vinfo.nx5; + const hsize_t nx4 = vinfo.nx4; + + local_count[1] = global_count[1] = nx6; + local_count[2] = global_count[2] = nx5; + local_count[3] = global_count[3] = nx4; + + std::vector alldims({nx6, nx5, nx4, static_cast(vinfo.nx3), + static_cast(vinfo.nx2), + static_cast(vinfo.nx1)}); + + int ndim = -1; +#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION + // we need chunks to enable compression + std::array chunk_size({1, 1, 1, 1, 1, 1, 1}); +#endif + if (vinfo.where == MetadataFlag(Metadata::Cell)) { + ndim = 3 + vinfo.tensor_rank + 1; + for (int i = 0; i < vinfo.tensor_rank; i++) { + local_count[1 + i] = global_count[1 + i] = alldims[3 - vinfo.tensor_rank + i]; + } + local_count[vinfo.tensor_rank + 1] = global_count[vinfo.tensor_rank + 1] = nx3; + local_count[vinfo.tensor_rank + 2] = global_count[vinfo.tensor_rank + 2] = nx2; + local_count[vinfo.tensor_rank + 3] = global_count[vinfo.tensor_rank + 3] = nx1; + +#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION + if (output_params.hdf5_compression_level > 0) { + for (int i = ndim - 3; i < ndim; i++) { + chunk_size[i] = local_count[i]; + } + } +#endif + } else if (vinfo.where == MetadataFlag(Metadata::None)) { + ndim = vinfo.tensor_rank + 1; + for (int i = 0; i < vinfo.tensor_rank; i++) { + local_count[1 + i] = global_count[1 + i] = alldims[6 - vinfo.tensor_rank + i]; + } + +#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION + if (output_params.hdf5_compression_level > 0) { + int nchunk_indices = std::min(vinfo.tensor_rank, 3); + for (int i = ndim - nchunk_indices; i < ndim; i++) { + chunk_size[i] = alldims[6 - nchunk_indices + i]; + } + } +#endif + } else { + PARTHENON_THROW("Only Cell and None locations supported!"); + } + +#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION + PARTHENON_HDF5_CHECK(H5Pset_chunk(pl_dcreate, ndim, chunk_size.data())); + // Do not run the pipeline if compression is soft disabled. + // By default data would still be passed, which may result in slower output. + if (output_params.hdf5_compression_level > 0) { + PARTHENON_HDF5_CHECK( + H5Pset_deflate(pl_dcreate, std::min(9, output_params.hdf5_compression_level))); + } +#endif + + // load up data + hsize_t index = 0; + + Kokkos::Profiling::pushRegion("fill host output buffer"); + // for each local mesh block + for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { + const auto &pmb = pm->block_list[b_idx]; + bool is_allocated = false; + + // for each variable that this local meshblock actually has + const auto vars = get_vars(pmb); + for (auto &v : vars) { + // For reference, if we update the logic here, there's also + // a similar block in parthenon_manager.cpp + if (v->IsAllocated() && (var_name == v->label())) { + auto v_h = v->data.GetHostMirrorAndCopy(); + for (int t = 0; t < nx6; ++t) { + for (int u = 0; u < nx5; ++u) { + for (int v = 0; v < nx4; ++v) { + if (vinfo.where == MetadataFlag(Metadata::Cell)) { + for (int k = out_kb.s; k <= out_kb.e; ++k) { + for (int j = out_jb.s; j <= out_jb.e; ++j) { + for (int i = out_ib.s; i <= out_ib.e; ++i) { + tmpData[index++] = static_cast(v_h(t, u, v, k, j, i)); + } + } + } + } else { + for (int k = 0; k < vinfo.nx3; ++k) { + for (int j = 0; j < vinfo.nx2; ++j) { + for (int i = 0; i < vinfo.nx1; ++i) { + tmpData[index++] = static_cast(v_h(t, u, v, k, j, i)); + } + } + } + } + } + } + } + + is_allocated = true; + break; + } + } + + if (vinfo.is_sparse) { + size_t sparse_idx = sparse_field_idx.at(vinfo.label); + sparse_allocated[b_idx * num_sparse + sparse_idx] = is_allocated; + } + + if (!is_allocated) { + if (vinfo.is_sparse) { + hsize_t varSize{}; + if (vinfo.where == MetadataFlag(Metadata::Cell)) { + varSize = vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * (out_kb.e - out_kb.s + 1) * + (out_jb.e - out_jb.s + 1) * (out_ib.e - out_ib.s + 1); + } else { + varSize = + vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * vinfo.nx3 * vinfo.nx2 * vinfo.nx1; + } + auto fill_val = + output_params.sparse_seed_nans ? std::numeric_limits::quiet_NaN() : 0; + std::fill(tmpData.data() + index, tmpData.data() + index + varSize, fill_val); + index += varSize; + } else { + std::stringstream msg; + msg << "### ERROR: Unable to find dense variable " << var_name << std::endl; + PARTHENON_FAIL(msg); + } + } + } + Kokkos::Profiling::popRegion(); // fill host output buffer + + Kokkos::Profiling::pushRegion("write variable data"); + // write data to file + HDF5WriteND(file, var_name, tmpData.data(), ndim, p_loc_offset, p_loc_cnt, p_glob_cnt, + pl_xfer, pl_dcreate); + Kokkos::Profiling::popRegion(); // write variable data + Kokkos::Profiling::popRegion(); // write variable loop + } + Kokkos::Profiling::popRegion(); // write all variable data + + // names of variables + std::vector var_names; + var_names.reserve(all_vars_info.size()); + + // number of components within each dataset + std::vector num_components; + num_components.reserve(all_vars_info.size()); + + // names of components within each dataset + std::vector component_names; + component_names.reserve(all_vars_info.size()); // may be larger + + for (const auto &vi : all_vars_info) { + var_names.push_back(vi.label); + + const auto &component_labels = vi.component_labels; + PARTHENON_REQUIRE_THROWS(component_labels.size() > 0, "Got 0 component labels"); + + num_components.push_back(component_labels.size()); + for (const auto &label : component_labels) { + component_names.push_back(label); + } + } + + HDF5WriteAttribute("NumComponents", num_components, info_group); + HDF5WriteAttribute("ComponentNames", component_names, info_group); + HDF5WriteAttribute("OutputDatasetNames", var_names, info_group); + + // write SparseInfo and SparseFields (we can't write a zero-size dataset, so only write + // this if we have sparse fields) + if (num_sparse > 0) { + Kokkos::Profiling::pushRegion("write sparse info"); + local_count[1] = global_count[1] = num_sparse; + + HDF5Write2D(file, "SparseInfo", sparse_allocated.get(), p_loc_offset, p_loc_cnt, + p_glob_cnt, pl_xfer); + + // write names of sparse fields as attribute, first convert to vector of const char* + std::vector names(num_sparse); + for (size_t i = 0; i < num_sparse; ++i) + names[i] = sparse_names[i].c_str(); + + const H5D dset = H5D::FromHIDCheck(H5Dopen2(file, "SparseInfo", H5P_DEFAULT)); + HDF5WriteAttribute("SparseFields", names, dset); + Kokkos::Profiling::popRegion(); // write sparse info + } // SparseInfo and SparseFields sections + + Kokkos::Profiling::popRegion(); // WriteOutputFile???Prec +} + +std::string UserOutput::GenerateFilename_(ParameterInput *pin, SimTime *tm, + const SignalHandler::OutputSignal signal) { + using namespace HDF5; + + auto filename = std::string(output_params.file_basename); + filename.append("."); + filename.append(output_params.file_id); + filename.append("."); + if (signal == SignalHandler::OutputSignal::now) { + filename.append("now"); + } else if (signal == SignalHandler::OutputSignal::final && + output_params.file_label_final) { + filename.append("final"); + // default time based data dump + } else { + std::stringstream file_number; + file_number << std::setw(output_params.file_number_width) << std::setfill('0') + << output_params.file_number; + filename.append(file_number.str()); + } + filename.append(".hdf"); + + if (signal == SignalHandler::OutputSignal::none) { + // After file has been opened with the current number, already advance output + // parameters so that for restarts the file is not immediatly overwritten again. + // Only applies to default time-based data dumps, so that writing "now" and "final" + // outputs does not change the desired output numbering. + output_params.file_number++; + output_params.next_time += output_params.dt; + pin->SetInteger(output_params.block_name, "file_number", output_params.file_number); + pin->SetReal(output_params.block_name, "next_time", output_params.next_time); + } + return filename; +} + +} // namespace parthenon + +#endif // ifdef ENABLE_HDF5 From 07c5f5720b130703b7cc6965a72a4a18ef0b3fc1 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 15 Nov 2023 17:59:41 +0100 Subject: [PATCH 36/48] Dump block center vol locs --- src/outputs/per_block.cpp | 69 +++++++++++++-------------------------- 1 file changed, 22 insertions(+), 47 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 5b0ffcdc..680d4b8b 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -83,22 +83,17 @@ void ComputeIDsAndFlags_(Mesh *pm, std::vector &data) { } // TODO(JMM): Should this live in the base class or output_utils? // TODO(pgete): yes, as they can be reused -void ComputeCoords_(Mesh *pm, bool face, const IndexRange &ib, const IndexRange &jb, - const IndexRange &kb, std::vector &x, std::vector &y, - std::vector &z) { - std::size_t idx_x = 0, idx_y = 0, idx_z = 0; +void ComputeBlockCenterCoords_(Mesh *pm, const IndexRange &ib, const IndexRange &jb, + const IndexRange &kb, std::vector &x, + std::vector &y, std::vector &z) { + std::size_t idx = 0; // note relies on casting of bool to int for (auto &pmb : pm->block_list) { - for (int i = ib.s; i <= ib.e + face; ++i) { - x[idx_x++] = face ? pmb->coords.Xf<1>(i) : pmb->coords.Xc<1>(i); - } - for (int j = jb.s; j <= jb.e + face; ++j) { - y[idx_y++] = face ? pmb->coords.Xf<2>(j) : pmb->coords.Xc<2>(j); - } - for (int k = kb.s; k <= kb.e + face; ++k) { - z[idx_z++] = face ? pmb->coords.Xf<3>(k) : pmb->coords.Xc<3>(k); - } + x[idx] = (pmb->coords.Xf<1>(ib.e + 1) + pmb->coords.Xf<1>(ib.s)) / 2.0; + y[idx] = (pmb->coords.Xf<2>(jb.e + 1) + pmb->coords.Xf<2>(jb.s)) / 2.0; + z[idx] = (pmb->coords.Xf<3>(kb.e + 1) + pmb->coords.Xf<3>(kb.s)) / 2.0; + idx += 1; } } } // namespace UserOutputHelper @@ -115,7 +110,6 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // writes all graphics variables to hdf file // HDF5 structures - // Also writes companion xdmf file const int max_blocks_global = pm->nbtotal; const int num_blocks_local = static_cast(pm->block_list.size()); @@ -139,7 +133,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // open HDF5 file // Define output filename - std::string filename = "bla"; // GenerateFilename_(pin, tm, signal); + std::string filename = GenerateFilename_(pin, tm, signal); // set file access property list H5P const acc_file = H5P::FromHIDCheck(HDF5::GenerateFileAccessProps()); @@ -320,26 +314,26 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // Write mesh coordinates to file Kokkos::Profiling::pushRegion("write mesh coords"); - for (const bool face : {true, false}) { - const H5G gLocations = MakeGroup(file, face ? "/Locations" : "/VolumeLocations"); + { + const H5G gLocations = MakeGroup(file, "/VolumeLocations"); // write X coordinates - std::vector loc_x((nx1 + face) * num_blocks_local); - std::vector loc_y((nx2 + face) * num_blocks_local); - std::vector loc_z((nx3 + face) * num_blocks_local); + std::vector loc_x(num_blocks_local); + std::vector loc_y(num_blocks_local); + std::vector loc_z(num_blocks_local); - UserOutputHelper::ComputeCoords_(pm, face, out_ib, out_jb, out_kb, loc_x, loc_y, - loc_z); + UserOutputHelper::ComputeBlockCenterCoords_(pm, out_ib, out_jb, out_kb, loc_x, loc_y, + loc_z); - local_count[1] = global_count[1] = nx1 + face; + local_count[1] = global_count[1] = 1; HDF5Write2D(gLocations, "x", loc_x.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, pl_xfer); - local_count[1] = global_count[1] = nx2 + face; + local_count[1] = global_count[1] = 1; HDF5Write2D(gLocations, "y", loc_y.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, pl_xfer); - local_count[1] = global_count[1] = nx3 + face; + local_count[1] = global_count[1] = 1; HDF5Write2D(gLocations, "z", loc_z.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, pl_xfer); } @@ -388,7 +382,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, auto get_vars = [=](const std::shared_ptr pmb) { auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); - return GetAnyVariables(var_vec, output_params.variables); + return GetAnyVariables(var_vec, output_params.variables); }; // get list of all vars, just use the first block since the list is the same for all @@ -628,30 +622,11 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, HDF5WriteAttribute("ComponentNames", component_names, info_group); HDF5WriteAttribute("OutputDatasetNames", var_names, info_group); - // write SparseInfo and SparseFields (we can't write a zero-size dataset, so only write - // this if we have sparse fields) - if (num_sparse > 0) { - Kokkos::Profiling::pushRegion("write sparse info"); - local_count[1] = global_count[1] = num_sparse; - - HDF5Write2D(file, "SparseInfo", sparse_allocated.get(), p_loc_offset, p_loc_cnt, - p_glob_cnt, pl_xfer); - - // write names of sparse fields as attribute, first convert to vector of const char* - std::vector names(num_sparse); - for (size_t i = 0; i < num_sparse; ++i) - names[i] = sparse_names[i].c_str(); - - const H5D dset = H5D::FromHIDCheck(H5Dopen2(file, "SparseInfo", H5P_DEFAULT)); - HDF5WriteAttribute("SparseFields", names, dset); - Kokkos::Profiling::popRegion(); // write sparse info - } // SparseInfo and SparseFields sections - - Kokkos::Profiling::popRegion(); // WriteOutputFile???Prec + Kokkos::Profiling::popRegion(); // WriteOutputFile } std::string UserOutput::GenerateFilename_(ParameterInput *pin, SimTime *tm, - const SignalHandler::OutputSignal signal) { + const SignalHandler::OutputSignal signal) { using namespace HDF5; auto filename = std::string(output_params.file_basename); From 59ea93ae8408514a2d36b2ced7824aeaa927c483 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 15 Nov 2023 18:18:53 +0100 Subject: [PATCH 37/48] Add infrastructure to dump per-block data --- src/outputs/per_block.cpp | 253 ++------------------------------------ 1 file changed, 13 insertions(+), 240 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 680d4b8b..77adf628 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -185,10 +185,6 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, HDF5WriteAttribute("NumMeshBlocks", pm->nbtotal, info_group); HDF5WriteAttribute("MaxLevel", max_level, info_group); // write whether we include ghost cells or not - HDF5WriteAttribute("IncludesGhost", output_params.include_ghost_zones ? 1 : 0, - info_group); - // write number of ghost cells in simulation - HDF5WriteAttribute("NGhost", Globals::nghost, info_group); HDF5WriteAttribute("Coordinates", std::string(first_block.coords.Name()).c_str(), info_group); @@ -376,250 +372,27 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // WRITING VARIABLES DATA // // -------------------------------------------------------------------------------- // Kokkos::Profiling::pushRegion("write all variable data"); - - // All blocks have the same list of variable metadata that exist in the entire - // simulation, but not all variables may be allocated on all blocks - - auto get_vars = [=](const std::shared_ptr pmb) { - auto &var_vec = pmb->meshblock_data.Get()->GetVariableVector(); - return GetAnyVariables(var_vec, output_params.variables); - }; - - // get list of all vars, just use the first block since the list is the same for all - // blocks - std::vector all_vars_info; - const auto vars = get_vars(pm->block_list.front()); - for (auto &v : vars) { - all_vars_info.emplace_back(v); - } - - // sort alphabetically - std::sort(all_vars_info.begin(), all_vars_info.end(), - [](const VarInfo &a, const VarInfo &b) { return a.label < b.label; }); - - // We need to add information about the sparse variables to the HDF5 file, namely: - // 1) Which variables are sparse - // 2) Is a sparse id of a particular sparse variable allocated on a given block - // - // This information is stored in the dataset called "SparseInfo". The data set - // contains an attribute "SparseFields" that is a vector of strings with the names - // of the sparse fields (field name with sparse id, i.e. "bar_28", "bar_7", foo_1", - // "foo_145"). The field names are in alphabetical order, which is the same order - // they show up in all_unique_vars (because it's a sorted set). - // - // The dataset SparseInfo itself is a 2D array of bools. The first index is the - // global block index and the second index is the sparse field (same order as the - // SparseFields attribute). SparseInfo[b][v] is true if the sparse field with index - // v is allocated on the block with index b, otherwise the value is false - - std::vector sparse_names; - std::unordered_map sparse_field_idx; - for (auto &vinfo : all_vars_info) { - if (vinfo.is_sparse) { - sparse_field_idx.insert({vinfo.label, sparse_names.size()}); - sparse_names.push_back(vinfo.label); - } - } - - hsize_t num_sparse = sparse_names.size(); - // can't use std::vector here because std::vector is the same as - // std::vector and it doesn't have .data() member - std::unique_ptr sparse_allocated(new hbool_t[num_blocks_local * num_sparse]); - - // allocate space for largest size variable - int varSize_max = 0; - for (auto &vinfo : all_vars_info) { - const int varSize = - vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * vinfo.nx3 * vinfo.nx2 * vinfo.nx1; - varSize_max = std::max(varSize_max, varSize); - } - - using OutT = Real; - std::vector tmpData(varSize_max * num_blocks_local); - - // create persistent spaces - local_count[0] = num_blocks_local; - global_count[0] = max_blocks_global; - local_count[4] = global_count[4] = nx3; - local_count[5] = global_count[5] = nx2; - local_count[6] = global_count[6] = nx1; - - // for each variable we write - for (auto &vinfo : all_vars_info) { - Kokkos::Profiling::pushRegion("write variable loop"); - // not really necessary, but doesn't hurt - memset(tmpData.data(), 0, tmpData.size() * sizeof(OutT)); - - const std::string var_name = vinfo.label; - const hsize_t nx6 = vinfo.nx6; - const hsize_t nx5 = vinfo.nx5; - const hsize_t nx4 = vinfo.nx4; - - local_count[1] = global_count[1] = nx6; - local_count[2] = global_count[2] = nx5; - local_count[3] = global_count[3] = nx4; - - std::vector alldims({nx6, nx5, nx4, static_cast(vinfo.nx3), - static_cast(vinfo.nx2), - static_cast(vinfo.nx1)}); - - int ndim = -1; -#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION - // we need chunks to enable compression - std::array chunk_size({1, 1, 1, 1, 1, 1, 1}); -#endif - if (vinfo.where == MetadataFlag(Metadata::Cell)) { - ndim = 3 + vinfo.tensor_rank + 1; - for (int i = 0; i < vinfo.tensor_rank; i++) { - local_count[1 + i] = global_count[1 + i] = alldims[3 - vinfo.tensor_rank + i]; - } - local_count[vinfo.tensor_rank + 1] = global_count[vinfo.tensor_rank + 1] = nx3; - local_count[vinfo.tensor_rank + 2] = global_count[vinfo.tensor_rank + 2] = nx2; - local_count[vinfo.tensor_rank + 3] = global_count[vinfo.tensor_rank + 3] = nx1; - -#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION - if (output_params.hdf5_compression_level > 0) { - for (int i = ndim - 3; i < ndim; i++) { - chunk_size[i] = local_count[i]; - } - } -#endif - } else if (vinfo.where == MetadataFlag(Metadata::None)) { - ndim = vinfo.tensor_rank + 1; - for (int i = 0; i < vinfo.tensor_rank; i++) { - local_count[1 + i] = global_count[1 + i] = alldims[6 - vinfo.tensor_rank + i]; - } - -#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION - if (output_params.hdf5_compression_level > 0) { - int nchunk_indices = std::min(vinfo.tensor_rank, 3); - for (int i = ndim - nchunk_indices; i < ndim; i++) { - chunk_size[i] = alldims[6 - nchunk_indices + i]; - } - } -#endif - } else { - PARTHENON_THROW("Only Cell and None locations supported!"); - } - -#ifndef PARTHENON_DISABLE_HDF5_COMPRESSION - PARTHENON_HDF5_CHECK(H5Pset_chunk(pl_dcreate, ndim, chunk_size.data())); - // Do not run the pipeline if compression is soft disabled. - // By default data would still be passed, which may result in slower output. - if (output_params.hdf5_compression_level > 0) { - PARTHENON_HDF5_CHECK( - H5Pset_deflate(pl_dcreate, std::min(9, output_params.hdf5_compression_level))); + { + const H5G gLocations = MakeGroup(file, "/stats"); + std::vector loc_x(num_blocks_local * 2); + local_count[1] = global_count[1] = 2; + + size_t b = 0; + for (auto &pmb : pm->block_list) { + loc_x[2 * b] = Globals::my_rank; + loc_x[2 * b + 1] = 10 * Globals::my_rank; + b++; } -#endif - // load up data - hsize_t index = 0; - - Kokkos::Profiling::pushRegion("fill host output buffer"); - // for each local mesh block - for (size_t b_idx = 0; b_idx < num_blocks_local; ++b_idx) { - const auto &pmb = pm->block_list[b_idx]; - bool is_allocated = false; - - // for each variable that this local meshblock actually has - const auto vars = get_vars(pmb); - for (auto &v : vars) { - // For reference, if we update the logic here, there's also - // a similar block in parthenon_manager.cpp - if (v->IsAllocated() && (var_name == v->label())) { - auto v_h = v->data.GetHostMirrorAndCopy(); - for (int t = 0; t < nx6; ++t) { - for (int u = 0; u < nx5; ++u) { - for (int v = 0; v < nx4; ++v) { - if (vinfo.where == MetadataFlag(Metadata::Cell)) { - for (int k = out_kb.s; k <= out_kb.e; ++k) { - for (int j = out_jb.s; j <= out_jb.e; ++j) { - for (int i = out_ib.s; i <= out_ib.e; ++i) { - tmpData[index++] = static_cast(v_h(t, u, v, k, j, i)); - } - } - } - } else { - for (int k = 0; k < vinfo.nx3; ++k) { - for (int j = 0; j < vinfo.nx2; ++j) { - for (int i = 0; i < vinfo.nx1; ++i) { - tmpData[index++] = static_cast(v_h(t, u, v, k, j, i)); - } - } - } - } - } - } - } - - is_allocated = true; - break; - } - } - - if (vinfo.is_sparse) { - size_t sparse_idx = sparse_field_idx.at(vinfo.label); - sparse_allocated[b_idx * num_sparse + sparse_idx] = is_allocated; - } - - if (!is_allocated) { - if (vinfo.is_sparse) { - hsize_t varSize{}; - if (vinfo.where == MetadataFlag(Metadata::Cell)) { - varSize = vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * (out_kb.e - out_kb.s + 1) * - (out_jb.e - out_jb.s + 1) * (out_ib.e - out_ib.s + 1); - } else { - varSize = - vinfo.nx6 * vinfo.nx5 * vinfo.nx4 * vinfo.nx3 * vinfo.nx2 * vinfo.nx1; - } - auto fill_val = - output_params.sparse_seed_nans ? std::numeric_limits::quiet_NaN() : 0; - std::fill(tmpData.data() + index, tmpData.data() + index + varSize, fill_val); - index += varSize; - } else { - std::stringstream msg; - msg << "### ERROR: Unable to find dense variable " << var_name << std::endl; - PARTHENON_FAIL(msg); - } - } - } - Kokkos::Profiling::popRegion(); // fill host output buffer - - Kokkos::Profiling::pushRegion("write variable data"); - // write data to file - HDF5WriteND(file, var_name, tmpData.data(), ndim, p_loc_offset, p_loc_cnt, p_glob_cnt, - pl_xfer, pl_dcreate); - Kokkos::Profiling::popRegion(); // write variable data - Kokkos::Profiling::popRegion(); // write variable loop + HDF5Write2D(gLocations, "x", loc_x.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, + pl_xfer); } Kokkos::Profiling::popRegion(); // write all variable data // names of variables std::vector var_names; - var_names.reserve(all_vars_info.size()); - - // number of components within each dataset - std::vector num_components; - num_components.reserve(all_vars_info.size()); - - // names of components within each dataset - std::vector component_names; - component_names.reserve(all_vars_info.size()); // may be larger - - for (const auto &vi : all_vars_info) { - var_names.push_back(vi.label); - - const auto &component_labels = vi.component_labels; - PARTHENON_REQUIRE_THROWS(component_labels.size() > 0, "Got 0 component labels"); - - num_components.push_back(component_labels.size()); - for (const auto &label : component_labels) { - component_names.push_back(label); - } - } + var_names.reserve(5); - HDF5WriteAttribute("NumComponents", num_components, info_group); - HDF5WriteAttribute("ComponentNames", component_names, info_group); HDF5WriteAttribute("OutputDatasetNames", var_names, info_group); Kokkos::Profiling::popRegion(); // WriteOutputFile From af99f2fb5582755a55d30bf00489a81eaa153781 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 16 Nov 2023 17:04:45 +0100 Subject: [PATCH 38/48] Add vel stats --- src/outputs/per_block.cpp | 81 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 75 insertions(+), 6 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 77adf628..e2c1cc38 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -16,9 +16,14 @@ //======================================================================================== // options for building +#include "Kokkos_MathematicalFunctions.hpp" #include "config.hpp" #include "globals.hpp" +#include "mesh/domain.hpp" #include "utils/error_checking.hpp" +#include +#include +#include // Only proceed if HDF5 output enabled #ifdef ENABLE_HDF5 @@ -42,6 +47,8 @@ #include "outputs/parthenon_xdmf.hpp" #include "utils/string_utils.hpp" +#include "../main.hpp" + namespace parthenon { namespace UserOutputHelper { // TODO(JMM): Should this live in the base class or output_utils? @@ -373,19 +380,81 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // -------------------------------------------------------------------------------- // Kokkos::Profiling::pushRegion("write all variable data"); { + const std::string stat_name = "stat_name"; + const std::vector stat_types = { + "min", "max", "absmin", "absmax", "mean", "rms", "stddev", "skew", "kurt"}; const H5G gLocations = MakeGroup(file, "/stats"); - std::vector loc_x(num_blocks_local * 2); - local_count[1] = global_count[1] = 2; + std::vector stat_results(num_blocks_local * stat_types.size()); + local_count[1] = global_count[1] = stat_types.size(); size_t b = 0; for (auto &pmb : pm->block_list) { - loc_x[2 * b] = Globals::my_rank; - loc_x[2 * b + 1] = 10 * Globals::my_rank; + const auto offset = stat_types.size() * b; + const auto ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + const auto jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + const auto kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // TODO(pgrete) consider using new packing machinery here for cleaner interface + const auto prim = pmb->meshblock_data.Get()->Get("prim").data; + + // Central moments about the origin (or non-central moments) + Real muprime1, muprime2, muprime3, muprime4; + + Kokkos::parallel_reduce( + "CalcVelStats", + Kokkos::MDRangePolicy>(DevExecSpace(), {kb.s, jb.s, ib.s}, + {kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &k, const int &j, const int &i, Real &lmin, Real &lmax, + Real &labsmin, Real &labsmax, Real &lsum1, Real &lsum2, + Real &lsum3, Real &lsum4) { + const auto vel = + Kokkos::sqrt(SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + + SQR(prim(IV3, k, j, i))); + + lmin = std::min(vel, lmin); + lmax = std::max(vel, lmax); + // not quite necessary for the velocity *magnitude* + labsmin = std::min(Kokkos::abs(vel), labsmin); + labsmax = std::max(Kokkos::abs(vel), labsmax); + + lsum1 += vel; + lsum2 += vel * vel; + lsum3 += vel * vel * vel; + lsum4 += vel * vel * vel * vel; + }, + Kokkos::Min(stat_results[offset + 0]), + Kokkos::Max(stat_results[offset + 1]), + Kokkos::Min(stat_results[offset + 2]), + Kokkos::Max(stat_results[offset + 3]), Kokkos::Sum(muprime1), + Kokkos::Sum(muprime2), Kokkos::Sum(muprime3), + Kokkos::Sum(muprime4)); + + muprime1 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + muprime2 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + muprime3 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + muprime4 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + + // Central moments about the mean. + // Being verbose here for better readibility + const auto mu = muprime1; // expected value or mean + const auto mu2 = muprime2 - SQR(mu); // variance + const auto mu3 = muprime3 - 3.0 * mu * muprime2 + 2 * std::pow(mu, 3.0); + const auto mu4 = + muprime4 - 4 * mu * muprime3 + 6 * SQR(mu) * muprime2 - 3 * std::pow(mu, 4.0); + + stat_results[offset + 4] = mu; // mean + stat_results[offset + 5] = std::sqrt(muprime2); // rms + const auto stddev = std::sqrt(mu2); // standard deviation + stat_results[offset + 6] = stddev; + stat_results[offset + 7] = mu3 / std::pow(stddev, 3.0); // skewness + stat_results[offset + 8] = mu4 / std::pow(stddev, 4.0); // kurtosis + b++; } - HDF5Write2D(gLocations, "x", loc_x.data(), p_loc_offset, p_loc_cnt, p_glob_cnt, - pl_xfer); + HDF5Write2D(gLocations, stat_name.c_str(), stat_results.data(), p_loc_offset, + p_loc_cnt, p_glob_cnt, pl_xfer); } Kokkos::Profiling::popRegion(); // write all variable data From 7a18c72d4603abf7bfc064e938ca66735df217df Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 16 Nov 2023 18:54:11 +0100 Subject: [PATCH 39/48] Add support for vector if stats and scalar stats --- src/outputs/per_block.cpp | 167 ++++++++++++++++++++++---------------- 1 file changed, 99 insertions(+), 68 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index e2c1cc38..496b37a6 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -23,6 +23,7 @@ #include "utils/error_checking.hpp" #include #include +#include #include // Only proceed if HDF5 output enabled @@ -380,81 +381,111 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // -------------------------------------------------------------------------------- // Kokkos::Profiling::pushRegion("write all variable data"); { - const std::string stat_name = "stat_name"; + struct Stats { + const std::string name; + const std::string field_name; + Kokkos::Array field_components{}; + Stats(std::string name_, std::string field_name_, + Kokkos::Array field_components_) + : name(std::move(name_)), field_name(std::move(field_name_)), + field_components(field_components_) {} + Stats(std::string name_, std::string field_name_, int field_component_) + : name(std::move(name_)), field_name(std::move(field_name_)) { + field_components[0] = field_component_; + field_components[1] = -1; + field_components[2] = -1; + } + }; + + std::vector stats; + stats.emplace_back(Stats("vel_mag", "prim", {IV1, IV2, IV3})); + stats.emplace_back(Stats("rho", "prim", 0)); + stats.emplace_back(Stats("vx", "prim", 1)); + const std::vector stat_types = { "min", "max", "absmin", "absmax", "mean", "rms", "stddev", "skew", "kurt"}; const H5G gLocations = MakeGroup(file, "/stats"); std::vector stat_results(num_blocks_local * stat_types.size()); local_count[1] = global_count[1] = stat_types.size(); - size_t b = 0; - for (auto &pmb : pm->block_list) { - const auto offset = stat_types.size() * b; - const auto ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - const auto jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - const auto kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // TODO(pgrete) consider using new packing machinery here for cleaner interface - const auto prim = pmb->meshblock_data.Get()->Get("prim").data; - - // Central moments about the origin (or non-central moments) - Real muprime1, muprime2, muprime3, muprime4; - - Kokkos::parallel_reduce( - "CalcVelStats", - Kokkos::MDRangePolicy>(DevExecSpace(), {kb.s, jb.s, ib.s}, - {kb.e + 1, jb.e + 1, ib.e + 1}, - {1, 1, ib.e + 1 - ib.s}), - KOKKOS_LAMBDA(const int &k, const int &j, const int &i, Real &lmin, Real &lmax, - Real &labsmin, Real &labsmax, Real &lsum1, Real &lsum2, - Real &lsum3, Real &lsum4) { - const auto vel = - Kokkos::sqrt(SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + - SQR(prim(IV3, k, j, i))); - - lmin = std::min(vel, lmin); - lmax = std::max(vel, lmax); - // not quite necessary for the velocity *magnitude* - labsmin = std::min(Kokkos::abs(vel), labsmin); - labsmax = std::max(Kokkos::abs(vel), labsmax); - - lsum1 += vel; - lsum2 += vel * vel; - lsum3 += vel * vel * vel; - lsum4 += vel * vel * vel * vel; - }, - Kokkos::Min(stat_results[offset + 0]), - Kokkos::Max(stat_results[offset + 1]), - Kokkos::Min(stat_results[offset + 2]), - Kokkos::Max(stat_results[offset + 3]), Kokkos::Sum(muprime1), - Kokkos::Sum(muprime2), Kokkos::Sum(muprime3), - Kokkos::Sum(muprime4)); - - muprime1 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - muprime2 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - muprime3 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - muprime4 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - - // Central moments about the mean. - // Being verbose here for better readibility - const auto mu = muprime1; // expected value or mean - const auto mu2 = muprime2 - SQR(mu); // variance - const auto mu3 = muprime3 - 3.0 * mu * muprime2 + 2 * std::pow(mu, 3.0); - const auto mu4 = - muprime4 - 4 * mu * muprime3 + 6 * SQR(mu) * muprime2 - 3 * std::pow(mu, 4.0); - - stat_results[offset + 4] = mu; // mean - stat_results[offset + 5] = std::sqrt(muprime2); // rms - const auto stddev = std::sqrt(mu2); // standard deviation - stat_results[offset + 6] = stddev; - stat_results[offset + 7] = mu3 / std::pow(stddev, 3.0); // skewness - stat_results[offset + 8] = mu4 / std::pow(stddev, 4.0); // kurtosis - - b++; + for (const auto &stat : stats) { + size_t b = 0; + for (auto &pmb : pm->block_list) { + const auto offset = stat_types.size() * b; + const auto ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + const auto jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + const auto kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // TODO(pgrete) consider using new packing machinery here for cleaner interface + const auto data = pmb->meshblock_data.Get()->Get(stat.field_name).data; + + const auto components = stat.field_components; + + // Central moments about the origin (or non-central moments) + Real muprime1, muprime2, muprime3, muprime4; + + Kokkos::parallel_reduce( + "CalcStats", + Kokkos::MDRangePolicy>(DevExecSpace(), {kb.s, jb.s, ib.s}, + {kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &k, const int &j, const int &i, Real &lmin, + Real &lmax, Real &labsmin, Real &labsmax, Real &lsum1, + Real &lsum2, Real &lsum3, Real &lsum4) { + Real val; + // check the desired field is a scalar + if (components[1] == -1) { + val = data(components[0], k, j, i); + // else is a vector + } else { + val = Kokkos::sqrt(SQR(data(components[0], k, j, i)) + + SQR(data(components[1], k, j, i)) + + SQR(data(components[2], k, j, i))); + } + + lmin = std::min(val, lmin); + lmax = std::max(val, lmax); + // not quite necessary for the velocity *magnitude* + labsmin = std::min(Kokkos::abs(val), labsmin); + labsmax = std::max(Kokkos::abs(val), labsmax); + + lsum1 += val; + lsum2 += val * val; + lsum3 += val * val * val; + lsum4 += val * val * val * val; + }, + Kokkos::Min(stat_results[offset + 0]), + Kokkos::Max(stat_results[offset + 1]), + Kokkos::Min(stat_results[offset + 2]), + Kokkos::Max(stat_results[offset + 3]), Kokkos::Sum(muprime1), + Kokkos::Sum(muprime2), Kokkos::Sum(muprime3), + Kokkos::Sum(muprime4)); + + muprime1 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + muprime2 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + muprime3 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + muprime4 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + + // Central moments about the mean. + // Being verbose here for better readibility + const auto mu = muprime1; // expected value or mean + const auto mu2 = muprime2 - SQR(mu); // variance + const auto mu3 = muprime3 - 3.0 * mu * muprime2 + 2 * std::pow(mu, 3.0); + const auto mu4 = + muprime4 - 4 * mu * muprime3 + 6 * SQR(mu) * muprime2 - 3 * std::pow(mu, 4.0); + + stat_results[offset + 4] = mu; // mean + stat_results[offset + 5] = std::sqrt(muprime2); // rms + const auto stddev = std::sqrt(mu2); // standard deviation + stat_results[offset + 6] = stddev; + stat_results[offset + 7] = mu3 / std::pow(stddev, 3.0); // skewness + stat_results[offset + 8] = mu4 / std::pow(stddev, 4.0); // kurtosis + + b++; + } + HDF5Write2D(gLocations, stat.name, stat_results.data(), p_loc_offset, p_loc_cnt, + p_glob_cnt, pl_xfer); } - - HDF5Write2D(gLocations, stat_name.c_str(), stat_results.data(), p_loc_offset, - p_loc_cnt, p_glob_cnt, pl_xfer); } Kokkos::Profiling::popRegion(); // write all variable data From b990cf6b3f75d3e0f397935905bada3168dcc399 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Thu, 16 Nov 2023 21:58:20 +0100 Subject: [PATCH 40/48] Fix higher order moments --- src/outputs/per_block.cpp | 71 ++++++++++++++++++++++++--------------- 1 file changed, 44 insertions(+), 27 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 496b37a6..6593b4c2 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -421,17 +421,17 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, const auto components = stat.field_components; - // Central moments about the origin (or non-central moments) - Real muprime1, muprime2, muprime3, muprime4; + Real mu; // expected value or mean + Real rms; Kokkos::parallel_reduce( - "CalcStats", + "CalcStatsMean", Kokkos::MDRangePolicy>(DevExecSpace(), {kb.s, jb.s, ib.s}, {kb.e + 1, jb.e + 1, ib.e + 1}, {1, 1, ib.e + 1 - ib.s}), KOKKOS_LAMBDA(const int &k, const int &j, const int &i, Real &lmin, Real &lmax, Real &labsmin, Real &labsmax, Real &lsum1, - Real &lsum2, Real &lsum3, Real &lsum4) { + Real &lsum2) { Real val; // check the desired field is a scalar if (components[1] == -1) { @@ -450,33 +450,50 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, labsmax = std::max(Kokkos::abs(val), labsmax); lsum1 += val; - lsum2 += val * val; - lsum3 += val * val * val; - lsum4 += val * val * val * val; + lsum2 += SQR(val); }, Kokkos::Min(stat_results[offset + 0]), Kokkos::Max(stat_results[offset + 1]), Kokkos::Min(stat_results[offset + 2]), - Kokkos::Max(stat_results[offset + 3]), Kokkos::Sum(muprime1), - Kokkos::Sum(muprime2), Kokkos::Sum(muprime3), - Kokkos::Sum(muprime4)); - - muprime1 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - muprime2 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - muprime3 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - muprime4 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - - // Central moments about the mean. - // Being verbose here for better readibility - const auto mu = muprime1; // expected value or mean - const auto mu2 = muprime2 - SQR(mu); // variance - const auto mu3 = muprime3 - 3.0 * mu * muprime2 + 2 * std::pow(mu, 3.0); - const auto mu4 = - muprime4 - 4 * mu * muprime3 + 6 * SQR(mu) * muprime2 - 3 * std::pow(mu, 4.0); - - stat_results[offset + 4] = mu; // mean - stat_results[offset + 5] = std::sqrt(muprime2); // rms - const auto stddev = std::sqrt(mu2); // standard deviation + Kokkos::Max(stat_results[offset + 3]), Kokkos::Sum(mu), + Kokkos::Sum(rms)); + + mu /= pmb->cellbounds.GetTotal(IndexDomain::interior); + rms = std::sqrt(rms / pmb->cellbounds.GetTotal(IndexDomain::interior)); + + // n-th moments about the mean or central moments + Real mu2, mu3, mu4; + Kokkos::parallel_reduce( + "CalcStatsHigherOrder", + Kokkos::MDRangePolicy>(DevExecSpace(), {kb.s, jb.s, ib.s}, + {kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &k, const int &j, const int &i, Real &lsum2, + Real &lsum3, Real &lsum4) { + Real val; + // check the desired field is a scalar + if (components[1] == -1) { + val = data(components[0], k, j, i); + // else is a vector + } else { + val = Kokkos::sqrt(SQR(data(components[0], k, j, i)) + + SQR(data(components[1], k, j, i)) + + SQR(data(components[2], k, j, i))); + } + + lsum2 += SQR(val - mu); + lsum3 += Kokkos::pow(val - mu, 3.0); + lsum4 += Kokkos::pow(val - mu, 4.0); + }, + Kokkos::Sum(mu2), Kokkos::Sum(mu3), Kokkos::Sum(mu4)); + + mu2 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + mu3 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + mu4 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + + stat_results[offset + 4] = mu; // mean + stat_results[offset + 5] = rms; // rms + const auto stddev = std::sqrt(mu2); // standard deviation stat_results[offset + 6] = stddev; stat_results[offset + 7] = mu3 / std::pow(stddev, 3.0); // skewness stat_results[offset + 8] = mu4 / std::pow(stddev, 4.0); // kurtosis From dfd665025127ad35ca7115a38f9d70dfd2d3429d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 17 Nov 2023 13:53:00 +0100 Subject: [PATCH 41/48] Add mass weighted per-block stats --- src/outputs/per_block.cpp | 60 ++++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 16 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 6593b4c2..815d9566 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -381,16 +381,25 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // -------------------------------------------------------------------------------- // Kokkos::Profiling::pushRegion("write all variable data"); { + PARTHENON_REQUIRE_THROWS( + typeid(Coordinates_t) == typeid(UniformCartesian), + "Stats in per-block output currently assume uniform coordinates. Cell volumes " + "should properly taken into account for other coordinate systems."); + + enum class Weight { None, Mass }; struct Stats { const std::string name; const std::string field_name; Kokkos::Array field_components{}; + Weight weight; + Stats(std::string name_, std::string field_name_, - Kokkos::Array field_components_) + Kokkos::Array field_components_, Weight weight_ = Weight::None) : name(std::move(name_)), field_name(std::move(field_name_)), - field_components(field_components_) {} - Stats(std::string name_, std::string field_name_, int field_component_) - : name(std::move(name_)), field_name(std::move(field_name_)) { + field_components(field_components_), weight(weight_) {} + Stats(std::string name_, std::string field_name_, int field_component_, + Weight weight_ = Weight::None) + : name(std::move(name_)), field_name(std::move(field_name_)), weight(weight_) { field_components[0] = field_component_; field_components[1] = -1; field_components[2] = -1; @@ -399,6 +408,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::vector stats; stats.emplace_back(Stats("vel_mag", "prim", {IV1, IV2, IV3})); + stats.emplace_back(Stats("vel_mag_mw", "prim", {IV1, IV2, IV3}, Weight::Mass)); stats.emplace_back(Stats("rho", "prim", 0)); stats.emplace_back(Stats("vx", "prim", 1)); @@ -418,11 +428,14 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, // TODO(pgrete) consider using new packing machinery here for cleaner interface const auto data = pmb->meshblock_data.Get()->Get(stat.field_name).data; + const auto prim = pmb->meshblock_data.Get()->Get("prim").data; const auto components = stat.field_components; + const auto weight = stat.weight; Real mu; // expected value or mean Real rms; + Real total_weight; Kokkos::parallel_reduce( "CalcStatsMean", @@ -431,7 +444,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, {1, 1, ib.e + 1 - ib.s}), KOKKOS_LAMBDA(const int &k, const int &j, const int &i, Real &lmin, Real &lmax, Real &labsmin, Real &labsmax, Real &lsum1, - Real &lsum2) { + Real &lsum2, Real &lsumweight) { Real val; // check the desired field is a scalar if (components[1] == -1) { @@ -449,17 +462,26 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, labsmin = std::min(Kokkos::abs(val), labsmin); labsmax = std::max(Kokkos::abs(val), labsmax); - lsum1 += val; - lsum2 += SQR(val); + Real w = 1.0; + if (weight == Weight::Mass) { + w = prim(IDN, k, j, i); + lsumweight += w; + } + lsum1 += val * w; + lsum2 += SQR(val) * w; }, Kokkos::Min(stat_results[offset + 0]), Kokkos::Max(stat_results[offset + 1]), Kokkos::Min(stat_results[offset + 2]), Kokkos::Max(stat_results[offset + 3]), Kokkos::Sum(mu), - Kokkos::Sum(rms)); + Kokkos::Sum(rms), Kokkos::Sum(total_weight)); - mu /= pmb->cellbounds.GetTotal(IndexDomain::interior); - rms = std::sqrt(rms / pmb->cellbounds.GetTotal(IndexDomain::interior)); + auto norm = weight == Weight::None + ? pmb->cellbounds.GetTotal(IndexDomain::interior) + : total_weight; + + mu /= norm; + rms = std::sqrt(rms / norm); // n-th moments about the mean or central moments Real mu2, mu3, mu4; @@ -481,15 +503,21 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, SQR(data(components[2], k, j, i))); } - lsum2 += SQR(val - mu); - lsum3 += Kokkos::pow(val - mu, 3.0); - lsum4 += Kokkos::pow(val - mu, 4.0); + Real w = 1.0; + if (weight == Weight::Mass) { + w = prim(IDN, k, j, i); + // no need to calculate the total weight, as it's the same as in the + // previous loop + } + lsum2 += SQR(val - mu) * w; + lsum3 += Kokkos::pow(val - mu, 3.0) * w; + lsum4 += Kokkos::pow(val - mu, 4.0) * w; }, Kokkos::Sum(mu2), Kokkos::Sum(mu3), Kokkos::Sum(mu4)); - mu2 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - mu3 /= pmb->cellbounds.GetTotal(IndexDomain::interior); - mu4 /= pmb->cellbounds.GetTotal(IndexDomain::interior); + mu2 /= norm; + mu3 /= norm; + mu4 /= norm; stat_results[offset + 4] = mu; // mean stat_results[offset + 5] = rms; // rms From a8a81e1534c0c62b007a41510b789b08a0030f4d Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Fri, 17 Nov 2023 14:59:34 +0100 Subject: [PATCH 42/48] Log edotcool --- src/hydro/hydro_driver.cpp | 5 +++ src/hydro/srcterms/tabular_cooling.cpp | 52 ++++++++++++++++++++------ src/outputs/per_block.cpp | 1 - src/pgen/turbulence.cpp | 13 ++++++- 4 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/hydro/hydro_driver.cpp b/src/hydro/hydro_driver.cpp index e33b1b0d..0db16b22 100644 --- a/src/hydro/hydro_driver.cpp +++ b/src/hydro/hydro_driver.cpp @@ -230,6 +230,11 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { #endif } + // Reset per cycle cooling logger + if (stage == 1 && hydro_pkg->AllParams().hasKey("cooling/total_deltaE_this_cycle")) { + hydro_pkg->UpdateParam("cooling/total_deltaE_this_cycle", 0.0); + } + // First add split sources before the main time integration if (stage == 1) { TaskRegion &strang_init_region = tc.AddRegion(num_partitions); diff --git a/src/hydro/srcterms/tabular_cooling.cpp b/src/hydro/srcterms/tabular_cooling.cpp index 9164f4cc..9176213c 100644 --- a/src/hydro/srcterms/tabular_cooling.cpp +++ b/src/hydro/srcterms/tabular_cooling.cpp @@ -22,6 +22,7 @@ // AthenaPK headers #include "../../units.hpp" #include "tabular_cooling.hpp" + #include "utils/error_checking.hpp" namespace cooling { @@ -264,6 +265,9 @@ TabularCooling::TabularCooling(ParameterInput *pin, cooling_table_obj_ = CoolingTableObj(log_lambdas_, log_temp_start_, log_temp_final_, d_log_temp_, n_temp_, mbar_over_kb, adiabatic_index, 1.0 - He_mass_fraction, units); + + // Add mutable param to keep track of total energy lost + hydro_pkg->AddParam("cooling/total_deltaE_this_cycle", 0.0, true); } void TabularCooling::SrcTerm(MeshData *md, const Real dt) const { @@ -313,12 +317,15 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - par_for( - DEFAULT_LOOP_PATTERN, "TabularCooling::SubcyclingSplitSrcTerm", DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + Real total_deltaE = NAN; // Total radiated energy (not a density) + + md->GetBlockData(0)->GetBlockPointer()->par_reduce( + "TabularCooling::SubcyclingSplitSrcTerm", 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, + jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, Real &ledot) { auto &cons = cons_pack(b); auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); // Need to use `cons` here as prim may still contain state at t_0; const Real rho = cons(IDN, k, j, i); // TODO(pgrete) with potentially more EOS, a separate get_pressure (or similar) @@ -470,11 +477,20 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt internal_e = (internal_e > internal_e_floor) ? internal_e : internal_e_floor; // Remove the cooling from the total energy density - cons(IEN, k, j, i) += rho * (internal_e - internal_e_initial); + const auto edotdensity = rho * (internal_e - internal_e_initial); + cons(IEN, k, j, i) += edotdensity; + ledot = edotdensity * coords.CellVolume(k, j, i); // Latter technically not required if no other tasks follows before // ConservedToPrim conversion, but keeping it for now (better safe than sorry). prim(IPR, k, j, i) = rho * internal_e * gm1; - }); + }, + Kokkos::Sum(total_deltaE)); + + // Updating current value as routine might be called multiple times (e.g. Strang split) + auto total_deltaE_this_cycle = + hydro_pkg->Param("cooling/total_deltaE_this_cycle"); + hydro_pkg->UpdateParam("cooling/total_deltaE_this_cycle", + total_deltaE_this_cycle + total_deltaE); } void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, @@ -514,12 +530,15 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, const auto temp_final = std::pow(10.0, log_temp_final_); const auto lambda_final = lambda_final_; - par_for( - DEFAULT_LOOP_PATTERN, "TabularCooling::TownsendSrcTerm", DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + Real total_deltaE = NAN; // Total radiated energy (not a density) + + md->GetBlockData(0)->GetBlockPointer()->par_reduce( + "TabularCooling::TownsendSrcTerm", 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, + jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, Real &ledot) { auto &cons = cons_pack(b); auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); // Need to use `cons` here as prim may still contain state at t_0; const auto rho = cons(IDN, k, j, i); // TODO(pgrete) with potentially more EOS, a separate get_pressure (or similar) @@ -587,11 +606,20 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, const auto internal_e_new = temp_new > temp_cool_floor ? temp_new / mbar_gm1_over_kb : temp_cool_floor / mbar_gm1_over_kb; - cons(IEN, k, j, i) += rho * (internal_e_new - internal_e); + const auto edotdensity = rho * (internal_e_new - internal_e); + cons(IEN, k, j, i) += edotdensity; + ledot = edotdensity * coords.CellVolume(k, j, i); // Latter technically not required if no other tasks follows before // ConservedToPrim conversion, but keeping it for now (better safe than sorry). prim(IPR, k, j, i) = rho * internal_e_new * gm1; - }); + }, + Kokkos::Sum(total_deltaE)); + + // Updating current value as routine might be called multiple times (e.g. Strang split) + auto total_deltaE_this_cycle = + hydro_pkg->Param("cooling/total_deltaE_this_cycle"); + hydro_pkg->UpdateParam("cooling/total_deltaE_this_cycle", + total_deltaE_this_cycle + total_deltaE); } Real TabularCooling::EstimateTimeStep(MeshData *md) const { diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 815d9566..b30bd6c7 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -410,7 +410,6 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, stats.emplace_back(Stats("vel_mag", "prim", {IV1, IV2, IV3})); stats.emplace_back(Stats("vel_mag_mw", "prim", {IV1, IV2, IV3}, Weight::Mass)); stats.emplace_back(Stats("rho", "prim", 0)); - stats.emplace_back(Stats("vx", "prim", 1)); const std::vector stat_types = { "min", "max", "absmin", "absmax", "mean", "rms", "stddev", "skew", "kurt"}; diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index ce97db87..560c6126 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -46,7 +46,7 @@ using utils::few_modes_ft::FewModesFT; // TODO(?) until we are able to process multiple variables in a single hst function call // we'll use this enum to identify the various vars. -enum class HstQuan { Ms, Ma, pb }; +enum class HstQuan { Ms, Ma, pb, DeltaEcool }; // Compute the local sum of either the sonic Mach number, // alfvenic Mach number, or plasma beta as specified by `hst_quan`. @@ -55,6 +55,14 @@ Real TurbulenceHst(MeshData *md) { auto pmb = md->GetBlockData(0)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); const auto gamma = hydro_pkg->Param("AdiabaticIndex"); + + if (hst_quan == HstQuan::DeltaEcool && + hydro_pkg->AllParams().hasKey("cooling/total_deltaE_this_cycle")) { + return hydro_pkg->Param("cooling/total_deltaE_this_cycle"); + } else { + return 0; + } + const auto fluid = hydro_pkg->Param("fluid"); const auto &prim_pack = md->PackVariables(std::vector{"prim"}); @@ -112,6 +120,9 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg hst_vars.emplace_back(parthenon::HistoryOutputVar(parthenon::UserHistoryOperation::sum, TurbulenceHst, "Ms")); + hst_vars.emplace_back(parthenon::HistoryOutputVar(parthenon::UserHistoryOperation::sum, + TurbulenceHst, + "DeltaEcool")); if (fluid == Fluid::glmmhd) { hst_vars.emplace_back(parthenon::HistoryOutputVar( parthenon::UserHistoryOperation::sum, TurbulenceHst, "Ma")); From 4525aaf7dd568673625237eb45ea56bd388ee789 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 22 Nov 2023 10:05:09 +0100 Subject: [PATCH 43/48] Add total weight to per-block output --- src/outputs/per_block.cpp | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index b30bd6c7..6d120608 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -377,9 +377,9 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, } // -------------------------------------------------------------------------------- // - // WRITING VARIABLES DATA // + // WRITING STATISTICS // // -------------------------------------------------------------------------------- // - Kokkos::Profiling::pushRegion("write all variable data"); + Kokkos::Profiling::pushRegion("write all stats data"); { PARTHENON_REQUIRE_THROWS( typeid(Coordinates_t) == typeid(UniformCartesian), @@ -409,10 +409,12 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, std::vector stats; stats.emplace_back(Stats("vel_mag", "prim", {IV1, IV2, IV3})); stats.emplace_back(Stats("vel_mag_mw", "prim", {IV1, IV2, IV3}, Weight::Mass)); - stats.emplace_back(Stats("rho", "prim", 0)); + stats.emplace_back(Stats("rho", "prim", IDN)); + stats.emplace_back(Stats("pressure", "prim", IPR)); const std::vector stat_types = { - "min", "max", "absmin", "absmax", "mean", "rms", "stddev", "skew", "kurt"}; + "min", "max", "absmin", "absmax", "mean", + "rms", "stddev", "skew", "kurt", "total_weight"}; const H5G gLocations = MakeGroup(file, "/stats"); std::vector stat_results(num_blocks_local * stat_types.size()); local_count[1] = global_count[1] = stat_types.size(); @@ -524,6 +526,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, stat_results[offset + 6] = stddev; stat_results[offset + 7] = mu3 / std::pow(stddev, 3.0); // skewness stat_results[offset + 8] = mu4 / std::pow(stddev, 4.0); // kurtosis + stat_results[offset + 9] = norm; // total_weight used for normalization b++; } @@ -531,13 +534,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, p_glob_cnt, pl_xfer); } } - Kokkos::Profiling::popRegion(); // write all variable data - - // names of variables - std::vector var_names; - var_names.reserve(5); - - HDF5WriteAttribute("OutputDatasetNames", var_names, info_group); + Kokkos::Profiling::popRegion(); // write all stats data Kokkos::Profiling::popRegion(); // WriteOutputFile } From 71004d5b6299f2c5aca0cfa80f84881e8f229edc Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 22 Nov 2023 10:24:31 +0100 Subject: [PATCH 44/48] Add vorticity mag to turb driver --- src/pgen/turbulence.cpp | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index 560c6126..7d804cc0 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -138,6 +138,9 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg "Using temperature fields requires units or mbar_over_kb."); pkg->AddField("temperature", m); } + if (pin->GetOrAddBoolean("problem/turbulence", "calc_vorticity_mag", false)) { + pkg->AddField("vorticity_mag", m); + } // Step 2. Add appropriate fields required by this pgen // Using OneCopy here to save memory. We typically don't need to update/evolve the @@ -800,6 +803,36 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { temperature(k, j, i) = mbar_over_kb * P / rho; }); } + if (pin->GetOrAddBoolean("problem/turbulence", "calc_vorticity_mag", false)) { + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; + auto &vorticity_mag = data->Get("vorticity_mag").data; + const auto &coords = pmb->coords; + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + // Loop bounds are adjusted below to take the derivative stencil into account. + // We chose to extend the calculation to the ghost zones (rather than the center + // only), because ghost cells are not exchanged again prior to output. + // So this allows, additional derived fields to use the vorticity magnitude in the + // ghost zones except for the outermost layer. + pmb->par_for( + "Turbulence::UserWorkBeforeOutput calc vorticity", kb.s + 1, kb.e - 1, jb.s + 1, + jb.e - 1, ib.s + 1, ib.e - 1, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + const auto vort_x = + (prim(IV3, k, j + 1, i) - prim(IV3, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (prim(IV2, k + 1, j, i) - prim(IV2, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + const auto vort_y = + (prim(IV1, k + 1, j, i) - prim(IV1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (prim(IV3, k, j, i + 1) - prim(IV3, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + const auto vort_z = + (prim(IV2, k, j, i + 1) - prim(IV2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (prim(IV1, k, j + 1, i) - prim(IV1, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + vorticity_mag(k, j, i) = Kokkos::sqrt(SQR(vort_x) + SQR(vort_y) + SQR(vort_z)); + }); + } } } // namespace turbulence From 07a8ee9805e206887f58572c0d889a76584d0dbf Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Wed, 22 Nov 2023 13:57:15 +0100 Subject: [PATCH 45/48] Add vort_mag to stats --- src/outputs/per_block.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 6d120608..636764ed 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -411,6 +411,10 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, stats.emplace_back(Stats("vel_mag_mw", "prim", {IV1, IV2, IV3}, Weight::Mass)); stats.emplace_back(Stats("rho", "prim", IDN)); stats.emplace_back(Stats("pressure", "prim", IPR)); + if (pin->GetOrAddBoolean("problem/turbulence", "calc_vorticity_mag", false)) { + stats.emplace_back(Stats("vort_mag", "vorticity_mag", 0)); + stats.emplace_back(Stats("vort_mag_mw", "vorticity_mag", 0), Weight::Mass); + } const std::vector stat_types = { "min", "max", "absmin", "absmax", "mean", From 7941b10208e8bba54a416140c4e0523d997a7247 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 16 Jan 2024 15:43:22 +0200 Subject: [PATCH 46/48] Bump parth --- external/Kokkos | 2 +- external/parthenon | 2 +- src/hydro/prolongation/custom_ops.hpp | 9 ++++++--- src/outputs/per_block.cpp | 2 +- src/pgen/cloud.cpp | 2 +- src/pgen/cluster.cpp | 6 +++--- src/pgen/cluster/agn_feedback.cpp | 8 ++++---- 7 files changed, 17 insertions(+), 14 deletions(-) diff --git a/external/Kokkos b/external/Kokkos index 62d2b6c8..1a3ea28f 160000 --- a/external/Kokkos +++ b/external/Kokkos @@ -1 +1 @@ -Subproject commit 62d2b6c879b74b6ae7bd06eb3e5e80139c4708e6 +Subproject commit 1a3ea28f6e97b4c9dd2c8ceed53ad58ed5f94dfe diff --git a/external/parthenon b/external/parthenon index f84dd6a1..4899b6c4 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit f84dd6a1bc08ad3144b29a78bf166f22c0d4437c +Subproject commit 4899b6c459b7f762bfa6f17ff8a762685466ece0 diff --git a/src/hydro/prolongation/custom_ops.hpp b/src/hydro/prolongation/custom_ops.hpp index 02d2b15f..c7a4eb6e 100644 --- a/src/hydro/prolongation/custom_ops.hpp +++ b/src/hydro/prolongation/custom_ops.hpp @@ -81,34 +81,37 @@ struct ProlongateCellMinModMultiD { Real dx1fm = 0; Real dx1fp = 0; + [[maybe_unused]] Real gx1m = 0, gx1p = 0; Real gx1c = 0; if constexpr (INCLUDE_X1) { Real dx1m, dx1p; GetGridSpacings<1, el>(coords, coarse_coords, cib, ib, i, fi, &dx1m, &dx1p, &dx1fm, &dx1fp); gx1c = GradMinMod(fc, coarse(element_idx, l, m, n, k, j, i - 1), - coarse(element_idx, l, m, n, k, j, i + 1), dx1m, dx1p); + coarse(element_idx, l, m, n, k, j, i + 1), dx1m, dx1p, gx1m, gx1p); } Real dx2fm = 0; Real dx2fp = 0; + [[maybe_unused]] Real gx2m = 0, gx2p = 0; Real gx2c = 0; if constexpr (INCLUDE_X2) { Real dx2m, dx2p; GetGridSpacings<2, el>(coords, coarse_coords, cjb, jb, j, fj, &dx2m, &dx2p, &dx2fm, &dx2fp); gx2c = GradMinMod(fc, coarse(element_idx, l, m, n, k, j - 1, i), - coarse(element_idx, l, m, n, k, j + 1, i), dx2m, dx2p); + coarse(element_idx, l, m, n, k, j + 1, i), dx2m, dx2p, gx2m, gx2p); } Real dx3fm = 0; Real dx3fp = 0; + [[maybe_unused]] Real gx3m = 0, gx3p = 0; Real gx3c = 0; if constexpr (INCLUDE_X3) { Real dx3m, dx3p; GetGridSpacings<3, el>(coords, coarse_coords, ckb, kb, k, fk, &dx3m, &dx3p, &dx3fm, &dx3fp); gx3c = GradMinMod(fc, coarse(element_idx, l, m, n, k - 1, j, i), - coarse(element_idx, l, m, n, k + 1, j, i), dx3m, dx3p); + coarse(element_idx, l, m, n, k + 1, j, i), dx3m, dx3p, gx3m, dx3p); } // Max. expected total difference. (dx#fm/p are positive by construction) diff --git a/src/outputs/per_block.cpp b/src/outputs/per_block.cpp index 636764ed..3955c600 100644 --- a/src/outputs/per_block.cpp +++ b/src/outputs/per_block.cpp @@ -413,7 +413,7 @@ void UserOutput::WriteOutputFile(Mesh *pm, ParameterInput *pin, SimTime *tm, stats.emplace_back(Stats("pressure", "prim", IPR)); if (pin->GetOrAddBoolean("problem/turbulence", "calc_vorticity_mag", false)) { stats.emplace_back(Stats("vort_mag", "vorticity_mag", 0)); - stats.emplace_back(Stats("vort_mag_mw", "vorticity_mag", 0), Weight::Mass); + stats.emplace_back(Stats("vort_mag_mw", "vorticity_mag", 0, Weight::Mass)); } const std::vector stat_types = { diff --git a/src/pgen/cloud.cpp b/src/pgen/cloud.cpp index bc9fed25..d8fd3939 100644 --- a/src/pgen/cloud.cpp +++ b/src/pgen/cloud.cpp @@ -213,7 +213,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { } void InflowWindX2(std::shared_ptr> &mbd, bool coarse) { - std::shared_ptr pmb = mbd->GetBlockPointer(); + auto* pmb = mbd->GetBlockPointer(); auto cons = mbd->PackVariables(std::vector{"cons"}, coarse); // TODO(pgrete) Add par_for_bndry to Parthenon without requiring nb const auto nb = IndexRange{0, 0}; diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 3a0cfb9a..291d51bc 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -567,7 +567,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { ************************************************************/ const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + magnetic_tower.AddInitialFieldToPotential(pmb, a_kb, a_jb, a_ib, A); /************************************************************ * Add dipole magnetic field to the magnetic potential @@ -670,7 +670,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { // Init phases on all blocks for (int b = 0; b < md->NumBlocks(); b++) { auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); + few_modes_ft.SetPhases(pmb, pin); } // As for t_corr in few_modes_ft, the choice for dt is // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain @@ -738,7 +738,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { // Init phases on all blocks for (int b = 0; b < md->NumBlocks(); b++) { auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); + few_modes_ft.SetPhases(pmb, pin); } // As for t_corr in few_modes_ft, the choice for dt is // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain diff --git a/src/pgen/cluster/agn_feedback.cpp b/src/pgen/cluster/agn_feedback.cpp index bcf63c24..30b09747 100644 --- a/src/pgen/cluster/agn_feedback.cpp +++ b/src/pgen/cluster/agn_feedback.cpp @@ -381,10 +381,10 @@ void AGNFeedback::FeedbackSrcTerm(parthenon::MeshData *md, eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); const Real new_specific_internal_e = prim(IPR, k, j, i) / (prim(IDN, k, j, i) * (eos.GetGamma() - 1.)); - PARTHENON_REQUIRE( - new_specific_internal_e > jet_specific_internal_e || - new_specific_internal_e > old_specific_internal_e, - "Kinetic injection leads to temperature below jet and existing gas"); + // PARTHENON_REQUIRE( + // new_specific_internal_e > jet_specific_internal_e || + // new_specific_internal_e > old_specific_internal_e, + // "Kinetic injection leads to temperature below jet and existing gas"); } // Apply velocity ceiling From 931ec8f6ce951e9be2ed6525bcde6303be09a0fb Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 16 Jan 2024 15:32:22 +0100 Subject: [PATCH 47/48] Add temp based recon --- src/recon/ppm_simple.hpp | 39 +++++++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/src/recon/ppm_simple.hpp b/src/recon/ppm_simple.hpp index 8792428e..dedde211 100644 --- a/src/recon/ppm_simple.hpp +++ b/src/recon/ppm_simple.hpp @@ -180,16 +180,43 @@ Reconstruct(parthenon::team_mbr_t const &member, const int k, const int j, const parthenon::par_for_inner(member, il, iu, [&](const int i) { if constexpr (XNDIR == parthenon::X1DIR) { // ql is ql_ip1 and qr is qr_i - PPM(q(n, k, j, i - 2), q(n, k, j, i - 1), q(n, k, j, i), q(n, k, j, i + 1), - q(n, k, j, i + 2), ql(n, i + 1), qr(n, i)); + if (n == 4) { + PPM(q(n, k, j, i - 2) / q(0, k, j, i - 2), + q(n, k, j, i - 1) / q(0, k, j, i - 1), q(n, k, j, i) / q(0, k, j, i), + q(n, k, j, i + 1) / q(0, k, j, i + 1), + q(n, k, j, i + 2) / q(0, k, j, i + 2), ql(n, i + 1), qr(n, i)); + ql(n, i + 1) *= q(0, k, j, i); + qr(n, i) *= q(0, k, j, i); + } else { + PPM(q(n, k, j, i - 2), q(n, k, j, i - 1), q(n, k, j, i), q(n, k, j, i + 1), + q(n, k, j, i + 2), ql(n, i + 1), qr(n, i)); + } } else if constexpr (XNDIR == parthenon::X2DIR) { // ql is ql_jp1 and qr is qr_j - PPM(q(n, k, j - 2, i), q(n, k, j - 1, i), q(n, k, j, i), q(n, k, j + 1, i), - q(n, k, j + 2, i), ql(n, i), qr(n, i)); + if (n == 4) { + PPM(q(n, k, j - 2, i) / q(0, k, j - 2, i), + q(n, k, j - 1, i) / q(0, k, j - 1, i), q(n, k, j, i) / q(0, k, j, i), + q(n, k, j + 1, i) / q(0, k, j + 1, i), + q(n, k, j + 2, i) / q(0, k, j + 2, i), ql(n, i), qr(n, i)); + ql(n, i) *= q(0, k, j, i); + qr(n, i) *= q(0, k, j, i); + } else { + PPM(q(n, k, j - 2, i), q(n, k, j - 1, i), q(n, k, j, i), q(n, k, j + 1, i), + q(n, k, j + 2, i), ql(n, i), qr(n, i)); + } } else if constexpr (XNDIR == parthenon::X3DIR) { // ql is ql_kp1 and qr is qr_k - PPM(q(n, k - 2, j, i), q(n, k - 1, j, i), q(n, k, j, i), q(n, k + 1, j, i), - q(n, k + 2, j, i), ql(n, i), qr(n, i)); + if (n == 4) { + PPM(q(n, k - 2, j, i) / q(0, k - 2, j, i), + q(n, k - 1, j, i) / q(0, k - 1, j, i), q(n, k, j, i) / q(0, k, j, i), + q(n, k + 1, j, i) / q(0, k + 1, j, i), + q(n, k + 2, j, i) / q(0, k + 2, j, i), ql(n, i), qr(n, i)); + ql(n, i) *= q(0, k, j, i); + qr(n, i) *= q(0, k, j, i); + } else { + PPM(q(n, k - 2, j, i), q(n, k - 1, j, i), q(n, k, j, i), q(n, k + 1, j, i), + q(n, k + 2, j, i), ql(n, i), qr(n, i)); + } } else { PARTHENON_FAIL("Unknow direction for PPM reconstruction.") } From 0f1c03a27f5183ef7f3f40185a7dca598ffe7e83 Mon Sep 17 00:00:00 2001 From: Philipp Grete Date: Tue, 16 Jan 2024 15:48:09 +0100 Subject: [PATCH 48/48] Fix formatting. Disable per-block output --- src/CMakeLists.txt | 2 +- src/hydro/prolongation/custom_ops.hpp | 15 +++++++++------ src/pgen/cloud.cpp | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 51a0e4b7..6da22e20 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -16,7 +16,7 @@ add_executable( hydro/srcterms/gravitational_field.hpp hydro/srcterms/tabular_cooling.hpp hydro/srcterms/tabular_cooling.cpp - outputs/per_block.cpp + #outputs/per_block.cpp refinement/gradient.cpp refinement/other.cpp utils/few_modes_ft.cpp diff --git a/src/hydro/prolongation/custom_ops.hpp b/src/hydro/prolongation/custom_ops.hpp index c7a4eb6e..4693e755 100644 --- a/src/hydro/prolongation/custom_ops.hpp +++ b/src/hydro/prolongation/custom_ops.hpp @@ -87,8 +87,9 @@ struct ProlongateCellMinModMultiD { Real dx1m, dx1p; GetGridSpacings<1, el>(coords, coarse_coords, cib, ib, i, fi, &dx1m, &dx1p, &dx1fm, &dx1fp); - gx1c = GradMinMod(fc, coarse(element_idx, l, m, n, k, j, i - 1), - coarse(element_idx, l, m, n, k, j, i + 1), dx1m, dx1p, gx1m, gx1p); + gx1c = + GradMinMod(fc, coarse(element_idx, l, m, n, k, j, i - 1), + coarse(element_idx, l, m, n, k, j, i + 1), dx1m, dx1p, gx1m, gx1p); } Real dx2fm = 0; @@ -99,8 +100,9 @@ struct ProlongateCellMinModMultiD { Real dx2m, dx2p; GetGridSpacings<2, el>(coords, coarse_coords, cjb, jb, j, fj, &dx2m, &dx2p, &dx2fm, &dx2fp); - gx2c = GradMinMod(fc, coarse(element_idx, l, m, n, k, j - 1, i), - coarse(element_idx, l, m, n, k, j + 1, i), dx2m, dx2p, gx2m, gx2p); + gx2c = + GradMinMod(fc, coarse(element_idx, l, m, n, k, j - 1, i), + coarse(element_idx, l, m, n, k, j + 1, i), dx2m, dx2p, gx2m, gx2p); } Real dx3fm = 0; Real dx3fp = 0; @@ -110,8 +112,9 @@ struct ProlongateCellMinModMultiD { Real dx3m, dx3p; GetGridSpacings<3, el>(coords, coarse_coords, ckb, kb, k, fk, &dx3m, &dx3p, &dx3fm, &dx3fp); - gx3c = GradMinMod(fc, coarse(element_idx, l, m, n, k - 1, j, i), - coarse(element_idx, l, m, n, k + 1, j, i), dx3m, dx3p, gx3m, dx3p); + gx3c = + GradMinMod(fc, coarse(element_idx, l, m, n, k - 1, j, i), + coarse(element_idx, l, m, n, k + 1, j, i), dx3m, dx3p, gx3m, dx3p); } // Max. expected total difference. (dx#fm/p are positive by construction) diff --git a/src/pgen/cloud.cpp b/src/pgen/cloud.cpp index d8fd3939..15a43cec 100644 --- a/src/pgen/cloud.cpp +++ b/src/pgen/cloud.cpp @@ -213,7 +213,7 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { } void InflowWindX2(std::shared_ptr> &mbd, bool coarse) { - auto* pmb = mbd->GetBlockPointer(); + auto *pmb = mbd->GetBlockPointer(); auto cons = mbd->PackVariables(std::vector{"cons"}, coarse); // TODO(pgrete) Add par_for_bndry to Parthenon without requiring nb const auto nb = IndexRange{0, 0};