From b9564147dcf88031459ea7da152f4e8ef4a0e3eb Mon Sep 17 00:00:00 2001 From: HPC user Date: Wed, 13 Dec 2023 23:55:41 +0100 Subject: [PATCH] Formating --- src/CMakeLists.txt | 1 - src/eos/adiabatic_glmmhd.cpp | 7 +- src/eos/adiabatic_glmmhd.hpp | 63 +- src/eos/adiabatic_hydro.cpp | 13 +- src/eos/adiabatic_hydro.hpp | 26 +- src/hydro/srcterms/gravitational_field.hpp | 18 +- .../srcterms/old/tabular_cooling_edit.cpp | 743 --------- .../srcterms/old/tabular_cooling_edit.hpp | 314 ---- src/hydro/srcterms/tabular_cooling.cpp | 82 +- src/hydro/srcterms/tabular_cooling.hpp | 35 +- .../srcterms/tabular_cooling_working.cpp | 733 --------- src/pgen/cluster/cluster_gravity_original.hpp | 240 --- src/pgen/cluster/cluster_gravity_working.hpp | 266 --- .../cluster/entropy_isothermal_profiles.hpp | 42 - src/pgen/old_cluster/cluster_16nov.cpp | 1415 ---------------- src/pgen/old_cluster/cluster_17nov.cpp | 1427 ----------------- src/pgen/old_cluster/cluster_22nov.cpp | 1324 --------------- src/pgen/old_cluster/cluster_divartifact.cpp | 1324 --------------- src/pgen/old_cluster/cluster_modified.cpp | 1020 ------------ src/pgen/old_cluster/cluster_temp.cpp | 1427 ----------------- src/pgen/old_cluster/cluster_working.cpp | 1127 ------------- src/refinement/other.cpp | 18 +- src/utils/few_modes_ft_lognormal.cpp | 446 ------ src/utils/few_modes_ft_lognormal.hpp | 72 - 24 files changed, 115 insertions(+), 12068 deletions(-) delete mode 100644 src/hydro/srcterms/old/tabular_cooling_edit.cpp delete mode 100644 src/hydro/srcterms/old/tabular_cooling_edit.hpp delete mode 100644 src/hydro/srcterms/tabular_cooling_working.cpp delete mode 100644 src/pgen/cluster/cluster_gravity_original.hpp delete mode 100644 src/pgen/cluster/cluster_gravity_working.hpp delete mode 100644 src/pgen/cluster/entropy_isothermal_profiles.hpp delete mode 100644 src/pgen/old_cluster/cluster_16nov.cpp delete mode 100644 src/pgen/old_cluster/cluster_17nov.cpp delete mode 100644 src/pgen/old_cluster/cluster_22nov.cpp delete mode 100644 src/pgen/old_cluster/cluster_divartifact.cpp delete mode 100644 src/pgen/old_cluster/cluster_modified.cpp delete mode 100644 src/pgen/old_cluster/cluster_temp.cpp delete mode 100644 src/pgen/old_cluster/cluster_working.cpp delete mode 100644 src/utils/few_modes_ft_lognormal.cpp delete mode 100644 src/utils/few_modes_ft_lognormal.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 386aff0d..5d8fd375 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -19,7 +19,6 @@ add_executable( refinement/gradient.cpp refinement/other.cpp utils/few_modes_ft.cpp - utils/few_modes_ft_lognormal.cpp ) add_subdirectory(pgen) diff --git a/src/eos/adiabatic_glmmhd.cpp b/src/eos/adiabatic_glmmhd.cpp index 28c0984a..a2e843da 100644 --- a/src/eos/adiabatic_glmmhd.cpp +++ b/src/eos/adiabatic_glmmhd.cpp @@ -41,7 +41,6 @@ void AdiabaticGLMMHDEOS::ConservedToPrimitive(MeshData *md) const { const auto nhydro = pkg->Param("nhydro"); const auto nscalars = pkg->Param("nscalars"); - auto this_on_device = (*this); parthenon::par_for( @@ -51,15 +50,13 @@ void AdiabaticGLMMHDEOS::ConservedToPrimitive(MeshData *md) const { const auto &cons = cons_pack(b); auto &prim = prim_pack(b); // auto &nu = entropy_pack(b); - - + // Getting the global indexing - + auto pmb = md->GetBlockData(b)->GetBlockPointer(); auto pm = pmb->pmy_mesh; auto hydro_pkg = pmb->packages.Get("Hydro"); - return this_on_device.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); }); } diff --git a/src/eos/adiabatic_glmmhd.hpp b/src/eos/adiabatic_glmmhd.hpp index ae073d97..93e48c6a 100644 --- a/src/eos/adiabatic_glmmhd.hpp +++ b/src/eos/adiabatic_glmmhd.hpp @@ -30,10 +30,10 @@ class AdiabaticGLMMHDEOS : public EquationOfState { gamma_{gamma} {} void ConservedToPrimitive(MeshData *md) const override; - + KOKKOS_INLINE_FUNCTION Real GetGamma() const { return gamma_; } - + //---------------------------------------------------------------------------------------- // \!fn Real EquationOfState::SoundSpeed(Real prim[NHYDRO]) // \brief returns adiabatic sound speed given vector of primitive variables @@ -53,7 +53,7 @@ class AdiabaticGLMMHDEOS : public EquationOfState { return std::sqrt(0.5 * (qsq + std::sqrt(tmp * tmp + 4.0 * asq * ct2)) / d); } // - + //---------------------------------------------------------------------------------------- // \!fn Real EquationOfState::ConsToPrim(View4D cons, View4D prim, const int& k, const // int& j, const int& i) \brief Fills an array of primitives given an array of @@ -70,51 +70,51 @@ class AdiabaticGLMMHDEOS : public EquationOfState { auto velocity_ceiling_ = GetVelocityCeiling(); auto e_ceiling_ = GetInternalECeiling(); - - Real &u_d = cons(IDN, k, j, i); - Real &u_m1 = cons(IM1, k, j, i); - Real &u_m2 = cons(IM2, k, j, i); - Real &u_m3 = cons(IM3, k, j, i); - Real &u_e = cons(IEN, k, j, i); - Real &u_b1 = cons(IB1, k, j, i); - Real &u_b2 = cons(IB2, k, j, i); - Real &u_b3 = cons(IB3, k, j, i); + + Real &u_d = cons(IDN, k, j, i); + Real &u_m1 = cons(IM1, k, j, i); + Real &u_m2 = cons(IM2, k, j, i); + Real &u_m3 = cons(IM3, k, j, i); + Real &u_e = cons(IEN, k, j, i); + Real &u_b1 = cons(IB1, k, j, i); + Real &u_b2 = cons(IB2, k, j, i); + Real &u_b3 = cons(IB3, k, j, i); Real &u_psi = cons(IPS, k, j, i); - - Real &w_d = prim(IDN, k, j, i); - Real &w_vx = prim(IV1, k, j, i); - Real &w_vy = prim(IV2, k, j, i); - Real &w_vz = prim(IV3, k, j, i); - Real &w_p = prim(IPR, k, j, i); - Real &w_Bx = prim(IB1, k, j, i); - Real &w_By = prim(IB2, k, j, i); - Real &w_Bz = prim(IB3, k, j, i); + + Real &w_d = prim(IDN, k, j, i); + Real &w_vx = prim(IV1, k, j, i); + Real &w_vy = prim(IV2, k, j, i); + Real &w_vz = prim(IV3, k, j, i); + Real &w_p = prim(IPR, k, j, i); + Real &w_Bx = prim(IB1, k, j, i); + Real &w_By = prim(IB2, k, j, i); + Real &w_Bz = prim(IB3, k, j, i); Real &w_psi = prim(IPS, k, j, i); - + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative density is encountered. - //PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, + // PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, // "Got negative density. Consider enabling first-order flux " // "correction or setting a reasonable density floor."); - + // apply density floor, without changing momentum or energy u_d = (u_d > density_floor_) ? u_d : density_floor_; w_d = u_d; - + Real di = 1.0 / u_d; w_vx = u_m1 * di; w_vy = u_m2 * di; w_vz = u_m3 * di; - + w_Bx = u_b1; w_By = u_b2; w_Bz = u_b3; w_psi = u_psi; - + Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); Real e_B = 0.5 * (SQR(u_b1) + SQR(u_b2) + SQR(u_b3)); w_p = gm1 * (u_e - e_k - e_B); - + // apply velocity ceiling. By default ceiling is std::numeric_limits::infinity() const Real w_v2 = SQR(w_vx) + SQR(w_vy) + SQR(w_vz); if (w_v2 > SQR(velocity_ceiling_)) { @@ -131,12 +131,13 @@ class AdiabaticGLMMHDEOS : public EquationOfState { u_e -= e_k - e_k_new; e_k = e_k_new; } - + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative pressure is encountered. - //PARTHENON_REQUIRE(w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, + // PARTHENON_REQUIRE(w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, // "Got negative pressure. Consider enabling first-order flux " - // "correction or setting a reasonable pressure or temperature floor."); + // "correction or setting a reasonable pressure or temperature + // floor."); // Pressure floor (if present) takes precedence over temperature floor if ((pressure_floor_ > 0.0) && (w_p < pressure_floor_)) { diff --git a/src/eos/adiabatic_hydro.cpp b/src/eos/adiabatic_hydro.cpp index 651000e4..fa0af7a7 100644 --- a/src/eos/adiabatic_hydro.cpp +++ b/src/eos/adiabatic_hydro.cpp @@ -37,27 +37,26 @@ void AdiabaticHydroEOS::ConservedToPrimitive(MeshData *md) const { auto ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); auto jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); auto kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - + auto pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const auto nhydro = pkg->Param("nhydro"); const auto nscalars = pkg->Param("nscalars"); - + auto this_on_device = (*this); - + parthenon::par_for( DEFAULT_LOOP_PATTERN, "ConservedToPrimitive", parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - // Getting the global indexing - + auto pmb = md->GetBlockData(b)->GetBlockPointer(); auto pm = pmb->pmy_mesh; auto hydro_pkg = pmb->packages.Get("Hydro"); - + const auto &cons = cons_pack(b); auto &prim = prim_pack(b); - + return this_on_device.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); }); } diff --git a/src/eos/adiabatic_hydro.hpp b/src/eos/adiabatic_hydro.hpp index b42196a4..f4d714f4 100644 --- a/src/eos/adiabatic_hydro.hpp +++ b/src/eos/adiabatic_hydro.hpp @@ -52,27 +52,27 @@ class AdiabaticHydroEOS : public EquationOfState { KOKKOS_INLINE_FUNCTION void ConsToPrim(View4D cons, View4D prim, const int &nhydro, const int &nscalars, const int &k, const int &j, const int &i) const { - + Real gm1 = GetGamma() - 1.0; auto density_floor_ = GetDensityFloor(); auto pressure_floor_ = GetPressureFloor(); auto e_floor_ = GetInternalEFloor(); - + auto velocity_ceiling_ = GetVelocityCeiling(); auto e_ceiling_ = GetInternalECeiling(); - - Real &u_d = cons(IDN, k, j, i); + + Real &u_d = cons(IDN, k, j, i); Real &u_m1 = cons(IM1, k, j, i); Real &u_m2 = cons(IM2, k, j, i); Real &u_m3 = cons(IM3, k, j, i); - Real &u_e = cons(IEN, k, j, i); - - Real &w_d = prim(IDN, k, j, i); + Real &u_e = cons(IEN, k, j, i); + + Real &w_d = prim(IDN, k, j, i); Real &w_vx = prim(IV1, k, j, i); Real &w_vy = prim(IV2, k, j, i); Real &w_vz = prim(IV3, k, j, i); - Real &w_p = prim(IPR, k, j, i); - + Real &w_p = prim(IPR, k, j, i); + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative density is encountered. PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, @@ -86,10 +86,10 @@ class AdiabaticHydroEOS : public EquationOfState { w_vx = u_m1 * di; w_vy = u_m2 * di; w_vz = u_m3 * di; - + Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); w_p = gm1 * (u_e - e_k); - + // apply velocity ceiling. By default ceiling is std::numeric_limits::infinity() const Real w_v2 = SQR(w_vx) + SQR(w_vy) + SQR(w_vz); if (w_v2 > SQR(velocity_ceiling_)) { @@ -106,10 +106,10 @@ class AdiabaticHydroEOS : public EquationOfState { u_e -= e_k - e_k_new; e_k = e_k_new; } - + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative pressure is encountered. - + // The first argument check whether one of the conditions is true // By default, floors are deactivated, ie. pressure floor and e_floor // are -1. The code will eventually crash when w_p turns < 0. diff --git a/src/hydro/srcterms/gravitational_field.hpp b/src/hydro/srcterms/gravitational_field.hpp index fd3bb3ca..e0ae98af 100644 --- a/src/hydro/srcterms/gravitational_field.hpp +++ b/src/hydro/srcterms/gravitational_field.hpp @@ -36,18 +36,22 @@ void GravitationalFieldSrcTerm(parthenon::MeshData *md, IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - + /* std::cout << "Values of gravitationnal field" << std::endl; - std::cout + std::cout << gravitationalField.g_from_r(1e-4) << ", " << gravitationalField.g_from_r(3.16e-4) - << ", " << gravitationalField.g_from_r(1e-3) << ", " << gravitationalField.g_from_r(3.16e-3) - << ", " << gravitationalField.g_from_r(1e-2) << ", " << gravitationalField.g_from_r(3.16e-2) - << ", " << gravitationalField.g_from_r(1e-1) << ", " << gravitationalField.g_from_r(3.16e-1) - << ", " << gravitationalField.g_from_r(1.0) << ", " << gravitationalField.g_from_r(3.16) + << ", " << gravitationalField.g_from_r(1e-3) << ", " << + gravitationalField.g_from_r(3.16e-3) + << ", " << gravitationalField.g_from_r(1e-2) << ", " << + gravitationalField.g_from_r(3.16e-2) + << ", " << gravitationalField.g_from_r(1e-1) << ", " << + gravitationalField.g_from_r(3.16e-1) + << ", " << gravitationalField.g_from_r(1.0) << ", " << + gravitationalField.g_from_r(3.16) << ", " << gravitationalField.g_from_r(10.0) << std::endl; */ - + parthenon::par_for( DEFAULT_LOOP_PATTERN, "GravitationalFieldSrcTerm", parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, diff --git a/src/hydro/srcterms/old/tabular_cooling_edit.cpp b/src/hydro/srcterms/old/tabular_cooling_edit.cpp deleted file mode 100644 index b8d7a0fd..00000000 --- a/src/hydro/srcterms/old/tabular_cooling_edit.cpp +++ /dev/null @@ -1,743 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file tabular_cooling.cpp -// \brief Applies tabular cooling -// -//======================================================================================== - -// C++ headers -#include -#include -#include - -// Parthenon headers -#include -#include -#include -#include - -// AthenaPK headers -#include "../../units.hpp" -#include "tabular_cooling.hpp" -#include "utils/error_checking.hpp" - -namespace cooling { -using namespace parthenon; - -TabularCooling::TabularCooling(ParameterInput *pin, - std::shared_ptr hydro_pkg) { - auto units = hydro_pkg->Param("units"); - - const std::string table_filename = pin->GetString("cooling", "table_filename"); - - const int log_temp_col = pin->GetOrAddInteger("cooling", "log_temp_col", 0); - const int log_lambda_col = pin->GetOrAddInteger("cooling", "log_lambda_col", 1); - - // Heating function as defined in Wladimir et al. 2021, ie. insure thermal eq. at n=0.1cm-3 - // and T = 1e4. By default, set to zero (heating deactivated) - // Provided in cgs units, ie. erg / s, then rescaled to code units - const Real heating_const_cgs = pin->GetOrAddReal("cooling", "heating_const", 0.0); - const Real gamma_units = heating_const_cgs * (units.erg() / units.s()); - - // Just for verification, check what units.s() is - //std::cout << "units.s()=" << units.s(); - - // Convert erg cm^3/s to code units - const Real lambda_units_cgs = pin->GetReal("cooling", "lambda_units_cgs"); - const Real lambda_units = - lambda_units_cgs / (units.erg() * pow(units.cm(), 3) / units.s()); - - // Heating parameters - const Real - - std::cout << "Entering lambda cout loop\n" - - const auto integrator_str = pin->GetOrAddString("cooling", "integrator", "rk12"); - if (integrator_str == "rk12") { - //std::cout << "Cooling integrator is rk12"; - integrator_ = CoolIntegrator::rk12; - } else if (integrator_str == "rk45") - //std::cout << "Cooling integrator is rk45"; - integrator_ = CoolIntegrator::rk45; - } else if (integrator_str == "townsend") { - //std::cout << "Cooling integrator is Townsend"; - integrator_ = CoolIntegrator::townsend; - } else { - integrator_ = CoolIntegrator::undefined; - } - - max_iter_ = pin->GetOrAddInteger("cooling", "max_iter", 100); - cooling_time_cfl_ = pin->GetOrAddReal("cooling", "cfl", 0.1); - d_log_temp_tol_ = pin->GetOrAddReal("cooling", "d_log_temp_tol", 1e-8); - d_e_tol_ = pin->GetOrAddReal("cooling", "d_e_tol", 1e-8); - // negative means disabled - T_floor_ = pin->GetOrAddReal("hydro", "Tfloor", -1.0); - - std::stringstream msg; - - /**************************************** - * Read tab file with IOWrapper - ****************************************/ - IOWrapper input; - input.Open(table_filename.c_str(), IOWrapper::FileMode::read); - - /**************************************** - * Read tab file from IOWrapper into a stringstream tab - ****************************************/ - std::stringstream tab_ss; - const int bufsize = 4096; - char *buf = new char[bufsize]; - std::ptrdiff_t ret; - parthenon::IOWrapperSizeT word_size = sizeof(char); - - do { - if (Globals::my_rank == 0) { // only the master process reads the cooling table - ret = input.Read(buf, word_size, bufsize); - } -#ifdef MPI_PARALLEL - // then broadcasts it - // no need for fence as cooling table is independent of execution/memory space - MPI_Bcast(&ret, sizeof(std::ptrdiff_t), MPI_BYTE, 0, MPI_COMM_WORLD); - MPI_Bcast(buf, ret, MPI_BYTE, 0, MPI_COMM_WORLD); -#endif - tab_ss.write(buf, ret); // add the buffer into the stream - } while (ret == bufsize); // till EOF (or par_end is found) - - delete[] buf; - input.Close(); - - /**************************************** - * Determine log_temps and and log_lambdas vectors - ****************************************/ - std::vector log_temps, log_lambdas; - std::string line; - std::size_t first_char; - while (tab_ss.good()) { - getline(tab_ss, line); - if (line.empty()) continue; // skip blank line - first_char = line.find_first_not_of(" "); // skip white space - if (first_char == std::string::npos) continue; // line is all white space - if (line.compare(first_char, 1, "#") == 0) continue; // skip comments - - // Parse the numbers on the line - std::istringstream iss(line); - std::vector line_data{std::istream_iterator{iss}, - std::istream_iterator{}}; - // Check size - if (line_data.empty() || line_data.size() <= std::max(log_temp_col, log_lambda_col)) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "Index " << std::max(log_temp_col, log_lambda_col) << " out of range on \"" - << line << "\"" << std::endl; - PARTHENON_FAIL(msg); - } - - try { - const Real log_temp = std::stod(line_data[log_temp_col]); - const Real log_lambda = std::stod(line_data[log_lambda_col]); - - // Add to growing list - log_temps.push_back(log_temp); - log_lambdas.push_back(log_lambda - std::log10(lambda_units)); - - } catch (const std::invalid_argument &ia) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "Number: \"" << ia.what() << "\" could not be parsed as double" << std::endl; - PARTHENON_FAIL(msg); - } - } - - /**************************************** - * Check some assumtions about the cooling table - ****************************************/ - - // Reconstructing the cooling function in new coordinate system - std::cout << "Entering lambda cout loop\n" - for (int i = 0; i < log_temps.size(); i++) { - std::cout << "log_temps" << log_temps[i] << ", log_lambdas" << log_lambdas[i] << "\n"; - } - - // Ensure at least two data points in the table to interpolate from - if (log_temps.size() < 2 || log_lambdas.size() < 2) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "Not enough data to interpolate cooling" << std::endl; - PARTHENON_FAIL(msg); - } - - // Ensure that the first log_temp is increasing - const Real log_temp_start = log_temps[0]; - const Real d_log_temp = log_temps[1] - log_temp_start; - - if (d_log_temp <= 0) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "second log_temp in table is descreasing" << std::endl; - PARTHENON_FAIL(msg); - } - - // Ensure that log_temps is evenly spaced - for (size_t i = 1; i < log_temps.size(); i++) { - const Real d_log_temp_i = log_temps[i] - log_temps[i - 1]; - - if (d_log_temp_i < 0) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "log_temp in table is descreasing at i= " << i - << " log_temp= " << log_temps[i] << std::endl; - PARTHENON_FAIL(msg); - } - - // The Dedt() function currently relies on an equally spaced cooling table for faster - // lookup (direct indexing). It is used in the subcycling method and in order to - // restrict `dt` by a cooling cfl. - if (((integrator_ != CoolIntegrator::townsend) || (cooling_time_cfl_ > 0.0)) && - (fabs(d_log_temp_i - d_log_temp) / d_log_temp > d_log_temp_tol_)) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "d_log_temp in table is uneven at i=" << i << " log_temp=" << log_temps[i] - << " d_log_temp= " << d_log_temp << " d_log_temp_i= " << d_log_temp_i - << " diff= " << d_log_temp_i - d_log_temp - << " rel_diff= " << fabs(d_log_temp_i - d_log_temp) / d_log_temp - << " tol= " << d_log_temp_tol_ << std::endl; - PARTHENON_FAIL(msg); - } - } - - /**************************************** - * Move values read into the data table - ****************************************/ - - n_temp_ = log_temps.size(); - log_temp_start_ = log_temps[0]; - log_temp_final_ = log_temps[n_temp_ - 1]; - d_log_temp_ = d_log_temp; - lambda_final_ = std::pow(10.0, log_lambdas[n_temp_ - 1]); - - // Setup log_lambdas_ used in Dedt() - { - // log_lambdas is used if the integrator isn't Townsend, if the cooling CFL - // is set, or if cooling time is a extra derived field. Since we don't have - // a good way to check the last condition we always initialize log_lambdas_ - log_lambdas_ = ParArray1D("log_lambdas_", n_temp_); - - // Read log_lambdas in host_log_lambdas, changing to code units along the way - auto host_log_lambdas = Kokkos::create_mirror_view(log_lambdas_); - for (unsigned int i = 0; i < n_temp_; i++) { - host_log_lambdas(i) = log_lambdas[i]; - } - // Copy host_log_lambdas into device memory - Kokkos::deep_copy(log_lambdas_, host_log_lambdas); - } - // Setup Townsend cooling, i.e., precalulcate piecewise powerlaw approx. - if (integrator_ == CoolIntegrator::townsend) { - lambdas_ = ParArray1D("lambdas_", n_temp_); - temps_ = ParArray1D("temps_", n_temp_); - - // Read log_lambdas in host_lambdas, changing to code units along the way - auto host_lambdas = Kokkos::create_mirror_view(lambdas_); - auto host_temps = Kokkos::create_mirror_view(temps_); - for (unsigned int i = 0; i < n_temp_; i++) { - host_lambdas(i) = std::pow(10.0, log_lambdas[i]); - host_temps(i) = std::pow(10.0, log_temps[i]); - } - // Copy host_lambdas into device memory - Kokkos::deep_copy(lambdas_, host_lambdas); - Kokkos::deep_copy(temps_, host_temps); - - // Coeffs are for intervals, i.e., only n_temp_ - 1 entries - const auto n_bins = n_temp_ - 1; - townsend_Y_k_ = ParArray1D("townsend_Y_k_", n_bins); - townsend_alpha_k_ = ParArray1D("townsend_alpha_k_", n_bins); - - // Initialize on host (make *this* recursion simpler) - auto host_townsend_Y_k = Kokkos::create_mirror_view(townsend_Y_k_); - auto host_townsend_alpha_k = Kokkos::create_mirror_view(townsend_alpha_k_); - - // Initialize piecewise power law indices - for (unsigned int i = 0; i < n_bins; i++) { - // Could use log_lambdas_ here, but using lambdas_ instead as they've already been - // converted to code units. - host_townsend_alpha_k(i) = - (std::log10(host_lambdas(i + 1)) - std::log10(host_lambdas(i))) / - (log_temps[i + 1] - log_temps[i]); - PARTHENON_REQUIRE(host_townsend_alpha_k(i) != 1.0, - "Need to implement special case for Townsend piecewise fits."); - } - - // Calculate TEF (temporal evolution functions Y_k recursively), (Eq. A6) - host_townsend_Y_k(n_bins - 1) = 0.0; // Last Y_N = Y(T_ref) = 0 - - for (int i = n_bins - 2; i >= 0; i--) { - const auto alpha_k_m1 = host_townsend_alpha_k(i) - 1.0; - const auto step = (host_lambdas(n_bins) / host_lambdas(i)) * - (host_temps(i) / host_temps(n_bins)) * - (std::pow(host_temps(i) / host_temps(i + 1), alpha_k_m1) - 1.0) / - alpha_k_m1; - - host_townsend_Y_k(i) = host_townsend_Y_k(i + 1) - step; - } - - Kokkos::deep_copy(townsend_alpha_k_, host_townsend_alpha_k); - Kokkos::deep_copy(townsend_Y_k_, host_townsend_Y_k); - } - - // Create a lightweight object for computing cooling rates within kernels - const auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - const auto adiabatic_index = hydro_pkg->Param("AdiabaticIndex"); - const auto He_mass_fraction = hydro_pkg->Param("He_mass_fraction"); - - cooling_table_obj_ = CoolingTableObj(log_lambdas_, log_temp_start_, log_temp_final_, - d_log_temp_, n_temp_, mbar_over_kb, - adiabatic_index, 1.0 - He_mass_fraction, units); -} - -void TabularCooling::SrcTerm(MeshData *md, const Real dt) const { - if (integrator_ == CoolIntegrator::rk12) { - SubcyclingFixedIntSrcTerm(md, dt, RK12Stepper()); - } else if (integrator_ == CoolIntegrator::rk45) { - SubcyclingFixedIntSrcTerm(md, dt, RK45Stepper()); - } else if (integrator_ == CoolIntegrator::townsend) { - TownsendSrcTerm(md, dt); - } else { - PARTHENON_FAIL("Unknown cooling integrator."); - } -} - -template -void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt_, - const RKStepper rk_stepper) const { - - const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - const bool mhd_enabled = hydro_pkg->Param("fluid") == Fluid::glmmhd; - // Grab member variables for compiler - - const CoolingTableObj cooling_table_obj = cooling_table_obj_; - const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - - const unsigned int max_iter = max_iter_; - - const Real min_sub_dt = dt / max_iter; - - const Real d_e_tol = d_e_tol_; - - // Determine the cooling floor, whichever is higher of the cooling table floor - // or fluid solver floor - const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - - const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. - - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - // need to include ghost zones as this source is called prior to the other fluxes when - // split - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - - par_for( - DEFAULT_LOOP_PATTERN, "TabularCooling::SubcyclingSplitSrcTerm", DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { - auto &cons = cons_pack(b); - auto &prim = prim_pack(b); - // Need to use `cons` here as prim may still contain state at t_0; - const Real rho = cons(IDN, k, j, i); - // TODO(pgrete) with potentially more EOS, a separate get_pressure (or similar) - // function could be useful. - Real internal_e = - cons(IEN, k, j, i) - 0.5 * - (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + - SQR(cons(IM3, k, j, i))) / - rho; - if (mhd_enabled) { - internal_e -= 0.5 * (SQR(cons(IB1, k, j, i)) + SQR(cons(IB2, k, j, i)) + - SQR(cons(IB3, k, j, i))); - } - internal_e /= rho; - const Real internal_e_initial = internal_e; - - bool dedt_valid = true; - - // Wrap DeDt into a functor for the RKStepper - auto DeDt_wrapper = [&](const Real t, const Real e, bool &valid) { - return cooling_table_obj.DeDt(e, rho, valid); - }; - - Real sub_t = 0; // current subcycle time - // Try full dt. If error is too large adaptive timestepping will reduce sub_dt - Real sub_dt = dt; - - // Check if cooling is actually happening, e.g., when T below T_cool_min or if - // temperature is already below floor. - const Real dedt_initial = DeDt_wrapper(0.0, internal_e_initial, dedt_valid); - if (dedt_initial == 0.0 || internal_e_initial <= internal_e_floor) { - return; // Function end here if no cooling needed, ie. if initially edot = 0 - } - - // Use minumum subcycle timestep when d_e_tol == 0 - if (d_e_tol == 0) { - sub_dt = min_sub_dt; - } - - unsigned int sub_iter = 0; - // check for dedt != 0.0 required in case cooling floor it hit during subcycling - while ((sub_t * (1 + KEpsilon_) < dt) && - (DeDt_wrapper(sub_t, internal_e, dedt_valid) != 0.0)) { - - if (sub_iter > max_iter) { - // Due to sub_dt >= min_dt, this error should never happen - PARTHENON_FAIL( - "FATAL ERROR in [TabularCooling::SubcyclingFixedIntSrcTerm]: Sub " - "cycles exceed max_iter (This should be impossible)"); - } - - // Next higher order estimate - Real internal_e_next_h; - // Error in estimate of higher order - Real d_e_err; - // Number of attempts on this subcycle - unsigned int sub_attempt = 0; - // Whether to reattempt this subcycle - bool reattempt_sub = true; - do { - // Next lower order estimate - Real internal_e_next_l; - // Do one dual order RK step - dedt_valid = true; - RKStepper::Step(sub_t, sub_dt, internal_e, DeDt_wrapper, internal_e_next_h, - internal_e_next_l, dedt_valid); - - sub_attempt++; - - if (!dedt_valid) { - if (sub_dt == min_sub_dt) { - // Cooling is so fast that even the minimum subcycle dt would lead to - // negative internal energy -- so just cool to the floor of the cooling - // table - sub_dt = (dt - sub_t); - internal_e_next_h = internal_e_floor; - reattempt_sub = false; - } else { - reattempt_sub = true; - sub_dt = min_sub_dt; - } - } else { - - // Compute error - d_e_err = fabs((internal_e_next_h - internal_e_next_l) / internal_e_next_h); - - reattempt_sub = false; - // Accepting or reattempting the subcycle: - // - // -If the error is small, accept the subcycle - // - // -If the error on the subcycle is too high, compute a new time - // step to reattempt the subcycle - // -But if the new time step is smaller than the minimum subcycle - // time step (total step duration/ max iterations), just use the - // minimum subcycle time step instead - - if (std::isnan(d_e_err)) { - reattempt_sub = true; - sub_dt = min_sub_dt; - } else if (d_e_err >= d_e_tol && sub_dt > min_sub_dt) { - // Reattempt this subcycle - reattempt_sub = true; - // Error was too high, shrink the timestep - if (d_e_tol == 0) { - sub_dt = min_sub_dt; - } else { - sub_dt = RKStepper::OptimalStep(sub_dt, d_e_err, d_e_tol); - } - // Don't drop timestep under maximum iteration count - if (sub_dt < min_sub_dt || sub_attempt >= max_iter) { - sub_dt = min_sub_dt; - } - } - } - - } while (reattempt_sub); - - // Accept this subcycle - sub_t += sub_dt; - - internal_e = internal_e_next_h; // !!!! internal_e updated here !!!! - - // skip to the end of subcycling if error is 0 (very unlikely) - if (d_e_err == 0) { - sub_dt = dt - sub_t; - } else { - // Grow the timestep - // (or shrink in case d_e_err >= d_e_tol and sub_dt is already at min_sub_dt) - sub_dt = RKStepper::OptimalStep(sub_dt, d_e_err, d_e_tol); - } - - if (d_e_tol == 0) { - sub_dt = min_sub_dt; - } - - // Don't drop timestep under the minimum step size - sub_dt = std::max(sub_dt, min_sub_dt); - - // Limit by end time - sub_dt = std::min(sub_dt, dt - sub_t); - - sub_iter++; - } - - // If cooled below floor, reset to floor value. - // This could happen if the floor value is larger than the lower end of the - // cooling table or if they are close and the last subcycle in the cooling above - // the lower end pushed the temperature below the lower end (and the floor). - // Literally do: if internal_e > internal_e_floor, internal_e remain unchanged - // otherwise, internal_e get internal_e_floor. - // internal_e is already updated before (see above) - internal_e = (internal_e > internal_e_floor) ? internal_e : internal_e_floor; - - // Remove the cooling from the total energy density - cons(IEN, k, j, i) += rho * (internal_e - internal_e_initial); - // Latter technically not required if no other tasks follows before - // ConservedToPrim conversion, but keeping it for now (better safe than sorry). - prim(IPR, k, j, i) = rho * internal_e * gm1; - }); -} - -void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, - const parthenon::Real dt_) const { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - const bool mhd_enabled = hydro_pkg->Param("fluid") == Fluid::glmmhd; - - // Grab member variables for compiler - const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - - const auto units = hydro_pkg->Param("units"); - const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - const Real X_by_mh2 = - std::pow((1 - hydro_pkg->Param("He_mass_fraction")) / units.mh(), 2); - - const auto lambdas = lambdas_; - const auto temps = temps_; - const auto alpha_k = townsend_alpha_k_; - const auto Y_k = townsend_Y_k_; - - const auto internal_e_floor = T_floor_ / mbar_gm1_over_kb; - const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - // need to include ghost zones as this source is called prior to the other fluxes when - // split - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - - const auto nbins = alpha_k.extent_int(0); - - // Get reference values - const auto temp_final = std::pow(10.0, log_temp_final_); - const auto lambda_final = lambda_final_; - - par_for( - DEFAULT_LOOP_PATTERN, "TabularCooling::TownsendSrcTerm", DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { - auto &cons = cons_pack(b); - auto &prim = prim_pack(b); - // Need to use `cons` here as prim may still contain state at t_0; - const auto rho = cons(IDN, k, j, i); - // TODO(pgrete) with potentially more EOS, a separate get_pressure (or similar) - // function could be useful. - auto internal_e = - cons(IEN, k, j, i) - 0.5 * - (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + - SQR(cons(IM3, k, j, i))) / - rho; - if (mhd_enabled) { - internal_e -= 0.5 * (SQR(cons(IB1, k, j, i)) + SQR(cons(IB2, k, j, i)) + - SQR(cons(IB3, k, j, i))); - } - internal_e /= rho; - - // If temp is below floor, reset and return - if (internal_e <= internal_e_floor) { - // Remove the cooling from the total energy density - cons(IEN, k, j, i) += rho * (internal_e_floor - internal_e); - // Latter technically not required if no other tasks follows before - // ConservedToPrim conversion, but keeping it for now (better safe than sorry). - prim(IPR, k, j, i) = rho * internal_e_floor * gm1; - return; - } - - auto temp = mbar_gm1_over_kb * internal_e; - // Temperature is above floor (see conditional above) but below cooling table: - // -> no cooling - if (temp < temp_cool_floor) { - return; - } - const Real n_h2_by_rho = rho * X_by_mh2; - - // Get the index of the right temperature bin - // TODO(?) this could be optimized for using a binary search - auto idx = 0; - while ((idx < nbins - 1) && (temps(idx + 1) < temp)) { - idx += 1; - } - - // Compute the Temporal Evolution Function Y(T) (Eq. A5) - const auto alpha_k_m1 = alpha_k(idx) - 1.0; - const auto tef = - Y_k(idx) + (lambda_final / lambdas(idx)) * (temps(idx) / temp_final) * - (std::pow(temps(idx) / temp, alpha_k_m1) - 1.0) / alpha_k_m1; - - // Compute the adjusted TEF for new timestep (Eqn. 26) (term in brackets) - const auto tef_adj = - tef + lambda_final * dt / temp_final * mbar_gm1_over_kb * n_h2_by_rho; - - // TEF is a strictly decreasing function and new_tef > tef - // Check if the new TEF falls into a lower bin, i.e., find the right bin for A7 - // If so, update slopes and coefficients - while ((idx > 0) && (tef_adj > Y_k(idx))) { - idx -= 1; - } - - // Compute the Inverse Temporal Evolution Function Y^{-1}(Y) (Eq. A7) - const auto temp_new = - temps(idx) * - std::pow(1 - (1.0 - alpha_k(idx)) * (lambdas(idx) / lambda_final) * - (temp_final / temps(idx)) * (tef_adj - Y_k(idx)), - 1.0 / (1.0 - alpha_k(idx))); - // Set new temp (at the lowest to the lower end of the cooling table) - const auto internal_e_new = temp_new > temp_cool_floor - ? temp_new / mbar_gm1_over_kb - : temp_cool_floor / mbar_gm1_over_kb; - cons(IEN, k, j, i) += rho * (internal_e_new - internal_e); - // Latter technically not required if no other tasks follows before - // ConservedToPrim conversion, but keeping it for now (better safe than sorry). - prim(IPR, k, j, i) = rho * internal_e_new * gm1; - }); -} - -Real TabularCooling::EstimateTimeStep(MeshData *md) const { - if (cooling_time_cfl_ <= 0.0) { - return std::numeric_limits::max(); - } - - if (isnan(cooling_time_cfl_) || isinf(cooling_time_cfl_)) { - return std::numeric_limits::infinity(); - } - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const CoolingTableObj cooling_table_obj = cooling_table_obj_; - const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - - // Determine the cooling floor, whichever is higher of the cooling table floor - // or fluid solver floor - const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - - const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. - - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - - Real min_cooling_time = std::numeric_limits::infinity(); - Kokkos::Min reducer_min(min_cooling_time); - - Kokkos::parallel_reduce( - "TabularCooling::TimeStep", - Kokkos::MDRangePolicy>( - {0, kb.s, jb.s, ib.s}, {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, - {1, 1, 1, ib.e + 1 - ib.s}), - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, - Real &thread_min_cooling_time) { - auto &prim = prim_pack(b); - - const Real rho = prim(IDN, k, j, i); - const Real pres = prim(IPR, k, j, i); - - const Real internal_e = pres / (rho * gm1); - - const Real de_dt = cooling_table_obj.DeDt(internal_e, rho); - - // Compute cooling time - // If de_dt is zero (temperature is smaller than lower end of cooling table) or - // current temp is below floor, use infinite cooling time - const Real cooling_time = ((de_dt == 0) || (internal_e < internal_e_floor)) - ? std::numeric_limits::infinity() - : fabs(internal_e / de_dt); - - thread_min_cooling_time = std::min(cooling_time, thread_min_cooling_time); - }, - reducer_min); - - return cooling_time_cfl_ * min_cooling_time; -} - -void TabularCooling::TestCoolingTable(ParameterInput *pin) const { - - const std::string test_filename = pin->GetString("cooling", "test_filename"); - - const auto rho0 = pin->GetReal("cooling", "test_rho0"); - const auto rho1 = pin->GetReal("cooling", "test_rho1"); - const auto n_rho = pin->GetInteger("cooling", "test_n_rho"); - - const auto pres0 = pin->GetReal("cooling", "test_pres0"); - const auto pres1 = pin->GetReal("cooling", "test_pres1"); - const auto n_pres = pin->GetInteger("cooling", "test_n_pres"); - - // Grab member variables for compiler - - // Everything needed by DeDt - const CoolingTableObj cooling_table_obj = cooling_table_obj_; - const auto gm1 = pin->GetReal("hydro", "gamma") - 1.0; - - // Make some device arrays to store the test data - ParArray2D d_rho("d_rho", n_rho, n_pres), d_pres("d_pres", n_rho, n_pres), - d_internal_e("d_internal_e", n_rho, n_pres), d_de_dt("d_de_dt", n_rho, n_pres); - - par_for( - loop_pattern_mdrange_tag, "TabularCooling::TestCoolingTable", DevExecSpace(), 0, - n_rho - 1, 0, n_pres - 1, KOKKOS_LAMBDA(const int &j, const int &i) { - const Real rho = rho0 * pow(rho1 / rho0, static_cast(j) / (n_rho - 1)); - const Real pres = pres0 * pow(pres1 / pres0, static_cast(i) / (n_pres - 1)); - - d_rho(j, i) = rho; - d_pres(j, i) = pres; - - const Real internal_e = pres / (rho * gm1); - - d_internal_e(j, i) = internal_e; - - const Real de_dt = cooling_table_obj.DeDt(internal_e, rho); - - d_de_dt(j, i) = de_dt; - }); - - // Copy Device arrays to host - auto h_rho = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_rho); - auto h_pres = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_pres); - auto h_internal_e = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_internal_e); - auto h_de_dt = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_de_dt); - - // Write to file - std::ofstream file(test_filename); - file << "#rho pres internal_e de_dt" << std::endl; - for (int j = 0; j < n_rho; j++) { - for (int i = 0; i < n_pres; i++) { - file << h_rho(j, i) << " " << h_pres(j, i) << " " << h_internal_e(j, i) << " " - << h_de_dt(j, i) << " " << std::endl; - } - } -} - -} // namespace cooling diff --git a/src/hydro/srcterms/old/tabular_cooling_edit.hpp b/src/hydro/srcterms/old/tabular_cooling_edit.hpp deleted file mode 100644 index 5caea98e..00000000 --- a/src/hydro/srcterms/old/tabular_cooling_edit.hpp +++ /dev/null @@ -1,314 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file tabular_cooling.hpp -// \brief Class for defining tabular cooling -#ifndef HYDRO_SRCTERMS_TABULAR_COOLING_HPP_ -#define HYDRO_SRCTERMS_TABULAR_COOLING_HPP_ - -// C++ headers -#include // stringstream -#include // istream_iterator -#include // stringstream -#include // runtime_error -#include // string -#include // vector - -// Parthenon headers -#include -#include -#include -#include -#include - -// AthenaPK headers -#include "../../main.hpp" -#include "../../units.hpp" - -#ifdef MPI_PARALLEL -#include -#endif - -namespace cooling { - -// Struct to take one RK step using heun's method to compute 2nd and 1st order estimations -// in y1_h and y1_l -struct RK12Stepper { - template - static KOKKOS_INLINE_FUNCTION void - Step(const parthenon::Real t0, const parthenon::Real h, const parthenon::Real y0, - Function f, parthenon::Real &y1_h, parthenon::Real &y1_l, bool &valid) { - const parthenon::Real f_t0_y0 = f(t0, y0, valid); - y1_l = y0 + h * f_t0_y0; // 1st order - y1_h = y0 + h / 2. * (f_t0_y0 + f(t0 + h, y1_l, valid)); // 2nd order - } - static KOKKOS_INLINE_FUNCTION parthenon::Real OptimalStep(const parthenon::Real h, - const parthenon::Real err, - const parthenon::Real tol) { - return 0.95 * h * pow(tol / err, 2); - } -}; - -// Struct to take a 5th and 4th order RK step to compute 5th and 4th order estimations in -// y1_h and y1_l -struct RK45Stepper { - template - static KOKKOS_INLINE_FUNCTION void - Step(const parthenon::Real t0, const parthenon::Real h, const parthenon::Real y0, - Function f, parthenon::Real &y1_h, parthenon::Real &y1_l, bool &valid) { - const parthenon::Real k1 = h * f(t0, y0, valid); - const parthenon::Real k2 = h * f(t0 + 1. / 4. * h, y0 + 1. / 4. * k1, valid); - const parthenon::Real k3 = - h * f(t0 + 3. / 8. * h, y0 + 3. / 32. * k1 + 9. / 32. * k2, valid); - const parthenon::Real k4 = - h * f(t0 + 12. / 13. * h, - y0 + 1932. / 2197. * k1 - 7200. / 2197. * k2 + 7296. / 2197. * k3, valid); - const parthenon::Real k5 = - h * f(t0 + h, - y0 + 439. / 216. * k1 - 8. * k2 + 3680. / 513. * k3 - 845. / 4104. * k4, - valid); - const parthenon::Real k6 = h * f(t0 + 1. / 2. * h, - y0 - 8. / 27. * k1 + 2. * k2 - 3544. / 2565. * k3 + - 1859. / 4104. * k4 - 11. / 40. * k5, - valid); // TODO(forrestglines): Check k2? - y1_l = y0 + 25. / 216. * k1 + 1408. / 2565. * k3 + 2197. / 4104. * k4 - - 1. / 5. * k5; // 4th order - y1_h = y0 + 16. / 135. * k1 + 6656. / 12825. * k3 + 28561. / 56430. * k4 - - 9. / 50. * k5 + 2. / 55. * k6; // 5th order - } - static KOKKOS_INLINE_FUNCTION parthenon::Real OptimalStep(const parthenon::Real h, - const parthenon::Real err, - const parthenon::Real tol) { - return 0.95 * h * pow(tol / err, 5); - } -}; - -enum class CoolIntegrator { undefined, rk12, rk45, townsend }; - -class CoolingTableObj { - /************************************************************ - * Cooling Table Object, for interpolating a cooling rate out of a cooling - * table. Currently assumes evenly space log_temperatures in cooling table - * - * Lightweight object intended for inlined computation within kernels - ************************************************************/ - private: - // Log cooling rate/ne^3 - parthenon::ParArray1D log_lambdas_; - - // Spacing of cooling table - // TODO: assumes evenly spaced cooling table - parthenon::Real log_temp_start_, log_temp_final_, d_log_temp_; - unsigned int n_temp_; - - // Mean molecular mass * ( adiabatic_index -1) / boltzmann_constant - parthenon::Real mbar_gm1_over_k_B_; - - // (Hydrogen mass fraction / hydrogen atomic mass)^2 - parthenon::Real x_H_over_m_h2_; - - public: - CoolingTableObj() - : log_lambdas_(), log_temp_start_(NAN), log_temp_final_(NAN), d_log_temp_(NAN), - n_temp_(0), mbar_gm1_over_k_B_(NAN), x_H_over_m_h2_(NAN) {} - CoolingTableObj(const parthenon::ParArray1D log_lambdas, - const parthenon::Real log_temp_start, - const parthenon::Real log_temp_final, const parthenon::Real d_log_temp, - const unsigned int n_temp, const parthenon::Real mbar_over_kb, - const parthenon::Real adiabatic_index, const parthenon::Real x_H, - const Units units) - : log_lambdas_(log_lambdas), log_temp_start_(log_temp_start), - log_temp_final_(log_temp_final), d_log_temp_(d_log_temp), n_temp_(n_temp), - mbar_gm1_over_k_B_(mbar_over_kb * (adiabatic_index - 1)), - x_H_over_m_h2_(SQR(x_H / units.mh())) {} - - // Interpolate a cooling rate from the table - // from internal energy density and density - KOKKOS_INLINE_FUNCTION parthenon::Real - DeDt(const parthenon::Real &e, const parthenon::Real &rho, bool &is_valid) const { - using namespace parthenon; - - if (e < 0 || std::isnan(e)) { - is_valid = false; - return 0; - } - - const Real temp = mbar_gm1_over_k_B_ * e; - const Real log_temp = log10(temp); - Real log_lambda; - if (log_temp < log_temp_start_) { - return 0; // Return no variation - - } else if (log_temp > log_temp_final_) { - // Cooling insured by free-free cooling ie. Bremsstrahlung - // Above table - // Return de/dt - // TODO(forrestglines):Currently free-free cooling is used for - // temperatures above the table. This behavior could be generalized via - // templates - log_lambda = 0.5 * log_temp - 0.5 * log_temp_final_ + log_lambdas_(n_temp_ - 1); - - // NEED TO BE MODIFIED HERE: ADD THE HEATING FUNCTION AND CHECK FOR CONTINUITY - - } else { - // Inside table, interpolate assuming log spaced temperatures - - // Determine where temp is in the table - const unsigned int i_temp = - static_cast((log_temp - log_temp_start_) / d_log_temp_); - const Real log_temp_i = log_temp_start_ + d_log_temp_ * i_temp; - - // log_temp should be between log_temps[i_temp] and log_temps[i_temp+1] - PARTHENON_REQUIRE(log_temp >= log_temp_i && log_temp <= log_temp_i + d_log_temp_, - "FATAL ERROR in [CoolingTable::DeDt]: Failed to find log_temp"); - - const Real log_lambda_i = log_lambdas_(i_temp); - const Real log_lambda_ip1 = log_lambdas_(i_temp + 1); - - // Linearly interpolate lambda at log_temp - log_lambda = log_lambda_i + (log_temp - log_temp_i) * - (log_lambda_ip1 - log_lambda_i) / d_log_temp_; - } - // Return de/dt - const Real lambda = pow(10., log_lambda); - const Real de_dt = -lambda * x_H_over_m_h2_ * rho; - return de_dt; - } - - // Interpolate a cooling rate from the table - // from internal energy density and density - - KOKKOS_INLINE_FUNCTION parthenon::Real - DeDt(const parthenon::Real &e, const parthenon::Real &rho, bool &is_valid) const { - using namespace parthenon; - - if (e < 0 || std::isnan(e)) { - is_valid = false; - return 0; - } - - const Real temp = mbar_gm1_over_k_B_ * e; - const Real log_temp = log10(temp); - Real log_lambda; - if (log_temp < log_temp_start_) { - return 0; // Return no variation - - } else if (log_temp > log_temp_final_) { - // Cooling insured by free-free cooling ie. Bremsstrahlung - // Above table - // Return de/dt - // TODO(forrestglines):Currently free-free cooling is used for - // temperatures above the table. This behavior could be generalized via - // templates - log_lambda = 0.5 * log_temp - 0.5 * log_temp_final_ + log_lambdas_(n_temp_ - 1); - - // NEED TO BE MODIFIED HERE: ADD THE HEATING FUNCTION AND CHECK FOR CONTINUITY - - } else { - // Inside table, interpolate assuming log spaced temperatures - - // Determine where temp is in the table - const unsigned int i_temp = - static_cast((log_temp - log_temp_start_) / d_log_temp_); - const Real log_temp_i = log_temp_start_ + d_log_temp_ * i_temp; - - // log_temp should be between log_temps[i_temp] and log_temps[i_temp+1] - PARTHENON_REQUIRE(log_temp >= log_temp_i && log_temp <= log_temp_i + d_log_temp_, - "FATAL ERROR in [CoolingTable::DeDt]: Failed to find log_temp"); - - const Real log_lambda_i = log_lambdas_(i_temp); - const Real log_lambda_ip1 = log_lambdas_(i_temp + 1); - - // Linearly interpolate lambda at log_temp - log_lambda = log_lambda_i + (log_temp - log_temp_i) * - (log_lambda_ip1 - log_lambda_i) / d_log_temp_; - } - // Return de/dt - const Real lambda = pow(10., log_lambda); - const Real de_dt = -lambda * x_H_over_m_h2_ * rho; - return de_dt; - } - - - - KOKKOS_INLINE_FUNCTION parthenon::Real DeDt(const parthenon::Real &e, - const parthenon::Real &rho) const { - bool is_valid = true; - return DeDt(e, rho, is_valid); - } -}; - -class TabularCooling { - private: - // Defines uniformly spaced log temperature range of the table - unsigned int n_temp_; - parthenon::Real log_temp_start_, log_temp_final_, d_log_temp_, lambda_final_; - - // Table of log cooling rates - // TODO(forrestglines): Make log_lambdas_ explicitly a texture cache array, use CUDA to - // interpolate directly - // Log versions are used in subcyling cooling where cooling rates are interpolated - // Non-log versions are used for Townsend cooling - parthenon::ParArray1D log_lambdas_; - parthenon::ParArray1D lambdas_; - parthenon::ParArray1D temps_; - // Townsend cooling temporal evolution function - parthenon::ParArray1D townsend_Y_k_; - // Townsend cooling power law indices - parthenon::ParArray1D townsend_alpha_k_; - - CoolIntegrator integrator_; - - // Temperature floor (assumed in Kelvin and only used in cooling function) - // This is either the temperature floor used by the hydro method or the - // lowest temperature in the cooling table (assuming zero cooling below the - // table), whichever temperature is higher - parthenon::Real T_floor_; - - // Maximum number of iterations/subcycles - unsigned int max_iter_; - - // Cooling CFL - parthenon::Real cooling_time_cfl_; - - // Minimum timestep that the cooling may limit the simulation timestep - // Use nonpositive values to disable - parthenon::Real min_cooling_timestep_; - - // Tolerances - parthenon::Real d_log_temp_tol_, d_e_tol_; - - // Used for roundoff as subcycle approaches end of timestep - static constexpr parthenon::Real KEpsilon_ = 1e-12; - - CoolingTableObj cooling_table_obj_; - - public: - TabularCooling(parthenon::ParameterInput *pin, - std::shared_ptr hydro_pkg); - - void SrcTerm(parthenon::MeshData *md, const parthenon::Real dt) const; - - // Townsend 2009 exact integration scheme - void TownsendSrcTerm(parthenon::MeshData *md, - const parthenon::Real dt) const; - - // (Adaptive) subcyling using a fixed integration scheme - template - void SubcyclingFixedIntSrcTerm(parthenon::MeshData *md, - const parthenon::Real dt, - const RKStepper rk_stepper) const; - - parthenon::Real EstimateTimeStep(parthenon::MeshData *md) const; - - // Get a lightweight object for computing cooling rate from the cooling table - const CoolingTableObj GetCoolingTableObj() const { return cooling_table_obj_; } - - void TestCoolingTable(parthenon::ParameterInput *pin) const; -}; - -} // namespace cooling - -#endif // HYDRO_SRCTERMS_TABULAR_COOLING_HPP_ diff --git a/src/hydro/srcterms/tabular_cooling.cpp b/src/hydro/srcterms/tabular_cooling.cpp index 4c2728d2..9164f4cc 100644 --- a/src/hydro/srcterms/tabular_cooling.cpp +++ b/src/hydro/srcterms/tabular_cooling.cpp @@ -36,29 +36,17 @@ TabularCooling::TabularCooling(ParameterInput *pin, const int log_temp_col = pin->GetOrAddInteger("cooling", "log_temp_col", 0); const int log_lambda_col = pin->GetOrAddInteger("cooling", "log_lambda_col", 1); - // Heating constants - const Real heating_const_cgs = pin->GetOrAddReal("cooling", "heating_const", 0.0); - const Real gamma_units = heating_const_cgs * (units.erg() / units.s()); - - // Checkpoint - std::cout << "heating_const_cgs=" << heating_const_cgs << "\n"; - std::cout << "units.s()=" << units.s() << "\n"; - std::cout << "gamma_units=" << gamma_units << "\n"; - const Real lambda_units_cgs = pin->GetReal("cooling", "lambda_units_cgs"); // Convert erg cm^3/s to code units const Real lambda_units = lambda_units_cgs / (units.erg() * pow(units.cm(), 3) / units.s()); - + const auto integrator_str = pin->GetOrAddString("cooling", "integrator", "rk12"); if (integrator_str == "rk12") { - std::cout << "Cooling integrator is rk12\n"; integrator_ = CoolIntegrator::rk12; } else if (integrator_str == "rk45") { - std::cout << "Cooling integrator is rk45\n"; integrator_ = CoolIntegrator::rk45; } else if (integrator_str == "townsend") { - std::cout << "Cooling integrator is Townsend\n"; integrator_ = CoolIntegrator::townsend; } else { integrator_ = CoolIntegrator::undefined; @@ -127,49 +115,43 @@ TabularCooling::TabularCooling(ParameterInput *pin, << line << "\"" << std::endl; PARTHENON_FAIL(msg); } - + try { const Real log_temp = std::stod(line_data[log_temp_col]); const Real log_lambda = std::stod(line_data[log_lambda_col]); - + // Add to growing list log_temps.push_back(log_temp); log_lambdas.push_back(log_lambda - std::log10(lambda_units)); - + } catch (const std::invalid_argument &ia) { msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl << "Number: \"" << ia.what() << "\" could not be parsed as double" << std::endl; PARTHENON_FAIL(msg); } } - + /**************************************** * Check some assumtions about the cooling table ****************************************/ - - // Reconstructing the cooling function in new coordinate system - //std::cout << "Entering lambda cout loop\n"; - //for (int i = 0; i < log_temps.size(); i++) { - //std::cout << "log_temps" << log_temps[i] << ", log_lambdas" << log_lambdas[i] << "\n"; - //} - + // Ensure at least two data points in the table to interpolate from if (log_temps.size() < 2 || log_lambdas.size() < 2) { msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl << "Not enough data to interpolate cooling" << std::endl; PARTHENON_FAIL(msg); } - + // Ensure that the first log_temp is increasing const Real log_temp_start = log_temps[0]; const Real d_log_temp = log_temps[1] - log_temp_start; - + if (d_log_temp <= 0) { msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl << "second log_temp in table is descreasing" << std::endl; PARTHENON_FAIL(msg); } - + // Ensure that log_temps is evenly spaced for (size_t i = 1; i < log_temps.size(); i++) { const Real d_log_temp_i = log_temps[i] - log_temps[i - 1]; @@ -180,7 +162,7 @@ TabularCooling::TabularCooling(ParameterInput *pin, << " log_temp= " << log_temps[i] << std::endl; PARTHENON_FAIL(msg); } - + // The Dedt() function currently relies on an equally spaced cooling table for faster // lookup (direct indexing). It is used in the subcycling method and in order to // restrict `dt` by a cooling cfl. @@ -195,18 +177,17 @@ TabularCooling::TabularCooling(ParameterInput *pin, PARTHENON_FAIL(msg); } } - + /**************************************** * Move values read into the data table ****************************************/ - + n_temp_ = log_temps.size(); log_temp_start_ = log_temps[0]; log_temp_final_ = log_temps[n_temp_ - 1]; d_log_temp_ = d_log_temp; lambda_final_ = std::pow(10.0, log_lambdas[n_temp_ - 1]); - gamma_units_ = gamma_units; - + // Setup log_lambdas_ used in Dedt() { // log_lambdas is used if the integrator isn't Townsend, if the cooling CFL @@ -226,7 +207,7 @@ TabularCooling::TabularCooling(ParameterInput *pin, if (integrator_ == CoolIntegrator::townsend) { lambdas_ = ParArray1D("lambdas_", n_temp_); temps_ = ParArray1D("temps_", n_temp_); - + // Read log_lambdas in host_lambdas, changing to code units along the way auto host_lambdas = Kokkos::create_mirror_view(lambdas_); auto host_temps = Kokkos::create_mirror_view(temps_); @@ -234,20 +215,19 @@ TabularCooling::TabularCooling(ParameterInput *pin, host_lambdas(i) = std::pow(10.0, log_lambdas[i]); host_temps(i) = std::pow(10.0, log_temps[i]); } - // Copy host_lambdas into device memory Kokkos::deep_copy(lambdas_, host_lambdas); Kokkos::deep_copy(temps_, host_temps); - + // Coeffs are for intervals, i.e., only n_temp_ - 1 entries const auto n_bins = n_temp_ - 1; townsend_Y_k_ = ParArray1D("townsend_Y_k_", n_bins); townsend_alpha_k_ = ParArray1D("townsend_alpha_k_", n_bins); - + // Initialize on host (make *this* recursion simpler) auto host_townsend_Y_k = Kokkos::create_mirror_view(townsend_Y_k_); auto host_townsend_alpha_k = Kokkos::create_mirror_view(townsend_alpha_k_); - + // Initialize piecewise power law indices for (unsigned int i = 0; i < n_bins; i++) { // Could use log_lambdas_ here, but using lambdas_ instead as they've already been @@ -282,7 +262,7 @@ TabularCooling::TabularCooling(ParameterInput *pin, const auto He_mass_fraction = hydro_pkg->Param("He_mass_fraction"); cooling_table_obj_ = CoolingTableObj(log_lambdas_, log_temp_start_, log_temp_final_, - d_log_temp_, n_temp_, gamma_units_, mbar_over_kb, + d_log_temp_, n_temp_, mbar_over_kb, adiabatic_index, 1.0 - He_mass_fraction, units); } @@ -310,20 +290,20 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt const CoolingTableObj cooling_table_obj = cooling_table_obj_; const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - + const unsigned int max_iter = max_iter_; - + const Real min_sub_dt = dt / max_iter; - + const Real d_e_tol = d_e_tol_; - + // Determine the cooling floor, whichever is higher of the cooling table floor // or fluid solver floor const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - + const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. - + // Grab some necessary variables const auto &prim_pack = md->PackVariables(std::vector{"prim"}); const auto &cons_pack = md->PackVariables(std::vector{"cons"}); @@ -332,7 +312,7 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - + par_for( DEFAULT_LOOP_PATTERN, "TabularCooling::SubcyclingSplitSrcTerm", DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, @@ -504,7 +484,7 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, // Grab member variables for compiler const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - + const auto units = hydro_pkg->Param("units"); const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; @@ -515,10 +495,10 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, const auto temps = temps_; const auto alpha_k = townsend_alpha_k_; const auto Y_k = townsend_Y_k_; - + const auto internal_e_floor = T_floor_ / mbar_gm1_over_kb; const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - + // Grab some necessary variables const auto &prim_pack = md->PackVariables(std::vector{"prim"}); const auto &cons_pack = md->PackVariables(std::vector{"cons"}); @@ -527,13 +507,13 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - + const auto nbins = alpha_k.extent_int(0); - + // Get reference values const auto temp_final = std::pow(10.0, log_temp_final_); const auto lambda_final = lambda_final_; - + par_for( DEFAULT_LOOP_PATTERN, "TabularCooling::TownsendSrcTerm", DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, diff --git a/src/hydro/srcterms/tabular_cooling.hpp b/src/hydro/srcterms/tabular_cooling.hpp index 24f5de28..c1ffa8b2 100644 --- a/src/hydro/srcterms/tabular_cooling.hpp +++ b/src/hydro/srcterms/tabular_cooling.hpp @@ -108,25 +108,19 @@ class CoolingTableObj { // (Hydrogen mass fraction / hydrogen atomic mass)^2 parthenon::Real x_H_over_m_h2_; - - // Heating constant unit - parthenon::Real gamma_units_; public: CoolingTableObj() : log_lambdas_(), log_temp_start_(NAN), log_temp_final_(NAN), d_log_temp_(NAN), - n_temp_(0), mbar_gm1_over_k_B_(NAN), x_H_over_m_h2_(NAN), gamma_units_(NAN) {} + n_temp_(0), mbar_gm1_over_k_B_(NAN), x_H_over_m_h2_(NAN) {} CoolingTableObj(const parthenon::ParArray1D log_lambdas, const parthenon::Real log_temp_start, const parthenon::Real log_temp_final, const parthenon::Real d_log_temp, - const unsigned int n_temp, - const parthenon::Real gamma_units, - const parthenon::Real mbar_over_kb, + const unsigned int n_temp, const parthenon::Real mbar_over_kb, const parthenon::Real adiabatic_index, const parthenon::Real x_H, const Units units) : log_lambdas_(log_lambdas), log_temp_start_(log_temp_start), log_temp_final_(log_temp_final), d_log_temp_(d_log_temp), n_temp_(n_temp), - gamma_units_(gamma_units), mbar_gm1_over_k_B_(mbar_over_kb * (adiabatic_index - 1)), x_H_over_m_h2_(SQR(x_H / units.mh())) {} @@ -135,12 +129,12 @@ class CoolingTableObj { KOKKOS_INLINE_FUNCTION parthenon::Real DeDt(const parthenon::Real &e, const parthenon::Real &rho, bool &is_valid) const { using namespace parthenon; - + if (e < 0 || std::isnan(e)) { is_valid = false; return 0; } - + const Real temp = mbar_gm1_over_k_B_ * e; const Real log_temp = log10(temp); Real log_lambda; @@ -155,34 +149,29 @@ class CoolingTableObj { log_lambda = 0.5 * log_temp - 0.5 * log_temp_final_ + log_lambdas_(n_temp_ - 1); } else { // Inside table, interpolate assuming log spaced temperatures - + // Determine where temp is in the table const unsigned int i_temp = static_cast((log_temp - log_temp_start_) / d_log_temp_); const Real log_temp_i = log_temp_start_ + d_log_temp_ * i_temp; - + // log_temp should be between log_temps[i_temp] and log_temps[i_temp+1] PARTHENON_REQUIRE(log_temp >= log_temp_i && log_temp <= log_temp_i + d_log_temp_, "FATAL ERROR in [CoolingTable::DeDt]: Failed to find log_temp"); - + const Real log_lambda_i = log_lambdas_(i_temp); const Real log_lambda_ip1 = log_lambdas_(i_temp + 1); - + // Linearly interpolate lambda at log_temp log_lambda = log_lambda_i + (log_temp - log_temp_i) * (log_lambda_ip1 - log_lambda_i) / d_log_temp_; } // Return de/dt - const Real lambda = pow(10., log_lambda); - const Real gamma_heat = gamma_units_; - const Real de_dt = -lambda * x_H_over_m_h2_ * rho + gamma_heat*pow(x_H_over_m_h2_,0.5); - //const Real de_dt = -lambda * x_H_over_m_h2_ * rho; - - //std::cout << "cool = " << -lambda * x_H_over_m_h2_ * rho << "\t heat = " << gamma*pow(x_H_over_m_h2_,0.5) << "\t e = " << e << "\t rho = " << rho << "\n" ; - + const Real lambda = pow(10., log_lambda); + const Real de_dt = -lambda * x_H_over_m_h2_ * rho; return de_dt; } - + KOKKOS_INLINE_FUNCTION parthenon::Real DeDt(const parthenon::Real &e, const parthenon::Real &rho) const { bool is_valid = true; @@ -194,7 +183,7 @@ class TabularCooling { private: // Defines uniformly spaced log temperature range of the table unsigned int n_temp_; - parthenon::Real log_temp_start_, log_temp_final_, d_log_temp_, lambda_final_, gamma_units_; + parthenon::Real log_temp_start_, log_temp_final_, d_log_temp_, lambda_final_; // Table of log cooling rates // TODO(forrestglines): Make log_lambdas_ explicitly a texture cache array, use CUDA to diff --git a/src/hydro/srcterms/tabular_cooling_working.cpp b/src/hydro/srcterms/tabular_cooling_working.cpp deleted file mode 100644 index 31ed7f3e..00000000 --- a/src/hydro/srcterms/tabular_cooling_working.cpp +++ /dev/null @@ -1,733 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file tabular_cooling.cpp -// \brief Applies tabular cooling -// -//======================================================================================== - -// C++ headers -#include -#include -#include - -// Parthenon headers -#include -#include -#include -#include - -// AthenaPK headers -#include "../../units.hpp" -#include "tabular_cooling.hpp" -#include "utils/error_checking.hpp" - -namespace cooling { -using namespace parthenon; - -TabularCooling::TabularCooling(ParameterInput *pin, - std::shared_ptr hydro_pkg) { - auto units = hydro_pkg->Param("units"); - - const std::string table_filename = pin->GetString("cooling", "table_filename"); - - const int log_temp_col = pin->GetOrAddInteger("cooling", "log_temp_col", 0); - const int log_lambda_col = pin->GetOrAddInteger("cooling", "log_lambda_col", 1); - - // Heating constants - const Real heating_const_cgs = pin->GetOrAddReal("cooling", "heating_const", 0.0); - const Real gamma_units = heating_const_cgs * (units.erg() / units.s()); - - // Checkpoint - std::cout << "heating_const_cgs=" << heating_const_cgs << "\n"; - std::cout << "units.s()=" << units.s() << "\n"; - std::cout << "gamma_units=" << gamma_units << "\n"; - - const Real lambda_units_cgs = pin->GetReal("cooling", "lambda_units_cgs"); - // Convert erg cm^3/s to code units - const Real lambda_units = - lambda_units_cgs / (units.erg() * pow(units.cm(), 3) / units.s()); - - const auto integrator_str = pin->GetOrAddString("cooling", "integrator", "rk12"); - if (integrator_str == "rk12") { - std::cout << "Cooling integrator is rk12\n"; - integrator_ = CoolIntegrator::rk12; - } else if (integrator_str == "rk45") { - std::cout << "Cooling integrator is rk45\n"; - integrator_ = CoolIntegrator::rk45; - } else if (integrator_str == "townsend\n") { - std::cout << "Cooling integrator is Townsend"; - integrator_ = CoolIntegrator::townsend; - } else { - integrator_ = CoolIntegrator::undefined; - } - max_iter_ = pin->GetOrAddInteger("cooling", "max_iter", 100); - cooling_time_cfl_ = pin->GetOrAddReal("cooling", "cfl", 0.1); - d_log_temp_tol_ = pin->GetOrAddReal("cooling", "d_log_temp_tol", 1e-8); - d_e_tol_ = pin->GetOrAddReal("cooling", "d_e_tol", 1e-8); - // negative means disabled - T_floor_ = pin->GetOrAddReal("hydro", "Tfloor", -1.0); - - std::stringstream msg; - - /**************************************** - * Read tab file with IOWrapper - ****************************************/ - IOWrapper input; - input.Open(table_filename.c_str(), IOWrapper::FileMode::read); - - /**************************************** - * Read tab file from IOWrapper into a stringstream tab - ****************************************/ - std::stringstream tab_ss; - const int bufsize = 4096; - char *buf = new char[bufsize]; - std::ptrdiff_t ret; - parthenon::IOWrapperSizeT word_size = sizeof(char); - - do { - if (Globals::my_rank == 0) { // only the master process reads the cooling table - ret = input.Read(buf, word_size, bufsize); - } -#ifdef MPI_PARALLEL - // then broadcasts it - // no need for fence as cooling table is independent of execution/memory space - MPI_Bcast(&ret, sizeof(std::ptrdiff_t), MPI_BYTE, 0, MPI_COMM_WORLD); - MPI_Bcast(buf, ret, MPI_BYTE, 0, MPI_COMM_WORLD); -#endif - tab_ss.write(buf, ret); // add the buffer into the stream - } while (ret == bufsize); // till EOF (or par_end is found) - - delete[] buf; - input.Close(); - - /**************************************** - * Determine log_temps and and log_lambdas vectors - ****************************************/ - std::vector log_temps, log_lambdas; - std::string line; - std::size_t first_char; - while (tab_ss.good()) { - getline(tab_ss, line); - if (line.empty()) continue; // skip blank line - first_char = line.find_first_not_of(" "); // skip white space - if (first_char == std::string::npos) continue; // line is all white space - if (line.compare(first_char, 1, "#") == 0) continue; // skip comments - - // Parse the numbers on the line - std::istringstream iss(line); - std::vector line_data{std::istream_iterator{iss}, - std::istream_iterator{}}; - // Check size - if (line_data.empty() || line_data.size() <= std::max(log_temp_col, log_lambda_col)) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "Index " << std::max(log_temp_col, log_lambda_col) << " out of range on \"" - << line << "\"" << std::endl; - PARTHENON_FAIL(msg); - } - - try { - const Real log_temp = std::stod(line_data[log_temp_col]); - const Real log_lambda = std::stod(line_data[log_lambda_col]); - - // Add to growing list - log_temps.push_back(log_temp); - log_lambdas.push_back(log_lambda - std::log10(lambda_units)); - - } catch (const std::invalid_argument &ia) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "Number: \"" << ia.what() << "\" could not be parsed as double" << std::endl; - PARTHENON_FAIL(msg); - } - } - - /**************************************** - * Check some assumtions about the cooling table - ****************************************/ - - // Reconstructing the cooling function in new coordinate system - //std::cout << "Entering lambda cout loop\n"; - //for (int i = 0; i < log_temps.size(); i++) { - //std::cout << "log_temps" << log_temps[i] << ", log_lambdas" << log_lambdas[i] << "\n"; - //} - - // Ensure at least two data points in the table to interpolate from - if (log_temps.size() < 2 || log_lambdas.size() < 2) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "Not enough data to interpolate cooling" << std::endl; - PARTHENON_FAIL(msg); - } - - // Ensure that the first log_temp is increasing - const Real log_temp_start = log_temps[0]; - const Real d_log_temp = log_temps[1] - log_temp_start; - - if (d_log_temp <= 0) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "second log_temp in table is descreasing" << std::endl; - PARTHENON_FAIL(msg); - } - - // Ensure that log_temps is evenly spaced - for (size_t i = 1; i < log_temps.size(); i++) { - const Real d_log_temp_i = log_temps[i] - log_temps[i - 1]; - - if (d_log_temp_i < 0) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "log_temp in table is descreasing at i= " << i - << " log_temp= " << log_temps[i] << std::endl; - PARTHENON_FAIL(msg); - } - - // The Dedt() function currently relies on an equally spaced cooling table for faster - // lookup (direct indexing). It is used in the subcycling method and in order to - // restrict `dt` by a cooling cfl. - if (((integrator_ != CoolIntegrator::townsend) || (cooling_time_cfl_ > 0.0)) && - (fabs(d_log_temp_i - d_log_temp) / d_log_temp > d_log_temp_tol_)) { - msg << "### FATAL ERROR in function [TabularCooling::TabularCooling]" << std::endl - << "d_log_temp in table is uneven at i=" << i << " log_temp=" << log_temps[i] - << " d_log_temp= " << d_log_temp << " d_log_temp_i= " << d_log_temp_i - << " diff= " << d_log_temp_i - d_log_temp - << " rel_diff= " << fabs(d_log_temp_i - d_log_temp) / d_log_temp - << " tol= " << d_log_temp_tol_ << std::endl; - PARTHENON_FAIL(msg); - } - } - - /**************************************** - * Move values read into the data table - ****************************************/ - - n_temp_ = log_temps.size(); - log_temp_start_ = log_temps[0]; - log_temp_final_ = log_temps[n_temp_ - 1]; - d_log_temp_ = d_log_temp; - lambda_final_ = std::pow(10.0, log_lambdas[n_temp_ - 1]); - - // Setup log_lambdas_ used in Dedt() - { - // log_lambdas is used if the integrator isn't Townsend, if the cooling CFL - // is set, or if cooling time is a extra derived field. Since we don't have - // a good way to check the last condition we always initialize log_lambdas_ - log_lambdas_ = ParArray1D("log_lambdas_", n_temp_); - - // Read log_lambdas in host_log_lambdas, changing to code units along the way - auto host_log_lambdas = Kokkos::create_mirror_view(log_lambdas_); - for (unsigned int i = 0; i < n_temp_; i++) { - host_log_lambdas(i) = log_lambdas[i]; - } - // Copy host_log_lambdas into device memory - Kokkos::deep_copy(log_lambdas_, host_log_lambdas); - } - // Setup Townsend cooling, i.e., precalulcate piecewise powerlaw approx. - if (integrator_ == CoolIntegrator::townsend) { - lambdas_ = ParArray1D("lambdas_", n_temp_); - temps_ = ParArray1D("temps_", n_temp_); - - // Read log_lambdas in host_lambdas, changing to code units along the way - auto host_lambdas = Kokkos::create_mirror_view(lambdas_); - auto host_temps = Kokkos::create_mirror_view(temps_); - for (unsigned int i = 0; i < n_temp_; i++) { - host_lambdas(i) = std::pow(10.0, log_lambdas[i]); - host_temps(i) = std::pow(10.0, log_temps[i]); - } - // Copy host_lambdas into device memory - Kokkos::deep_copy(lambdas_, host_lambdas); - Kokkos::deep_copy(temps_, host_temps); - - // Coeffs are for intervals, i.e., only n_temp_ - 1 entries - const auto n_bins = n_temp_ - 1; - townsend_Y_k_ = ParArray1D("townsend_Y_k_", n_bins); - townsend_alpha_k_ = ParArray1D("townsend_alpha_k_", n_bins); - - // Initialize on host (make *this* recursion simpler) - auto host_townsend_Y_k = Kokkos::create_mirror_view(townsend_Y_k_); - auto host_townsend_alpha_k = Kokkos::create_mirror_view(townsend_alpha_k_); - - // Initialize piecewise power law indices - for (unsigned int i = 0; i < n_bins; i++) { - // Could use log_lambdas_ here, but using lambdas_ instead as they've already been - // converted to code units. - host_townsend_alpha_k(i) = - (std::log10(host_lambdas(i + 1)) - std::log10(host_lambdas(i))) / - (log_temps[i + 1] - log_temps[i]); - PARTHENON_REQUIRE(host_townsend_alpha_k(i) != 1.0, - "Need to implement special case for Townsend piecewise fits."); - } - - // Calculate TEF (temporal evolution functions Y_k recursively), (Eq. A6) - host_townsend_Y_k(n_bins - 1) = 0.0; // Last Y_N = Y(T_ref) = 0 - - for (int i = n_bins - 2; i >= 0; i--) { - const auto alpha_k_m1 = host_townsend_alpha_k(i) - 1.0; - const auto step = (host_lambdas(n_bins) / host_lambdas(i)) * - (host_temps(i) / host_temps(n_bins)) * - (std::pow(host_temps(i) / host_temps(i + 1), alpha_k_m1) - 1.0) / - alpha_k_m1; - - host_townsend_Y_k(i) = host_townsend_Y_k(i + 1) - step; - } - - Kokkos::deep_copy(townsend_alpha_k_, host_townsend_alpha_k); - Kokkos::deep_copy(townsend_Y_k_, host_townsend_Y_k); - } - - // Create a lightweight object for computing cooling rates within kernels - const auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - const auto adiabatic_index = hydro_pkg->Param("AdiabaticIndex"); - const auto He_mass_fraction = hydro_pkg->Param("He_mass_fraction"); - - cooling_table_obj_ = CoolingTableObj(log_lambdas_, log_temp_start_, log_temp_final_, - d_log_temp_, n_temp_, mbar_over_kb, - adiabatic_index, 1.0 - He_mass_fraction, units); -} - -void TabularCooling::SrcTerm(MeshData *md, const Real dt) const { - if (integrator_ == CoolIntegrator::rk12) { - SubcyclingFixedIntSrcTerm(md, dt, RK12Stepper()); - } else if (integrator_ == CoolIntegrator::rk45) { - SubcyclingFixedIntSrcTerm(md, dt, RK45Stepper()); - } else if (integrator_ == CoolIntegrator::townsend) { - TownsendSrcTerm(md, dt); - } else { - PARTHENON_FAIL("Unknown cooling integrator."); - } -} - -template -void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt_, - const RKStepper rk_stepper) const { - - const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - const bool mhd_enabled = hydro_pkg->Param("fluid") == Fluid::glmmhd; - // Grab member variables for compiler - - const CoolingTableObj cooling_table_obj = cooling_table_obj_; - const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - - const unsigned int max_iter = max_iter_; - - const Real min_sub_dt = dt / max_iter; - - const Real d_e_tol = d_e_tol_; - - // Determine the cooling floor, whichever is higher of the cooling table floor - // or fluid solver floor - const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - - const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. - - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - // need to include ghost zones as this source is called prior to the other fluxes when - // split - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - - par_for( - DEFAULT_LOOP_PATTERN, "TabularCooling::SubcyclingSplitSrcTerm", DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { - auto &cons = cons_pack(b); - auto &prim = prim_pack(b); - // Need to use `cons` here as prim may still contain state at t_0; - const Real rho = cons(IDN, k, j, i); - // TODO(pgrete) with potentially more EOS, a separate get_pressure (or similar) - // function could be useful. - Real internal_e = - cons(IEN, k, j, i) - 0.5 * - (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + - SQR(cons(IM3, k, j, i))) / - rho; - if (mhd_enabled) { - internal_e -= 0.5 * (SQR(cons(IB1, k, j, i)) + SQR(cons(IB2, k, j, i)) + - SQR(cons(IB3, k, j, i))); - } - internal_e /= rho; - const Real internal_e_initial = internal_e; - - bool dedt_valid = true; - - // Wrap DeDt into a functor for the RKStepper - auto DeDt_wrapper = [&](const Real t, const Real e, bool &valid) { - return cooling_table_obj.DeDt(e, rho, valid); - }; - - Real sub_t = 0; // current subcycle time - // Try full dt. If error is too large adaptive timestepping will reduce sub_dt - Real sub_dt = dt; - - // Check if cooling is actually happening, e.g., when T below T_cool_min or if - // temperature is already below floor. - const Real dedt_initial = DeDt_wrapper(0.0, internal_e_initial, dedt_valid); - if (dedt_initial == 0.0 || internal_e_initial <= internal_e_floor) { - return; - } - - // Use minumum subcycle timestep when d_e_tol == 0 - if (d_e_tol == 0) { - sub_dt = min_sub_dt; - } - - unsigned int sub_iter = 0; - // check for dedt != 0.0 required in case cooling floor it hit during subcycling - while ((sub_t * (1 + KEpsilon_) < dt) && - (DeDt_wrapper(sub_t, internal_e, dedt_valid) != 0.0)) { - - if (sub_iter > max_iter) { - // Due to sub_dt >= min_dt, this error should never happen - PARTHENON_FAIL( - "FATAL ERROR in [TabularCooling::SubcyclingFixedIntSrcTerm]: Sub " - "cycles exceed max_iter (This should be impossible)"); - } - - // Next higher order estimate - Real internal_e_next_h; - // Error in estimate of higher order - Real d_e_err; - // Number of attempts on this subcycle - unsigned int sub_attempt = 0; - // Whether to reattempt this subcycle - bool reattempt_sub = true; - do { - // Next lower order estimate - Real internal_e_next_l; - // Do one dual order RK step - dedt_valid = true; - RKStepper::Step(sub_t, sub_dt, internal_e, DeDt_wrapper, internal_e_next_h, - internal_e_next_l, dedt_valid); - - sub_attempt++; - - if (!dedt_valid) { - if (sub_dt == min_sub_dt) { - // Cooling is so fast that even the minimum subcycle dt would lead to - // negative internal energy -- so just cool to the floor of the cooling - // table - sub_dt = (dt - sub_t); - internal_e_next_h = internal_e_floor; - reattempt_sub = false; - } else { - reattempt_sub = true; - sub_dt = min_sub_dt; - } - } else { - - // Compute error - d_e_err = fabs((internal_e_next_h - internal_e_next_l) / internal_e_next_h); - - reattempt_sub = false; - // Accepting or reattempting the subcycle: - // - // -If the error is small, accept the subcycle - // - // -If the error on the subcycle is too high, compute a new time - // step to reattempt the subcycle - // -But if the new time step is smaller than the minimum subcycle - // time step (total step duration/ max iterations), just use the - // minimum subcycle time step instead - - if (std::isnan(d_e_err)) { - reattempt_sub = true; - sub_dt = min_sub_dt; - } else if (d_e_err >= d_e_tol && sub_dt > min_sub_dt) { - // Reattempt this subcycle - reattempt_sub = true; - // Error was too high, shrink the timestep - if (d_e_tol == 0) { - sub_dt = min_sub_dt; - } else { - sub_dt = RKStepper::OptimalStep(sub_dt, d_e_err, d_e_tol); - } - // Don't drop timestep under maximum iteration count - if (sub_dt < min_sub_dt || sub_attempt >= max_iter) { - sub_dt = min_sub_dt; - } - } - } - - } while (reattempt_sub); - // Accept this subcycle - sub_t += sub_dt; - - internal_e = internal_e_next_h; - - // skip to the end of subcycling if error is 0 (very unlikely) - if (d_e_err == 0) { - sub_dt = dt - sub_t; - } else { - // Grow the timestep - // (or shrink in case d_e_err >= d_e_tol and sub_dt is already at min_sub_dt) - sub_dt = RKStepper::OptimalStep(sub_dt, d_e_err, d_e_tol); - } - - if (d_e_tol == 0) { - sub_dt = min_sub_dt; - } - - // Don't drop timestep under the minimum step size - sub_dt = std::max(sub_dt, min_sub_dt); - - // Limit by end time - sub_dt = std::min(sub_dt, dt - sub_t); - - sub_iter++; - } - - // If cooled below floor, reset to floor value. - // This could happen if the floor value is larger than the lower end of the - // cooling table or if they are close and the last subcycle in the cooling above - // the lower end pushed the temperature below the lower end (and the floor). - internal_e = (internal_e > internal_e_floor) ? internal_e : internal_e_floor; - - // Remove the cooling from the total energy density - cons(IEN, k, j, i) += rho * (internal_e - internal_e_initial); - // Latter technically not required if no other tasks follows before - // ConservedToPrim conversion, but keeping it for now (better safe than sorry). - prim(IPR, k, j, i) = rho * internal_e * gm1; - }); -} - -void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, - const parthenon::Real dt_) const { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - const bool mhd_enabled = hydro_pkg->Param("fluid") == Fluid::glmmhd; - - // Grab member variables for compiler - const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - - const auto units = hydro_pkg->Param("units"); - const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - const Real X_by_mh2 = - std::pow((1 - hydro_pkg->Param("He_mass_fraction")) / units.mh(), 2); - - const auto lambdas = lambdas_; - const auto temps = temps_; - const auto alpha_k = townsend_alpha_k_; - const auto Y_k = townsend_Y_k_; - - const auto internal_e_floor = T_floor_ / mbar_gm1_over_kb; - const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - // need to include ghost zones as this source is called prior to the other fluxes when - // split - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - - const auto nbins = alpha_k.extent_int(0); - - // Get reference values - const auto temp_final = std::pow(10.0, log_temp_final_); - const auto lambda_final = lambda_final_; - - par_for( - DEFAULT_LOOP_PATTERN, "TabularCooling::TownsendSrcTerm", DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { - auto &cons = cons_pack(b); - auto &prim = prim_pack(b); - // Need to use `cons` here as prim may still contain state at t_0; - const auto rho = cons(IDN, k, j, i); - // TODO(pgrete) with potentially more EOS, a separate get_pressure (or similar) - // function could be useful. - auto internal_e = - cons(IEN, k, j, i) - 0.5 * - (SQR(cons(IM1, k, j, i)) + SQR(cons(IM2, k, j, i)) + - SQR(cons(IM3, k, j, i))) / - rho; - if (mhd_enabled) { - internal_e -= 0.5 * (SQR(cons(IB1, k, j, i)) + SQR(cons(IB2, k, j, i)) + - SQR(cons(IB3, k, j, i))); - } - internal_e /= rho; - - // If temp is below floor, reset and return - if (internal_e <= internal_e_floor) { - // Remove the cooling from the total energy density - cons(IEN, k, j, i) += rho * (internal_e_floor - internal_e); - // Latter technically not required if no other tasks follows before - // ConservedToPrim conversion, but keeping it for now (better safe than sorry). - prim(IPR, k, j, i) = rho * internal_e_floor * gm1; - return; - } - - auto temp = mbar_gm1_over_kb * internal_e; - // Temperature is above floor (see conditional above) but below cooling table: - // -> no cooling - if (temp < temp_cool_floor) { - return; - } - const Real n_h2_by_rho = rho * X_by_mh2; - - // Get the index of the right temperature bin - // TODO(?) this could be optimized for using a binary search - auto idx = 0; - while ((idx < nbins - 1) && (temps(idx + 1) < temp)) { - idx += 1; - } - - // Compute the Temporal Evolution Function Y(T) (Eq. A5) - const auto alpha_k_m1 = alpha_k(idx) - 1.0; - const auto tef = - Y_k(idx) + (lambda_final / lambdas(idx)) * (temps(idx) / temp_final) * - (std::pow(temps(idx) / temp, alpha_k_m1) - 1.0) / alpha_k_m1; - - // Compute the adjusted TEF for new timestep (Eqn. 26) (term in brackets) - const auto tef_adj = - tef + lambda_final * dt / temp_final * mbar_gm1_over_kb * n_h2_by_rho; - - // TEF is a strictly decreasing function and new_tef > tef - // Check if the new TEF falls into a lower bin, i.e., find the right bin for A7 - // If so, update slopes and coefficients - while ((idx > 0) && (tef_adj > Y_k(idx))) { - idx -= 1; - } - - // Compute the Inverse Temporal Evolution Function Y^{-1}(Y) (Eq. A7) - const auto temp_new = - temps(idx) * - std::pow(1 - (1.0 - alpha_k(idx)) * (lambdas(idx) / lambda_final) * - (temp_final / temps(idx)) * (tef_adj - Y_k(idx)), - 1.0 / (1.0 - alpha_k(idx))); - // Set new temp (at the lowest to the lower end of the cooling table) - const auto internal_e_new = temp_new > temp_cool_floor - ? temp_new / mbar_gm1_over_kb - : temp_cool_floor / mbar_gm1_over_kb; - cons(IEN, k, j, i) += rho * (internal_e_new - internal_e); - // Latter technically not required if no other tasks follows before - // ConservedToPrim conversion, but keeping it for now (better safe than sorry). - prim(IPR, k, j, i) = rho * internal_e_new * gm1; - }); -} - -Real TabularCooling::EstimateTimeStep(MeshData *md) const { - if (cooling_time_cfl_ <= 0.0) { - return std::numeric_limits::max(); - } - - if (isnan(cooling_time_cfl_) || isinf(cooling_time_cfl_)) { - return std::numeric_limits::infinity(); - } - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const CoolingTableObj cooling_table_obj = cooling_table_obj_; - const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; - - // Determine the cooling floor, whichever is higher of the cooling table floor - // or fluid solver floor - const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - - const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. - - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - - Real min_cooling_time = std::numeric_limits::infinity(); - Kokkos::Min reducer_min(min_cooling_time); - - Kokkos::parallel_reduce( - "TabularCooling::TimeStep", - Kokkos::MDRangePolicy>( - {0, kb.s, jb.s, ib.s}, {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, - {1, 1, 1, ib.e + 1 - ib.s}), - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, - Real &thread_min_cooling_time) { - auto &prim = prim_pack(b); - - const Real rho = prim(IDN, k, j, i); - const Real pres = prim(IPR, k, j, i); - - const Real internal_e = pres / (rho * gm1); - - const Real de_dt = cooling_table_obj.DeDt(internal_e, rho); - - // Compute cooling time - // If de_dt is zero (temperature is smaller than lower end of cooling table) or - // current temp is below floor, use infinite cooling time - const Real cooling_time = ((de_dt == 0) || (internal_e < internal_e_floor)) - ? std::numeric_limits::infinity() - : fabs(internal_e / de_dt); - - thread_min_cooling_time = std::min(cooling_time, thread_min_cooling_time); - }, - reducer_min); - - return cooling_time_cfl_ * min_cooling_time; -} - -void TabularCooling::TestCoolingTable(ParameterInput *pin) const { - - const std::string test_filename = pin->GetString("cooling", "test_filename"); - - const auto rho0 = pin->GetReal("cooling", "test_rho0"); - const auto rho1 = pin->GetReal("cooling", "test_rho1"); - const auto n_rho = pin->GetInteger("cooling", "test_n_rho"); - - const auto pres0 = pin->GetReal("cooling", "test_pres0"); - const auto pres1 = pin->GetReal("cooling", "test_pres1"); - const auto n_pres = pin->GetInteger("cooling", "test_n_pres"); - - // Grab member variables for compiler - - // Everything needed by DeDt - const CoolingTableObj cooling_table_obj = cooling_table_obj_; - const auto gm1 = pin->GetReal("hydro", "gamma") - 1.0; - - // Make some device arrays to store the test data - ParArray2D d_rho("d_rho", n_rho, n_pres), d_pres("d_pres", n_rho, n_pres), - d_internal_e("d_internal_e", n_rho, n_pres), d_de_dt("d_de_dt", n_rho, n_pres); - - par_for( - loop_pattern_mdrange_tag, "TabularCooling::TestCoolingTable", DevExecSpace(), 0, - n_rho - 1, 0, n_pres - 1, KOKKOS_LAMBDA(const int &j, const int &i) { - const Real rho = rho0 * pow(rho1 / rho0, static_cast(j) / (n_rho - 1)); - const Real pres = pres0 * pow(pres1 / pres0, static_cast(i) / (n_pres - 1)); - - d_rho(j, i) = rho; - d_pres(j, i) = pres; - - const Real internal_e = pres / (rho * gm1); - - d_internal_e(j, i) = internal_e; - - const Real de_dt = cooling_table_obj.DeDt(internal_e, rho); - - d_de_dt(j, i) = de_dt; - }); - - // Copy Device arrays to host - auto h_rho = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_rho); - auto h_pres = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_pres); - auto h_internal_e = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_internal_e); - auto h_de_dt = Kokkos::create_mirror_view_and_copy(HostMemSpace(), d_de_dt); - - // Write to file - std::ofstream file(test_filename); - file << "#rho pres internal_e de_dt" << std::endl; - for (int j = 0; j < n_rho; j++) { - for (int i = 0; i < n_pres; i++) { - file << h_rho(j, i) << " " << h_pres(j, i) << " " << h_internal_e(j, i) << " " - << h_de_dt(j, i) << " " << std::endl; - } - } -} - -} // namespace cooling diff --git a/src/pgen/cluster/cluster_gravity_original.hpp b/src/pgen/cluster/cluster_gravity_original.hpp deleted file mode 100644 index 54192d63..00000000 --- a/src/pgen/cluster/cluster_gravity_original.hpp +++ /dev/null @@ -1,240 +0,0 @@ -#ifndef CLUSTER_CLUSTER_GRAVITY_HPP_ -#define CLUSTER_CLUSTER_GRAVITY_HPP_ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster_gravity.hpp -// \brief Class for defining gravitational acceleration for a cluster+bcg+smbh - -// Parthenon headers -#include - -// AthenaPK headers -#include "../../units.hpp" - -namespace cluster { - -// Types of BCG's -enum class BCG { NONE, HERNQUIST }; -// Hernquiest BCG: Hernquist 1990 DOI:10.1086/168845 - -/************************************************************ - * Cluster Gravity Class, for computing gravitational acceleration - * Lightweight object for inlined computation within kernels - ************************************************************/ -class ClusterGravity { - - // Parameters for which gravity sources to include - bool include_nfw_g_; - BCG which_bcg_g_; - bool include_smbh_g_; - - // NFW Parameters - parthenon::Real r_nfw_s_; - // G , Mass, and Constants rolled into one - parthenon::Real g_const_nfw_; - parthenon::Real rho_const_nfw_; - - // BCG Parameters - parthenon::Real alpha_bcg_s_; - parthenon::Real beta_bcg_s_; - parthenon::Real r_bcg_s_; - // G , Mass, and Constants rolled into one - parthenon::Real g_const_bcg_; - parthenon::Real rho_const_bcg_; - - // SMBH Parameters - // G , Mass, and Constants rolled into one - parthenon::Real g_const_smbh_; - - // Radius underwhich to truncate - parthenon::Real smoothing_r_; - - // Static Helper functions to calculate constants to minimize in-kernel work - // ie. compute the constants characterizing the profile (M_nfw, etc...) - - static parthenon::Real calc_R_nfw_s(const parthenon::Real rho_crit, - const parthenon::Real m_nfw_200, - const parthenon::Real c_nfw) { - const parthenon::Real rho_nfw_0 = - 200 / 3. * rho_crit * pow(c_nfw, 3.) / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); - const parthenon::Real R_nfw_s = - pow(m_nfw_200 / (4 * M_PI * rho_nfw_0 * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))), - 1. / 3.); - return R_nfw_s; - } - static parthenon::Real calc_g_const_nfw(const parthenon::Real gravitational_constant, - const parthenon::Real m_nfw_200, - const parthenon::Real c_nfw) { - return gravitational_constant * m_nfw_200 / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); - } - static parthenon::Real calc_rho_const_nfw(const parthenon::Real gravitational_constant, - const parthenon::Real m_nfw_200, - const parthenon::Real c_nfw) { - return m_nfw_200 / (4 * M_PI * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))); - } - static parthenon::Real calc_g_const_bcg(const parthenon::Real gravitational_constant, - BCG which_bcg_g, const parthenon::Real m_bcg_s, - const parthenon::Real r_bcg_s, - const parthenon::Real alpha_bcg_s, - const parthenon::Real beta_bcg_s) { - switch (which_bcg_g) { - case BCG::NONE: - return 0; - case BCG::HERNQUIST: - return gravitational_constant * m_bcg_s / (r_bcg_s * r_bcg_s); - } - return NAN; - } - static parthenon::Real calc_rho_const_bcg(const parthenon::Real gravitational_constant, - BCG which_bcg_g, - const parthenon::Real m_bcg_s, - const parthenon::Real r_bcg_s, - const parthenon::Real alpha_bcg_s, - const parthenon::Real beta_bcg_s) { - switch (which_bcg_g) { - case BCG::NONE: - return 0; - case BCG::HERNQUIST: - return m_bcg_s * r_bcg_s / (2 * M_PI); - } - return NAN; - } - static KOKKOS_INLINE_FUNCTION parthenon::Real - calc_g_const_smbh(const parthenon::Real gravitational_constant, - const parthenon::Real m_smbh) { - return gravitational_constant * m_smbh; - } - - public: - // ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) - // is called from cluster.cpp to add the ClusterGravity object to hydro_pkg - // - // ClusterGravity(parthenon::ParameterInput *pin) is used in SNIAFeedback to - // calculate the BCG density profile - ClusterGravity(parthenon::ParameterInput *pin) { - Units units(pin); - - // Determine which element to include - include_nfw_g_ = - pin->GetOrAddBoolean("problem/cluster/gravity", "include_nfw_g", false); - const std::string which_bcg_g_str = - pin->GetOrAddString("problem/cluster/gravity", "which_bcg_g", "NONE"); - if (which_bcg_g_str == "NONE") { - which_bcg_g_ = BCG::NONE; - } else if (which_bcg_g_str == "HERNQUIST") { - which_bcg_g_ = BCG::HERNQUIST; - } else { - std::stringstream msg; - msg << "### FATAL ERROR in function [InitUserMeshData]" << std::endl - << "Unknown BCG type " << which_bcg_g_str << std::endl; - PARTHENON_FAIL(msg); - } - - include_smbh_g_ = - pin->GetOrAddBoolean("problem/cluster/gravity", "include_smbh_g", false); - - // Initialize the NFW Profile - const parthenon::Real hubble_parameter = pin->GetOrAddReal( - "problem/cluster", "hubble_parameter", 70 * units.km_s() / units.mpc()); - const parthenon::Real rho_crit = 3 * hubble_parameter * hubble_parameter / - (8 * M_PI * units.gravitational_constant()); - - const parthenon::Real M_nfw_200 = - pin->GetOrAddReal("problem/cluster/gravity", "m_nfw_200", 8.5e14 * units.msun()); - const parthenon::Real c_nfw = - pin->GetOrAddReal("problem/cluster/gravity", "c_nfw", 6.81); - r_nfw_s_ = calc_R_nfw_s(rho_crit, M_nfw_200, c_nfw); - g_const_nfw_ = calc_g_const_nfw(units.gravitational_constant(), M_nfw_200, c_nfw); - - // Initialize the BCG Profile - alpha_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "alpha_bcg_s", 0.1); - beta_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "beta_bcg_s", 1.43); - const parthenon::Real M_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - r_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - g_const_bcg_ = calc_g_const_bcg(units.gravitational_constant(), which_bcg_g_, M_bcg_s, - r_bcg_s_, alpha_bcg_s_, beta_bcg_s_); - - const parthenon::Real m_smbh = - pin->GetOrAddReal("problem/cluster/gravity", "m_smbh", 3.4e8 * units.msun()); - g_const_smbh_ = calc_g_const_smbh(units.gravitational_constant(), m_smbh), - - smoothing_r_ = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - } - - ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) - : ClusterGravity(pin) { - hydro_pkg->AddParam<>("cluster_gravity", *this); - } - - // Inline functions to compute gravitational acceleration - KOKKOS_INLINE_FUNCTION parthenon::Real g_from_r(const parthenon::Real r_in) const - __attribute__((always_inline)) { - - const parthenon::Real r = std::max(r_in, smoothing_r_); - const parthenon::Real r2 = r * r; - - parthenon::Real g_r = 0; - - // Add NFW gravity - if (include_nfw_g_) { - g_r += g_const_nfw_ * (log(1 + r / r_nfw_s_) - r / (r + r_nfw_s_)) / r2; - } - - // Add BCG gravity - switch (which_bcg_g_) { - case BCG::NONE: - break; - case BCG::HERNQUIST: - g_r += g_const_bcg_ / ((1 + r / r_bcg_s_) * (1 + r / r_bcg_s_)); - break; - } - - // Add SMBH, point mass gravity - if (include_smbh_g_) { - g_r += g_const_smbh_ / r2; - } - - return g_r; - } - // Inline functions to compute density - KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r_in) const - __attribute__((always_inline)) { - - const parthenon::Real r = std::max(r_in, smoothing_r_); - - parthenon::Real rho = 0; - - // Add NFW gravity - if (include_nfw_g_) { - rho += rho_const_nfw_ / (r * pow(r + r_nfw_s_, 2)); - } - - // Add BCG gravity - switch (which_bcg_g_) { - case BCG::NONE: - break; - case BCG::HERNQUIST: - rho += rho_const_bcg_ / (r * pow(r + r_bcg_s_, 3)); - break; - } - - // SMBH, point mass gravity -- density is not defined. Throw an error - if (include_smbh_g_ && r <= smoothing_r_) { - Kokkos::abort("ClusterGravity::SMBH density is not defined"); - } - - return rho; - } - - // SNIAFeedback needs to be a friend to disable the SMBH and NFW - friend class SNIAFeedback; -}; - -} // namespace cluster - -#endif // CLUSTER_CLUSTER_GRAVITY_HPP_ diff --git a/src/pgen/cluster/cluster_gravity_working.hpp b/src/pgen/cluster/cluster_gravity_working.hpp deleted file mode 100644 index 0e988134..00000000 --- a/src/pgen/cluster/cluster_gravity_working.hpp +++ /dev/null @@ -1,266 +0,0 @@ -#ifndef CLUSTER_CLUSTER_GRAVITY_HPP_ -#define CLUSTER_CLUSTER_GRAVITY_HPP_ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster_gravity.hpp -// \brief Class for defining gravitational acceleration for a cluster+bcg+smbh - -// Parthenon headers -#include - -// AthenaPK headers -#include "../../units.hpp" - -namespace cluster { - -// Types of BCG's -enum class BCG { NONE, HERNQUIST, ISOTHERMAL }; -// Hernquiest BCG: Hernquist 1990 DOI:10.1086/168845 - -/************************************************************ - * Cluster Gravity Class, for computing gravitational acceleration - * Lightweight object for inlined computation within kernels - ************************************************************/ -class ClusterGravity { - - // Parameters for which gravity sources to include - bool include_nfw_g_; - BCG which_bcg_g_; - bool include_smbh_g_; - - // NFW Parameters - parthenon::Real r_nfw_s_; - // G , Mass, and Constants rolled into one - parthenon::Real gravitationnal_const_; - parthenon::Real g_const_nfw_; - parthenon::Real rho_const_nfw_; - - // BCG Parameters - parthenon::Real alpha_bcg_s_; - parthenon::Real beta_bcg_s_; - parthenon::Real r_bcg_s_; - parthenon::Real T_bcg_s_; - // G , Mass, and Constants rolled into one - parthenon::Real g_const_bcg_; - parthenon::Real rho_const_bcg_; - - // SMBH Parameters - // G , Mass, and Constants rolled into one - parthenon::Real g_const_smbh_; - - // Radius underwhich to truncate - parthenon::Real smoothing_r_; - - // Static Helper functions to calculate constants to minimize in-kernel work - // ie. compute the constants characterizing the profile (M_nfw, etc...) - - static parthenon::Real calc_R_nfw_s(const parthenon::Real rho_crit, - const parthenon::Real m_nfw_200, - const parthenon::Real c_nfw) { - const parthenon::Real rho_nfw_0 = - 200 / 3. * rho_crit * pow(c_nfw, 3.) / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); - const parthenon::Real R_nfw_s = - pow(m_nfw_200 / (4 * M_PI * rho_nfw_0 * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))), - 1. / 3.); - return R_nfw_s; - } - static parthenon::Real calc_g_const_nfw(const parthenon::Real gravitational_constant, - const parthenon::Real m_nfw_200, - const parthenon::Real c_nfw) { - return gravitational_constant * m_nfw_200 / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); - } - static parthenon::Real calc_rho_const_nfw(const parthenon::Real gravitational_constant, - const parthenon::Real m_nfw_200, - const parthenon::Real c_nfw) { - return m_nfw_200 / (4 * M_PI * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))); - } - static parthenon::Real calc_g_const_bcg(const parthenon::Real gravitational_constant, - BCG which_bcg_g, const parthenon::Real m_bcg_s, - const parthenon::Real r_bcg_s, - const parthenon::Real alpha_bcg_s, - const parthenon::Real beta_bcg_s) { - switch (which_bcg_g) { - case BCG::NONE: - return 0; - case BCG::HERNQUIST: - return gravitational_constant * m_bcg_s / (r_bcg_s * r_bcg_s); - } - return NAN; - } - static parthenon::Real calc_rho_const_bcg(const parthenon::Real gravitational_constant, - BCG which_bcg_g, - const parthenon::Real m_bcg_s, - const parthenon::Real r_bcg_s, - const parthenon::Real alpha_bcg_s, - const parthenon::Real beta_bcg_s) { - switch (which_bcg_g) { - case BCG::NONE: - return 0; - case BCG::HERNQUIST: - return m_bcg_s * r_bcg_s / (2 * M_PI); - } - return NAN; - } - static KOKKOS_INLINE_FUNCTION parthenon::Real - calc_g_const_smbh(const parthenon::Real gravitational_constant, - const parthenon::Real m_smbh) { - return gravitational_constant * m_smbh; - } - - public: - // ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) - // is called from cluster.cpp to add the ClusterGravity object to hydro_pkg - // - // ClusterGravity(parthenon::ParameterInput *pin) is used in SNIAFeedback to - // calculate the BCG density profile - ClusterGravity(parthenon::ParameterInput *pin) { - Units units(pin); - - gravitationnal_const_ = units.gravitational_constant(); - - // Determine which element to include - include_nfw_g_ = - pin->GetOrAddBoolean("problem/cluster/gravity", "include_nfw_g", false); - const std::string which_bcg_g_str = - pin->GetOrAddString("problem/cluster/gravity", "which_bcg_g", "NONE"); - if (which_bcg_g_str == "NONE") { - which_bcg_g_ = BCG::NONE; - } else if (which_bcg_g_str == "HERNQUIST") { - which_bcg_g_ = BCG::HERNQUIST; - } else if (which_bcg_g_str == "ISOTHERMAL") { - which_bcg_g_ = BCG::ISOTHERMAL; - } - - else { - std::stringstream msg; - msg << "### FATAL ERROR in function [InitUserMeshData]" << std::endl - << "Unknown BCG type " << which_bcg_g_str << std::endl; - PARTHENON_FAIL(msg); - } - - include_smbh_g_ = - pin->GetOrAddBoolean("problem/cluster/gravity", "include_smbh_g", false); - - // Initialize the NFW Profile - const parthenon::Real hubble_parameter = pin->GetOrAddReal( - "problem/cluster", "hubble_parameter", 70 * units.km_s() / units.mpc()); - const parthenon::Real rho_crit = 3 * hubble_parameter * hubble_parameter / - (8 * M_PI * units.gravitational_constant()); - - const auto He_mass_fraction = pin->GetReal("hydro", "He_mass_fraction"); - const auto mu = 1 / (He_mass_fraction * 3. / 4. + (1 - He_mass_fraction) * 2); - const parthenon::Real M_nfw_200 = - pin->GetOrAddReal("problem/cluster/gravity", "m_nfw_200", 8.5e14 * units.msun()); - const parthenon::Real c_nfw = - pin->GetOrAddReal("problem/cluster/gravity", "c_nfw", 6.81); - r_nfw_s_ = calc_R_nfw_s(rho_crit, M_nfw_200, c_nfw); - g_const_nfw_ = calc_g_const_nfw(units.gravitational_constant(), M_nfw_200, c_nfw); - - // Initialize the BCG Profile - alpha_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "alpha_bcg_s", 0.1); - beta_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "beta_bcg_s", 1.43); - const parthenon::Real M_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - r_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - T_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", - 10000); // Temperature in the isothermal case - - if (which_bcg_g_str == "ISOTHERMAL") { - - g_const_bcg_ = 2 * units.k_boltzmann() * T_bcg_s_ / (mu * units.mh()); - - } - - else { - - g_const_bcg_ = calc_g_const_bcg(units.gravitational_constant(), which_bcg_g_, - M_bcg_s, r_bcg_s_, alpha_bcg_s_, beta_bcg_s_); - } - - const parthenon::Real m_smbh = - pin->GetOrAddReal("problem/cluster/gravity", "m_smbh", 3.4e8 * units.msun()); - g_const_smbh_ = calc_g_const_smbh(units.gravitational_constant(), m_smbh), - - smoothing_r_ = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - } - - ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) - : ClusterGravity(pin) { - hydro_pkg->AddParam<>("cluster_gravity", *this); - } - - // Inline functions to compute gravitational acceleration - KOKKOS_INLINE_FUNCTION parthenon::Real g_from_r(const parthenon::Real r_in) const - __attribute__((always_inline)) { - - const parthenon::Real r = std::max(r_in, smoothing_r_); - const parthenon::Real r2 = r * r; - - parthenon::Real g_r = 0; - - // Add NFW gravity - if (include_nfw_g_) { - g_r += g_const_nfw_ * (log(1 + r / r_nfw_s_) - r / (r + r_nfw_s_)) / r2; - } - - // Add BCG gravity - switch (which_bcg_g_) { - case BCG::NONE: - break; - case BCG::HERNQUIST: - g_r += g_const_bcg_ / ((1 + r / r_bcg_s_) * (1 + r / r_bcg_s_)); - case BCG::ISOTHERMAL: - g_r += g_const_bcg_ / r; - break; - } - - // Add SMBH, point mass gravity - if (include_smbh_g_) { - g_r += g_const_smbh_ / r2; - } - - return g_r; - } - // Inline functions to compute density - KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r_in) const - __attribute__((always_inline)) { - - const parthenon::Real r = std::max(r_in, smoothing_r_); - - parthenon::Real rho = 0; - - // Add NFW gravity - if (include_nfw_g_) { - rho += rho_const_nfw_ / (r * pow(r + r_nfw_s_, 2)); - } - - // Add BCG gravity - switch (which_bcg_g_) { - case BCG::NONE: - break; - case BCG::HERNQUIST: - rho += rho_const_bcg_ / (r * pow(r + r_bcg_s_, 3)); - break; - case BCG::ISOTHERMAL: - rho += g_const_bcg_ * 1 / (4 * M_PI * gravitationnal_const_ * r * r); - } - - // SMBH, point mass gravity -- density is not defined. Throw an error - if (include_smbh_g_ && r <= smoothing_r_) { - Kokkos::abort("ClusterGravity::SMBH density is not defined"); - } - - return rho; - } - - // SNIAFeedback needs to be a friend to disable the SMBH and NFW - friend class SNIAFeedback; -}; - -} // namespace cluster - -#endif // CLUSTER_CLUSTER_GRAVITY_HPP_ diff --git a/src/pgen/cluster/entropy_isothermal_profiles.hpp b/src/pgen/cluster/entropy_isothermal_profiles.hpp deleted file mode 100644 index db254fc4..00000000 --- a/src/pgen/cluster/entropy_isothermal_profiles.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#ifndef CLUSTER_ENTROPY_PROFILES_HPP_ -#define CLUSTER_ENTROPY_PROFILES_HPP_ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file entropy profiles.hpp -// \brief Classes defining initial entropy profile - -// Parthenon headers -#include - -// AthenaPK headers -#include "../../units.hpp" - -namespace cluster { - -template -class ISOEntropyProfile { - - public: - // Entropy Profile - parthenon::T_isothermal_; - - ISOEntropyProfile(parthenon::ParameterInput *pin, - GravitationalField gravitational_field) { - - Units units(pin); - } - - // Get entropy from radius, using broken power law profile for entropy - KOKKOS_INLINE_FUNCTION parthenon::Real K_from_r(const parthenon::Real r) const { - - const parthenon::Real k = 1; - return k; - } -}; - -} // namespace cluster - -#endif // CLUSTER_ENTROPY_PROFILES_HPP_ diff --git a/src/pgen/old_cluster/cluster_16nov.cpp b/src/pgen/old_cluster/cluster_16nov.cpp deleted file mode 100644 index 13a079a9..00000000 --- a/src/pgen/old_cluster/cluster_16nov.cpp +++ /dev/null @@ -1,1415 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() - -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" -#include "../utils/few_modes_ft_lognormal.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_clips.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/cluster_reductions.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "cluster/stellar_feedback.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; -using utils::few_modes_ft_log::FewModesFTLog; - -void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); -}; -void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); - stellar_feedback.FeedbackSrcTerm(md, dt, tm); - - ApplyClusterClips(md, tm, dt); -} - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Create hydrostatic sphere with ACCEPT entropy profile - ACCEPTEntropyProfile entropy_profile(pin); - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - // Create hydrostatic sphere with ISOTHERMAL entropy profile - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Stellar Feedback - ************************************************************/ - - StellarFeedback stellar_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Start running reductions into history outputs for clips, stellar mass, cold - * gas, and AGN extent - ************************************************************/ - - /* FIXME(forrestglines) This implementation with a reduction into Params might - be broken in several ways. - 1. Each reduction in params is Rank local. Multiple meshblocks packs per - rank adding to these params is not thread-safe - 2. These Params are not carried over between restarts. If a restart dump is - made and a history output is not, then the mass/energy between the last - history output and the restart dump is lost - */ - std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", - "removed_eceil_energy", "removed_vceil_energy", - "added_vAceil_mass"}; - - // Add a param for each reduction, then add it as a summation reduction for - // history outputs - auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - - for (auto reduction_str : reduction_strs) { - hydro_pkg->AddParam(reduction_str, 0.0, true); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, - [reduction_str](MeshData *md) { - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - const Real reduction = hydro_pkg->Param(reduction_str); - // Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str, 0.0); - return reduction; - }, - reduction_str)); - } - - // Add history reduction for total cold gas using stellar mass threshold - const Real cold_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if (cold_thresh > 0) { - hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); - } - const Real agn_tracer_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if (agn_tracer_thresh >= 0) { - PARTHENON_REQUIRE( - pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); - hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); - } - - hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", - 0.0); // Mean density of perturbations - - if (mu_rho != 0.0) { - - auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", - "k_min_rho"); // Minimum wavenumber of perturbation - auto num_modes_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); - auto sol_weight_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); - uint32_t rseed_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); - - // Computing the kmax ie. the Nyquist limit - auto grid_ni = pin->GetOrAddInteger( - "parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis - auto k_max_rho = grid_ni / 2; - - const auto t_corr_rho = 1e-10; - auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog( - num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes - - auto few_modes_ft_rho = - FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, k_vec_rho, - k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); - - hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); - auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); - - PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || - (k_peak_v > 0.0 && l_peak_v <= 0.0), - "Setting initial velocity perturbation requires a single " - "length scale by either setting l_peak_v or k_peak_v."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_v > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_v = Lx / l_peak_v; - } - - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Magnetic field perturbation - ************************************************************/ - - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init magnetic field perturb - auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); - auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); - PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || - (k_peak_b > 0.0 && l_peak_b <= 0.0), - "Setting initial B perturbation requires a single " - "length scale by either setting l_peak_b or k_peak_b."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_b > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_b = Lx / l_peak_b; - } - - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - - // Defining a table within which the values of the hydrostatic density profile will be - // stored - auto pmc = md->GetBlockData(0)->GetBlockPointer(); - const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); - - // Here, the pmc-> contains the number of cells of each MESHBLOCK, including the - // 2*n_ghost ghost cells - parthenon::ParArray4D hydrostatic_rho( - "hydrostatic_rho", md->NumBlocks(), pmc->cellbounds.ncellsk(IndexDomain::entire), - pmc->cellbounds.ncellsj(IndexDomain::entire), - pmc->cellbounds.ncellsi(IndexDomain::entire)); - - for (int b = 0; b < md->NumBlocks(); b++) { - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - const auto isothermal_sphere = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); - const auto isothermal_hernquist = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); - - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else if (isothermal_sphere) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real mu = hydro_pkg->Param("mu"); - const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); - const Real grav_const = units.gravitational_constant(); - - std::cout << "Entering isothermal sphere generation \n"; - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real rho_r = - prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); - const Real P_r = (prefactor / 2) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } else if (isothermal_hernquist) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real m_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - const Real r_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - const Real mu = hydro_pkg->Param("mu"); - const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); - const Real grav_const = units.gravitational_constant(); - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real phi_r = -grav_const * m_bcg_s / (r_effective + r_bcg_s); - const Real rho_r = rho_0 * std::exp(-mu * units.mh() / - (units.k_boltzmann() * T_bcg_s) * phi_r); - const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } - - else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - - // Updating hydrostatic_rho table - hydrostatic_rho(b, k, j, i) = rho_r; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - // To check whether there is some component before initiating perturbations - std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations (read from HDF5 file) - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool spherical_collapse = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - Real passive_scalar = 0.0; // Not useful here - - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - // Create an homogeneous sphere of a density superior or inferior to the background - // density - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal( - "problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to - // foreground_density - if (r < overdensity_radius) { - u(IDN, k, j, i) = foreground_density; - } - }, - passive_scalar); - } - - /* -------------- Setting up a clumpy atmosphere -------------- - - 1) Extract the values of the density from an input hdf5 file using H5Easy - 2) Initiate the associated density field - 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) - 4) Optionnaly, add a central overdensity - - */ - - if (init_perturb_rho == true) { - - auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", - "init_perturb_rho_file", "none"); - - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real perturb_amplitude = - pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); - - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - - const int rho_init_size = 256; - auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>( - file, keys_rho); - - Real passive_scalar = 0.0; // Useless - - std::cout << "Entering initialisation of rho field"; - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Case where the box is filled with perturbations of equal mean amplitude - if (full_box) { - - u(IDN, k, j, i) += perturb_amplitude * - rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * - (u(IDN, k, j, i) / 29.6); - - } - - // Mean amplitude of the perturbations is modulated by the - else { - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); - const Real r_effective = std::max(r, r_smoothing); - - // u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; - } - }, - passive_scalar); - } - - /************************************************************ - * Setting up a perturbed density field (hardcoded version) - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); - const auto background_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); - - if (mu_rho != 0.0) { - - auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); - - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft_rho.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); - - Real v2_sum_rho = 0.0; // used for normalization - - auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - Real perturb = 0.0; - - perturb = std::sqrt(SQR(perturb_pack_rho(b, 0, k, j, i)) + - SQR(perturb_pack_rho(b, 1, k, j, i)) + - SQR(perturb_pack_rho(b, 2, k, j, i))); - u(IDN, k, j, i) = background_rho + mu_rho * perturb; - }, - v2_sum_rho); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - const auto alpha_b = - pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0 / 3.0); - const auto density_scale = - pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); - const auto standard_B = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); - const auto r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); - - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - // Defining a new table that will contains the values of the perturbed magnetic field, - // and magnetic energy - parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, - pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - if (standard_B) { - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && - u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - - } else { - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - auto pmbl = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkgl = pmbl->packages.Get("Hydro"); - - const Real x = coords.Xc<1>(i); - const Real xp = coords.Xc<1>(i + 1); - const Real xm = coords.Xc<1>(i - 1); - const Real y = coords.Xc<1>(j); - const Real yp = coords.Xc<1>(j + 1); - const Real ym = coords.Xc<1>(j - 1); - const Real z = coords.Xc<1>(k); - const Real zp = coords.Xc<1>(k + 1); - const Real zm = coords.Xc<1>(k - 1); - - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); - const Real r_ip = std::sqrt(SQR(xp) + SQR(y) + SQR(z)); - const Real r_im = std::sqrt(SQR(xm) + SQR(y) + SQR(z)); - const Real r_jp = std::sqrt(SQR(x) + SQR(yp) + SQR(z)); - const Real r_jm = std::sqrt(SQR(x) + SQR(ym) + SQR(z)); - const Real r_kp = std::sqrt(SQR(x) + SQR(y) + SQR(zp)); - const Real r_km = std::sqrt(SQR(x) + SQR(y) + SQR(zm)); - - // Re-generating the hydrostatic sphere - const auto &he_sphere = hydro_pkgl->Param< - HydrostaticEquilibriumSphere>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = - he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - const auto rho_central = P_rho_profile.rho_from_r(r_smoothing); - const auto rho_ip = P_rho_profile.rho_from_r(r_ip); - const auto rho_im = P_rho_profile.rho_from_r(r_im); - const auto rho_jp = P_rho_profile.rho_from_r(r_jp); - const auto rho_jm = P_rho_profile.rho_from_r(r_jm); - const auto rho_kp = P_rho_profile.rho_from_r(r_kp); - const auto rho_km = P_rho_profile.rho_from_r(r_km); - - // const Real rho_r = P_rho_profile.rho_from_r(r); - - // Normalization condition here, which we want to lift. To do so, we'll need - // to copy the values of the magnetic field into a new array, which we will - // use to store the perturbed magnetic field and then rescale it, until values - // are added to u = cons(b) - - // PARTHENON_REQUIRE( - // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == - // 0.0, "Found existing non-zero B when setting magnetic field - // perturbations."); - - // First, needs to rescale the perturb_pack, ie. the magnetic field potential - Real perturb2_k_jp_i, perturb2_k_jm_i, perturb1_kp_j_i, perturb1_km_j_i; - Real perturb0_kp_j_i, perturb0_km_j_i, perturb2_k_j_ip, perturb2_k_j_im; - Real perturb1_k_j_ip, perturb1_k_j_im, perturb0_k_jp_i, perturb0_k_jm_i; - - // Rescaling the magnetic potential - - perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i) * std::pow(rho_jp, alpha_b); - perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i) * std::pow(rho_jm, alpha_b); - perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i) * std::pow(rho_kp, alpha_b); - perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i) * std::pow(rho_km, alpha_b); - perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i) * std::pow(rho_kp, alpha_b); - perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i) * std::pow(rho_km, alpha_b); - perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1) * std::pow(rho_ip, alpha_b); - perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1) * std::pow(rho_im, alpha_b); - perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1) * std::pow(rho_ip, alpha_b); - perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1) * std::pow(rho_im, alpha_b); - perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i) * std::pow(rho_jp, alpha_b); - perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i) * std::pow(rho_jm, alpha_b); - - // Then, compute the curl of the magnetic field - - Real curlBx, curlBy, curlBz; - - curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - - (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0; - curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - - (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0; - curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - - (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0; - - dB(b, 0, k, j, i) = curlBx; - dB(b, 1, k, j, i) = curlBy; - dB(b, 2, k, j, i) = curlBz; - - lsum += - (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); - - /* - gradBx = (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 ; - - gradBy = (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 ; - - gradBz = (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 ; - - dB(b,0, k, j, i) = gradBx * std::pow(hydrostatic_rho(b,k,j,i) / 29.6, - alpha_b); dB(b,1, k, j, i) = gradBy * std::pow(hydrostatic_rho(b,k,j,i) - / 29.6, alpha_b); dB(b,2, k, j, i) = gradBz * - std::pow(hydrostatic_rho(b,k,j,i) / 29.6, alpha_b); - - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(gradBx) + SQR(gradBy) + SQR(gradBz)) * - coords.CellVolume(k, j, i); - */ - }, - b2_sum); - } - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - if (standard_B) { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - } else { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - dB(b, 0, k, j, i) /= b_norm; - dB(b, 1, k, j, i) /= b_norm; - dB(b, 2, k, j, i) /= b_norm; - - // Computing energy - dB(b, 3, k, j, i) += 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + - SQR(dB(b, 2, k, j, i))); - - // Updating the MHD vector - - u(IB1, k, j, i) += dB(b, 0, k, j, i); - u(IB2, k, j, i) += dB(b, 1, k, j, i); - u(IB3, k, j, i) += dB(b, 2, k, j, i); - u(IEN, k, j, i) += dB(b, 3, k, j, i); - }); - } - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_17nov.cpp b/src/pgen/old_cluster/cluster_17nov.cpp deleted file mode 100644 index 97b75d81..00000000 --- a/src/pgen/old_cluster/cluster_17nov.cpp +++ /dev/null @@ -1,1427 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() - -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" -#include "../utils/few_modes_ft_lognormal.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_clips.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/cluster_reductions.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "cluster/stellar_feedback.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; -using utils::few_modes_ft_log::FewModesFTLog; - -void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); -}; -void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); - stellar_feedback.FeedbackSrcTerm(md, dt, tm); - - ApplyClusterClips(md, tm, dt); -} - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Create hydrostatic sphere with ACCEPT entropy profile - ACCEPTEntropyProfile entropy_profile(pin); - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - // Create hydrostatic sphere with ISOTHERMAL entropy profile - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Stellar Feedback - ************************************************************/ - - StellarFeedback stellar_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Start running reductions into history outputs for clips, stellar mass, cold - * gas, and AGN extent - ************************************************************/ - - /* FIXME(forrestglines) This implementation with a reduction into Params might - be broken in several ways. - 1. Each reduction in params is Rank local. Multiple meshblocks packs per - rank adding to these params is not thread-safe - 2. These Params are not carried over between restarts. If a restart dump is - made and a history output is not, then the mass/energy between the last - history output and the restart dump is lost - */ - std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", - "removed_eceil_energy", "removed_vceil_energy", - "added_vAceil_mass"}; - - // Add a param for each reduction, then add it as a summation reduction for - // history outputs - auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - - for (auto reduction_str : reduction_strs) { - hydro_pkg->AddParam(reduction_str, 0.0, true); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, - [reduction_str](MeshData *md) { - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - const Real reduction = hydro_pkg->Param(reduction_str); - // Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str, 0.0); - return reduction; - }, - reduction_str)); - } - - // Add history reduction for total cold gas using stellar mass threshold - const Real cold_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if (cold_thresh > 0) { - hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); - } - const Real agn_tracer_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if (agn_tracer_thresh >= 0) { - PARTHENON_REQUIRE( - pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); - hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); - } - - hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", - 0.0); // Mean density of perturbations - - if (mu_rho != 0.0) { - - auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", - "k_min_rho"); // Minimum wavenumber of perturbation - auto num_modes_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); - auto sol_weight_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); - uint32_t rseed_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); - - // Computing the kmax ie. the Nyquist limit - auto grid_ni = pin->GetOrAddInteger( - "parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis - auto k_max_rho = grid_ni / 2; - - const auto t_corr_rho = 1e-10; - auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog( - num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes - - auto few_modes_ft_rho = - FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, k_vec_rho, - k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); - - hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); - auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); - - PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || - (k_peak_v > 0.0 && l_peak_v <= 0.0), - "Setting initial velocity perturbation requires a single " - "length scale by either setting l_peak_v or k_peak_v."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_v > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_v = Lx / l_peak_v; - } - - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Magnetic field perturbation - ************************************************************/ - - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init magnetic field perturb - auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); - auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); - PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || - (k_peak_b > 0.0 && l_peak_b <= 0.0), - "Setting initial B perturbation requires a single " - "length scale by either setting l_peak_b or k_peak_b."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_b > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_b = Lx / l_peak_b; - } - - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - - // Defining a table within which the values of the hydrostatic density profile will be - // stored - auto pmc = md->GetBlockData(0)->GetBlockPointer(); - const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); - - // Here, the pmc-> contains the number of cells of each MESHBLOCK, including the - // 2*n_ghost ghost cells - parthenon::ParArray4D hydrostatic_rho( - "hydrostatic_rho", md->NumBlocks(), pmc->cellbounds.ncellsk(IndexDomain::entire), - pmc->cellbounds.ncellsj(IndexDomain::entire), - pmc->cellbounds.ncellsi(IndexDomain::entire)); - - for (int b = 0; b < md->NumBlocks(); b++) { - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - const auto isothermal_sphere = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); - const auto isothermal_hernquist = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); - - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else if (isothermal_sphere) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real mu = hydro_pkg->Param("mu"); - const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); - const Real grav_const = units.gravitational_constant(); - - std::cout << "Entering isothermal sphere generation \n"; - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real rho_r = - prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); - const Real P_r = (prefactor / 2) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } else if (isothermal_hernquist) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real m_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - const Real r_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - const Real mu = hydro_pkg->Param("mu"); - const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); - const Real grav_const = units.gravitational_constant(); - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real phi_r = -grav_const * m_bcg_s / (r_effective + r_bcg_s); - const Real rho_r = rho_0 * std::exp(-mu * units.mh() / - (units.k_boltzmann() * T_bcg_s) * phi_r); - const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } - - else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - - // Updating hydrostatic_rho table - hydrostatic_rho(b, k, j, i) = rho_r; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - // To check whether there is some component before initiating perturbations - std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations (read from HDF5 file) - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool spherical_collapse = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - Real passive_scalar = 0.0; // Not useful here - - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - // Create an homogeneous sphere of a density superior or inferior to the background - // density - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal( - "problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to - // foreground_density - if (r < overdensity_radius) { - u(IDN, k, j, i) = foreground_density; - } - }, - passive_scalar); - } - - /* -------------- Setting up a clumpy atmosphere -------------- - - 1) Extract the values of the density from an input hdf5 file using H5Easy - 2) Initiate the associated density field - 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) - 4) Optionnaly, add a central overdensity - - */ - - if (init_perturb_rho == true) { - - auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", - "init_perturb_rho_file", "none"); - - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real perturb_amplitude = - pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); - - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - - const int rho_init_size = 256; - auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>( - file, keys_rho); - - Real passive_scalar = 0.0; // Useless - - std::cout << "Entering initialisation of rho field"; - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Case where the box is filled with perturbations of equal mean amplitude - if (full_box) { - - u(IDN, k, j, i) += perturb_amplitude * - rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * - (u(IDN, k, j, i) / 29.6); - - } - - // Mean amplitude of the perturbations is modulated by the - else { - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); - const Real r_effective = std::max(r, r_smoothing); - - // u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; - } - }, - passive_scalar); - } - - /************************************************************ - * Setting up a perturbed density field (hardcoded version) - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); - const auto background_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); - - if (mu_rho != 0.0) { - - auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); - - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft_rho.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); - - Real v2_sum_rho = 0.0; // used for normalization - - auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - Real perturb = 0.0; - - perturb = std::sqrt(SQR(perturb_pack_rho(b, 0, k, j, i)) + - SQR(perturb_pack_rho(b, 1, k, j, i)) + - SQR(perturb_pack_rho(b, 2, k, j, i))); - u(IDN, k, j, i) = background_rho + mu_rho * perturb; - }, - v2_sum_rho); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - const auto alpha_b = - pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0 / 3.0); - const auto density_scale = - pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); - const auto standard_B = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); - const auto r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); - const auto rho_type = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rho_type", 0); // test - - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - // Defining a new table that will contains the values of the perturbed magnetic field, - // and magnetic energy - parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, - pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - if (standard_B) { - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && - u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - - } else { - - // Idea: separate into two loops, one par_for and one par_reduce - - std::cout << "Setting up perturbed magnetic field." << std::endl; - - for (int b = 0; b < md->NumBlocks(); b++) { - - std::cout << "Treating block number " << b << " out of " << md->NumBlocks() - << std::endl; - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Computation of the P_rho_profile require entire domain of the block (boundary - // conditions for curl computation) - IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - auto &coords = pmb->coords; - - const auto &he_sphere = hydro_pkg->Param< - HydrostaticEquilibriumSphere>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = - he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::MagneticPerturbations", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real x = coords.Xc<1>(i); - const Real xp = coords.Xc<1>(i + 1); - const Real xm = coords.Xc<1>(i - 1); - const Real y = coords.Xc<2>(j); - const Real yp = coords.Xc<2>(j + 1); - const Real ym = coords.Xc<2>(j - 1); - const Real z = coords.Xc<3>(k); - const Real zp = coords.Xc<3>(k + 1); - const Real zm = coords.Xc<3>(k - 1); - - const Real r = sqrt(SQR(x) + SQR(y) + SQR(z)); - const Real r_ip = sqrt(SQR(xp) + SQR(y) + SQR(z)); - const Real r_im = sqrt(SQR(xm) + SQR(y) + SQR(z)); - const Real r_jp = sqrt(SQR(x) + SQR(yp) + SQR(z)); - const Real r_jm = sqrt(SQR(x) + SQR(ym) + SQR(z)); - const Real r_kp = sqrt(SQR(x) + SQR(y) + SQR(zp)); - const Real r_km = sqrt(SQR(x) + SQR(y) + SQR(zm)); - - const auto rho = P_rho_profile.rho_from_r(r); - const auto rho_ip = P_rho_profile.rho_from_r(r_ip); - const auto rho_im = P_rho_profile.rho_from_r(r_im); - const auto rho_jp = P_rho_profile.rho_from_r(r_jp); - const auto rho_jm = P_rho_profile.rho_from_r(r_jm); - const auto rho_kp = P_rho_profile.rho_from_r(r_kp); - const auto rho_km = P_rho_profile.rho_from_r(r_km); - - // const Real rho_r = P_rho_profile.rho_from_r(r); - - // Normalization condition here, which we want to lift. To do so, we'll need - // to copy the values of the magnetic field into a new array, which we will - // use to store the perturbed magnetic field and then rescale it, until - // values are added to u = cons(b) - - // PARTHENON_REQUIRE( - // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) - // == 0.0, "Found existing non-zero B when setting magnetic field - // perturbations."); - - // First, needs to rescale the perturb_pack, ie. the magnetic field - // potential - Real perturb2_k_jp_i, perturb2_k_jm_i, perturb1_kp_j_i, perturb1_km_j_i; - Real perturb0_kp_j_i, perturb0_km_j_i, perturb2_k_j_ip, perturb2_k_j_im; - Real perturb1_k_j_ip, perturb1_k_j_im, perturb0_k_jp_i, perturb0_k_jm_i; - - // Rescaling the magnetic potential - - perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i) * std::pow(r_jp, alpha_b); - perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i) * std::pow(r_jm, alpha_b); - perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i) * std::pow(r_kp, alpha_b); - perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i) * std::pow(r_km, alpha_b); - perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i) * std::pow(r_kp, alpha_b); - perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i) * std::pow(r_km, alpha_b); - perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1) * std::pow(r_ip, alpha_b); - perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1) * std::pow(r_im, alpha_b); - perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1) * std::pow(r_ip, alpha_b); - perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1) * std::pow(r_im, alpha_b); - perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i) * std::pow(r_jp, alpha_b); - perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i) * std::pow(r_jm, alpha_b); - - // Then, compute the curl of the magnetic field - - Real curlBx, curlBy, curlBz; - - curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - - (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0; - curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - - (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0; - curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - - (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0; - - dB(b, 0, k, j, i) = curlBx; - dB(b, 1, k, j, i) = curlBy; - dB(b, 2, k, j, i) = curlBz; - }); - } - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - - Real curlBx, curlBy, curlBz; - - curlBx = dB(b, 0, k, j, i); - curlBy = dB(b, 1, k, j, i); - curlBz = dB(b, 2, k, j, i); - - lsum += - (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); - }, - b2_sum); - } - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - if (standard_B) { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - } else { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - dB(b, 0, k, j, i) /= b_norm; - dB(b, 1, k, j, i) /= b_norm; - dB(b, 2, k, j, i) /= b_norm; - - // Computing energy - dB(b, 3, k, j, i) += 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + - SQR(dB(b, 2, k, j, i))); - - // Updating the MHD vector - - u(IB1, k, j, i) += dB(b, 0, k, j, i); - u(IB2, k, j, i) += dB(b, 1, k, j, i); - u(IB3, k, j, i) += dB(b, 2, k, j, i); - u(IEN, k, j, i) += dB(b, 3, k, j, i); - }); - } - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_22nov.cpp b/src/pgen/old_cluster/cluster_22nov.cpp deleted file mode 100644 index 64880be7..00000000 --- a/src/pgen/old_cluster/cluster_22nov.cpp +++ /dev/null @@ -1,1324 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() - -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" -#include "../utils/few_modes_ft_lognormal.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_clips.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/cluster_reductions.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "cluster/stellar_feedback.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; -using utils::few_modes_ft_log::FewModesFTLog; - -void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); -}; -void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); - stellar_feedback.FeedbackSrcTerm(md, dt, tm); - - ApplyClusterClips(md, tm, dt); -} - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Create hydrostatic sphere with ACCEPT entropy profile - ACCEPTEntropyProfile entropy_profile(pin); - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - // Create hydrostatic sphere with ISOTHERMAL entropy profile - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Stellar Feedback - ************************************************************/ - - StellarFeedback stellar_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Start running reductions into history outputs for clips, stellar mass, cold - * gas, and AGN extent - ************************************************************/ - - /* FIXME(forrestglines) This implementation with a reduction into Params might - be broken in several ways. - 1. Each reduction in params is Rank local. Multiple meshblocks packs per - rank adding to these params is not thread-safe - 2. These Params are not carried over between restarts. If a restart dump is - made and a history output is not, then the mass/energy between the last - history output and the restart dump is lost - */ - std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", - "removed_eceil_energy", "removed_vceil_energy", - "added_vAceil_mass"}; - - // Add a param for each reduction, then add it as a summation reduction for - // history outputs - auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - - for (auto reduction_str : reduction_strs) { - hydro_pkg->AddParam(reduction_str, 0.0, true); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, - [reduction_str](MeshData *md) { - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - const Real reduction = hydro_pkg->Param(reduction_str); - // Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str, 0.0); - return reduction; - }, - reduction_str)); - } - - // Add history reduction for total cold gas using stellar mass threshold - const Real cold_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if (cold_thresh > 0) { - hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); - } - const Real agn_tracer_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if (agn_tracer_thresh >= 0) { - PARTHENON_REQUIRE( - pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); - hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); - } - - hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", - 0.0); // Mean density of perturbations - - if (mu_rho != 0.0) { - - auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", - "k_min_rho"); // Minimum wavenumber of perturbation - auto num_modes_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); - auto sol_weight_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); - uint32_t rseed_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); - - // Computing the kmax ie. the Nyquist limit - auto grid_ni = pin->GetOrAddInteger( - "parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis - auto k_max_rho = grid_ni / 2; - - const auto t_corr_rho = 1e-10; - auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog( - num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes - - auto few_modes_ft_rho = - FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, k_vec_rho, - k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); - - hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); - auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); - - PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || - (k_peak_v > 0.0 && l_peak_v <= 0.0), - "Setting initial velocity perturbation requires a single " - "length scale by either setting l_peak_v or k_peak_v."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_v > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_v = Lx / l_peak_v; - } - - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Magnetic field perturbation - ************************************************************/ - - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init magnetic field perturb - auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); - auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); - PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || - (k_peak_b > 0.0 && l_peak_b <= 0.0), - "Setting initial B perturbation requires a single " - "length scale by either setting l_peak_b or k_peak_b."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_b > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_b = Lx / l_peak_b; - } - - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - - // Defining a table within which the values of the hydrostatic density profile will be - // stored - auto pmc = md->GetBlockData(0)->GetBlockPointer(); - const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); - - parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho", grid_size + 4, - grid_size + 4, grid_size + 4); - - // parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho", md->NumBlocks(), - // pmc->cellbounds.ncellsk(IndexDomain::entire), - // pmc->cellbounds.ncellsj(IndexDomain::entire), - // pmc->cellbounds.ncellsi(IndexDomain::entire)); - - for (int b = 0; b < md->NumBlocks(); b++) { - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - const auto isothermal_sphere = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); - const auto isothermal_hernquist = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); - - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else if (isothermal_sphere) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real mu = hydro_pkg->Param("mu"); - const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); - const Real grav_const = units.gravitational_constant(); - - std::cout << "Entering isothermal sphere generation \n"; - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real rho_r = - prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); - const Real P_r = (prefactor / 2) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } else if (isothermal_hernquist) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real m_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - const Real r_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - const Real mu = hydro_pkg->Param("mu"); - const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); - const Real grav_const = units.gravitational_constant(); - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real phi_r = -grav_const * m_bcg_s / (r_effective + r_bcg_s); - const Real rho_r = rho_0 * std::exp(-mu * units.mh() / - (units.k_boltzmann() * T_bcg_s) * phi_r); - const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } - - else { - - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = - he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // u(IDN, k, j, i) = rho_r; - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - - // Updating hydrostatic_rho table - hydrostatic_rho(gks + k, gjs + j, gis + i) = rho_r; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - // To check whether there is some component before initiating perturbations - std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations (read from HDF5 file) - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool spherical_collapse = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - Real passive_scalar = 0.0; // Not useful here - - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - // Create an homogeneous sphere of a density superior or inferior to the background - // density - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal( - "problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to - // foreground_density - if (r < overdensity_radius) { - u(IDN, k, j, i) = foreground_density; - } - }, - passive_scalar); - } - - /* -------------- Setting up a clumpy atmosphere -------------- - - 1) Extract the values of the density from an input hdf5 file using H5Easy - 2) Initiate the associated density field - 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) - 4) Optionnaly, add a central overdensity - - */ - - if (init_perturb_rho == true) { - - auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", - "init_perturb_rho_file", "none"); - - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real perturb_amplitude = - pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); - - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - - const int rho_init_size = 260; - auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>( - file, keys_rho); - - Real passive_scalar = 0.0; // Useless - - std::cout << "Entering initialisation of rho field"; - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Case where the box is filled with perturbations of equal mean amplitude - if (full_box) { - - u(IDN, k, j, i) += perturb_amplitude * rho_init[gks + k][gjs + j][gis + i] * - (u(IDN, k, j, i) / 29.6); - hydrostatic_rho(gks + k, gjs + j, gis + i) += - perturb_amplitude * rho_init[gks + k][gjs + j][gis + i] * - (u(IDN, k, j, i) / 29.6); - } - }, - passive_scalar); - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - const auto alpha_b = - pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0 / 3.0); - const auto density_scale = - pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); - const auto standard_B = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); - const auto r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); - - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - // Defining a new table that will contains the values of the perturbed magnetic field, - // and magnetic energy - parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, - pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - // Modifying the magnetic potential so that it follows the rho profile - pmb->par_for( - "Init sigma_b", 0, num_blocks - 1, kb.s - 1, kb.e + 1, jb.s - 1, jb.e + 1, - ib.s - 1, ib.e + 1, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; - - perturb_pack(b, 0, k, j, i) *= u(IDN, k, j, i); - perturb_pack(b, 1, k, j, i) *= u(IDN, k, j, i); - perturb_pack(b, 2, k, j, i) *= u(IDN, k, j, i); - - // perturb_pack(b, 0, k, j, i) *= std::pow(hydrostatic_rho(gks + k, gjs + j, gis - // + i),alpha_b); perturb_pack(b, 1, k, j, i) *= std::pow(hydrostatic_rho(gks + - // k, gjs + j, gis + i),alpha_b); perturb_pack(b, 2, k, j, i) *= - // std::pow(hydrostatic_rho(gks + k, gjs + j, gis + i),alpha_b); - - // perturb_pack(b, 0, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); - // perturb_pack(b, 1, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); - // perturb_pack(b, 2, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); - }); - - if (standard_B) { - - std::cout << "Entering standard B" << std::endl; - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && - u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - - } else { - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // First, needs to rescale the perturb_pack, ie. the magnetic field potential - Real perturb2_k_jp_i, perturb2_k_jm_i, perturb1_kp_j_i, perturb1_km_j_i; - Real perturb0_kp_j_i, perturb0_km_j_i, perturb2_k_j_ip, perturb2_k_j_im; - Real perturb1_k_j_ip, perturb1_k_j_im, perturb0_k_jp_i, perturb0_k_jm_i; - - perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i); - perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i); - perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i); - perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i); - perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i); - perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i); - perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1); - perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1); - perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1); - perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1); - perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i); - perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i); - - // Then, compute the curl of the magnetic field - - Real curlBx, curlBy, curlBz; - - curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - - (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0; - curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - - (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0; - curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - - (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0; - - dB(b, 0, k, j, i) = curlBx; - dB(b, 1, k, j, i) = curlBy; - dB(b, 2, k, j, i) = curlBz; - - lsum += - (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); - }, - b2_sum); - } - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - if (standard_B) { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - } else { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - dB(b, 0, k, j, i) /= b_norm; - dB(b, 1, k, j, i) /= b_norm; - dB(b, 2, k, j, i) /= b_norm; - - // Computing energy - dB(b, 3, k, j, i) += 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + - SQR(dB(b, 2, k, j, i))); - - // Updating the MHD vector - - u(IB1, k, j, i) += dB(b, 0, k, j, i); - u(IB2, k, j, i) += dB(b, 1, k, j, i); - u(IB3, k, j, i) += dB(b, 2, k, j, i); - u(IEN, k, j, i) += dB(b, 3, k, j, i); - }); - } - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_divartifact.cpp b/src/pgen/old_cluster/cluster_divartifact.cpp deleted file mode 100644 index 2d5a1036..00000000 --- a/src/pgen/old_cluster/cluster_divartifact.cpp +++ /dev/null @@ -1,1324 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() - -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" -#include "../utils/few_modes_ft_lognormal.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_clips.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/cluster_reductions.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "cluster/stellar_feedback.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; -using utils::few_modes_ft_log::FewModesFTLog; - -void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); -}; -void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); - stellar_feedback.FeedbackSrcTerm(md, dt, tm); - - ApplyClusterClips(md, tm, dt); -} - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Create hydrostatic sphere with ACCEPT entropy profile - ACCEPTEntropyProfile entropy_profile(pin); - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - // Create hydrostatic sphere with ISOTHERMAL entropy profile - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Stellar Feedback - ************************************************************/ - - StellarFeedback stellar_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Start running reductions into history outputs for clips, stellar mass, cold - * gas, and AGN extent - ************************************************************/ - - /* FIXME(forrestglines) This implementation with a reduction into Params might - be broken in several ways. - 1. Each reduction in params is Rank local. Multiple meshblocks packs per - rank adding to these params is not thread-safe - 2. These Params are not carried over between restarts. If a restart dump is - made and a history output is not, then the mass/energy between the last - history output and the restart dump is lost - */ - std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", - "removed_eceil_energy", "removed_vceil_energy", - "added_vAceil_mass"}; - - // Add a param for each reduction, then add it as a summation reduction for - // history outputs - auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - - for (auto reduction_str : reduction_strs) { - hydro_pkg->AddParam(reduction_str, 0.0, true); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, - [reduction_str](MeshData *md) { - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - const Real reduction = hydro_pkg->Param(reduction_str); - // Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str, 0.0); - return reduction; - }, - reduction_str)); - } - - // Add history reduction for total cold gas using stellar mass threshold - const Real cold_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if (cold_thresh > 0) { - hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); - } - const Real agn_tracer_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if (agn_tracer_thresh >= 0) { - PARTHENON_REQUIRE( - pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); - hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); - } - - hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", - 0.0); // Mean density of perturbations - - if (mu_rho != 0.0) { - - auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", - "k_min_rho"); // Minimum wavenumber of perturbation - auto num_modes_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); - auto sol_weight_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); - uint32_t rseed_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); - - // Computing the kmax ie. the Nyquist limit - auto grid_ni = pin->GetOrAddInteger( - "parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis - auto k_max_rho = grid_ni / 2; - - const auto t_corr_rho = 1e-10; - auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog( - num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes - - auto few_modes_ft_rho = - FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, k_vec_rho, - k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); - - hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); - auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); - - PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || - (k_peak_v > 0.0 && l_peak_v <= 0.0), - "Setting initial velocity perturbation requires a single " - "length scale by either setting l_peak_v or k_peak_v."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_v > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_v = Lx / l_peak_v; - } - - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Magnetic field perturbation - ************************************************************/ - - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init magnetic field perturb - auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); - auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); - PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || - (k_peak_b > 0.0 && l_peak_b <= 0.0), - "Setting initial B perturbation requires a single " - "length scale by either setting l_peak_b or k_peak_b."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_b > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_b = Lx / l_peak_b; - } - - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - - // Defining a table within which the values of the hydrostatic density profile will be - // stored - auto pmc = md->GetBlockData(0)->GetBlockPointer(); - const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); - - // parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho",grid_size + 4,grid_size - // + 4,grid_size + 4); - - parthenon::ParArray4D hydrostatic_rho( - "hydrostatic_rho", md->NumBlocks(), pmc->cellbounds.ncellsk(IndexDomain::entire), - pmc->cellbounds.ncellsj(IndexDomain::entire), - pmc->cellbounds.ncellsi(IndexDomain::entire)); - - for (int b = 0; b < md->NumBlocks(); b++) { - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - const auto isothermal_sphere = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); - const auto isothermal_hernquist = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); - - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else if (isothermal_sphere) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real mu = hydro_pkg->Param("mu"); - const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); - const Real grav_const = units.gravitational_constant(); - - std::cout << "Entering isothermal sphere generation \n"; - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real rho_r = - prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); - const Real P_r = (prefactor / 2) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } else if (isothermal_hernquist) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real m_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - const Real r_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - const Real mu = hydro_pkg->Param("mu"); - const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); - const Real grav_const = units.gravitational_constant(); - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real phi_r = -grav_const * m_bcg_s / (r_effective + r_bcg_s); - const Real rho_r = rho_0 * std::exp(-mu * units.mh() / - (units.k_boltzmann() * T_bcg_s) * phi_r); - const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } - - else { - - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = - he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s - 1, kb.e + 1, jb.s - 1, jb.e + 1, ib.s - 1, - ib.e + 1, KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // u(IDN, k, j, i) = rho_r; - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - - // Updating hydrostatic_rho table - hydrostatic_rho(b, k, j, i) = rho_r; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - // To check whether there is some component before initiating perturbations - std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations (read from HDF5 file) - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool spherical_collapse = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - Real passive_scalar = 0.0; // Not useful here - - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - // Create an homogeneous sphere of a density superior or inferior to the background - // density - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal( - "problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to - // foreground_density - if (r < overdensity_radius) { - u(IDN, k, j, i) = foreground_density; - } - }, - passive_scalar); - } - - /* -------------- Setting up a clumpy atmosphere -------------- - - 1) Extract the values of the density from an input hdf5 file using H5Easy - 2) Initiate the associated density field - 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) - 4) Optionnaly, add a central overdensity - - */ - - if (init_perturb_rho == true) { - - auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", - "init_perturb_rho_file", "none"); - - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real perturb_amplitude = - pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); - - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - - const int rho_init_size = 260; - auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>( - file, keys_rho); - - Real passive_scalar = 0.0; // Useless - - std::cout << "Entering initialisation of rho field"; - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Case where the box is filled with perturbations of equal mean amplitude - if (full_box) { - - u(IDN, k, j, i) += perturb_amplitude * rho_init[gks + k][gjs + j][gis + i] * - (u(IDN, k, j, i) / 29.6); - hydrostatic_rho(gks + k, gjs + j, gis + i) += - perturb_amplitude * rho_init[gks + k][gjs + j][gis + i] * - (u(IDN, k, j, i) / 29.6); - } - }, - passive_scalar); - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - const auto alpha_b = - pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0 / 3.0); - const auto density_scale = - pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); - const auto standard_B = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); - const auto r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); - - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - // Defining a new table that will contains the values of the perturbed magnetic field, - // and magnetic energy - parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, - pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - // Modifying the magnetic potential so that it follows the rho profile - pmb->par_for( - "Init sigma_b", 0, num_blocks - 1, kb.s - 1, kb.e + 1, jb.s - 1, jb.e + 1, - ib.s - 1, ib.e + 1, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; - - perturb_pack(b, 0, k, j, i) *= hydrostatic_rho(b, k, j, i); - perturb_pack(b, 1, k, j, i) *= hydrostatic_rho(b, k, j, i); - perturb_pack(b, 2, k, j, i) *= hydrostatic_rho(b, k, j, i); - - // perturb_pack(b, 0, k, j, i) *= std::pow(hydrostatic_rho(gks + k, gjs + j, gis - // + i),alpha_b); perturb_pack(b, 1, k, j, i) *= std::pow(hydrostatic_rho(gks + - // k, gjs + j, gis + i),alpha_b); perturb_pack(b, 2, k, j, i) *= - // std::pow(hydrostatic_rho(gks + k, gjs + j, gis + i),alpha_b); - - // perturb_pack(b, 0, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); - // perturb_pack(b, 1, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); - // perturb_pack(b, 2, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); - }); - - if (standard_B) { - - std::cout << "Entering standard B" << std::endl; - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && - u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - - } else { - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // First, needs to rescale the perturb_pack, ie. the magnetic field potential - Real perturb2_k_jp_i, perturb2_k_jm_i, perturb1_kp_j_i, perturb1_km_j_i; - Real perturb0_kp_j_i, perturb0_km_j_i, perturb2_k_j_ip, perturb2_k_j_im; - Real perturb1_k_j_ip, perturb1_k_j_im, perturb0_k_jp_i, perturb0_k_jm_i; - - perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i); - perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i); - perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i); - perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i); - perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i); - perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i); - perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1); - perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1); - perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1); - perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1); - perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i); - perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i); - - // Then, compute the curl of the magnetic field - - Real curlBx, curlBy, curlBz; - - curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - - (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0; - curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - - (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0; - curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - - (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0; - - dB(b, 0, k, j, i) = curlBx; - dB(b, 1, k, j, i) = curlBy; - dB(b, 2, k, j, i) = curlBz; - - lsum += - (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); - }, - b2_sum); - } - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - if (standard_B) { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - } else { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - dB(b, 0, k, j, i) /= b_norm; - dB(b, 1, k, j, i) /= b_norm; - dB(b, 2, k, j, i) /= b_norm; - - // Computing energy - dB(b, 3, k, j, i) += 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + - SQR(dB(b, 2, k, j, i))); - - // Updating the MHD vector - - u(IB1, k, j, i) += dB(b, 0, k, j, i); - u(IB2, k, j, i) += dB(b, 1, k, j, i); - u(IB3, k, j, i) += dB(b, 2, k, j, i); - u(IEN, k, j, i) += dB(b, 3, k, j, i); - }); - } - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_modified.cpp b/src/pgen/old_cluster/cluster_modified.cpp deleted file mode 100644 index feca9e8a..00000000 --- a/src/pgen/old_cluster/cluster_modified.cpp +++ /dev/null @@ -1,1020 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback(TODO) -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; - -template -void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt, const EOS eos) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // Apply clips -- ceilings on temperature, velocity, alfven velocity, and - // density floor -- within a radius of the AGN - const auto &dfloor = hydro_pkg->Param("cluster_dfloor"); - const auto &eceil = hydro_pkg->Param("cluster_eceil"); - const auto &vceil = hydro_pkg->Param("cluster_vceil"); - const auto &vAceil = hydro_pkg->Param("cluster_vAceil"); - const auto &clip_r = hydro_pkg->Param("cluster_clip_r"); - - if (clip_r > 0 && (dfloor > 0 || eceil < std::numeric_limits::infinity() || - vceil < std::numeric_limits::infinity() || - vAceil < std::numeric_limits::infinity())) { - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - const auto nhydro = hydro_pkg->Param("nhydro"); - const auto nscalars = hydro_pkg->Param("nscalars"); - - const Real clip_r2 = SQR(clip_r); - const Real vceil2 = SQR(vceil); - const Real vAceil2 = SQR(vAceil); - const Real gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ApplyClusterClips", parthenon::DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { - auto &cons = cons_pack(b); - auto &prim = prim_pack(b); - const auto &coords = cons_pack.GetCoords(b); - - const Real r2 = - SQR(coords.Xc<1>(i)) + SQR(coords.Xc<2>(j)) + SQR(coords.Xc<3>(k)); - - if (r2 < clip_r2) { - // Cell falls within clipping radius - eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); - - if (dfloor > 0) { - const Real rho = prim(IDN, k, j, i); - if (rho < dfloor) { - cons(IDN, k, j, i) = dfloor; - prim(IDN, k, j, i) = dfloor; - } - } - - if (vceil < std::numeric_limits::infinity()) { - // Apply velocity ceiling - const Real v2 = SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + - SQR(prim(IV3, k, j, i)); - if (v2 > vceil2) { - // Fix the velocity to the velocity ceiling - const Real v = sqrt(v2); - cons(IM1, k, j, i) *= vceil / v; - cons(IM2, k, j, i) *= vceil / v; - cons(IM3, k, j, i) *= vceil / v; - prim(IV1, k, j, i) *= vceil / v; - prim(IV2, k, j, i) *= vceil / v; - prim(IV3, k, j, i) *= vceil / v; - - // Remove kinetic energy - cons(IEN, k, j, i) -= 0.5 * prim(IDN, k, j, i) * (v2 - vceil2); - } - } - - if (vAceil2 < std::numeric_limits::infinity()) { - // Apply Alfven velocity ceiling by raising density - const Real rho = prim(IDN, k, j, i); - const Real B2 = (SQR(prim(IB1, k, j, i)) + SQR(prim(IB2, k, j, i)) + - SQR(prim(IB3, k, j, i))); - - // compute Alfven mach number - const Real va2 = (B2 / rho); - - if (va2 > vAceil2) { - // Increase the density to match the alfven velocity ceiling - const Real rho_new = std::sqrt(B2 / vAceil2); - cons(IDN, k, j, i) = rho_new; - prim(IDN, k, j, i) = rho_new; - } - } - - if (eceil < std::numeric_limits::infinity()) { - // Apply internal energy ceiling as a pressure ceiling - const Real internal_e = prim(IPR, k, j, i) / (gm1 * prim(IDN, k, j, i)); - if (internal_e > eceil) { - cons(IEN, k, j, i) -= prim(IDN, k, j, i) * (internal_e - eceil); - prim(IPR, k, j, i) = gm1 * prim(IDN, k, j, i) * eceil; - } - } - } - }); - } -} - -void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - auto fluid = hydro_pkg->Param("fluid"); - if (fluid == Fluid::euler) { - ApplyClusterClips(md, tm, beta_dt, hydro_pkg->Param("eos")); - } else if (fluid == Fluid::glmmhd) { - ApplyClusterClips(md, tm, beta_dt, hydro_pkg->Param("eos")); - } else { - PARTHENON_FAIL("Cluster::ApplyClusterClips: Unknown EOS"); - } -} - -void ClusterSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - ApplyClusterClips(md, tm, beta_dt); -}; - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - // hydro_pkg->AddParam<>("cluster_gravity", cluster_gravity); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Build entropy_profile object - ACCEPTEntropyProfile entropy_profile(pin); - - /************************************************************ - * Build Hydrostatic Equilibrium Sphere - ************************************************************/ - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - // const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", - // "init_perturb_rho", false); hydro_pkg->AddParam<>("init_perturb_rho", - // init_perturb_rho); - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto k_peak_v = pin->GetReal("problem/cluster/init_perturb", "k_peak_v"); - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init vel perturb - auto k_peak_b = pin->GetReal("problem/cluster/init_perturb", "k_peak_b"); - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - std::cout << "Setting uniform gas"; - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - if (init_perturb_rho == true) { - - // File load - - auto init_perturb_rho_file = - pin->GetString("problem/cluster/init_perturb", "init_perturb_rho_file"); - auto init_perturb_rho_keys = - pin->GetString("problem/cluster/init_perturb", "init_perturb_rho_keys"); - - hydro_pkg->AddParam<>("cluster/init_perturb_rho_file", init_perturb_rho_file); - hydro_pkg->AddParam<>("cluster/init_perturb_rho_keys", init_perturb_rho_keys); - - std::cout << "Setting density perturbation"; - - // Read HDF5 file containing the density - std::string filename_rho = "/work/bbd0833/test/rho.h5"; - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - auto rho_init = H5Easy::load, 64>, 64>>( - file, keys_rho); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho_init[k - 2][j - 2][i - 2]; - }); - - /* - auto init_perturb_rho_file = pin->GetString("problem/cluster/init_perturb", - "init_perturb_rho_file"); auto init_perturb_rho_keys = - pin->GetString("problem/cluster/init_perturb", "init_perturb_rho_keys"); - - hydro_pkg->AddParam<>("cluster/init_perturb_rho_file", init_perturb_rho_file); - hydro_pkg->AddParam<>("cluster/init_perturb_rho_keys", init_perturb_rho_keys); - - std::cout << "Setting density perturbation"; - - // Read HDF5 file containing the density - std::string filename_rho = "/work/bbd0833/test/rho.h5"; - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - auto rho_init = H5Easy::load, 64>, - 64>>(file, keys_rho); - - Real passive_scalar = 0.0; // Useless - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Adding the value of the density field - - u(IDN, k, j, i) = rho_init[k-2][j-2][i-2]; - - if (rho_init[k-2][j-2][i-2] < 0.0) { - std::cout << "Negative density for rho_init[" << k-2 << "][" << j-2 << "][" << i-2 << - "]: " << rho_init[k-2][j-2][i-2] << "\n";} - - }, - passive_scalar); - - */ - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); - }); - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_temp.cpp b/src/pgen/old_cluster/cluster_temp.cpp deleted file mode 100644 index 97b75d81..00000000 --- a/src/pgen/old_cluster/cluster_temp.cpp +++ /dev/null @@ -1,1427 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() - -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" -#include "../utils/few_modes_ft_lognormal.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_clips.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/cluster_reductions.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "cluster/stellar_feedback.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; -using utils::few_modes_ft_log::FewModesFTLog; - -void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); -}; -void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); - stellar_feedback.FeedbackSrcTerm(md, dt, tm); - - ApplyClusterClips(md, tm, dt); -} - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Create hydrostatic sphere with ACCEPT entropy profile - ACCEPTEntropyProfile entropy_profile(pin); - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - // Create hydrostatic sphere with ISOTHERMAL entropy profile - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Stellar Feedback - ************************************************************/ - - StellarFeedback stellar_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Start running reductions into history outputs for clips, stellar mass, cold - * gas, and AGN extent - ************************************************************/ - - /* FIXME(forrestglines) This implementation with a reduction into Params might - be broken in several ways. - 1. Each reduction in params is Rank local. Multiple meshblocks packs per - rank adding to these params is not thread-safe - 2. These Params are not carried over between restarts. If a restart dump is - made and a history output is not, then the mass/energy between the last - history output and the restart dump is lost - */ - std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", - "removed_eceil_energy", "removed_vceil_energy", - "added_vAceil_mass"}; - - // Add a param for each reduction, then add it as a summation reduction for - // history outputs - auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); - - for (auto reduction_str : reduction_strs) { - hydro_pkg->AddParam(reduction_str, 0.0, true); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, - [reduction_str](MeshData *md) { - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - const Real reduction = hydro_pkg->Param(reduction_str); - // Reset the running count for this reduction between history outputs - hydro_pkg->UpdateParam(reduction_str, 0.0); - return reduction; - }, - reduction_str)); - } - - // Add history reduction for total cold gas using stellar mass threshold - const Real cold_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); - if (cold_thresh > 0) { - hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); - } - const Real agn_tracer_thresh = - pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); - if (agn_tracer_thresh >= 0) { - PARTHENON_REQUIRE( - pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), - "AGN Tracer must be enabled to reduce AGN tracer extent"); - hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); - hst_vars.emplace_back(parthenon::HistoryOutputVar( - parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); - } - - hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", - 0.0); // Mean density of perturbations - - if (mu_rho != 0.0) { - - auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", - "k_min_rho"); // Minimum wavenumber of perturbation - auto num_modes_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); - auto sol_weight_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); - uint32_t rseed_rho = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); - - // Computing the kmax ie. the Nyquist limit - auto grid_ni = pin->GetOrAddInteger( - "parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis - auto k_max_rho = grid_ni / 2; - - const auto t_corr_rho = 1e-10; - auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog( - num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes - - auto few_modes_ft_rho = - FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, k_vec_rho, - k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); - - hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); - auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); - - PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || - (k_peak_v > 0.0 && l_peak_v <= 0.0), - "Setting initial velocity perturbation requires a single " - "length scale by either setting l_peak_v or k_peak_v."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_v > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_v = Lx / l_peak_v; - } - - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - - /************************************************************ - * Read Magnetic field perturbation - ************************************************************/ - - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init magnetic field perturb - auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); - auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); - PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || - (k_peak_b > 0.0 && l_peak_b <= 0.0), - "Setting initial B perturbation requires a single " - "length scale by either setting l_peak_b or k_peak_b."); - // Set peak wavemode as required by few_modes_fft when not directly given - if (l_peak_b > 0) { - const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - - pin->GetReal("parthenon/mesh", "x1min"); - // Note that this assumes a cubic box - k_peak_b = Lx / l_peak_b; - } - - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - - // Defining a table within which the values of the hydrostatic density profile will be - // stored - auto pmc = md->GetBlockData(0)->GetBlockPointer(); - const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); - - // Here, the pmc-> contains the number of cells of each MESHBLOCK, including the - // 2*n_ghost ghost cells - parthenon::ParArray4D hydrostatic_rho( - "hydrostatic_rho", md->NumBlocks(), pmc->cellbounds.ncellsk(IndexDomain::entire), - pmc->cellbounds.ncellsj(IndexDomain::entire), - pmc->cellbounds.ncellsi(IndexDomain::entire)); - - for (int b = 0; b < md->NumBlocks(); b++) { - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - const auto isothermal_sphere = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); - const auto isothermal_hernquist = - pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); - - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else if (isothermal_sphere) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real mu = hydro_pkg->Param("mu"); - const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); - const Real grav_const = units.gravitational_constant(); - - std::cout << "Entering isothermal sphere generation \n"; - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real rho_r = - prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); - const Real P_r = (prefactor / 2) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } else if (isothermal_hernquist) { - - const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); - const Real m_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); - const Real r_bcg_s = - pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); - const Real mu = hydro_pkg->Param("mu"); - const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); - const Real grav_const = units.gravitational_constant(); - - // Generating the profile - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - const Real r_effective = std::max(r_smoothing, r); - - const Real phi_r = -grav_const * m_bcg_s / (r_effective + r_bcg_s); - const Real rho_r = rho_0 * std::exp(-mu * units.mh() / - (units.k_boltzmann() * T_bcg_s) * phi_r); - const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - - } - - else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - - // Updating hydrostatic_rho table - hydrostatic_rho(b, k, j, i) = rho_r; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - // To check whether there is some component before initiating perturbations - std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations (read from HDF5 file) - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool spherical_collapse = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - Real passive_scalar = 0.0; // Not useful here - - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - // Create an homogeneous sphere of a density superior or inferior to the background - // density - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal( - "problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to - // foreground_density - if (r < overdensity_radius) { - u(IDN, k, j, i) = foreground_density; - } - }, - passive_scalar); - } - - /* -------------- Setting up a clumpy atmosphere -------------- - - 1) Extract the values of the density from an input hdf5 file using H5Easy - 2) Initiate the associated density field - 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) - 4) Optionnaly, add a central overdensity - - */ - - if (init_perturb_rho == true) { - - auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", - "init_perturb_rho_file", "none"); - - const Real r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); - const Real perturb_amplitude = - pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); - - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - - const int rho_init_size = 256; - auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>( - file, keys_rho); - - Real passive_scalar = 0.0; // Useless - - std::cout << "Entering initialisation of rho field"; - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Case where the box is filled with perturbations of equal mean amplitude - if (full_box) { - - u(IDN, k, j, i) += perturb_amplitude * - rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * - (u(IDN, k, j, i) / 29.6); - - } - - // Mean amplitude of the perturbations is modulated by the - else { - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); - const Real r_effective = std::max(r, r_smoothing); - - // u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; - } - }, - passive_scalar); - } - - /************************************************************ - * Setting up a perturbed density field (hardcoded version) - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); - const auto background_rho = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); - - if (mu_rho != 0.0) { - - auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); - - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft_rho.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); - - Real v2_sum_rho = 0.0; // used for normalization - - auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - Real perturb = 0.0; - - perturb = std::sqrt(SQR(perturb_pack_rho(b, 0, k, j, i)) + - SQR(perturb_pack_rho(b, 1, k, j, i)) + - SQR(perturb_pack_rho(b, 2, k, j, i))); - u(IDN, k, j, i) = background_rho + mu_rho * perturb; - }, - v2_sum_rho); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - const auto alpha_b = - pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0 / 3.0); - const auto density_scale = - pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); - const auto standard_B = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); - const auto r_smoothing = - pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); - const auto rho_type = - pin->GetOrAddInteger("problem/cluster/init_perturb", "rho_type", 0); // test - - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - // Defining a new table that will contains the values of the perturbed magnetic field, - // and magnetic energy - parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, - pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - if (standard_B) { - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && - u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - - } else { - - // Idea: separate into two loops, one par_for and one par_reduce - - std::cout << "Setting up perturbed magnetic field." << std::endl; - - for (int b = 0; b < md->NumBlocks(); b++) { - - std::cout << "Treating block number " << b << " out of " << md->NumBlocks() - << std::endl; - - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - auto units = hydro_pkg->Param("units"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Computation of the P_rho_profile require entire domain of the block (boundary - // conditions for curl computation) - IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - auto &coords = pmb->coords; - - const auto &he_sphere = hydro_pkg->Param< - HydrostaticEquilibriumSphere>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = - he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::MagneticPerturbations", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real x = coords.Xc<1>(i); - const Real xp = coords.Xc<1>(i + 1); - const Real xm = coords.Xc<1>(i - 1); - const Real y = coords.Xc<2>(j); - const Real yp = coords.Xc<2>(j + 1); - const Real ym = coords.Xc<2>(j - 1); - const Real z = coords.Xc<3>(k); - const Real zp = coords.Xc<3>(k + 1); - const Real zm = coords.Xc<3>(k - 1); - - const Real r = sqrt(SQR(x) + SQR(y) + SQR(z)); - const Real r_ip = sqrt(SQR(xp) + SQR(y) + SQR(z)); - const Real r_im = sqrt(SQR(xm) + SQR(y) + SQR(z)); - const Real r_jp = sqrt(SQR(x) + SQR(yp) + SQR(z)); - const Real r_jm = sqrt(SQR(x) + SQR(ym) + SQR(z)); - const Real r_kp = sqrt(SQR(x) + SQR(y) + SQR(zp)); - const Real r_km = sqrt(SQR(x) + SQR(y) + SQR(zm)); - - const auto rho = P_rho_profile.rho_from_r(r); - const auto rho_ip = P_rho_profile.rho_from_r(r_ip); - const auto rho_im = P_rho_profile.rho_from_r(r_im); - const auto rho_jp = P_rho_profile.rho_from_r(r_jp); - const auto rho_jm = P_rho_profile.rho_from_r(r_jm); - const auto rho_kp = P_rho_profile.rho_from_r(r_kp); - const auto rho_km = P_rho_profile.rho_from_r(r_km); - - // const Real rho_r = P_rho_profile.rho_from_r(r); - - // Normalization condition here, which we want to lift. To do so, we'll need - // to copy the values of the magnetic field into a new array, which we will - // use to store the perturbed magnetic field and then rescale it, until - // values are added to u = cons(b) - - // PARTHENON_REQUIRE( - // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) - // == 0.0, "Found existing non-zero B when setting magnetic field - // perturbations."); - - // First, needs to rescale the perturb_pack, ie. the magnetic field - // potential - Real perturb2_k_jp_i, perturb2_k_jm_i, perturb1_kp_j_i, perturb1_km_j_i; - Real perturb0_kp_j_i, perturb0_km_j_i, perturb2_k_j_ip, perturb2_k_j_im; - Real perturb1_k_j_ip, perturb1_k_j_im, perturb0_k_jp_i, perturb0_k_jm_i; - - // Rescaling the magnetic potential - - perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i) * std::pow(r_jp, alpha_b); - perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i) * std::pow(r_jm, alpha_b); - perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i) * std::pow(r_kp, alpha_b); - perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i) * std::pow(r_km, alpha_b); - perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i) * std::pow(r_kp, alpha_b); - perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i) * std::pow(r_km, alpha_b); - perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1) * std::pow(r_ip, alpha_b); - perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1) * std::pow(r_im, alpha_b); - perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1) * std::pow(r_ip, alpha_b); - perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1) * std::pow(r_im, alpha_b); - perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i) * std::pow(r_jp, alpha_b); - perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i) * std::pow(r_jm, alpha_b); - - // Then, compute the curl of the magnetic field - - Real curlBx, curlBy, curlBz; - - curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - - (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0; - curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - - (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0; - curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - - (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0; - - dB(b, 0, k, j, i) = curlBx; - dB(b, 1, k, j, i) = curlBy; - dB(b, 2, k, j, i) = curlBz; - }); - } - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - - Real curlBx, curlBy, curlBz; - - curlBx = dB(b, 0, k, j, i); - curlBy = dB(b, 1, k, j, i); - curlBz = dB(b, 2, k, j, i); - - lsum += - (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); - }, - b2_sum); - } - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - if (standard_B) { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - } else { - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - dB(b, 0, k, j, i) /= b_norm; - dB(b, 1, k, j, i) /= b_norm; - dB(b, 2, k, j, i) /= b_norm; - - // Computing energy - dB(b, 3, k, j, i) += 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + - SQR(dB(b, 2, k, j, i))); - - // Updating the MHD vector - - u(IB1, k, j, i) += dB(b, 0, k, j, i); - u(IB2, k, j, i) += dB(b, 1, k, j, i); - u(IB3, k, j, i) += dB(b, 2, k, j, i); - u(IEN, k, j, i) += dB(b, 3, k, j, i); - }); - } - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_working.cpp b/src/pgen/old_cluster/cluster_working.cpp deleted file mode 100644 index 3a2d1618..00000000 --- a/src/pgen/old_cluster/cluster_working.cpp +++ /dev/null @@ -1,1127 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//! \file cluster.cpp -// \brief Idealized galaxy cluster problem generator -// -// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in -// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes AGN feedback, AGN -// triggering via cold gas, simple SNIA Feedback(TODO) -//======================================================================================== - -// C headers - -// C++ headers -#include // min, max -#include // sqrt() -#include // fopen(), fprintf(), freopen() -#include // endl -#include -#include // stringstream -#include // runtime_error -#include // c_str() - -// #include // HDF5 -#include "../../external/HighFive/include/highfive/H5Easy.hpp" - -// Parthenon headers -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/mesh.hpp" -#include -#include - -// AthenaPK headers -#include "../eos/adiabatic_glmmhd.hpp" -#include "../eos/adiabatic_hydro.hpp" -#include "../hydro/hydro.hpp" -#include "../hydro/srcterms/gravitational_field.hpp" -#include "../hydro/srcterms/tabular_cooling.hpp" -#include "../main.hpp" -#include "../utils/few_modes_ft.hpp" - -// Cluster headers -#include "cluster/agn_feedback.hpp" -#include "cluster/agn_triggering.hpp" -#include "cluster/cluster_gravity.hpp" -#include "cluster/entropy_profiles.hpp" -#include "cluster/hydrostatic_equilibrium_sphere.hpp" -#include "cluster/magnetic_tower.hpp" -#include "cluster/snia_feedback.hpp" -#include "parthenon_array_generic.hpp" -#include "utils/error_checking.hpp" - -namespace cluster { -using namespace parthenon::driver::prelude; -using namespace parthenon::package::prelude; -using utils::few_modes_ft::FewModesFT; - -template -void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt, const EOS eos) { - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // Apply clips -- ceilings on temperature, velocity, alfven velocity, and - // density floor -- within a radius of the AGN - const auto &dfloor = hydro_pkg->Param("cluster_dfloor"); - const auto &eceil = hydro_pkg->Param("cluster_eceil"); - const auto &vceil = hydro_pkg->Param("cluster_vceil"); - const auto &vAceil = hydro_pkg->Param("cluster_vAceil"); - const auto &clip_r = hydro_pkg->Param("cluster_clip_r"); - - if (clip_r > 0 && (dfloor > 0 || eceil < std::numeric_limits::infinity() || - vceil < std::numeric_limits::infinity() || - vAceil < std::numeric_limits::infinity())) { - // Grab some necessary variables - const auto &prim_pack = md->PackVariables(std::vector{"prim"}); - const auto &cons_pack = md->PackVariables(std::vector{"cons"}); - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - const auto nhydro = hydro_pkg->Param("nhydro"); - const auto nscalars = hydro_pkg->Param("nscalars"); - - const Real clip_r2 = SQR(clip_r); - const Real vceil2 = SQR(vceil); - const Real vAceil2 = SQR(vAceil); - const Real gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ApplyClusterClips", parthenon::DevExecSpace(), 0, - cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { - auto &cons = cons_pack(b); - auto &prim = prim_pack(b); - const auto &coords = cons_pack.GetCoords(b); - - const Real r2 = - SQR(coords.Xc<1>(i)) + SQR(coords.Xc<2>(j)) + SQR(coords.Xc<3>(k)); - - if (r2 < clip_r2) { - // Cell falls within clipping radius - - const int gks = 0; - const int gjs = 0; - const int gis = 0; - - eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i, gks, gjs, - gis); // Three last parameters are just passive here - - if (dfloor > 0) { - const Real rho = prim(IDN, k, j, i); - if (rho < dfloor) { - cons(IDN, k, j, i) = dfloor; - prim(IDN, k, j, i) = dfloor; - } - } - - if (vceil < std::numeric_limits::infinity()) { - // Apply velocity ceiling - const Real v2 = SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + - SQR(prim(IV3, k, j, i)); - if (v2 > vceil2) { - // Fix the velocity to the velocity ceiling - const Real v = sqrt(v2); - cons(IM1, k, j, i) *= vceil / v; - cons(IM2, k, j, i) *= vceil / v; - cons(IM3, k, j, i) *= vceil / v; - prim(IV1, k, j, i) *= vceil / v; - prim(IV2, k, j, i) *= vceil / v; - prim(IV3, k, j, i) *= vceil / v; - - // Remove kinetic energy - cons(IEN, k, j, i) -= 0.5 * prim(IDN, k, j, i) * (v2 - vceil2); - } - } - - if (vAceil2 < std::numeric_limits::infinity()) { - // Apply Alfven velocity ceiling by raising density - const Real rho = prim(IDN, k, j, i); - const Real B2 = (SQR(prim(IB1, k, j, i)) + SQR(prim(IB2, k, j, i)) + - SQR(prim(IB3, k, j, i))); - - // compute Alfven mach number - const Real va2 = (B2 / rho); - - if (va2 > vAceil2) { - // Increase the density to match the alfven velocity ceiling - const Real rho_new = std::sqrt(B2 / vAceil2); - cons(IDN, k, j, i) = rho_new; - prim(IDN, k, j, i) = rho_new; - } - } - - if (eceil < std::numeric_limits::infinity()) { - // Apply internal energy ceiling as a pressure ceiling - const Real internal_e = prim(IPR, k, j, i) / (gm1 * prim(IDN, k, j, i)); - if (internal_e > eceil) { - cons(IEN, k, j, i) -= prim(IDN, k, j, i) * (internal_e - eceil); - prim(IPR, k, j, i) = gm1 * prim(IDN, k, j, i) * eceil; - } - } - } - }); - } -} - -void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - auto fluid = hydro_pkg->Param("fluid"); - if (fluid == Fluid::euler) { - ApplyClusterClips(md, tm, beta_dt, hydro_pkg->Param("eos")); - } else if (fluid == Fluid::glmmhd) { - ApplyClusterClips(md, tm, beta_dt, hydro_pkg->Param("eos")); - } else { - PARTHENON_FAIL("Cluster::ApplyClusterClips: Unknown EOS"); - } -} - -void ClusterSrcTerm(MeshData *md, const parthenon::SimTime &tm, - const Real beta_dt) { - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); - - if (gravity_srcterm) { - const ClusterGravity &cluster_gravity = - hydro_pkg->Param("cluster_gravity"); - - GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); - } - - const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); - agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); - - const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); - snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); - - ApplyClusterClips(md, tm, beta_dt); -}; - -Real ClusterEstimateTimestep(MeshData *md) { - Real min_dt = std::numeric_limits::max(); - - auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // TODO time constraints imposed by thermal AGN feedback, jet velocity, - // magnetic tower - const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); - const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); - min_dt = std::min(min_dt, agn_triggering_min_dt); - - return min_dt; -} - -//======================================================================================== -//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor -//! *hydro_pkg) \brief Init package data from parameter input -//======================================================================================== - -void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - - /************************************************************ - * Read Uniform Gas - ************************************************************/ - - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); - - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_uniform_b_field = pin->GetOrAddBoolean( - "problem/cluster/uniform_b_field", "init_uniform_b_field", false); - hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); - - if (init_uniform_b_field) { - const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); - const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); - const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); - - hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); - hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); - hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); - } - - /************************************************************ - * Read Uniform Magnetic Field - ************************************************************/ - - const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", - "init_dipole_b_field", false); - hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); - - if (init_dipole_b_field) { - const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); - const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); - const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); - - hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); - hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); - hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); - } - - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ - - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin, hydro_pkg); - // hydro_pkg->AddParam<>("cluster_gravity", cluster_gravity); - - // Include gravity as a source term during evolution - const bool gravity_srcterm = - pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ - - // Build entropy_profile object - ACCEPTEntropyProfile entropy_profile(pin); - - /************************************************************ - * Build Hydrostatic Equilibrium Sphere - ************************************************************/ - - HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, - entropy_profile); - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - - JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - - /************************************************************ - * Read AGN Feedback - ************************************************************/ - - AGNFeedback agn_feedback(pin, hydro_pkg); - - /************************************************************ - * Read AGN Triggering - ************************************************************/ - - AGNTriggering agn_triggering(pin, hydro_pkg); - - /************************************************************ - * Read Magnetic Tower - ************************************************************/ - - // Build Magnetic Tower - MagneticTower magnetic_tower(pin, hydro_pkg); - - // Determine if magnetic_tower_power_scaling is needed - // Is AGN Power and Magnetic fraction non-zero? - bool magnetic_tower_power_scaling = - (agn_feedback.magnetic_fraction_ != 0 && - (agn_feedback.fixed_power_ != 0 || - agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); - hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); - - /************************************************************ - * Read SNIA Feedback - ************************************************************/ - - SNIAFeedback snia_feedback(pin, hydro_pkg); - - /************************************************************ - * Read Clips (ceilings and floors) - ************************************************************/ - - // Disable all clips by default with a negative radius clip - Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); - - // By default disable floors by setting a negative value - Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); - - // By default disable ceilings by setting to infinity - Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", - std::numeric_limits::infinity()); - Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", - std::numeric_limits::infinity()); - Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", - std::numeric_limits::infinity()); - Real eceil = Tceil; - if (eceil < std::numeric_limits::infinity()) { - if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { - PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " - "Either set a 'units' block and the 'hydro/He_mass_fraction' in " - "input file or use a pressure floor " - "(defined code units) instead."); - } - auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); - eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); - } - hydro_pkg->AddParam("cluster_dfloor", dfloor); - hydro_pkg->AddParam("cluster_eceil", eceil); - hydro_pkg->AddParam("cluster_vceil", vceil); - hydro_pkg->AddParam("cluster_vAceil", vAceil); - hydro_pkg->AddParam("cluster_clip_r", clip_r); - - /************************************************************ - * Add derived fields - * NOTE: these must be filled in UserWorkBeforeOutput - ************************************************************/ - - auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - - // log10 of cell-centered radius - hydro_pkg->AddField("log10_cell_radius", m); - // entropy - hydro_pkg->AddField("entropy", m); - // sonic Mach number v/c_s - hydro_pkg->AddField("mach_sonic", m); - // temperature - hydro_pkg->AddField("temperature", m); - - if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { - // cooling time - hydro_pkg->AddField("cooling_time", m); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - // alfven Mach number v_A/c_s - hydro_pkg->AddField("mach_alfven", m); - - // plasma beta - hydro_pkg->AddField("plasma_beta", m); - } - - /************************************************************ - * Read Density perturbation - ************************************************************/ - - // const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", - // "init_perturb_rho", false); hydro_pkg->AddParam<>("init_perturb_rho", - // init_perturb_rho); - - /************************************************************ - * Read Velocity perturbation - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - if (sigma_v != 0.0) { - // peak of init vel perturb - auto k_peak_v = pin->GetReal("problem/cluster/init_perturb", "k_peak_v"); - auto num_modes_v = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); - auto sol_weight_v = - pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); - uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); - // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const auto t_corr = 1e-10; - - auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); - - auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, - k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); - hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now) - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, - "Requested initial magnetic field perturbations but not " - "solving the MHD equations.") - // peak of init vel perturb - auto k_peak_b = pin->GetReal("problem/cluster/init_perturb", "k_peak_b"); - auto num_modes_b = - pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); - uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); - // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const auto t_corr = 1e-10; - // This field should by construction have no compressive modes, so we fix the number. - const auto sol_weight_b = 1.0; - - auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); - - const bool fill_ghosts = true; // as we fill a vector potential to calc B - auto few_modes_ft = - FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, - sol_weight_b, t_corr, rseed_b, fill_ghosts); - hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); - - // Add field for initial perturation (must not need to be consistent but defining it - // this way is easier for now). Only add if not already done for the velocity. - if (sigma_v == 0.0) { - Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3})); - hydro_pkg->AddField("tmp_perturb", m); - } - } -} - -//======================================================================================== -//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) -//! \brief Generate problem data for all blocks on rank -// -// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that -// reductions work -//======================================================================================== - -void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { - // This could be more optimized, but require a refactor of init routines being called. - // However, given that it's just called during initial setup, this should not be a - // performance concern. - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - auto hydro_pkg = pmb->packages.Get("Hydro"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - // Initialize the conserved variables - auto &u = pmb->meshblock_data.Get()->Get("cons").data; - - auto &coords = pmb->coords; - - // Get Adiabatic Index - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - /************************************************************ - * Initialize the initial hydro state - ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - }); - - // end if(init_uniform_gas) - } else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - - // initialize conserved variables - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Calculate radius - const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + - coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - }); - } - - if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { - /************************************************************ - * Initialize the initial magnetic field state via a vector potential - ************************************************************/ - parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - IndexRange a_ib = ib; - a_ib.s -= 1; - a_ib.e += 1; - IndexRange a_jb = jb; - a_jb.s -= 1; - a_jb.e += 1; - IndexRange a_kb = kb; - a_kb.s -= 1; - a_kb.e += 1; - - /************************************************************ - * Initialize an initial magnetic tower - ************************************************************/ - const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - - magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - - /************************************************************ - * Add dipole magnetic field to the magnetic potential - ************************************************************/ - const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); - if (init_dipole_b_field) { - const Real mx = hydro_pkg->Param("dipole_b_field_mx"); - const Real my = hydro_pkg->Param("dipole_b_field_my"); - const Real mz = hydro_pkg->Param("dipole_b_field_mz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", - parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - // Compute and apply potential - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - - const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - - const Real m_cross_r_x = my * z - mz * y; - const Real m_cross_r_y = mz * x - mx * z; - const Real m_cross_r_z = mx * y - mx * y; - - A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); - A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); - A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); - }); - } - - /************************************************************ - * Apply the potential to the conserved variables - ************************************************************/ - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - u(IB1, k, j, i) = - (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - - (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - - (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - - (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; - - u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + - SQR(u(IB3, k, j, i))); - }); - - /************************************************************ - * Add uniform magnetic field to the conserved variables - ************************************************************/ - const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); - if (init_uniform_b_field) { - const Real bx = hydro_pkg->Param("uniform_b_field_bx"); - const Real by = hydro_pkg->Param("uniform_b_field_by"); - const Real bz = hydro_pkg->Param("uniform_b_field_bz"); - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", - parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { - const Real bx_i = u(IB1, k, j, i); - const Real by_i = u(IB2, k, j, i); - const Real bz_i = u(IB3, k, j, i); - - u(IB1, k, j, i) += bx; - u(IB2, k, j, i) += by; - u(IB3, k, j, i) += bz; - - // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + - // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that - u(IEN, k, j, i) += - bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); - }); - // end if(init_uniform_b_field) - } - - } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) - } - - /************************************************************ - * Initial parameters - ************************************************************/ - - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - - auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto fluid = hydro_pkg->Param("fluid"); - auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); - - /************************************************************ - * Set initial density perturbations - ************************************************************/ - - const bool init_perturb_rho = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); - const Real thickness_ism = - pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool overpressure_ring = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "overpressure_ring", false); - const bool spherical_collapse = - pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - - hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); - - Real passive_scalar = 0.0; // Not useful here - - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1 * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2 * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3 * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = - pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal( - "problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to - // foreground_density - if (r < overdensity_radius) { - u(IDN, k, j, i) = foreground_density; - } - - // u(IDN, k, j, i) += foreground_density * std::exp(- 2 * SQR(r) / - // SQR(overdensity_radius) ) ; - }, - passive_scalar); - } - - /* -------------- Setting up a clumpy atmosphere -------------- - - 1) Extract the values of the density from an input hdf5 file using H5Easy - 2) Initiate the associated density field - 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) - 4) Optionnaly, add a central overdensity - - */ - - if (init_perturb_rho == true) { - - auto init_perturb_rho_file = - pin->GetString("problem/cluster/init_perturb", "init_perturb_rho_file"); - auto init_perturb_rho_keys = - pin->GetString("problem/cluster/init_perturb", "init_perturb_rho_keys"); - - hydro_pkg->AddParam<>("cluster/init_perturb_rho_file", init_perturb_rho_file); - hydro_pkg->AddParam<>("cluster/init_perturb_rho_keys", init_perturb_rho_keys); - - std::cout << "Setting density perturbation"; - - // Read HDF5 file containing the density - std::string filename_rho = "/work/bbd0833/test/rho.h5"; - std::string keys_rho = "data"; - H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - auto rho_init = - H5Easy::load, 256>, 256>>(file, - keys_rho); - - Real passive_scalar = 0.0; // Useless - - std::cout << "entering initialisation of rho field"; - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1 * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2 * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3 * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - // Adding the value of the density field - if (full_box) { - - u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; - - } - - else { - - const Real z = coords.Xc<3>(k); - - // if (z > -thickness_ism / 2 && z < thickness_ism / 2){ - - u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * - std::exp(-SQR(z) / SQR(thickness_ism)); - } - - // Ring of overpressure - // compute radius - if (overpressure_ring) { - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real width = 0.002; - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - - if (r > 0 && r < overdensity_radius) { - - // u(IEN, k, j, i) *= 100; - - // Pushing a ring of gas outward (kinetic boost) - const Real u_x = x / r; - const Real u_y = y / r; - const Real u_z = z / r; - - const Real velocity_blast = pin->GetOrAddReal( - "problem/cluster/init_perturb", "velocity_blast", 0.0); - - // u(IEN, k, j, i) *= 100; - - u(IM1, k, j, i) = - u_x * velocity_blast * u(IDN, k, j, i); // p = (1 Mpc / Gyr) * rho - u(IM2, k, j, i) = - u_y * velocity_blast * u(IDN, k, j, i); // p = (1 Mpc / Gyr) * rho - u(IM3, k, j, i) = - u_z * velocity_blast * u(IDN, k, j, i); // p = (1 Mpc / Gyr) * rho - u(IEN, k, j, i) += 0.5 * u(IDN, k, j, i) * (pow(velocity_blast, 2)); - } - } - - if (spherical_collapse) { - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal( - "problem/cluster/init_perturb", "overdensity_radius", 0.01); - - u(IDN, k, j, i) = 3; - u(IDN, k, j, i) *= 50 * std::exp(-2 * SQR(r) / SQR(overdensity_radius)); - } - }, - passive_scalar); - } - - /************************************************************ - * Set initial velocity perturbations (requires no other velocities for now) - ************************************************************/ - - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); - - if (sigma_v != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real v2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - auto rho = u(IDN, k, j, i); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, - "Found existing non-zero velocity when setting velocity perturbations."); - - u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); - u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); - u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); - // No need to touch the energy yet as we'll normalize later - - lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * - coords.CellVolume(k, j, i) / SQR(rho); - }, - v2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - - pmb->par_for( - "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IM1, k, j, i) /= v_norm; - u(IM2, k, j, i) /= v_norm; - u(IM3, k, j, i) /= v_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / - u(IDN, k, j, i); - }); - } - - /************************************************************ - * Set initial magnetic field perturbations (resets magnetic field field) - ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - if (sigma_b != 0.0) { - auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft.SetPhases(pmb.get(), pin); - } - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_b) - const Real dt = 1.0; - few_modes_ft.Generate(md, dt, "tmp_perturb"); - - Real b2_sum = 0.0; // used for normalization - - auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - // The following restriction could be lifted, but requires refactoring of the - // logic for the normalization/reduction below - PARTHENON_REQUIRE( - u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, - "Found existing non-zero B when setting magnetic field perturbations."); - u(IB1, k, j, i) = - (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 - - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0; - u(IB2, k, j, i) = - (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 - - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0; - u(IB3, k, j, i) = - (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 - - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0; - - // No need to touch the energy yet as we'll normalize later - lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * - coords.CellVolume(k, j, i); - }, - b2_sum); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL - - const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; - const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; - const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; - auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - - pmb->par_for( - "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const auto &u = cons(b); - - u(IB1, k, j, i) /= b_norm; - u(IB2, k, j, i) /= b_norm; - u(IB3, k, j, i) /= b_norm; - - u(IEN, k, j, i) += - 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); - }); - } -} - -void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { - // get hydro - auto pkg = pmb->packages.Get("Hydro"); - const Real gam = pin->GetReal("hydro", "gamma"); - const Real gm1 = (gam - 1.0); - - // get prim vars - auto &data = pmb->meshblock_data.Get(); - auto const &prim = data->Get("prim").data; - - // get derived fields - auto &log10_radius = data->Get("log10_cell_radius").data; - auto &entropy = data->Get("entropy").data; - auto &mach_sonic = data->Get("mach_sonic").data; - auto &temperature = data->Get("temperature").data; - - // for computing temperature from primitives - auto units = pkg->Param("units"); - auto mbar_over_kb = pkg->Param("mbar_over_kb"); - auto mbar = mbar_over_kb * units.k_boltzmann(); - - // fill derived vars (*including ghost cells*) - auto &coords = pmb->coords; - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real v1 = prim(IV1, k, j, i); - const Real v2 = prim(IV2, k, j, i); - const Real v3 = prim(IV3, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute radius - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r2 = SQR(x) + SQR(y) + SQR(z); - log10_radius(k, j, i) = 0.5 * std::log10(r2); - - // compute entropy - const Real K = P / std::pow(rho / mbar, gam); - entropy(k, j, i) = K; - - const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - const Real M_s = v_mag / c_s; - mach_sonic(k, j, i) = M_s; - - // compute temperature - temperature(k, j, i) = mbar_over_kb * P / rho; - }); - - if (pkg->Param("enable_cooling") == Cooling::tabular) { - auto &cooling_time = data->Get("cooling_time").data; - - // get cooling function - const cooling::TabularCooling &tabular_cooling = - pkg->Param("tabular_cooling"); - const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - - // compute cooling time - const Real eint = P / (rho * gm1); - const Real edot = cooling_table_obj.DeDt(eint, rho); - cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; - }); - } - - if (pkg->Param("fluid") == Fluid::glmmhd) { - auto &plasma_beta = data->Get("plasma_beta").data; - auto &mach_alfven = data->Get("mach_alfven").data; - - pmb->par_for( - "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - // get gas properties - const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); - - // compute Alfven mach number - const Real v_A = std::sqrt(B2 / rho); - const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS - mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; - - // compute plasma beta - plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; - }); - } -} - -} // namespace cluster diff --git a/src/refinement/other.cpp b/src/refinement/other.cpp index fa1ffc94..3b16ac09 100644 --- a/src/refinement/other.cpp +++ b/src/refinement/other.cpp @@ -16,19 +16,19 @@ using parthenon::IndexRange; // refinement condition: check max density parthenon::AmrTag MaxDensity(MeshBlockData *rc) { - auto pmb = rc->GetBlockPointer(); - auto w = rc->Get("prim").data; - //auto &coords = pmb->coords; - + auto pmb = rc->GetBlockPointer(); + auto w = rc->Get("prim").data; + // auto &coords = pmb->coords; + const auto deref_below = pmb->packages.Get("Hydro")->Param("refinement/maxdensity_deref_below"); const auto refine_above = pmb->packages.Get("Hydro")->Param("refinement/maxdensity_refine_above"); - + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - + Real maxrho = 0.0; pmb->par_reduce( "overdens check refinement", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e + 1, @@ -36,12 +36,10 @@ parthenon::AmrTag MaxDensity(MeshBlockData *rc) { lmaxrho = std::max(lmaxrho, w(IDN, k, j, i)); }, Kokkos::Max(maxrho)); - + if (maxrho > refine_above) return parthenon::AmrTag::refine; if (maxrho < deref_below) return parthenon::AmrTag::derefine; - - - + return parthenon::AmrTag::same; } diff --git a/src/utils/few_modes_ft_lognormal.cpp b/src/utils/few_modes_ft_lognormal.cpp deleted file mode 100644 index f04d0fe3..00000000 --- a/src/utils/few_modes_ft_lognormal.cpp +++ /dev/null @@ -1,446 +0,0 @@ -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//======================================================================================== -//! \file few_modes_ft.cpp -// \brief Helper functions for an inverse (explicit complex to real) FT - -// C++ headers -#include -#include - -// Parthenon headers -#include "basic_types.hpp" -#include "config.hpp" -#include "globals.hpp" -#include "kokkos_abstraction.hpp" -#include "mesh/domain.hpp" -#include "mesh/meshblock_pack.hpp" - -// AthenaPK headers -#include "../main.hpp" -#include "few_modes_ft_lognormal.hpp" -#include "utils/error_checking.hpp" - -namespace utils::few_modes_ft_log { -using Complex = Kokkos::complex; -using parthenon::IndexRange; -using parthenon::Metadata; - -FewModesFTLog::FewModesFTLog(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, - std::string prefix, int num_modes, ParArray2D k_vec, - Real k_min, Real k_max, Real sol_weight, Real t_corr, uint32_t rseed, - bool fill_ghosts) - : prefix_(prefix), num_modes_(num_modes), k_vec_(k_vec), k_min_(k_min), k_max_(k_max), - t_corr_(t_corr), fill_ghosts_(fill_ghosts) { - - //std::cout << "FewModesFTLog: k_vec_(0,0)=" << - - if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { - std::cout << "### WARNING using more than 100 explicit modes will significantly " - << "increase the runtime." << std::endl - << "If many modes are required in the transform field consider using " - << "the driving mechanism based on full FFTs." << std::endl; - } - // Ensure that all all wavevectors can be represented on the root grid - const auto gnx1 = pin->GetInteger("parthenon/mesh", "nx1"); - const auto gnx2 = pin->GetInteger("parthenon/mesh", "nx2"); - const auto gnx3 = pin->GetInteger("parthenon/mesh", "nx3"); - // Need to make this comparison on the host as (for some reason) an extended cuda device - // lambda cannot live in the constructor of an object. - auto k_vec_host = k_vec.GetHostMirrorAndCopy(); - - for (int i = 0; i < num_modes; i++) { - - PARTHENON_REQUIRE(std::abs(k_vec_host(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); - PARTHENON_REQUIRE(std::abs(k_vec_host(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); - PARTHENON_REQUIRE(std::abs(k_vec_host(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); - } - - const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); - const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); - const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); - const auto ng_tot = fill_ghosts_ ? 2 * parthenon::Globals::nghost : 0; - auto m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx1 + ng_tot}), prefix + "_phases_i"); - pkg->AddField(prefix + "_phases_i", m); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx2 + ng_tot}), prefix + "_phases_j"); - pkg->AddField(prefix + "_phases_j", m); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx3 + ng_tot}), prefix + "_phases_k"); - pkg->AddField(prefix + "_phases_k", m); - - // Variable (e.g., acceleration field for turbulence driver) in Fourier space using - // complex to real transform. - var_hat_ = ParArray2D(prefix + "_var_hat", 3, num_modes); - var_hat_new_ = ParArray2D(prefix + "_var_hat_new", 3, num_modes); - - PARTHENON_REQUIRE((sol_weight == -1.0) || (sol_weight >= 0.0 && sol_weight <= 1.0), - "sol_weight for projection in few modes fft module needs to be " - "between 0.0 and 1.0 or set to -1.0 (to disable projection).") - sol_weight_ = sol_weight; - - random_num_ = Kokkos::View( - "random_num", 3, num_modes, 2); - random_num_host_ = Kokkos::create_mirror_view(random_num_); - - rng_.seed(rseed); - dist_ = std::uniform_real_distribution<>(-1.0, 1.0); -} - -void FewModesFTLog::SetPhases(MeshBlock *pmb, ParameterInput *pin) { - - auto pm = pmb->pmy_mesh; - auto hydro_pkg = pmb->packages.Get("Hydro"); - - // The following restriction could technically be lifted if the turbulence driver is - // directly embedded in the hydro driver rather than a user defined source as well as - // fixing the pack_size=-1 when using the Mesh- (not MeshBlock-)based problem generator. - // The restriction stems from requiring a collective MPI comm to normalize the - // acceleration and magnetic field, respectively. Note, that the restriction does not - // apply here, but for the ProblemGenerator() and Driving() function below. The check is - // just added here for convenience as this function is called during problem - // initializtion. From my (pgrete) point of view, it's currently cleaner to keep things - // separate and not touch the main driver at the expense of using one pack per rank -- - // which is typically fastest on devices anyway. - - const auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size"); - PARTHENON_REQUIRE_THROWS(pack_size == -1, - "Few modes FT currently needs parthenon/mesh/pack_size=-1 " - "to work because of global reductions.") - - auto Lx1 = pm->mesh_size.x1max - pm->mesh_size.x1min; - auto Lx2 = pm->mesh_size.x2max - pm->mesh_size.x2min; - auto Lx3 = pm->mesh_size.x3max - pm->mesh_size.x3min; - - // Adjust (logical) grid size at levels other than the root level. - // This is required for simulation with mesh refinement so that the phases calculated - // below take the logical grid size into account. For example, the local phases at level - // 1 should be calculated assuming a grid that is twice as large as the root grid. - const auto root_level = pm->GetRootLevel(); - auto gnx1 = pm->mesh_size.nx1 * std::pow(2, pmb->loc.level() - root_level); // Size of cell - auto gnx2 = pm->mesh_size.nx2 * std::pow(2, pmb->loc.level() - root_level); - auto gnx3 = pm->mesh_size.nx3 * std::pow(2, pmb->loc.level() - root_level); - - // Restriction should also be easily fixed, just need to double check transforms and - // volume weighting everywhere - PARTHENON_REQUIRE_THROWS(((gnx1 == gnx2) && (gnx2 == gnx3)) && - ((Lx1 == Lx2) && (Lx2 == Lx3)), - "FMFT has only been tested with cubic meshes and constant " - "dx/dy/dz. Remove this warning at your own risk.") - - const auto nx1 = pmb->block_size.nx1; - const auto nx2 = pmb->block_size.nx2; - const auto nx3 = pmb->block_size.nx3; - - const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; - - // make local ref to capure in lambda - const auto num_modes = num_modes_; - auto &k_vec = k_vec_; - - Complex I(0.0, 1.0); - - auto &base = pmb->meshblock_data.Get(); - auto &phases_i = base->Get(prefix_ + "_phases_i").data; - auto &phases_j = base->Get(prefix_ + "_phases_j").data; - auto &phases_k = base->Get(prefix_ + "_phases_k").data; - - const auto ng = fill_ghosts_ ? parthenon::Globals::nghost : 0; - pmb->par_for( - "FMFT: calc phases_i", 0, nx1 - 1 + 2 * ng, KOKKOS_LAMBDA(int i) { - Real gi = static_cast((i + gis - ng) % static_cast(gnx1)); - Real w_kx; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - - - w_kx = k_vec(0, m) * 2. * M_PI / static_cast(gnx1); - - // adjust phase factor to Complex->Real IFT: u_hat*(k) = u_hat(-k) - - if (k_vec(0, m) == 0.0) { - phase = 0.5 * Kokkos::exp(I * w_kx * gi); - } else { - phase = Kokkos::exp(I * w_kx * gi); - } - phases_i(i, m, 0) = phase.real(); - phases_i(i, m, 1) = phase.imag(); - - } - }); - - pmb->par_for( - "FMFT: calc phases_j", 0, nx2 - 1 + 2 * ng, KOKKOS_LAMBDA(int j) { - Real gj = static_cast((j + gjs - ng) % static_cast(gnx2)); - Real w_ky; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_ky = k_vec(1, m) * 2. * M_PI / static_cast(gnx2); - phase = Kokkos::exp(I * w_ky * gj); - phases_j(j, m, 0) = phase.real(); - phases_j(j, m, 1) = phase.imag(); - } - }); - - pmb->par_for( - "FMFT: calc phases_k", 0, nx3 - 1 + 2 * ng, KOKKOS_LAMBDA(int k) { - Real gk = static_cast((k + gks - ng) % static_cast(gnx3)); - Real w_kz; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_kz = k_vec(2, m) * 2. * M_PI / static_cast(gnx3); - phase = Kokkos::exp(I * w_kz * gk); - phases_k(k, m, 0) = phase.real(); - phases_k(k, m, 1) = phase.imag(); - } - }); -} - -void FewModesFTLog::Generate(MeshData *md, const Real dt, - const std::string &var_name) { - auto pmb = md->GetBlockData(0)->GetBlockPointer(); - - const auto num_modes = num_modes_; - - Complex I(0.0, 1.0); - auto &random_num = random_num_; - - // get a set of random numbers from the CPU so that they are deterministic - // when run on GPUs - Real v1, v2, v_sqr; - for (int n = 0; n < 3; n++) - for (int m = 0; m < num_modes; m++) { - do { - v1 = dist_(rng_); - v2 = dist_(rng_); - v_sqr = v1 * v1 + v2 * v2; - } while (v_sqr >= 1.0 || v_sqr == 0.0); - - random_num_host_(n, m, 0) = v1; - random_num_host_(n, m, 1) = v2; - } - Kokkos::deep_copy(random_num, random_num_host_); - - // make local ref to capure in lambda - auto &k_vec = k_vec_; - auto &var_hat = var_hat_; - auto &var_hat_new = var_hat_new_; - - const auto kmin = k_min_; - - // generate new power spectrum (injection) - pmb->par_for( - "FMFT: new power spec", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - Real kmag, tmp, norm, v_sqr; - - Real kx = k_vec(0, m); - Real ky = k_vec(1, m); - Real kz = k_vec(2, m); - - kmag = std::sqrt(kx * kx + ky * ky + kz * kz); - - var_hat_new(n, m) = Complex(0., 0.); - - tmp = std::pow(kmag / kmin, -5./3.); // AMPLITUDE, TO BE REDEFINED TO LOGNORMAL DISTRIBUTION - if (tmp < 0.) tmp = 0.; - v_sqr = SQR(random_num(n, m, 0)) + SQR(random_num(n, m, 1)); - norm = std::sqrt(-2.0 * std::log(v_sqr) / v_sqr); - - var_hat_new(n, m) = - Complex(tmp * norm * random_num(n, m, 0), tmp * norm * random_num(n, m, 1)); - }); - - // enforce symmetry of complex to real transform - pmb->par_for( - "forcing: enforce symmetry", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - if (k_vec(0, m) == 0.) { - for (int m2 = 0; m2 < m; m2++) { - if (k_vec(1, m) == -k_vec(1, m2) && k_vec(2, m) == -k_vec(2, m2)) - var_hat_new(n, m) = - Complex(var_hat_new(n, m2).real(), -var_hat_new(n, m2).imag()); - } - } - }); - - const auto sol_weight = sol_weight_; - if (sol_weight_ >= 0.0) { - // project - pmb->par_for( - "forcing: projection", 0, num_modes - 1, KOKKOS_LAMBDA(const int m) { - Real kmag; - - Real kx = k_vec(0, m); - Real ky = k_vec(1, m); - Real kz = k_vec(2, m); - - kmag = std::sqrt(kx * kx + ky * ky + kz * kz); - - // setting kmag to 1 as a "continue" doesn't work within the parallel_for - // construct and it doesn't affect anything (there should never be power in the - // k=0 mode) - if (kmag == 0.) kmag = 1.; - - // make it a unit vector - kx /= kmag; - ky /= kmag; - kz /= kmag; - - Complex dot(var_hat_new(0, m).real() * kx + var_hat_new(1, m).real() * ky + - var_hat_new(2, m).real() * kz, - var_hat_new(0, m).imag() * kx + var_hat_new(1, m).imag() * ky + - var_hat_new(2, m).imag() * kz); - - var_hat_new(0, m) = Complex(var_hat_new(0, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * kx, - var_hat_new(0, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * kx); - var_hat_new(1, m) = Complex(var_hat_new(1, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * ky, - var_hat_new(1, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * ky); - var_hat_new(2, m) = Complex(var_hat_new(2, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * kz, - var_hat_new(2, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * kz); - }); - } - - // evolve - const auto c_drift = std::exp(-dt / t_corr_); - const auto c_diff = std::sqrt(1.0 - c_drift * c_drift); - - pmb->par_for( - "FMFT: evolve spec", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - var_hat(n, m) = - Complex(var_hat(n, m).real() * c_drift + var_hat_new(n, m).real() * c_diff, - var_hat(n, m).imag() * c_drift + var_hat_new(n, m).imag() * c_diff); - }); - - auto domain = fill_ghosts_ ? IndexDomain::entire : IndexDomain::interior; - IndexRange ib = md->GetBlockData(0)->GetBoundsI(domain); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(domain); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(domain); - auto var_pack = md->PackVariables(std::vector{var_name}); - auto phases_i = md->PackVariables(std::vector{prefix_ + "_phases_i"}); - auto phases_j = md->PackVariables(std::vector{prefix_ + "_phases_j"}); - auto phases_k = md->PackVariables(std::vector{prefix_ + "_phases_k"}); - - // implictly assuming cubic box of size L=1 - parthenon::par_for( - DEFAULT_LOOP_PATTERN, "FMFT: Inverse FT", parthenon::DevExecSpace(), 0, - md->NumBlocks() - 1, 0, 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { - Complex phase, phase_i, phase_j, phase_k; - var_pack(b, n, k, j, i) = 0.0; - - for (int m = 0; m < num_modes; m++) { - phase_i = - Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); - phase_j = - Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); - phase_k = - Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); - phase = phase_i * phase_j * phase_k; - var_pack(b, n, k, j, i) += 2. * (var_hat(n, m).real() * phase.real() - - var_hat(n, m).imag() * phase.imag()); - } - }); -} - -// Creates a random set of wave vectors from k_min to k_max (limit imposed by cell size) -ParArray2D MakeRandomModesLog(const int num_modes, const Real k_min, const Real k_max, - uint32_t rseed = 31224) { - - auto k_vec = parthenon::ParArray2D("k_vec", 3, num_modes); - auto k_vec_h = Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), k_vec); - - const int k_low = k_min; - const int k_high = k_max; - - std::mt19937 rng; - rng.seed(rseed); - std::uniform_real_distribution<> dist(-k_high, k_high); - std::uniform_real_distribution<> distlog(std::log10(k_low), std::log10(k_high)); - - int n_mode = 0; - int n_attempt = 0; - constexpr int max_attempts = 1000000; - Real kx1, kx2, kx3, skx1, skx2, skx3, lkx1, lkx2, lkx3, k_mag, ampl; - bool mode_exists = false; - while (n_mode < num_modes && n_attempt < max_attempts) { - n_attempt += 1; - - // ============================= - // Uniformly taken in lin scaled - // ============================= - - //kx1 = dist(rng); - //kx2 = dist(rng); - //kx3 = dist(rng); - - // ============================= - // Uniformly taken in log scaled - // ============================= - - skx1 = dist(rng); - skx2 = dist(rng); - skx3 = dist(rng); - - skx1 = skx1 / std::abs(skx1); - skx2 = skx2 / std::abs(skx2); - skx3 = skx3 / std::abs(skx3); - - lkx1 = distlog(rng); - lkx2 = distlog(rng); - lkx3 = distlog(rng); - - kx1 = skx1 * std::floor(std::pow(10,lkx1)); - kx2 = skx2 * std::floor(std::pow(10,lkx2)); - kx3 = skx3 * std::floor(std::pow(10,lkx3)); - - k_mag = std::sqrt(SQR(kx1) + SQR(kx2) + SQR(kx3)); - - // Expected amplitude of the spectral function. If this is changed, it also needs to - // be changed in the FMFT class (or abstracted). - ampl = std::pow((k_mag / k_high), -5./3.); - - // Check is mode was already picked by chance - mode_exists = false; - for (int n_mode_exsist = 0; n_mode_exsist < n_mode; n_mode_exsist++) { - if (k_vec_h(0, n_mode_exsist) == kx1 && k_vec_h(1, n_mode_exsist) == kx2 && - k_vec_h(2, n_mode_exsist) == kx3) { - mode_exists = true; - } - } - - // kx1 < 0.0 because we use a explicit symmetric Complex to Real transform - if (ampl < 0 || k_mag < k_low || k_mag > k_high || mode_exists || kx1 < 0.0) { - continue; - } - k_vec_h(0, n_mode) = kx1; - k_vec_h(1, n_mode) = kx2; - k_vec_h(2, n_mode) = kx3; - n_mode++; - } - PARTHENON_REQUIRE_THROWS( - n_attempt < max_attempts, - "Cluster init did not succeed in calculating perturbation modes.") - Kokkos::deep_copy(k_vec, k_vec_h); - - return k_vec; -} - - -} // namespace utils::few_modes_ft \ No newline at end of file diff --git a/src/utils/few_modes_ft_lognormal.hpp b/src/utils/few_modes_ft_lognormal.hpp deleted file mode 100644 index a8823947..00000000 --- a/src/utils/few_modes_ft_lognormal.hpp +++ /dev/null @@ -1,72 +0,0 @@ - -//======================================================================================== -// AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. -// Licensed under the 3-clause BSD License, see LICENSE file for details -//======================================================================================== -//======================================================================================== -//! \file few_modes_ft.hpp -// \brief Helper functions for an inverse (explicit complex to real) FT - -// Parthenon headers -#include "basic_types.hpp" -#include "config.hpp" -#include -#include -#include - -// AthenaPK headers -#include "../main.hpp" -#include "mesh/domain.hpp" - -namespace utils::few_modes_ft_log { -using namespace parthenon::package::prelude; -using parthenon::Real; -using Complex = Kokkos::complex; -using parthenon::IndexRange; -using parthenon::ParArray2D; - -class FewModesFTLog { - private: - int num_modes_; - std::string prefix_; - ParArray2D var_hat_, var_hat_new_; - ParArray2D k_vec_; - Real k_min_; // minimum wave vector - Real k_max_; // maximum wave vector - Kokkos::View random_num_; - Kokkos::View random_num_host_; - std::mt19937 rng_; - std::uniform_real_distribution<> dist_; - Real sol_weight_; // power in solenoidal modes for projection. Set to negative to - // disable projection - Real t_corr_; // correlation time for evolution of Ornstein-Uhlenbeck process - bool fill_ghosts_; // if the inverse transform should also fill ghost zones - - public: - FewModesFTLog(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, - std::string prefix, int num_modes, ParArray2D k_vec, Real k_min, Real k_max, - Real sol_weight, Real t_corr, uint32_t rseed, bool fill_ghosts = false); - - ParArray2D GetVarHat() { return var_hat_; } - int GetNumModes() { return num_modes_; } - void SetPhases(MeshBlock *pmb, ParameterInput *pin); - void Generate(MeshData *md, const Real dt, const std::string &var_name); - void RestoreRNG(std::istringstream &iss) { iss >> rng_; } - void RestoreDist(std::istringstream &iss) { iss >> dist_; } - std::string GetRNGState() { - std::ostringstream oss; - oss << rng_; - return oss.str(); - } - std::string GetDistState() { - std::ostringstream oss; - oss << dist_; - return oss.str(); - } -}; - -// Creates a random set of wave vectors with k_mag within k_min and k_max (Nyquist lim.) -ParArray2D MakeRandomModesLog(const int num_modes, const Real k_min, const Real k_max, uint32_t rseed); - -} // namespace utils::few_modes_ft \ No newline at end of file