diff --git a/inputs/cluster/hse.in b/inputs/cluster/hse.in index 7bb5080d..17b3c60d 100644 --- a/inputs/cluster/hse.in +++ b/inputs/cluster/hse.in @@ -24,7 +24,6 @@ nlim = -1 # cycle limit tlim = 1e-3 # time limit integrator = vl2 # time integration algorithm - refinement = static nghost = 2 @@ -47,7 +46,6 @@ x3max = 0.1 # maximum value of X3 ix3_bc = outflow # inner-X3 boundary flag ox3_bc = outflow # outer-X3 boundary flag - x1min = -0.0125 x1max = 0.0125 @@ -57,7 +55,6 @@ x3min = -0.0125 x3max = 0.0125 level = 2 - nx1 = 32 # Number of zones in X1-direction nx2 = 32 # Number of zones in X2-direction @@ -106,7 +103,6 @@ g_smoothing_radius = 1e-6 #Include gravity as a source term gravity_srcterm = true - #Entropy profile parameters k_0 = 8.851337676479303e-121 @@ -115,7 +111,6 @@ r_k = 0.1 alpha_k = 1.1 - #Fix density at radius to close system of equations r_fix = 2.0 rho_fix = 0.01477557589278723 diff --git a/src/eos/adiabatic_glmmhd.hpp b/src/eos/adiabatic_glmmhd.hpp index 13b60373..0a4cbc9f 100644 --- a/src/eos/adiabatic_glmmhd.hpp +++ b/src/eos/adiabatic_glmmhd.hpp @@ -70,50 +70,51 @@ class AdiabaticGLMMHDEOS : public EquationOfState { auto velocity_ceiling_ = GetVelocityCeiling(); auto e_ceiling_ = GetInternalECeiling(); - - Real &u_d = cons(IDN, k, j, i); - Real &u_m1 = cons(IM1, k, j, i); - Real &u_m2 = cons(IM2, k, j, i); - Real &u_m3 = cons(IM3, k, j, i); - Real &u_e = cons(IEN, k, j, i); - Real &u_b1 = cons(IB1, k, j, i); - Real &u_b2 = cons(IB2, k, j, i); - Real &u_b3 = cons(IB3, k, j, i); + + Real &u_d = cons(IDN, k, j, i); + Real &u_m1 = cons(IM1, k, j, i); + Real &u_m2 = cons(IM2, k, j, i); + Real &u_m3 = cons(IM3, k, j, i); + Real &u_e = cons(IEN, k, j, i); + Real &u_b1 = cons(IB1, k, j, i); + Real &u_b2 = cons(IB2, k, j, i); + Real &u_b3 = cons(IB3, k, j, i); Real &u_psi = cons(IPS, k, j, i); - - Real &w_d = prim(IDN, k, j, i); - Real &w_vx = prim(IV1, k, j, i); - Real &w_vy = prim(IV2, k, j, i); - Real &w_vz = prim(IV3, k, j, i); - Real &w_p = prim(IPR, k, j, i); - Real &w_Bx = prim(IB1, k, j, i); - Real &w_By = prim(IB2, k, j, i); - Real &w_Bz = prim(IB3, k, j, i); + + Real &w_d = prim(IDN, k, j, i); + Real &w_vx = prim(IV1, k, j, i); + Real &w_vy = prim(IV2, k, j, i); + Real &w_vz = prim(IV3, k, j, i); + Real &w_p = prim(IPR, k, j, i); + Real &w_Bx = prim(IB1, k, j, i); + Real &w_By = prim(IB2, k, j, i); + Real &w_Bz = prim(IB3, k, j, i); Real &w_psi = prim(IPS, k, j, i); - + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative density is encountered. PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, "Got negative density. Consider enabling first-order flux " - "correction or setting a reasonble density floor."); + "correction or setting a reasonable density floor."); + // apply density floor, without changing momentum or energy u_d = (u_d > density_floor_) ? u_d : density_floor_; w_d = u_d; - + Real di = 1.0 / u_d; w_vx = u_m1 * di; w_vy = u_m2 * di; w_vz = u_m3 * di; - + w_Bx = u_b1; w_By = u_b2; w_Bz = u_b3; w_psi = u_psi; - + Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); Real e_B = 0.5 * (SQR(u_b1) + SQR(u_b2) + SQR(u_b3)); w_p = gm1 * (u_e - e_k - e_B); - + // apply velocity ceiling. By default ceiling is std::numeric_limits::infinity() const Real w_v2 = SQR(w_vx) + SQR(w_vy) + SQR(w_vz); if (w_v2 > SQR(velocity_ceiling_)) { @@ -130,12 +131,12 @@ class AdiabaticGLMMHDEOS : public EquationOfState { u_e -= e_k - e_k_new; e_k = e_k_new; } - + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative pressure is encountered. PARTHENON_REQUIRE(w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, "Got negative pressure. Consider enabling first-order flux " - "correction or setting a reasonble pressure or temperature floor."); + "correction or setting a reasonable pressure or temperature floor."); // Pressure floor (if present) takes precedence over temperature floor if ((pressure_floor_ > 0.0) && (w_p < pressure_floor_)) { diff --git a/src/eos/adiabatic_hydro.hpp b/src/eos/adiabatic_hydro.hpp index d51adf4b..b42196a4 100644 --- a/src/eos/adiabatic_hydro.hpp +++ b/src/eos/adiabatic_hydro.hpp @@ -61,18 +61,18 @@ class AdiabaticHydroEOS : public EquationOfState { auto velocity_ceiling_ = GetVelocityCeiling(); auto e_ceiling_ = GetInternalECeiling(); - Real &u_d = cons(IDN, k, j, i); + Real &u_d = cons(IDN, k, j, i); Real &u_m1 = cons(IM1, k, j, i); Real &u_m2 = cons(IM2, k, j, i); Real &u_m3 = cons(IM3, k, j, i); - Real &u_e = cons(IEN, k, j, i); - - Real &w_d = prim(IDN, k, j, i); + Real &u_e = cons(IEN, k, j, i); + + Real &w_d = prim(IDN, k, j, i); Real &w_vx = prim(IV1, k, j, i); Real &w_vy = prim(IV2, k, j, i); Real &w_vz = prim(IV3, k, j, i); - Real &w_p = prim(IPR, k, j, i); - + Real &w_p = prim(IPR, k, j, i); + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) // and the code will fail if a negative density is encountered. PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, @@ -86,7 +86,7 @@ class AdiabaticHydroEOS : public EquationOfState { w_vx = u_m1 * di; w_vy = u_m2 * di; w_vz = u_m3 * di; - + Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); w_p = gm1 * (u_e - e_k); diff --git a/src/hydro/srcterms/tabular_cooling.cpp b/src/hydro/srcterms/tabular_cooling.cpp index ee4645b5..4c2728d2 100644 --- a/src/hydro/srcterms/tabular_cooling.cpp +++ b/src/hydro/srcterms/tabular_cooling.cpp @@ -57,8 +57,8 @@ TabularCooling::TabularCooling(ParameterInput *pin, } else if (integrator_str == "rk45") { std::cout << "Cooling integrator is rk45\n"; integrator_ = CoolIntegrator::rk45; - } else if (integrator_str == "townsend\n") { - std::cout << "Cooling integrator is Townsend"; + } else if (integrator_str == "townsend") { + std::cout << "Cooling integrator is Townsend\n"; integrator_ = CoolIntegrator::townsend; } else { integrator_ = CoolIntegrator::undefined; @@ -226,7 +226,7 @@ TabularCooling::TabularCooling(ParameterInput *pin, if (integrator_ == CoolIntegrator::townsend) { lambdas_ = ParArray1D("lambdas_", n_temp_); temps_ = ParArray1D("temps_", n_temp_); - + // Read log_lambdas in host_lambdas, changing to code units along the way auto host_lambdas = Kokkos::create_mirror_view(lambdas_); auto host_temps = Kokkos::create_mirror_view(temps_); @@ -238,7 +238,7 @@ TabularCooling::TabularCooling(ParameterInput *pin, // Copy host_lambdas into device memory Kokkos::deep_copy(lambdas_, host_lambdas); Kokkos::deep_copy(temps_, host_temps); - + // Coeffs are for intervals, i.e., only n_temp_ - 1 entries const auto n_bins = n_temp_ - 1; townsend_Y_k_ = ParArray1D("townsend_Y_k_", n_bins); @@ -504,7 +504,7 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, // Grab member variables for compiler const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - + const auto units = hydro_pkg->Param("units"); const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; @@ -515,10 +515,10 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, const auto temps = temps_; const auto alpha_k = townsend_alpha_k_; const auto Y_k = townsend_Y_k_; - + const auto internal_e_floor = T_floor_ / mbar_gm1_over_kb; const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table - + // Grab some necessary variables const auto &prim_pack = md->PackVariables(std::vector{"prim"}); const auto &cons_pack = md->PackVariables(std::vector{"cons"}); @@ -527,13 +527,13 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - + const auto nbins = alpha_k.extent_int(0); - + // Get reference values const auto temp_final = std::pow(10.0, log_temp_final_); const auto lambda_final = lambda_final_; - + par_for( DEFAULT_LOOP_PATTERN, "TabularCooling::TownsendSrcTerm", DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, diff --git a/src/hydro/srcterms/tabular_cooling.hpp b/src/hydro/srcterms/tabular_cooling.hpp index 0e9f9c24..1a455fb3 100644 --- a/src/hydro/srcterms/tabular_cooling.hpp +++ b/src/hydro/srcterms/tabular_cooling.hpp @@ -176,6 +176,7 @@ class CoolingTableObj { const Real lambda = pow(10., log_lambda); const Real gamma = gamma_units_; const Real de_dt = -lambda * x_H_over_m_h2_ * rho + gamma*pow(x_H_over_m_h2_,0.5); + //const Real de_dt = -lambda * x_H_over_m_h2_ * rho; //std::cout << "cool = " << -lambda * x_H_over_m_h2_ * rho << "\t heat = " << gamma*pow(x_H_over_m_h2_,0.5) << "\t e = " << e << "\t rho = " << rho << "\n" ; diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 49c756ee..52d6eebe 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -179,43 +179,41 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd /************************************************************ * Read Cluster Gravity Parameters ************************************************************/ - + // Build cluster_gravity object ClusterGravity cluster_gravity(pin, hydro_pkg); - // hydro_pkg->AddParam<>("cluster_gravity", cluster_gravity); - + // Include gravity as a source term during evolution const bool gravity_srcterm = pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); - + /************************************************************ * Read Initial Entropy Profile ************************************************************/ - - // Build entropy_profile object + + // Create hydrostatic sphere with ACCEPT entropy profile ACCEPTEntropyProfile entropy_profile(pin); - - /************************************************************ - * Build Hydrostatic Equilibrium Sphere - ************************************************************/ - + HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, entropy_profile); - - /************************************************************ - * Read Precessing Jet Coordinate system - ************************************************************/ - + // Create hydrostatic sphere with ISOTHERMAL entropy profile + + + + /************************************************************ + * Read Precessing Jet Coordinate system + ************************************************************/ + JetCoordsFactory jet_coords_factory(pin, hydro_pkg); - + /************************************************************ * Read AGN Feedback ************************************************************/ - + AGNFeedback agn_feedback(pin, hydro_pkg); - + /************************************************************ * Read AGN Triggering ************************************************************/ @@ -229,7 +227,6 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // Build Magnetic Tower MagneticTower magnetic_tower(pin, hydro_pkg); - // Determine if magnetic_tower_power_scaling is needed // Is AGN Power and Magnetic fraction non-zero? bool magnetic_tower_power_scaling = @@ -283,7 +280,7 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd hydro_pkg->AddParam("cluster_vceil", vceil); hydro_pkg->AddParam("cluster_vAceil", vAceil); hydro_pkg->AddParam("cluster_clip_r", clip_r); - + /************************************************************ * Start running reductions into history outputs for clips, stellar mass, cold * gas, and AGN extent @@ -338,16 +335,16 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd hst_vars.emplace_back(parthenon::HistoryOutputVar( parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); } - + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - + /************************************************************ * Add derived fields * NOTE: these must be filled in UserWorkBeforeOutput ************************************************************/ - + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - + // log10 of cell-centered radius hydro_pkg->AddField("log10_cell_radius", m); // entropy @@ -361,7 +358,7 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // cooling time hydro_pkg->AddField("cooling_time", m); } - + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { // alfven Mach number v_A/c_s hydro_pkg->AddField("mach_alfven", m); @@ -409,7 +406,6 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd * Read Velocity perturbation ************************************************************/ - const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); if (sigma_v != 0.0) { // peak of init vel perturb @@ -427,6 +423,7 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // Note that this assumes a cubic box k_peak_v = Lx / l_peak_v; } + auto num_modes_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); auto sol_weight_v = @@ -472,6 +469,7 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd // Note that this assumes a cubic box k_peak_b = Lx / l_peak_b; } + auto num_modes_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); @@ -509,30 +507,42 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hyd //======================================================================================== void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + // This could be more optimized, but require a refactor of init routines being called. // However, given that it's just called during initial setup, this should not be a // performance concern. + + // Defining a table within which the values of the hydrostatic density profile will be stored + auto pmc = md->GetBlockData(0)->GetBlockPointer(); + const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); - - IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); - IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); - IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); - + auto units = hydro_pkg->Param("units"); + + const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + // Initialize the conserved variables auto &u = pmb->meshblock_data.Get()->Get("cons").data; - auto &coords = pmb->coords; - + // Get Adiabatic Index const Real gam = pin->GetReal("hydro", "gamma"); const Real gm1 = (gam - 1.0); - + /************************************************************ * Initialize the initial hydro state ************************************************************/ - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + if (init_uniform_gas) { const Real rho = hydro_pkg->Param("uniform_gas_rho"); const Real ux = hydro_pkg->Param("uniform_gas_ux"); @@ -544,7 +554,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const Real My = rho * uy; const Real Mz = rho * uz; const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - + parthenon::par_for( DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, @@ -555,9 +565,9 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { u(IM3, k, j, i) = Mz; u(IEN, k, j, i) = E; }); - - // end if(init_uniform_gas) + } else { + /************************************************************ * Initialize a HydrostaticEquilibriumSphere ************************************************************/ @@ -565,29 +575,31 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { hydro_pkg ->Param>( "hydrostatic_equilibirum_sphere"); - + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); - + // initialize conserved variables parthenon::par_for( DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Calculate radius const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + coords.Xc<2>(j) * coords.Xc<2>(j) + coords.Xc<3>(k) * coords.Xc<3>(k)); - + // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); + const Real P_r = P_rho_profile.P_from_r(r); const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity + + //u(IDN, k, j, i) = rho_r; u(IDN, k, j, i) = rho_r; u(IM1, k, j, i) = 0.0; u(IM2, k, j, i) = 0.0; u(IM3, k, j, i) = 0.0; u(IEN, k, j, i) = P_r / gm1; + }); } @@ -598,7 +610,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), pmb->cellbounds.ncellsj(IndexDomain::entire), pmb->cellbounds.ncellsi(IndexDomain::entire)); - + IndexRange a_ib = ib; a_ib.s -= 1; a_ib.e += 1; @@ -608,14 +620,14 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { IndexRange a_kb = kb; a_kb.s -= 1; a_kb.e += 1; - + /************************************************************ * Initialize an initial magnetic tower ************************************************************/ const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); - + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); - + /************************************************************ * Add dipole magnetic field to the magnetic potential ************************************************************/ @@ -632,13 +644,16 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { const Real x = coords.Xc<1>(i); const Real y = coords.Xc<2>(j); const Real z = coords.Xc<3>(k); - + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); - + const Real m_cross_r_x = my * z - mz * y; const Real m_cross_r_y = mz * x - mx * z; const Real m_cross_r_z = mx * y - mx * y; - + + // To check whether there is some component before initiating perturbations + std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); @@ -694,7 +709,7 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { }); // end if(init_uniform_b_field) } - + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) } @@ -710,63 +725,20 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { auto hydro_pkg = pmb->packages.Get("Hydro"); const auto fluid = hydro_pkg->Param("fluid"); auto const &cons = md->PackVariables(std::vector{"cons"}); - const auto num_blocks = md->NumBlocks(); + const auto num_blocks = md->NumBlocks(); /************************************************************ * Set initial density perturbations (read from HDF5 file) ************************************************************/ const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); - const bool full_box = pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); - const bool overpressure_ring = pin->GetOrAddBoolean("problem/cluster/init_perturb", "overpressure_ring", false); const bool spherical_collapse = pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); - + hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); Real passive_scalar = 0.0; // Not useful here - // Spherical collapse test with an initial overdensity - - if (spherical_collapse == true) { - - pmb->par_reduce( - "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - - auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); - const Real background_density = pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); - const Real foreground_density = pin->GetOrAddReal("problem/cluster/init_perturb", "foreground_density", 150); - - // Setting an initial - u(IDN, k, j, i) = background_density; - - // For any cell at a radius less then overdensity radius, set the density to foreground_density - if (r < overdensity_radius){ - u(IDN, k, j, i) = foreground_density; - } - - //u(IDN, k, j, i) += foreground_density * std::exp(- 2 * SQR(r) / SQR(overdensity_radius) ) ; - - - }, - passive_scalar); - - } - /* -------------- Setting up a clumpy atmosphere -------------- 1) Extract the values of the density from an input hdf5 file using H5Easy @@ -779,14 +751,15 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { if (init_perturb_rho == true) { auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", "init_perturb_rho_file","none"); - auto grid_size = pin->GetOrAddInteger("parthenon/mesh","nx1",256); + const Real perturb_amplitude = pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); + const Real box_size_over_two = pin->GetOrAddReal("parthenon/mesh", "x1max", 0.250); std::string keys_rho = "data"; H5Easy::File file(filename_rho, HighFive::File::ReadOnly); - auto rho_init = H5Easy::load, 256>, 256>>(file, keys_rho); - //std::vector rho_init(grid_size * grid_size * grid_size); - //std::vector rho_init(grid_size * grid_size * grid_size) = H5Easy::load(file, keys_rho); + const int rho_init_size = 512; + auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>(file, keys_rho); Real passive_scalar = 0.0; // Useless @@ -798,124 +771,26 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b - const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; - const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; - const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; - const auto &coords = cons.GetCoords(b); const auto &u = cons(b); - // Adding the value of the density field - if (full_box){ - - u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; - - } - - else { - - const Real z = coords.Xc<3>(k); - - // if (z > -thickness_ism / 2 && z < thickness_ism / 2){ - - u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * std::exp(- SQR(z) / SQR(thickness_ism)) ; - - - } - - // Ring of overpressure - // compute radius - if (overpressure_ring){ - - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real width = 0.002; - const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); - - if (r > 0 && r < overdensity_radius){ - - //u(IEN, k, j, i) *= 100; - - // Pushing a ring of gas outward (kinetic boost) - const Real u_x = x / r; - const Real u_y = y / r; - const Real u_z = z / r; - - const Real velocity_blast = pin->GetOrAddReal("problem/cluster/init_perturb", "velocity_blast", 0.0); - - //u(IEN, k, j, i) *= 100; - - u(IM1, k, j, i) = u_x * velocity_blast * u(IDN, k, j, i); // p = (1 Mpc / Gyr) * rho - u(IM2, k, j, i) = u_y * velocity_blast * u(IDN, k, j, i); // p = (1 Mpc / Gyr) * rho - u(IM3, k, j, i) = u_z * velocity_blast * u(IDN, k, j, i); // p = (1 Mpc / Gyr) * rho - u(IEN, k, j, i) += 0.5 * u(IDN, k, j, i) * (pow(velocity_blast,2)); - } - } - - if (spherical_collapse){ + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); - const Real x = coords.Xc<1>(i); - const Real y = coords.Xc<2>(j); - const Real z = coords.Xc<3>(k); - const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius - const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); - - u(IDN, k, j, i) = 3; - u(IDN, k, j, i) *= 50 * std::exp(- 2 * SQR(r) / SQR(overdensity_radius)); + // Getting the corresponding index in the + const Real rho_init_index_x = floor((x + box_size_over_two) / (2 * box_size_over_two) * (rho_init_size - 1)); + const Real rho_init_index_y = floor((y + box_size_over_two) / (2 * box_size_over_two) * (rho_init_size - 1)); + const Real rho_init_index_z = floor((z + box_size_over_two) / (2 * box_size_over_two) * (rho_init_size - 1)); - } + // Case where the box is filled with perturbations of equal mean amplitude + u(IDN, k, j, i) += perturb_amplitude * rho_init[rho_init_index_x][rho_init_index_y][rho_init_index_z] * (u(IDN, k, j, i) / 29.6); + }, passive_scalar); - - } - - /************************************************************ - * Setting up a perturbed density field (hardcoded version) - ************************************************************/ - - const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); - const auto background_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); - - if (mu_rho != 0.0) { - - auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); - - // Init phases on all blocks - for (int b = 0; b < md->NumBlocks(); b++) { - auto pmb = md->GetBlockData(b)->GetBlockPointer(); - few_modes_ft_rho.SetPhases(pmb.get(), pin); - - } - - // As for t_corr in few_modes_ft, the choice for dt is - // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain - // the perturbation (and is normalized in the following to get the desired sigma_v) - const Real dt = 1.0; - few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); - - Real v2_sum_rho = 0.0; // used for normalization - - auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); - - pmb->par_reduce( - "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { - const auto &coords = cons.GetCoords(b); - const auto &u = cons(b); - - u(IDN, k, j, i) = background_rho + mu_rho * std::pow(10,perturb_pack_rho(b, 0, k, j, i)); - - }, - v2_sum_rho); - -#ifdef MPI_PARALLEL - PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, - MPI_SUM, MPI_COMM_WORLD)); -#endif // MPI_PARALLEL + } /************************************************************ @@ -930,18 +805,18 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { for (int b = 0; b < md->NumBlocks(); b++) { auto pmb = md->GetBlockData(b)->GetBlockPointer(); few_modes_ft.SetPhases(pmb.get(), pin); - + } // As for t_corr in few_modes_ft, the choice for dt is // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain // the perturbation (and is normalized in the following to get the desired sigma_v) const Real dt = 1.0; few_modes_ft.Generate(md, dt, "tmp_perturb"); - + Real v2_sum = 0.0; // used for normalization - + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - + pmb->par_reduce( "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { @@ -968,12 +843,12 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); - + pmb->par_for( "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { @@ -992,8 +867,10 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { /************************************************************ * Set initial magnetic field perturbations (resets magnetic field field) ************************************************************/ - const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); - + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + const auto alpha_b = pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0/3.0); + const auto r_scale = pin->GetOrAddReal("problem/cluster/init_perturb", "r_scale", 0.1); + if (sigma_b != 0.0) { auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); // Init phases on all blocks @@ -1001,62 +878,70 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { auto pmb = md->GetBlockData(b)->GetBlockPointer(); few_modes_ft.SetPhases(pmb.get(), pin); } + // As for t_corr in few_modes_ft, the choice for dt is // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain // the perturbation (and is normalized in the following to get the desired sigma_b) const Real dt = 1.0; few_modes_ft.Generate(md, dt, "tmp_perturb"); - + Real b2_sum = 0.0; // used for normalization - + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); - // Defining a new table that will contains the values of the perturbed magnetic field, and magnetic energy - parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, - pmb->cellbounds.ncellsk(IndexDomain::entire), - pmb->cellbounds.ncellsj(IndexDomain::entire), - pmb->cellbounds.ncellsi(IndexDomain::entire)); - - - pmb->par_reduce( - "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + // Modifying the magnetic potential so that it follows the rho profile + pmb->par_for( + "Init sigma_b", 0, num_blocks - 1, kb.s-1, kb.e+1, jb.s-1, jb.e+1, ib.s-1, ib.e+1, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const auto &coords = cons.GetCoords(b); const auto &u = cons(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r = sqrt(SQR(x) + SQR(y) + SQR(z)); - // Normalization condition here, which we want to lift. To do so, we'll need - // to copy the values of the magnetic field into a new array, which we will - // use to store the perturbed magnetic field and then rescale it, until values - // are added to u = cons(b) - - //PARTHENON_REQUIRE( - // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, - // "Found existing non-zero B when setting magnetic field perturbations."); - - Real gradBx,gradBy,gradBz; + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; + + perturb_pack(b, 0, k, j, i) *= 1 / (1 + (r / r_scale)); + perturb_pack(b, 1, k, j, i) *= 1 / (1 + (r / r_scale)); + perturb_pack(b, 2, k, j, i) *= 1 / (1 + (r / r_scale)); - gradBx = (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + }); + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / - coords.Dxc<3>(k) / 2.0 ; - - gradBy = (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / - coords.Dxc<1>(i) / 2.0 ; - - gradBz = (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / - coords.Dxc<2>(j) / 2.0 ; - - dB(b,0, k, j, i) = gradBx; - dB(b,1, k, j, i) = gradBy; - dB(b,2, k, j, i) = gradBz; - + coords.Dxc<2>(j) / 2.0; + // No need to touch the energy yet as we'll normalize later - lsum += (SQR(gradBx) + SQR(gradBy) + SQR(gradBz)) * + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * coords.CellVolume(k, j, i); }, b2_sum); @@ -1065,30 +950,23 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, MPI_SUM, MPI_COMM_WORLD)); #endif // MPI_PARALLEL - + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); - + pmb->par_for( "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const auto &u = cons(b); - - dB(b, 0, k, j, i) /= b_norm; - dB(b, 1, k, j, i) /= b_norm; - dB(b, 2, k, j, i) /= b_norm; - - dB(b, 3, k, j, i) += - 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + SQR(dB(b, 2, k, j, i))); - - // Updating the MHD vector - - u(IB1, k, j, i) += dB(b, 0, k, j, i); - u(IB2, k, j, i) += dB(b, 1, k, j, i); - u(IB3, k, j, i) += dB(b, 2, k, j, i); - u(IEN, k, j, i) += dB(b, 3, k, j, i); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); }); } } @@ -1181,11 +1059,11 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { KOKKOS_LAMBDA(const int k, const int j, const int i) { // get gas properties const Real rho = prim(IDN, k, j, i); - const Real P = prim(IPR, k, j, i); - const Real Bx = prim(IB1, k, j, i); - const Real By = prim(IB2, k, j, i); - const Real Bz = prim(IB3, k, j, i); - const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + const Real P = prim(IPR, k, j, i); + const Real Bx = prim(IB1, k, j, i); + const Real By = prim(IB2, k, j, i); + const Real Bz = prim(IB3, k, j, i); + const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); // compute Alfven mach number const Real v_A = std::sqrt(B2 / rho); diff --git a/src/pgen/cluster/cluster_gravity.hpp b/src/pgen/cluster/cluster_gravity.hpp index e35e60e4..43cd4c69 100644 --- a/src/pgen/cluster/cluster_gravity.hpp +++ b/src/pgen/cluster/cluster_gravity.hpp @@ -17,7 +17,7 @@ namespace cluster { // Types of BCG's -enum class BCG { NONE, HERNQUIST }; +enum class BCG {NONE, HERNQUIST, ISOTHERMAL}; // Hernquiest BCG: Hernquist 1990 DOI:10.1086/168845 /************************************************************ @@ -34,6 +34,7 @@ class ClusterGravity { // NFW Parameters parthenon::Real r_nfw_s_; // G , Mass, and Constants rolled into one + parthenon::Real gravitationnal_const_; parthenon::Real g_const_nfw_; parthenon::Real rho_const_nfw_; @@ -41,6 +42,7 @@ class ClusterGravity { parthenon::Real alpha_bcg_s_; parthenon::Real beta_bcg_s_; parthenon::Real r_bcg_s_; + parthenon::Real T_bcg_s_; // G , Mass, and Constants rolled into one parthenon::Real g_const_bcg_; parthenon::Real rho_const_bcg_; @@ -53,6 +55,8 @@ class ClusterGravity { parthenon::Real smoothing_r_; // Static Helper functions to calculate constants to minimize in-kernel work + // ie. compute the constants characterizing the profile (M_nfw, etc...) + static parthenon::Real calc_R_nfw_s(const parthenon::Real rho_crit, const parthenon::Real m_nfw_200, const parthenon::Real c_nfw) { @@ -114,7 +118,9 @@ class ClusterGravity { // calculate the BCG density profile ClusterGravity(parthenon::ParameterInput *pin) { Units units(pin); - + + gravitationnal_const_ = units.gravitational_constant(); + // Determine which element to include include_nfw_g_ = pin->GetOrAddBoolean("problem/cluster/gravity", "include_nfw_g", false); @@ -124,13 +130,17 @@ class ClusterGravity { which_bcg_g_ = BCG::NONE; } else if (which_bcg_g_str == "HERNQUIST") { which_bcg_g_ = BCG::HERNQUIST; - } else { + } else if (which_bcg_g_str == "ISOTHERMAL") { + which_bcg_g_ = BCG::ISOTHERMAL; + } + + else { std::stringstream msg; msg << "### FATAL ERROR in function [InitUserMeshData]" << std::endl << "Unknown BCG type " << which_bcg_g_str << std::endl; PARTHENON_FAIL(msg); } - + include_smbh_g_ = pin->GetOrAddBoolean("problem/cluster/gravity", "include_smbh_g", false); @@ -139,23 +149,36 @@ class ClusterGravity { "problem/cluster", "hubble_parameter", 70 * units.km_s() / units.mpc()); const parthenon::Real rho_crit = 3 * hubble_parameter * hubble_parameter / (8 * M_PI * units.gravitational_constant()); - + + const auto He_mass_fraction = pin->GetReal("hydro", "He_mass_fraction"); + const auto mu = 1 / (He_mass_fraction * 3. / 4. + (1 - He_mass_fraction) * 2); const parthenon::Real M_nfw_200 = pin->GetOrAddReal("problem/cluster/gravity", "m_nfw_200", 8.5e14 * units.msun()); const parthenon::Real c_nfw = pin->GetOrAddReal("problem/cluster/gravity", "c_nfw", 6.81); r_nfw_s_ = calc_R_nfw_s(rho_crit, M_nfw_200, c_nfw); g_const_nfw_ = calc_g_const_nfw(units.gravitational_constant(), M_nfw_200, c_nfw); - + // Initialize the BCG Profile alpha_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "alpha_bcg_s", 0.1); - beta_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "beta_bcg_s", 1.43); + beta_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "beta_bcg_s", 1.43); const parthenon::Real M_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); r_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); - g_const_bcg_ = calc_g_const_bcg(units.gravitational_constant(), which_bcg_g_, M_bcg_s, - r_bcg_s_, alpha_bcg_s_, beta_bcg_s_); - + T_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); // Temperature in the isothermal case + + if (which_bcg_g_str == "ISOTHERMAL") { + + g_const_bcg_ = 2 * units.k_boltzmann() * T_bcg_s_ / (mu * units.mh()); + + } + + else { + + g_const_bcg_ = calc_g_const_bcg(units.gravitational_constant(), which_bcg_g_, M_bcg_s, + r_bcg_s_, alpha_bcg_s_, beta_bcg_s_); + } + const parthenon::Real m_smbh = pin->GetOrAddReal("problem/cluster/gravity", "m_smbh", 3.4e8 * units.msun()); g_const_smbh_ = calc_g_const_smbh(units.gravitational_constant(), m_smbh), @@ -182,21 +205,23 @@ class ClusterGravity { if (include_nfw_g_) { g_r += g_const_nfw_ * (log(1 + r / r_nfw_s_) - r / (r + r_nfw_s_)) / r2; } - + // Add BCG gravity switch (which_bcg_g_) { case BCG::NONE: break; case BCG::HERNQUIST: g_r += g_const_bcg_ / ((1 + r / r_bcg_s_) * (1 + r / r_bcg_s_)); + case BCG::ISOTHERMAL: + g_r += g_const_bcg_ / r; break; } - + // Add SMBH, point mass gravity if (include_smbh_g_) { g_r += g_const_smbh_ / r2; } - + return g_r; } // Inline functions to compute density @@ -219,6 +244,8 @@ class ClusterGravity { case BCG::HERNQUIST: rho += rho_const_bcg_ / (r * pow(r + r_bcg_s_, 3)); break; + case BCG::ISOTHERMAL: + rho += g_const_bcg_ * 1 / (4 * M_PI * gravitationnal_const_ * r * r); } // SMBH, point mass gravity -- density is not defined. Throw an error diff --git a/src/pgen/cluster/cluster_gravity_original.hpp b/src/pgen/cluster/cluster_gravity_original.hpp new file mode 100644 index 00000000..91acb6de --- /dev/null +++ b/src/pgen/cluster/cluster_gravity_original.hpp @@ -0,0 +1,240 @@ +#ifndef CLUSTER_CLUSTER_GRAVITY_HPP_ +#define CLUSTER_CLUSTER_GRAVITY_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster_gravity.hpp +// \brief Class for defining gravitational acceleration for a cluster+bcg+smbh + +// Parthenon headers +#include + +// AthenaPK headers +#include "../../units.hpp" + +namespace cluster { + +// Types of BCG's +enum class BCG { NONE, HERNQUIST }; +// Hernquiest BCG: Hernquist 1990 DOI:10.1086/168845 + +/************************************************************ + * Cluster Gravity Class, for computing gravitational acceleration + * Lightweight object for inlined computation within kernels + ************************************************************/ +class ClusterGravity { + + // Parameters for which gravity sources to include + bool include_nfw_g_; + BCG which_bcg_g_; + bool include_smbh_g_; + + // NFW Parameters + parthenon::Real r_nfw_s_; + // G , Mass, and Constants rolled into one + parthenon::Real g_const_nfw_; + parthenon::Real rho_const_nfw_; + + // BCG Parameters + parthenon::Real alpha_bcg_s_; + parthenon::Real beta_bcg_s_; + parthenon::Real r_bcg_s_; + // G , Mass, and Constants rolled into one + parthenon::Real g_const_bcg_; + parthenon::Real rho_const_bcg_; + + // SMBH Parameters + // G , Mass, and Constants rolled into one + parthenon::Real g_const_smbh_; + + // Radius underwhich to truncate + parthenon::Real smoothing_r_; + + // Static Helper functions to calculate constants to minimize in-kernel work + // ie. compute the constants characterizing the profile (M_nfw, etc...) + + static parthenon::Real calc_R_nfw_s(const parthenon::Real rho_crit, + const parthenon::Real m_nfw_200, + const parthenon::Real c_nfw) { + const parthenon::Real rho_nfw_0 = + 200 / 3. * rho_crit * pow(c_nfw, 3.) / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); + const parthenon::Real R_nfw_s = + pow(m_nfw_200 / (4 * M_PI * rho_nfw_0 * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))), + 1. / 3.); + return R_nfw_s; + } + static parthenon::Real calc_g_const_nfw(const parthenon::Real gravitational_constant, + const parthenon::Real m_nfw_200, + const parthenon::Real c_nfw) { + return gravitational_constant * m_nfw_200 / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); + } + static parthenon::Real calc_rho_const_nfw(const parthenon::Real gravitational_constant, + const parthenon::Real m_nfw_200, + const parthenon::Real c_nfw) { + return m_nfw_200 / (4 * M_PI * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))); + } + static parthenon::Real calc_g_const_bcg(const parthenon::Real gravitational_constant, + BCG which_bcg_g, const parthenon::Real m_bcg_s, + const parthenon::Real r_bcg_s, + const parthenon::Real alpha_bcg_s, + const parthenon::Real beta_bcg_s) { + switch (which_bcg_g) { + case BCG::NONE: + return 0; + case BCG::HERNQUIST: + return gravitational_constant * m_bcg_s / (r_bcg_s * r_bcg_s); + } + return NAN; + } + static parthenon::Real calc_rho_const_bcg(const parthenon::Real gravitational_constant, + BCG which_bcg_g, + const parthenon::Real m_bcg_s, + const parthenon::Real r_bcg_s, + const parthenon::Real alpha_bcg_s, + const parthenon::Real beta_bcg_s) { + switch (which_bcg_g) { + case BCG::NONE: + return 0; + case BCG::HERNQUIST: + return m_bcg_s * r_bcg_s / (2 * M_PI); + } + return NAN; + } + static KOKKOS_INLINE_FUNCTION parthenon::Real + calc_g_const_smbh(const parthenon::Real gravitational_constant, + const parthenon::Real m_smbh) { + return gravitational_constant * m_smbh; + } + + public: + // ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) + // is called from cluster.cpp to add the ClusterGravity object to hydro_pkg + // + // ClusterGravity(parthenon::ParameterInput *pin) is used in SNIAFeedback to + // calculate the BCG density profile + ClusterGravity(parthenon::ParameterInput *pin) { + Units units(pin); + + // Determine which element to include + include_nfw_g_ = + pin->GetOrAddBoolean("problem/cluster/gravity", "include_nfw_g", false); + const std::string which_bcg_g_str = + pin->GetOrAddString("problem/cluster/gravity", "which_bcg_g", "NONE"); + if (which_bcg_g_str == "NONE") { + which_bcg_g_ = BCG::NONE; + } else if (which_bcg_g_str == "HERNQUIST") { + which_bcg_g_ = BCG::HERNQUIST; + } else { + std::stringstream msg; + msg << "### FATAL ERROR in function [InitUserMeshData]" << std::endl + << "Unknown BCG type " << which_bcg_g_str << std::endl; + PARTHENON_FAIL(msg); + } + + include_smbh_g_ = + pin->GetOrAddBoolean("problem/cluster/gravity", "include_smbh_g", false); + + // Initialize the NFW Profile + const parthenon::Real hubble_parameter = pin->GetOrAddReal( + "problem/cluster", "hubble_parameter", 70 * units.km_s() / units.mpc()); + const parthenon::Real rho_crit = 3 * hubble_parameter * hubble_parameter / + (8 * M_PI * units.gravitational_constant()); + + const parthenon::Real M_nfw_200 = + pin->GetOrAddReal("problem/cluster/gravity", "m_nfw_200", 8.5e14 * units.msun()); + const parthenon::Real c_nfw = + pin->GetOrAddReal("problem/cluster/gravity", "c_nfw", 6.81); + r_nfw_s_ = calc_R_nfw_s(rho_crit, M_nfw_200, c_nfw); + g_const_nfw_ = calc_g_const_nfw(units.gravitational_constant(), M_nfw_200, c_nfw); + + // Initialize the BCG Profile + alpha_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "alpha_bcg_s", 0.1); + beta_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "beta_bcg_s", 1.43); + const parthenon::Real M_bcg_s = + pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); + r_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); + g_const_bcg_ = calc_g_const_bcg(units.gravitational_constant(), which_bcg_g_, M_bcg_s, + r_bcg_s_, alpha_bcg_s_, beta_bcg_s_); + + const parthenon::Real m_smbh = + pin->GetOrAddReal("problem/cluster/gravity", "m_smbh", 3.4e8 * units.msun()); + g_const_smbh_ = calc_g_const_smbh(units.gravitational_constant(), m_smbh), + + smoothing_r_ = + pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); + } + + ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) + : ClusterGravity(pin) { + hydro_pkg->AddParam<>("cluster_gravity", *this); + } + + // Inline functions to compute gravitational acceleration + KOKKOS_INLINE_FUNCTION parthenon::Real g_from_r(const parthenon::Real r_in) const + __attribute__((always_inline)) { + + const parthenon::Real r = std::max(r_in, smoothing_r_); + const parthenon::Real r2 = r * r; + + parthenon::Real g_r = 0; + + // Add NFW gravity + if (include_nfw_g_) { + g_r += g_const_nfw_ * (log(1 + r / r_nfw_s_) - r / (r + r_nfw_s_)) / r2; + } + + // Add BCG gravity + switch (which_bcg_g_) { + case BCG::NONE: + break; + case BCG::HERNQUIST: + g_r += g_const_bcg_ / ((1 + r / r_bcg_s_) * (1 + r / r_bcg_s_)); + break; + } + + // Add SMBH, point mass gravity + if (include_smbh_g_) { + g_r += g_const_smbh_ / r2; + } + + return g_r; + } + // Inline functions to compute density + KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r_in) const + __attribute__((always_inline)) { + + const parthenon::Real r = std::max(r_in, smoothing_r_); + + parthenon::Real rho = 0; + + // Add NFW gravity + if (include_nfw_g_) { + rho += rho_const_nfw_ / (r * pow(r + r_nfw_s_, 2)); + } + + // Add BCG gravity + switch (which_bcg_g_) { + case BCG::NONE: + break; + case BCG::HERNQUIST: + rho += rho_const_bcg_ / (r * pow(r + r_bcg_s_, 3)); + break; + } + + // SMBH, point mass gravity -- density is not defined. Throw an error + if (include_smbh_g_ && r <= smoothing_r_) { + Kokkos::abort("ClusterGravity::SMBH density is not defined"); + } + + return rho; + } + + // SNIAFeedback needs to be a friend to disable the SMBH and NFW + friend class SNIAFeedback; +}; + +} // namespace cluster + +#endif // CLUSTER_CLUSTER_GRAVITY_HPP_ diff --git a/src/pgen/cluster/entropy_isothermal_profiles.hpp b/src/pgen/cluster/entropy_isothermal_profiles.hpp new file mode 100644 index 00000000..8b5c805f --- /dev/null +++ b/src/pgen/cluster/entropy_isothermal_profiles.hpp @@ -0,0 +1,45 @@ +#ifndef CLUSTER_ENTROPY_PROFILES_HPP_ +#define CLUSTER_ENTROPY_PROFILES_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file entropy profiles.hpp +// \brief Classes defining initial entropy profile + +// Parthenon headers +#include + +// AthenaPK headers +#include "../../units.hpp" + +namespace cluster { + +template +class ISOEntropyProfile { + + public: + + // Entropy Profile + parthenon::T_isothermal_; + + ISOEntropyProfile(parthenon::ParameterInput *pin, GravitationalField gravitational_field) { + + Units units(pin); + + + } + + // Get entropy from radius, using broken power law profile for entropy + KOKKOS_INLINE_FUNCTION parthenon::Real K_from_r(const parthenon::Real r) const { + + const parthenon::Real k = 1; + return k; + } + +}; + +} // namespace cluster + +#endif // CLUSTER_ENTROPY_PROFILES_HPP_ diff --git a/src/pgen/cluster/entropy_profiles.hpp b/src/pgen/cluster/entropy_profiles.hpp index 1fd75580..9b6a09ba 100644 --- a/src/pgen/cluster/entropy_profiles.hpp +++ b/src/pgen/cluster/entropy_profiles.hpp @@ -21,23 +21,26 @@ class ACCEPTEntropyProfile { public: // Entropy Profile parthenon::Real k_0_, k_100_, r_k_, alpha_k_; - + ACCEPTEntropyProfile(parthenon::ParameterInput *pin) { + Units units(pin); - - k_0_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "k_0", - 20 * units.kev() * units.cm() * units.cm()); - k_100_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "k_100", + k_0_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "k_0", + 20 * units.kev() * units.cm() * units.cm()); + k_100_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "k_100", 120 * units.kev() * units.cm() * units.cm()); - r_k_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "r_k", 100 * units.kpc()); + r_k_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "r_k", 100 * units.kpc()); alpha_k_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "alpha_k", 1.75); } - + // Get entropy from radius, using broken power law profile for entropy - KOKKOS_INLINE_FUNCTION parthenon::Real K_from_r(const parthenon::Real r) const { - const parthenon::Real k = k_0_ + k_100_ * pow(r / r_k_, alpha_k_); - return k; + KOKKOS_INLINE_FUNCTION parthenon::Real K_from_r(const parthenon::Real r) const { + + const parthenon::Real k = k_0_ + k_100_ * pow(r / r_k_, alpha_k_); + return k; + } + }; } // namespace cluster diff --git a/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp b/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp index 9c88269d..3504cd35 100644 --- a/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp +++ b/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp @@ -121,6 +121,7 @@ std::ostream &PRhoProfile::write_to_ostream( /************************************************************ *HydrostaticEquilibriumSphere::generate_P_rho_profile(x,y,z) ************************************************************/ + template PRhoProfile HydrostaticEquilibriumSphere::generate_P_rho_profile( @@ -154,10 +155,23 @@ HydrostaticEquilibriumSphere::generate_P_rho // Make sure to include R_fix_ Real r_start = r_fix_; Real r_end = r_fix_; + for (int k = kb.s; k <= kb.e; k++) { for (int j = jb.s; j <= jb.e; j++) { for (int i = ib.s; i <= ib.e; i++) { - + + // Integrate from r_start to r_end. If r_fix out of the domain, integrate from r_fix to r_max. + // Either: + // ==================== Integration domain ==================== + + // r_end = r_max --------- r_min ------ r_fix + + // or: + + // r_end ------------------------------ r_fix --------- r_min. + + // ============================================================ + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + coords.Xc<2>(j) * coords.Xc<2>(j) + coords.Xc<3>(k) * coords.Xc<3>(k)); @@ -166,16 +180,16 @@ HydrostaticEquilibriumSphere::generate_P_rho } } } - + // Add some room for R_start and R_end r_start = std::max(0.0, r_start - r_sampling_ * dr); r_end += r_sampling_ * dr; - + // Compute number of cells needed const auto n_r = static_cast(ceil((r_end - r_start) / dr)); // Make R_end consistent r_end = r_start + dr * (n_r - 1); - + return generate_P_rho_profile(r_start, r_end, n_r); } @@ -186,31 +200,31 @@ template PRhoProfile HydrostaticEquilibriumSphere::generate_P_rho_profile( const Real r_start, const Real r_end, const unsigned int n_r) const { - + // Array of radii along which to compute the profile ParArray1D device_r("PRhoProfile r", n_r); auto r = Kokkos::create_mirror_view(device_r); const Real dr = (r_end - r_start) / (n_r - 1.0); - + // Use a linear R - possibly adapt if using a mesh with logrithmic r for (int i = 0; i < n_r; i++) { r(i) = r_start + i * dr; } - + /************************************************************ * Integrate Pressure inward and outward from virial radius ************************************************************/ // Create array for pressure ParArray1D device_p("PRhoProfile p", n_r); auto p = Kokkos::create_mirror_view(device_p); - + const Real k_fix = entropy_profile_.K_from_r(r_fix_); const Real p_fix = P_from_rho_K(rho_fix_, k_fix); - + // Integrate P inward from R_fix_ Real r_i = r_fix_; // Start Ri at R_fix_ first Real p_i = p_fix; // Start with pressure at R_fix_ - + // Find the index in R right before R_fix_ int i_fix = static_cast(floor((n_r - 1) / (r_end - r_start) * (r_fix_ - r_start))); if (r_fix_ < r(i_fix) - kRTol || r_fix_ > r(i_fix + 1) + kRTol) { diff --git a/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp b/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp index fe1ceb34..933182dc 100644 --- a/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp +++ b/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp @@ -21,7 +21,7 @@ template class PRhoProfile; /************************************************************ - * Hydrostatic Equilbrium Spnere Class, + * Hydrostatic Equilbrium Sphere Class, * for initializing a sphere in hydrostatic equiblibrium * * @@ -36,26 +36,26 @@ class HydrostaticEquilibriumSphere { // Graviational field and entropy profile const GravitationalField gravitational_field_; const EntropyProfile entropy_profile_; - + // Physical constants parthenon::Real mh_, k_boltzmann_; - + // Density to fix baryons at a radius (change to temperature?) parthenon::Real r_fix_, rho_fix_; - + // Molecular weights parthenon::Real mu_, mu_e_; - + // R mesh sampling parameter parthenon::Real r_sampling_; - + /************************************************************ * Functions to build the cluster model * * Using lambda functions to make picking up model parameters seamlessly * Read A_from_B as "A as a function of B" ************************************************************/ - + // Get pressure from density and entropy, using ideal gas law and definition // of entropy KOKKOS_INLINE_FUNCTION parthenon::Real P_from_rho_K(const parthenon::Real rho, @@ -63,7 +63,7 @@ class HydrostaticEquilibriumSphere { const parthenon::Real p = k * pow(rho / mh_, 5. / 3.) / (mu_ * pow(mu_e_, 2. / 3.)); return p; } - + // Get density from pressure and entropy, using ideal gas law and definition // of entropy KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_P_K(const parthenon::Real p, @@ -71,7 +71,7 @@ class HydrostaticEquilibriumSphere { const parthenon::Real rho = pow(mu_ * p / k, 3. / 5.) * mh_ * pow(mu_e_, 2. / 5); return rho; } - + // Get total number density from density KOKKOS_INLINE_FUNCTION parthenon::Real n_from_rho(const parthenon::Real rho) const { const parthenon::Real n = rho / (mu_ * mh_); @@ -114,7 +114,7 @@ class HydrostaticEquilibriumSphere { return dP_dr; } }; - + // Takes one rk4 step from t0 to t1, taking y0 and returning y1, using y'(t) = f(t,y) template parthenon::Real step_rk4(const parthenon::Real t0, const parthenon::Real t1, diff --git a/src/pgen/cluster/isothermal_sphere.cpp b/src/pgen/cluster/isothermal_sphere.cpp new file mode 100644 index 00000000..d51a1850 --- /dev/null +++ b/src/pgen/cluster/isothermal_sphere.cpp @@ -0,0 +1,262 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file hydrostatic_equilbirum_sphere.cpp +// \brief Creates pressure profile in hydrostatic equilbrium +// +// Setups up a pressure profile in hydrostatic equilbrium given an entropy +// profile and gravitational field +//======================================================================================== + +// C++ headers +#include +#include + +// Parthenon headers +#include +#include +#include +#include + +// AthenaPK headers +#include "../../units.hpp" + +// Cluster headers +#include "cluster_gravity.hpp" +#include "entropy_profiles.hpp" +#include "hydrostatic_equilibrium_sphere.hpp" + +namespace cluster { +using namespace parthenon; + +/************************************************************ + * HydrostaticEquilibriumSphere constructor + ************************************************************/ +template +HydrostaticEquilibriumSphere:: + HydrostaticEquilibriumSphere(ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg, + GravitationalField gravitational_field, + EntropyProfile entropy_profile) + : gravitational_field_(gravitational_field), entropy_profile_(entropy_profile) { + Units units(pin); + + mh_ = units.mh(); + k_boltzmann_ = units.k_boltzmann(); + + mu_ = hydro_pkg->Param("mu"); + mu_e_ = hydro_pkg->Param("mu_e"); + + r_fix_ = pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", "r_fix", + 1953.9724519818478 * units.kpc()); + rho_fix_ = pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", "rho_fix", + 8.607065015897638e-30 * units.g() / pow(units.kpc(), 3)); + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + r_sampling_ = + pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", "r_sampling", 4.0); + + // Test out the HSE sphere if requested + const bool test_he_sphere = pin->GetOrAddBoolean( + "problem/cluster/hydrostatic_equilibrium", "test_he_sphere", false); + if (test_he_sphere) { + const Real test_he_sphere_r_start = + pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", + "test_he_sphere_r_start", 1e-3 * units.kpc()); + const Real test_he_sphere_r_end = + pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", + "test_he_sphere_r_end", 4000 * units.kpc()); + const int test_he_sphere_n_r = pin->GetOrAddInteger( + "problem/cluster/hydrostatic_equilibrium", "test_he_sphere_n_r", 4000); + if (Globals::my_rank == 0) { + + auto P_rho_profile = generate_P_rho_profile( + test_he_sphere_r_start, test_he_sphere_r_end, test_he_sphere_n_r); + + std::ofstream test_he_file; + test_he_file.open("test_he_sphere.dat"); + P_rho_profile.write_to_ostream(test_he_file); + test_he_file.close(); + } + } + + hydro_pkg->AddParam<>("hydrostatic_equilibirum_sphere", *this); +} + +/************************************************************ + * PRhoProfile::write_to_ostream + ************************************************************/ +template +std::ostream &PRhoProfile::write_to_ostream( + std::ostream &os) const { + + const typename HydrostaticEquilibriumSphere< + GravitationalField, EntropyProfile>::dP_dr_from_r_P_functor dP_dr_func(sphere_); + + auto host_r = Kokkos::create_mirror_view(r_); + Kokkos::deep_copy(host_r, r_); + auto host_p = Kokkos::create_mirror_view(p_); + Kokkos::deep_copy(host_p, p_); + + for (int i = 0; i < host_r.extent(0); i++) { + const Real r = host_r(i); + const Real p = host_p(i); + const Real k = sphere_.entropy_profile_.K_from_r(r); + const Real rho = sphere_.rho_from_P_K(p, k); + const Real n = sphere_.n_from_rho(rho); + const Real ne = sphere_.ne_from_rho(rho); + const Real temp = sphere_.T_from_rho_P(rho, p); + const Real g = sphere_.gravitational_field_.g_from_r(r); + const Real dP_dr = dP_dr_func(r, p); + + os << r << " " << p << " " << k << " " << rho << " " << n << " " << ne << " " << temp + << " " << g << " " << dP_dr << std::endl; + } + return os; +} + +/************************************************************ + *HydrostaticEquilibriumSphere::generate_P_rho_profile(x,y,z) + ************************************************************/ + +template +PRhoProfile +HydrostaticEquilibriumSphere::generate_P_rho_profile( + IndexRange ib, IndexRange jb, IndexRange kb, + parthenon::UniformCartesian coords) const { + + /************************************************************ + * Define R mesh to integrate pressure along + * + * R mesh should adapt with requirements of MeshBlock + ************************************************************/ + + // Determine spacing of grid (WARNING assumes equispaced grid in x,y,z) + PARTHENON_REQUIRE(std::abs(coords.Dxc<1>(0) - coords.Dxc<1>(1)) < + 10 * std::numeric_limits::epsilon(), + "No equidistant grid in x1dir"); + PARTHENON_REQUIRE(std::abs(coords.Dxc<2>(0) - coords.Dxc<2>(1)) < + 10 * std::numeric_limits::epsilon(), + "No equidistant grid in x2dir"); + PARTHENON_REQUIRE(std::abs(coords.Dxc<3>(0) - coords.Dxc<3>(1)) < + 10 * std::numeric_limits::epsilon(), + "No equidistant grid in x3dir"); + // Resolution of profile on this meshbock -- use 1/r_sampling_ of resolution + // or 1/r_sampling_ of r_k, whichever is smaller + const Real dr = + std::min(std::min(coords.Dxc<1>(0), std::min(coords.Dxc<2>(0), coords.Dxc<3>(0))) / + r_sampling_, + entropy_profile_.r_k_ / r_sampling_); + + // Loop through mesh for minimum and maximum radius + // Make sure to include R_fix_ + Real r_start = r_fix_; + Real r_end = r_fix_; + for (int k = kb.s; k <= kb.e; k++) { + for (int j = jb.s; j <= jb.e; j++) { + for (int i = ib.s; i <= ib.e; i++) { + + const Real r = + sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + r_start = std::min(r, r_start); + r_end = std::max(r, r_end); + } + } + } + + // Add some room for R_start and R_end + r_start = std::max(0.0, r_start - r_sampling_ * dr); + r_end += r_sampling_ * dr; + + // Compute number of cells needed + const auto n_r = static_cast(ceil((r_end - r_start) / dr)); + // Make R_end consistent + r_end = r_start + dr * (n_r - 1); + + return generate_P_rho_profile(r_start, r_end, n_r); +} + +/************************************************************ + * HydrostaticEquilibriumSphere::generate_P_rho_profile(Ri,Re,nR) + ************************************************************/ +template +PRhoProfile +HydrostaticEquilibriumSphere::generate_P_rho_profile( + const Real r_start, const Real r_end, const unsigned int n_r) const { + + // Array of radii along which to compute the profile + ParArray1D device_r("PRhoProfile r", n_r); + auto r = Kokkos::create_mirror_view(device_r); + const Real dr = (r_end - r_start) / (n_r - 1.0); + + // Use a linear R - possibly adapt if using a mesh with logrithmic r + for (int i = 0; i < n_r; i++) { + r(i) = r_start + i * dr; + } + + /************************************************************ + * Integrate Pressure inward and outward from virial radius + ************************************************************/ + // Create array for pressure + ParArray1D device_p("PRhoProfile p", n_r); + auto p = Kokkos::create_mirror_view(device_p); + + const Real k_fix = entropy_profile_.K_from_r(r_fix_); + const Real p_fix = P_from_rho_K(rho_fix_, k_fix); + + // Integrate P inward from R_fix_ + Real r_i = r_fix_; // Start Ri at R_fix_ first + Real p_i = p_fix; // Start with pressure at R_fix_ + + // Find the index in R right before R_fix_ + int i_fix = static_cast(floor((n_r - 1) / (r_end - r_start) * (r_fix_ - r_start))); + if (r_fix_ < r(i_fix) - kRTol || r_fix_ > r(i_fix + 1) + kRTol) { + std::stringstream msg; + msg << "### FATAL ERROR in function " + "[HydrostaticEquilibriumSphere::generate_P_rho_profile]" + << std::endl + << "r(i_fix) to r_(i_fix+1) does not contain r_fix_" << std::endl + << "r(i_fix) r_fix_ r(i_fix+1):" << r(i_fix) << " " << r_fix_ << " " + << r(i_fix + 1) << std::endl; + PARTHENON_FAIL(msg); + } + + dP_dr_from_r_P_functor dP_dr_from_r_P(*this); + + // Make is the i right before R_fix_ + for (int i = i_fix + 1; i > 0; i--) { // Move is up one, to account for initial R_fix_ + p(i - 1) = step_rk4(r_i, r(i - 1), p_i, dP_dr_from_r_P); + r_i = r(i - 1); + p_i = p(i - 1); + } + + // Integrate P outward from R_fix_ + r_i = r_fix_; // Start Ri at R_fix_ first + p_i = p_fix; // Start with pressure at R_fix_ + + // Make is the i right after R_fix_ + for (int i = i_fix; i < n_r - 1; + i++) { // Move is back one, to account for initial R_fix_ + p(i + 1) = step_rk4(r_i, r(i + 1), p_i, dP_dr_from_r_P); + r_i = r(i + 1); + p_i = p(i + 1); + } + + Kokkos::deep_copy(device_r, r); + Kokkos::deep_copy(device_p, p); + + return PRhoProfile(device_r, device_p, r(0), + r(n_r - 1), *this); +} + +// Instantiate HydrostaticEquilibriumSphere +template class HydrostaticEquilibriumSphere; + +// Instantiate PRhoProfile +template class PRhoProfile; + +} // namespace cluster diff --git a/src/pgen/cluster/isothermal_sphere.hpp b/src/pgen/cluster/isothermal_sphere.hpp new file mode 100644 index 00000000..b99ef68b --- /dev/null +++ b/src/pgen/cluster/isothermal_sphere.hpp @@ -0,0 +1,127 @@ +#ifndef CLUSTER_ISOTHERMAL_SPHERE_HPP_ +#define CLUSTER_ISOTHERMAL_SPHERE_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file hydrostatic_equilbirum_sphere +// \brief Class for initializing a sphere in hydrostatic equiblibrium + +// Parthenon headers +#include +#include + +// AthenaPK headers +#include "../../units.hpp" + +namespace cluster { + +/************************************************************ + * Hydrostatic Equilbrium Sphere Class, + * for initializing a sphere in hydrostatic equiblibrium + * + * + * GravitationField: + * Graviational field class with member function g_from_r(parthenon::Real r) + * EntropyProfile: + * Entropy profile class with member function g_from_r(parthenon::Real r) + ************************************************************/ +template +class IsothermalSphere { + private: + // Graviational field and entropy profile + const GravitationalField gravitational_field_; + + // Physical constants + parthenon::Real mh_, k_boltzmann_; + + // Molecular weights + parthenon::Real mu_ + + /************************************************************ + * Functions to build the cluster model + * + * Using lambda functions to make picking up model parameters seamlessly + * Read A_from_B as "A as a function of B" + ************************************************************/ + + // Get the dP/dr from radius and pressure, which we will // + /************************************************************ + * dP_dr_from_r_P_functor + * + * Functor class giving dP/dr (r,P) + * which is used to compute the HSE profile + ************************************************************/ + + public: + KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r) const { + using parthenon::Real; + + + functor( + const IsothermalSphere &sphere) + : sphere_(sphere) {} + parthenon::Real operator()(const parthenon::Real r) const { + + const parthenon::Real rho = sphere_.gravitational_field_.rho_from_r(r); + return rho; + } + }; + + public: + IsothermalSphere(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg, + GravitationalField gravitational_field); + + PProfile + generate_P_profile(parthenon::IndexRange ib, parthenon::IndexRange jb, + parthenon::IndexRange kb, + parthenon::UniformCartesian coords) const; +}; + +template +class PProfile { + private: + const parthenon::ParArray1D r_; + const parthenon::ParArray1D p_; + const IsothermalSphere sphere_; + + public: + PProfile( + const IsothermalSphere &sphere) + : r_(r), p_(p), sphere_(sphere), n_r_(r_.extent(0)), r_start_(r_start), + r_end_(r_end) {} + + KOKKOS_INLINE_FUNCTION parthenon::Real P_from_r(const parthenon::Real r) const { + // Determine indices in R bounding r + const int i_r = + static_cast(floor((n_r_ - 1) / (r_end_ - r_start_) * (r - r_start_))); + + if (r < r_(i_r) - sphere_.kRTol || r > r_(i_r + 1) + sphere_.kRTol) { + Kokkos::abort("PProfile::P_from_r R(i_r) to R_(i_r+1) does not contain r"); + } + + // Linearly interpolate Pressure from P + const parthenon::Real P_r = + (p_(i_r) * (r_(i_r + 1) - r) + p_(i_r + 1) * (r - r_(i_r))) / + (r_(i_r + 1) - r_(i_r)); + + return P_r; + } + + KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r) const { + using parthenon::Real; + // Get pressure first + const Real p_r = P_from_r(r); + // Compute entropy and pressure here + const Real k_r = sphere_.entropy_profile_.K_from_r(r); + const Real rho_r = sphere_.rho_from_P_K(p_r, k_r); + return rho_r; + } + std::ostream &write_to_ostream(std::ostream &os) const; +}; + +} // namespace cluster + +#endif // CLUSTER_HYDROSTATIC_EQUILIBRIUM_SPHERE_HPP_ diff --git a/src/pgen/old_cluster/cluster_16nov.cpp b/src/pgen/old_cluster/cluster_16nov.cpp new file mode 100644 index 00000000..18657bf6 --- /dev/null +++ b/src/pgen/old_cluster/cluster_16nov.cpp @@ -0,0 +1,1395 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster.cpp +// \brief Idealized galaxy cluster problem generator +// +// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in +// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, +// optionally with an initial magnetic tower field. Includes AGN feedback, AGN +// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback +//======================================================================================== + +// C headers + +// C++ headers +#include // min, max +#include // sqrt() +#include // fopen(), fprintf(), freopen() +#include // endl +#include +#include // stringstream +#include // runtime_error +#include // c_str() + +// #include // HDF5 +#include "../../external/HighFive/include/highfive/H5Easy.hpp" + +// Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" +#include +#include + +// AthenaPK headers +#include "../eos/adiabatic_glmmhd.hpp" +#include "../eos/adiabatic_hydro.hpp" +#include "../hydro/hydro.hpp" +#include "../hydro/srcterms/gravitational_field.hpp" +#include "../hydro/srcterms/tabular_cooling.hpp" +#include "../main.hpp" +#include "../utils/few_modes_ft.hpp" +#include "../utils/few_modes_ft_lognormal.hpp" + +// Cluster headers +#include "cluster/agn_feedback.hpp" +#include "cluster/agn_triggering.hpp" +#include "cluster/cluster_clips.hpp" +#include "cluster/cluster_gravity.hpp" +#include "cluster/cluster_reductions.hpp" +#include "cluster/entropy_profiles.hpp" +#include "cluster/hydrostatic_equilibrium_sphere.hpp" +#include "cluster/magnetic_tower.hpp" +#include "cluster/snia_feedback.hpp" +#include "cluster/stellar_feedback.hpp" + +namespace cluster { +using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; +using utils::few_modes_ft::FewModesFT; +using utils::few_modes_ft_log::FewModesFTLog; + +void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); + + if (gravity_srcterm) { + const ClusterGravity &cluster_gravity = + hydro_pkg->Param("cluster_gravity"); + + GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); + } + + const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); + agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); + + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); + + const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); +}; +void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real dt) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); + stellar_feedback.FeedbackSrcTerm(md, dt, tm); + + ApplyClusterClips(md, tm, dt); +} + +Real ClusterEstimateTimestep(MeshData *md) { + Real min_dt = std::numeric_limits::max(); + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // TODO time constraints imposed by thermal AGN feedback, jet velocity, + // magnetic tower + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); + min_dt = std::min(min_dt, agn_triggering_min_dt); + + return min_dt; +} + +//======================================================================================== +//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor +//! *hydro_pkg) \brief Init package data from parameter input +//======================================================================================== + +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { + + /************************************************************ + * Read Uniform Gas + ************************************************************/ + + const bool init_uniform_gas = + pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); + hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); + + if (init_uniform_gas) { + const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); + const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); + const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); + const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); + const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); + + hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); + hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); + hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); + hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); + hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_uniform_b_field = pin->GetOrAddBoolean( + "problem/cluster/uniform_b_field", "init_uniform_b_field", false); + hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); + + if (init_uniform_b_field) { + const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); + const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); + const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); + + hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); + hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); + hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", + "init_dipole_b_field", false); + hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); + + if (init_dipole_b_field) { + const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); + const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); + const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); + + hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); + hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); + hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); + } + + /************************************************************ + * Read Cluster Gravity Parameters + ************************************************************/ + + // Build cluster_gravity object + ClusterGravity cluster_gravity(pin, hydro_pkg); + + // Include gravity as a source term during evolution + const bool gravity_srcterm = + pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); + hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); + + /************************************************************ + * Read Initial Entropy Profile + ************************************************************/ + + // Create hydrostatic sphere with ACCEPT entropy profile + ACCEPTEntropyProfile entropy_profile(pin); + + HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, + entropy_profile); + + // Create hydrostatic sphere with ISOTHERMAL entropy profile + + + + /************************************************************ + * Read Precessing Jet Coordinate system + ************************************************************/ + + JetCoordsFactory jet_coords_factory(pin, hydro_pkg); + + /************************************************************ + * Read AGN Feedback + ************************************************************/ + + AGNFeedback agn_feedback(pin, hydro_pkg); + + /************************************************************ + * Read AGN Triggering + ************************************************************/ + + AGNTriggering agn_triggering(pin, hydro_pkg); + + /************************************************************ + * Read Magnetic Tower + ************************************************************/ + + // Build Magnetic Tower + MagneticTower magnetic_tower(pin, hydro_pkg); + + // Determine if magnetic_tower_power_scaling is needed + // Is AGN Power and Magnetic fraction non-zero? + bool magnetic_tower_power_scaling = + (agn_feedback.magnetic_fraction_ != 0 && + (agn_feedback.fixed_power_ != 0 || + agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); + hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); + + /************************************************************ + * Read SNIA Feedback + ************************************************************/ + + SNIAFeedback snia_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Stellar Feedback + ************************************************************/ + + StellarFeedback stellar_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Clips (ceilings and floors) + ************************************************************/ + + // Disable all clips by default with a negative radius clip + Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); + + // By default disable floors by setting a negative value + Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); + + // By default disable ceilings by setting to infinity + Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", + std::numeric_limits::infinity()); + Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", + std::numeric_limits::infinity()); + Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", + std::numeric_limits::infinity()); + Real eceil = Tceil; + if (eceil < std::numeric_limits::infinity()) { + if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { + PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " + "Either set a 'units' block and the 'hydro/He_mass_fraction' in " + "input file or use a pressure floor " + "(defined code units) instead."); + } + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); + } + hydro_pkg->AddParam("cluster_dfloor", dfloor); + hydro_pkg->AddParam("cluster_eceil", eceil); + hydro_pkg->AddParam("cluster_vceil", vceil); + hydro_pkg->AddParam("cluster_vAceil", vAceil); + hydro_pkg->AddParam("cluster_clip_r", clip_r); + + /************************************************************ + * Start running reductions into history outputs for clips, stellar mass, cold + * gas, and AGN extent + ************************************************************/ + + /* FIXME(forrestglines) This implementation with a reduction into Params might + be broken in several ways. + 1. Each reduction in params is Rank local. Multiple meshblocks packs per + rank adding to these params is not thread-safe + 2. These Params are not carried over between restarts. If a restart dump is + made and a history output is not, then the mass/energy between the last + history output and the restart dump is lost + */ + std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", + "removed_eceil_energy", "removed_vceil_energy", + "added_vAceil_mass"}; + + // Add a param for each reduction, then add it as a summation reduction for + // history outputs + auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); + + for (auto reduction_str : reduction_strs) { + hydro_pkg->AddParam(reduction_str, 0.0, true); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, + [reduction_str](MeshData *md) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const Real reduction = hydro_pkg->Param(reduction_str); + // Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str, 0.0); + return reduction; + }, + reduction_str)); + } + + // Add history reduction for total cold gas using stellar mass threshold + const Real cold_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); + if (cold_thresh > 0) { + hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); + } + const Real agn_tracer_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); + if (agn_tracer_thresh >= 0) { + PARTHENON_REQUIRE( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); + hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); + } + + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); + + /************************************************************ + * Add derived fields + * NOTE: these must be filled in UserWorkBeforeOutput + ************************************************************/ + + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); + + // log10 of cell-centered radius + hydro_pkg->AddField("log10_cell_radius", m); + // entropy + hydro_pkg->AddField("entropy", m); + // sonic Mach number v/c_s + hydro_pkg->AddField("mach_sonic", m); + // temperature + hydro_pkg->AddField("temperature", m); + + if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { + // cooling time + hydro_pkg->AddField("cooling_time", m); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + // alfven Mach number v_A/c_s + hydro_pkg->AddField("mach_alfven", m); + + // plasma beta + hydro_pkg->AddField("plasma_beta", m); + } + + /************************************************************ + * Read Density perturbation + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); // Mean density of perturbations + + if (mu_rho != 0.0) { + + auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", "k_min_rho"); // Minimum wavenumber of perturbation + auto num_modes_rho = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); + auto sol_weight_rho = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); + uint32_t rseed_rho = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); + + // Computing the kmax ie. the Nyquist limit + auto grid_ni = pin->GetOrAddInteger("parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis + auto k_max_rho = grid_ni / 2; + + const auto t_corr_rho = 1e-10; + auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog(num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes + + auto few_modes_ft_rho = FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, + k_vec_rho, k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); + + hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + + } + + /************************************************************ + * Read Velocity perturbation + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + if (sigma_v != 0.0) { + // peak of init vel perturb + auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); + auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); + + PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || + (k_peak_v > 0.0 && l_peak_v <= 0.0), + "Setting initial velocity perturbation requires a single " + "length scale by either setting l_peak_v or k_peak_v."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_v > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_v = Lx / l_peak_v; + } + + auto num_modes_v = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); + auto sol_weight_v = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); + uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); + // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const auto t_corr = 1e-10; + + auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); + + auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, + k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); + hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + + /************************************************************ + * Read Magnetic field perturbation + ************************************************************/ + + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, + "Requested initial magnetic field perturbations but not " + "solving the MHD equations.") + // peak of init magnetic field perturb + auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); + auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); + PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || + (k_peak_b > 0.0 && l_peak_b <= 0.0), + "Setting initial B perturbation requires a single " + "length scale by either setting l_peak_b or k_peak_b."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_b > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_b = Lx / l_peak_b; + } + + auto num_modes_b = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); + uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); + // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const auto t_corr = 1e-10; + // This field should by construction have no compressive modes, so we fix the number. + const auto sol_weight_b = 1.0; + + auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); + + const bool fill_ghosts = true; // as we fill a vector potential to calc B + auto few_modes_ft = + FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, + sol_weight_b, t_corr, rseed_b, fill_ghosts); + hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now). Only add if not already done for the velocity. + if (sigma_v == 0.0) { + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + } + +} + +//======================================================================================== +//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) +//! \brief Generate problem data for all blocks on rank +// +// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that +// reductions work +//======================================================================================== + +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + + // This could be more optimized, but require a refactor of init routines being called. + // However, given that it's just called during initial setup, this should not be a + // performance concern. + + // Defining a table within which the values of the hydrostatic density profile will be stored + auto pmc = md->GetBlockData(0)->GetBlockPointer(); + const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); + + // Here, the pmc-> contains the number of cells of each MESHBLOCK, including the 2*n_ghost ghost cells + parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho", md->NumBlocks(), + pmc->cellbounds.ncellsk(IndexDomain::entire), + pmc->cellbounds.ncellsj(IndexDomain::entire), + pmc->cellbounds.ncellsi(IndexDomain::entire)); + + for (int b = 0; b < md->NumBlocks(); b++) { + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + + auto &coords = pmb->coords; + + // Get Adiabatic Index + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + /************************************************************ + * Initialize the initial hydro state + ************************************************************/ + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + const auto isothermal_sphere = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); + const auto isothermal_hernquist = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); + + if (init_uniform_gas) { + const Real rho = hydro_pkg->Param("uniform_gas_rho"); + const Real ux = hydro_pkg->Param("uniform_gas_ux"); + const Real uy = hydro_pkg->Param("uniform_gas_uy"); + const Real uz = hydro_pkg->Param("uniform_gas_uz"); + const Real pres = hydro_pkg->Param("uniform_gas_pres"); + + const Real Mx = rho * ux; + const Real My = rho * uy; + const Real Mz = rho * uz; + const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + u(IDN, k, j, i) = rho; + u(IM1, k, j, i) = Mx; + u(IM2, k, j, i) = My; + u(IM3, k, j, i) = Mz; + u(IEN, k, j, i) = E; + }); + + // end if(init_uniform_gas) + } else if (isothermal_sphere) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real mu = hydro_pkg->Param("mu"); + const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); + const Real grav_const = units.gravitational_constant(); + + std::cout << "Entering isothermal sphere generation \n" ; + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real rho_r = prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); + const Real P_r = (prefactor/2) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } else if (isothermal_hernquist) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real m_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); + const Real r_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); + const Real mu = hydro_pkg->Param("mu"); + const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); + const Real grav_const = units.gravitational_constant(); + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real phi_r = - grav_const * m_bcg_s / (r_effective + r_bcg_s); + const Real rho_r = rho_0 * std::exp( - mu * units.mh() / (units.k_boltzmann() * T_bcg_s) * phi_r); + const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } + + else { + /************************************************************ + * Initialize a HydrostaticEquilibriumSphere + ************************************************************/ + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + // Get pressure and density from generated profile + const Real P_r = P_rho_profile.P_from_r(r); + const Real rho_r = P_rho_profile.rho_from_r(r); + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + // Updating hydrostatic_rho table + hydrostatic_rho(b, k, j, i) = rho_r; + + }); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + /************************************************************ + * Initialize the initial magnetic field state via a vector potential + ************************************************************/ + parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + /************************************************************ + * Initialize an initial magnetic tower + ************************************************************/ + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + + /************************************************************ + * Add dipole magnetic field to the magnetic potential + ************************************************************/ + const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); + if (init_dipole_b_field) { + const Real mx = hydro_pkg->Param("dipole_b_field_mx"); + const Real my = hydro_pkg->Param("dipole_b_field_my"); + const Real mz = hydro_pkg->Param("dipole_b_field_mz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + + const Real m_cross_r_x = my * z - mz * y; + const Real m_cross_r_y = mz * x - mx * z; + const Real m_cross_r_z = mx * y - mx * y; + + // To check whether there is some component before initiating perturbations + std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; + + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); + A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); + A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + }); + } + + /************************************************************ + * Apply the potential to the conserved variables + ************************************************************/ + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + u(IB1, k, j, i) = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + + SQR(u(IB3, k, j, i))); + }); + + /************************************************************ + * Add uniform magnetic field to the conserved variables + ************************************************************/ + const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); + if (init_uniform_b_field) { + const Real bx = hydro_pkg->Param("uniform_b_field_bx"); + const Real by = hydro_pkg->Param("uniform_b_field_by"); + const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + const Real bx_i = u(IB1, k, j, i); + const Real by_i = u(IB2, k, j, i); + const Real bz_i = u(IB3, k, j, i); + + u(IB1, k, j, i) += bx; + u(IB2, k, j, i) += by; + u(IB3, k, j, i) += bz; + + // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + + // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that + u(IEN, k, j, i) += + bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + }); + // end if(init_uniform_b_field) + } + + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) + } + + /************************************************************ + * Initial parameters + ************************************************************/ + + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto fluid = hydro_pkg->Param("fluid"); + auto const &cons = md->PackVariables(std::vector{"cons"}); + const auto num_blocks = md->NumBlocks(); + + /************************************************************ + * Set initial density perturbations (read from HDF5 file) + ************************************************************/ + + const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); + const bool full_box = pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); + const bool spherical_collapse = pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); + + hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); + + Real passive_scalar = 0.0; // Not useful here + + // Spherical collapse test with an initial overdensity + + if (spherical_collapse == true) { + + // Create an homogeneous sphere of a density superior or inferior to the background density + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius + const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); + const Real background_density = pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); + const Real foreground_density = pin->GetOrAddReal("problem/cluster/init_perturb", "foreground_density", 150); + + // Setting an initial + u(IDN, k, j, i) = background_density; + + // For any cell at a radius less then overdensity radius, set the density to foreground_density + if (r < overdensity_radius){ + u(IDN, k, j, i) = foreground_density; + } + + }, + passive_scalar); + + } + + /* -------------- Setting up a clumpy atmosphere -------------- + + 1) Extract the values of the density from an input hdf5 file using H5Easy + 2) Initiate the associated density field + 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) + 4) Optionnaly, add a central overdensity + + */ + + if (init_perturb_rho == true) { + + auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", "init_perturb_rho_file","none"); + + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real perturb_amplitude = pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); + + std::string keys_rho = "data"; + H5Easy::File file(filename_rho, HighFive::File::ReadOnly); + + const int rho_init_size = 256; + auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>(file, keys_rho); + + Real passive_scalar = 0.0; // Useless + + std::cout << "Entering initialisation of rho field"; + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + // Case where the box is filled with perturbations of equal mean amplitude + if (full_box){ + + u(IDN, k, j, i) += perturb_amplitude * rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * (u(IDN, k, j, i) / 29.6); + + } + + // Mean amplitude of the perturbations is modulated by the + else { + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); + const Real r_effective = std::max(r,r_smoothing); + + //u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; + + } + + }, + passive_scalar); + + } + + /************************************************************ + * Setting up a perturbed density field (hardcoded version) + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); + const auto background_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); + + if (mu_rho != 0.0) { + + auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); + + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft_rho.SetPhases(pmb.get(), pin); + + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); + + Real v2_sum_rho = 0.0; // used for normalization + + auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + Real perturb = 0.0 ; + + perturb = std::sqrt(SQR(perturb_pack_rho(b, 0, k, j, i)) + SQR(perturb_pack_rho(b, 1, k, j, i)) + SQR(perturb_pack_rho(b, 2, k, j, i))); + u(IDN, k, j, i) = background_rho + mu_rho * perturb; + + }, + v2_sum_rho); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + } + + /************************************************************ + * Set initial velocity perturbations (requires no other velocities for now) + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + + if (sigma_v != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real v2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + auto rho = u(IDN, k, j, i); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, + "Found existing non-zero velocity when setting velocity perturbations."); + + u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); + u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); + u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); + // No need to touch the energy yet as we'll normalize later + + lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * + coords.CellVolume(k, j, i) / SQR(rho); + }, + v2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); + + pmb->par_for( + "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IM1, k, j, i) /= v_norm; + u(IM2, k, j, i) /= v_norm; + u(IM3, k, j, i) /= v_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / + u(IDN, k, j, i); + }); + } + + /************************************************************ + * Set initial magnetic field perturbations (resets magnetic field field) + ************************************************************/ + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + const auto alpha_b = pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0/3.0); + const auto density_scale = pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); + const auto standard_B = pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); + const auto r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); + + if (sigma_b != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real b2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + // Defining a new table that will contains the values of the perturbed magnetic field, and magnetic energy + parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, + pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + if (standard_B){ + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0; + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * + coords.CellVolume(k, j, i); + }, + b2_sum); + + } else { + + + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + auto pmbl = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkgl = pmbl->packages.Get("Hydro"); + + const Real x = coords.Xc<1>(i); + const Real xp = coords.Xc<1>(i+1); + const Real xm = coords.Xc<1>(i-1); + const Real y = coords.Xc<1>(j); + const Real yp = coords.Xc<1>(j+1); + const Real ym = coords.Xc<1>(j-1); + const Real z = coords.Xc<1>(k); + const Real zp = coords.Xc<1>(k+1); + const Real zm = coords.Xc<1>(k-1); + + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); + const Real r_ip = std::sqrt(SQR(xp) + SQR(y) + SQR(z)); + const Real r_im = std::sqrt(SQR(xm) + SQR(y) + SQR(z)); + const Real r_jp = std::sqrt(SQR(x) + SQR(yp) + SQR(z)); + const Real r_jm = std::sqrt(SQR(x) + SQR(ym) + SQR(z)); + const Real r_kp = std::sqrt(SQR(x) + SQR(y) + SQR(zp)); + const Real r_km = std::sqrt(SQR(x) + SQR(y) + SQR(zm)); + + // Re-generating the hydrostatic sphere + const auto &he_sphere = + hydro_pkgl + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); + const auto rho_central = P_rho_profile.rho_from_r(r_smoothing); + const auto rho_ip = P_rho_profile.rho_from_r(r_ip); + const auto rho_im = P_rho_profile.rho_from_r(r_im); + const auto rho_jp = P_rho_profile.rho_from_r(r_jp); + const auto rho_jm = P_rho_profile.rho_from_r(r_jm); + const auto rho_kp = P_rho_profile.rho_from_r(r_kp); + const auto rho_km = P_rho_profile.rho_from_r(r_km); + + // const Real rho_r = P_rho_profile.rho_from_r(r); + + // Normalization condition here, which we want to lift. To do so, we'll need + // to copy the values of the magnetic field into a new array, which we will + // use to store the perturbed magnetic field and then rescale it, until values + // are added to u = cons(b) + + //PARTHENON_REQUIRE( + // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + // "Found existing non-zero B when setting magnetic field perturbations."); + + // First, needs to rescale the perturb_pack, ie. the magnetic field potential + Real perturb2_k_jp_i,perturb2_k_jm_i,perturb1_kp_j_i,perturb1_km_j_i; + Real perturb0_kp_j_i,perturb0_km_j_i,perturb2_k_j_ip,perturb2_k_j_im; + Real perturb1_k_j_ip,perturb1_k_j_im,perturb0_k_jp_i,perturb0_k_jm_i; + + // Rescaling the magnetic potential + + perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i) * std::pow(rho_jp, alpha_b); + perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i) * std::pow(rho_jm, alpha_b); + perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i) * std::pow(rho_kp, alpha_b); + perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i) * std::pow(rho_km, alpha_b); + perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i) * std::pow(rho_kp, alpha_b); + perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i) * std::pow(rho_km, alpha_b); + perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1) * std::pow(rho_ip, alpha_b); + perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1) * std::pow(rho_im, alpha_b); + perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1) * std::pow(rho_ip, alpha_b); + perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1) * std::pow(rho_im, alpha_b); + perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i) * std::pow(rho_jp, alpha_b); + perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i) * std::pow(rho_jm, alpha_b); + + // Then, compute the curl of the magnetic field + + Real curlBx,curlBy,curlBz; + + curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - + (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0 ; + curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - + (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0 ; + curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - + (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0 ; + + dB(b,0, k, j, i) = curlBx; + dB(b,1, k, j, i) = curlBy; + dB(b,2, k, j, i) = curlBz; + + lsum += (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); + + /* + gradBx = (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 ; + + gradBy = (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 ; + + gradBz = (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 ; + + dB(b,0, k, j, i) = gradBx * std::pow(hydrostatic_rho(b,k,j,i) / 29.6, alpha_b); + dB(b,1, k, j, i) = gradBy * std::pow(hydrostatic_rho(b,k,j,i) / 29.6, alpha_b); + dB(b,2, k, j, i) = gradBz * std::pow(hydrostatic_rho(b,k,j,i) / 29.6, alpha_b); + + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(gradBx) + SQR(gradBy) + SQR(gradBz)) * + coords.CellVolume(k, j, i); + */ + }, + b2_sum); + } + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); + + if (standard_B){ + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); + }); + + } else { + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + dB(b, 0, k, j, i) /= b_norm; + dB(b, 1, k, j, i) /= b_norm; + dB(b, 2, k, j, i) /= b_norm; + + // Computing energy + dB(b, 3, k, j, i) += + 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + SQR(dB(b, 2, k, j, i))); + + // Updating the MHD vector + + u(IB1, k, j, i) += dB(b, 0, k, j, i); + u(IB2, k, j, i) += dB(b, 1, k, j, i); + u(IB3, k, j, i) += dB(b, 2, k, j, i); + u(IEN, k, j, i) += dB(b, 3, k, j, i); + + }); + } + } +} + +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { + // get hydro + auto pkg = pmb->packages.Get("Hydro"); + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + // get prim vars + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; + + // get derived fields + auto &log10_radius = data->Get("log10_cell_radius").data; + auto &entropy = data->Get("entropy").data; + auto &mach_sonic = data->Get("mach_sonic").data; + auto &temperature = data->Get("temperature").data; + + // for computing temperature from primitives + auto units = pkg->Param("units"); + auto mbar_over_kb = pkg->Param("mbar_over_kb"); + auto mbar = mbar_over_kb * units.k_boltzmann(); + + // fill derived vars (*including ghost cells*) + auto &coords = pmb->coords; + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real v1 = prim(IV1, k, j, i); + const Real v2 = prim(IV2, k, j, i); + const Real v3 = prim(IV3, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute radius + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r2 = SQR(x) + SQR(y) + SQR(z); + log10_radius(k, j, i) = 0.5 * std::log10(r2); + + // compute entropy + const Real K = P / std::pow(rho / mbar, gam); + entropy(k, j, i) = K; + + const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + const Real M_s = v_mag / c_s; + mach_sonic(k, j, i) = M_s; + + // compute temperature + temperature(k, j, i) = mbar_over_kb * P / rho; + }); + + if (pkg->Param("enable_cooling") == Cooling::tabular) { + auto &cooling_time = data->Get("cooling_time").data; + + // get cooling function + const cooling::TabularCooling &tabular_cooling = + pkg->Param("tabular_cooling"); + const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute cooling time + const Real eint = P / (rho * gm1); + const Real edot = cooling_table_obj.DeDt(eint, rho); + cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; + }); + } + + if (pkg->Param("fluid") == Fluid::glmmhd) { + auto &plasma_beta = data->Get("plasma_beta").data; + auto &mach_alfven = data->Get("mach_alfven").data; + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + const Real Bx = prim(IB1, k, j, i); + const Real By = prim(IB2, k, j, i); + const Real Bz = prim(IB3, k, j, i); + const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + + // compute Alfven mach number + const Real v_A = std::sqrt(B2 / rho); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; + + // compute plasma beta + plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; + }); + } +} + +} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_17nov.cpp b/src/pgen/old_cluster/cluster_17nov.cpp new file mode 100644 index 00000000..44f1bbfc --- /dev/null +++ b/src/pgen/old_cluster/cluster_17nov.cpp @@ -0,0 +1,1404 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster.cpp +// \brief Idealized galaxy cluster problem generator +// +// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in +// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, +// optionally with an initial magnetic tower field. Includes AGN feedback, AGN +// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback +//======================================================================================== + +// C headers + +// C++ headers +#include // min, max +#include // sqrt() +#include // fopen(), fprintf(), freopen() +#include // endl +#include +#include // stringstream +#include // runtime_error +#include // c_str() + +// #include // HDF5 +#include "../../external/HighFive/include/highfive/H5Easy.hpp" + +// Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" +#include +#include + +// AthenaPK headers +#include "../eos/adiabatic_glmmhd.hpp" +#include "../eos/adiabatic_hydro.hpp" +#include "../hydro/hydro.hpp" +#include "../hydro/srcterms/gravitational_field.hpp" +#include "../hydro/srcterms/tabular_cooling.hpp" +#include "../main.hpp" +#include "../utils/few_modes_ft.hpp" +#include "../utils/few_modes_ft_lognormal.hpp" + +// Cluster headers +#include "cluster/agn_feedback.hpp" +#include "cluster/agn_triggering.hpp" +#include "cluster/cluster_clips.hpp" +#include "cluster/cluster_gravity.hpp" +#include "cluster/cluster_reductions.hpp" +#include "cluster/entropy_profiles.hpp" +#include "cluster/hydrostatic_equilibrium_sphere.hpp" +#include "cluster/magnetic_tower.hpp" +#include "cluster/snia_feedback.hpp" +#include "cluster/stellar_feedback.hpp" + +namespace cluster { +using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; +using utils::few_modes_ft::FewModesFT; +using utils::few_modes_ft_log::FewModesFTLog; + +void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); + + if (gravity_srcterm) { + const ClusterGravity &cluster_gravity = + hydro_pkg->Param("cluster_gravity"); + + GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); + } + + const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); + agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); + + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); + + const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); +}; +void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real dt) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); + stellar_feedback.FeedbackSrcTerm(md, dt, tm); + + ApplyClusterClips(md, tm, dt); +} + +Real ClusterEstimateTimestep(MeshData *md) { + Real min_dt = std::numeric_limits::max(); + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // TODO time constraints imposed by thermal AGN feedback, jet velocity, + // magnetic tower + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); + min_dt = std::min(min_dt, agn_triggering_min_dt); + + return min_dt; +} + +//======================================================================================== +//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor +//! *hydro_pkg) \brief Init package data from parameter input +//======================================================================================== + +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { + + /************************************************************ + * Read Uniform Gas + ************************************************************/ + + const bool init_uniform_gas = + pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); + hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); + + if (init_uniform_gas) { + const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); + const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); + const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); + const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); + const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); + + hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); + hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); + hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); + hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); + hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_uniform_b_field = pin->GetOrAddBoolean( + "problem/cluster/uniform_b_field", "init_uniform_b_field", false); + hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); + + if (init_uniform_b_field) { + const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); + const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); + const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); + + hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); + hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); + hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", + "init_dipole_b_field", false); + hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); + + if (init_dipole_b_field) { + const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); + const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); + const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); + + hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); + hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); + hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); + } + + /************************************************************ + * Read Cluster Gravity Parameters + ************************************************************/ + + // Build cluster_gravity object + ClusterGravity cluster_gravity(pin, hydro_pkg); + + // Include gravity as a source term during evolution + const bool gravity_srcterm = + pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); + hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); + + /************************************************************ + * Read Initial Entropy Profile + ************************************************************/ + + // Create hydrostatic sphere with ACCEPT entropy profile + ACCEPTEntropyProfile entropy_profile(pin); + + HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, + entropy_profile); + + // Create hydrostatic sphere with ISOTHERMAL entropy profile + + + + /************************************************************ + * Read Precessing Jet Coordinate system + ************************************************************/ + + JetCoordsFactory jet_coords_factory(pin, hydro_pkg); + + /************************************************************ + * Read AGN Feedback + ************************************************************/ + + AGNFeedback agn_feedback(pin, hydro_pkg); + + /************************************************************ + * Read AGN Triggering + ************************************************************/ + + AGNTriggering agn_triggering(pin, hydro_pkg); + + /************************************************************ + * Read Magnetic Tower + ************************************************************/ + + // Build Magnetic Tower + MagneticTower magnetic_tower(pin, hydro_pkg); + + // Determine if magnetic_tower_power_scaling is needed + // Is AGN Power and Magnetic fraction non-zero? + bool magnetic_tower_power_scaling = + (agn_feedback.magnetic_fraction_ != 0 && + (agn_feedback.fixed_power_ != 0 || + agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); + hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); + + /************************************************************ + * Read SNIA Feedback + ************************************************************/ + + SNIAFeedback snia_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Stellar Feedback + ************************************************************/ + + StellarFeedback stellar_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Clips (ceilings and floors) + ************************************************************/ + + // Disable all clips by default with a negative radius clip + Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); + + // By default disable floors by setting a negative value + Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); + + // By default disable ceilings by setting to infinity + Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", + std::numeric_limits::infinity()); + Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", + std::numeric_limits::infinity()); + Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", + std::numeric_limits::infinity()); + Real eceil = Tceil; + if (eceil < std::numeric_limits::infinity()) { + if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { + PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " + "Either set a 'units' block and the 'hydro/He_mass_fraction' in " + "input file or use a pressure floor " + "(defined code units) instead."); + } + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); + } + hydro_pkg->AddParam("cluster_dfloor", dfloor); + hydro_pkg->AddParam("cluster_eceil", eceil); + hydro_pkg->AddParam("cluster_vceil", vceil); + hydro_pkg->AddParam("cluster_vAceil", vAceil); + hydro_pkg->AddParam("cluster_clip_r", clip_r); + + /************************************************************ + * Start running reductions into history outputs for clips, stellar mass, cold + * gas, and AGN extent + ************************************************************/ + + /* FIXME(forrestglines) This implementation with a reduction into Params might + be broken in several ways. + 1. Each reduction in params is Rank local. Multiple meshblocks packs per + rank adding to these params is not thread-safe + 2. These Params are not carried over between restarts. If a restart dump is + made and a history output is not, then the mass/energy between the last + history output and the restart dump is lost + */ + std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", + "removed_eceil_energy", "removed_vceil_energy", + "added_vAceil_mass"}; + + // Add a param for each reduction, then add it as a summation reduction for + // history outputs + auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); + + for (auto reduction_str : reduction_strs) { + hydro_pkg->AddParam(reduction_str, 0.0, true); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, + [reduction_str](MeshData *md) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const Real reduction = hydro_pkg->Param(reduction_str); + // Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str, 0.0); + return reduction; + }, + reduction_str)); + } + + // Add history reduction for total cold gas using stellar mass threshold + const Real cold_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); + if (cold_thresh > 0) { + hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); + } + const Real agn_tracer_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); + if (agn_tracer_thresh >= 0) { + PARTHENON_REQUIRE( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); + hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); + } + + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); + + /************************************************************ + * Add derived fields + * NOTE: these must be filled in UserWorkBeforeOutput + ************************************************************/ + + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); + + // log10 of cell-centered radius + hydro_pkg->AddField("log10_cell_radius", m); + // entropy + hydro_pkg->AddField("entropy", m); + // sonic Mach number v/c_s + hydro_pkg->AddField("mach_sonic", m); + // temperature + hydro_pkg->AddField("temperature", m); + + if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { + // cooling time + hydro_pkg->AddField("cooling_time", m); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + // alfven Mach number v_A/c_s + hydro_pkg->AddField("mach_alfven", m); + + // plasma beta + hydro_pkg->AddField("plasma_beta", m); + } + + /************************************************************ + * Read Density perturbation + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); // Mean density of perturbations + + if (mu_rho != 0.0) { + + auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", "k_min_rho"); // Minimum wavenumber of perturbation + auto num_modes_rho = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); + auto sol_weight_rho = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); + uint32_t rseed_rho = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); + + // Computing the kmax ie. the Nyquist limit + auto grid_ni = pin->GetOrAddInteger("parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis + auto k_max_rho = grid_ni / 2; + + const auto t_corr_rho = 1e-10; + auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog(num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes + + auto few_modes_ft_rho = FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, + k_vec_rho, k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); + + hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + + } + + /************************************************************ + * Read Velocity perturbation + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + if (sigma_v != 0.0) { + // peak of init vel perturb + auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); + auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); + + PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || + (k_peak_v > 0.0 && l_peak_v <= 0.0), + "Setting initial velocity perturbation requires a single " + "length scale by either setting l_peak_v or k_peak_v."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_v > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_v = Lx / l_peak_v; + } + + auto num_modes_v = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); + auto sol_weight_v = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); + uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); + // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const auto t_corr = 1e-10; + + auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); + + auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, + k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); + hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + + /************************************************************ + * Read Magnetic field perturbation + ************************************************************/ + + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, + "Requested initial magnetic field perturbations but not " + "solving the MHD equations.") + // peak of init magnetic field perturb + auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); + auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); + PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || + (k_peak_b > 0.0 && l_peak_b <= 0.0), + "Setting initial B perturbation requires a single " + "length scale by either setting l_peak_b or k_peak_b."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_b > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_b = Lx / l_peak_b; + } + + auto num_modes_b = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); + uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); + // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const auto t_corr = 1e-10; + // This field should by construction have no compressive modes, so we fix the number. + const auto sol_weight_b = 1.0; + + auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); + + const bool fill_ghosts = true; // as we fill a vector potential to calc B + auto few_modes_ft = + FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, + sol_weight_b, t_corr, rseed_b, fill_ghosts); + hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now). Only add if not already done for the velocity. + if (sigma_v == 0.0) { + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + } + +} + +//======================================================================================== +//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) +//! \brief Generate problem data for all blocks on rank +// +// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that +// reductions work +//======================================================================================== + +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + + // This could be more optimized, but require a refactor of init routines being called. + // However, given that it's just called during initial setup, this should not be a + // performance concern. + + // Defining a table within which the values of the hydrostatic density profile will be stored + auto pmc = md->GetBlockData(0)->GetBlockPointer(); + const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); + + // Here, the pmc-> contains the number of cells of each MESHBLOCK, including the 2*n_ghost ghost cells + parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho", md->NumBlocks(), + pmc->cellbounds.ncellsk(IndexDomain::entire), + pmc->cellbounds.ncellsj(IndexDomain::entire), + pmc->cellbounds.ncellsi(IndexDomain::entire)); + + for (int b = 0; b < md->NumBlocks(); b++) { + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + + auto &coords = pmb->coords; + + // Get Adiabatic Index + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + /************************************************************ + * Initialize the initial hydro state + ************************************************************/ + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + const auto isothermal_sphere = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); + const auto isothermal_hernquist = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); + + if (init_uniform_gas) { + const Real rho = hydro_pkg->Param("uniform_gas_rho"); + const Real ux = hydro_pkg->Param("uniform_gas_ux"); + const Real uy = hydro_pkg->Param("uniform_gas_uy"); + const Real uz = hydro_pkg->Param("uniform_gas_uz"); + const Real pres = hydro_pkg->Param("uniform_gas_pres"); + + const Real Mx = rho * ux; + const Real My = rho * uy; + const Real Mz = rho * uz; + const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + u(IDN, k, j, i) = rho; + u(IM1, k, j, i) = Mx; + u(IM2, k, j, i) = My; + u(IM3, k, j, i) = Mz; + u(IEN, k, j, i) = E; + }); + + // end if(init_uniform_gas) + } else if (isothermal_sphere) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real mu = hydro_pkg->Param("mu"); + const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); + const Real grav_const = units.gravitational_constant(); + + std::cout << "Entering isothermal sphere generation \n" ; + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real rho_r = prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); + const Real P_r = (prefactor/2) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } else if (isothermal_hernquist) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real m_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); + const Real r_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); + const Real mu = hydro_pkg->Param("mu"); + const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); + const Real grav_const = units.gravitational_constant(); + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real phi_r = - grav_const * m_bcg_s / (r_effective + r_bcg_s); + const Real rho_r = rho_0 * std::exp( - mu * units.mh() / (units.k_boltzmann() * T_bcg_s) * phi_r); + const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } + + else { + /************************************************************ + * Initialize a HydrostaticEquilibriumSphere + ************************************************************/ + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + // Get pressure and density from generated profile + const Real P_r = P_rho_profile.P_from_r(r); + const Real rho_r = P_rho_profile.rho_from_r(r); + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + // Updating hydrostatic_rho table + hydrostatic_rho(b, k, j, i) = rho_r; + + }); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + /************************************************************ + * Initialize the initial magnetic field state via a vector potential + ************************************************************/ + parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + /************************************************************ + * Initialize an initial magnetic tower + ************************************************************/ + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + + /************************************************************ + * Add dipole magnetic field to the magnetic potential + ************************************************************/ + const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); + if (init_dipole_b_field) { + const Real mx = hydro_pkg->Param("dipole_b_field_mx"); + const Real my = hydro_pkg->Param("dipole_b_field_my"); + const Real mz = hydro_pkg->Param("dipole_b_field_mz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + + const Real m_cross_r_x = my * z - mz * y; + const Real m_cross_r_y = mz * x - mx * z; + const Real m_cross_r_z = mx * y - mx * y; + + // To check whether there is some component before initiating perturbations + std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; + + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); + A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); + A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + }); + } + + /************************************************************ + * Apply the potential to the conserved variables + ************************************************************/ + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + u(IB1, k, j, i) = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + + SQR(u(IB3, k, j, i))); + }); + + /************************************************************ + * Add uniform magnetic field to the conserved variables + ************************************************************/ + const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); + if (init_uniform_b_field) { + const Real bx = hydro_pkg->Param("uniform_b_field_bx"); + const Real by = hydro_pkg->Param("uniform_b_field_by"); + const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + const Real bx_i = u(IB1, k, j, i); + const Real by_i = u(IB2, k, j, i); + const Real bz_i = u(IB3, k, j, i); + + u(IB1, k, j, i) += bx; + u(IB2, k, j, i) += by; + u(IB3, k, j, i) += bz; + + // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + + // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that + u(IEN, k, j, i) += + bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + }); + // end if(init_uniform_b_field) + } + + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) + } + + /************************************************************ + * Initial parameters + ************************************************************/ + + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto fluid = hydro_pkg->Param("fluid"); + auto const &cons = md->PackVariables(std::vector{"cons"}); + const auto num_blocks = md->NumBlocks(); + + /************************************************************ + * Set initial density perturbations (read from HDF5 file) + ************************************************************/ + + const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); + const bool full_box = pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); + const bool spherical_collapse = pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); + + hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); + + Real passive_scalar = 0.0; // Not useful here + + // Spherical collapse test with an initial overdensity + + if (spherical_collapse == true) { + + // Create an homogeneous sphere of a density superior or inferior to the background density + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius + const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); + const Real background_density = pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); + const Real foreground_density = pin->GetOrAddReal("problem/cluster/init_perturb", "foreground_density", 150); + + // Setting an initial + u(IDN, k, j, i) = background_density; + + // For any cell at a radius less then overdensity radius, set the density to foreground_density + if (r < overdensity_radius){ + u(IDN, k, j, i) = foreground_density; + } + + }, + passive_scalar); + + } + + /* -------------- Setting up a clumpy atmosphere -------------- + + 1) Extract the values of the density from an input hdf5 file using H5Easy + 2) Initiate the associated density field + 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) + 4) Optionnaly, add a central overdensity + + */ + + if (init_perturb_rho == true) { + + auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", "init_perturb_rho_file","none"); + + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real perturb_amplitude = pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); + + std::string keys_rho = "data"; + H5Easy::File file(filename_rho, HighFive::File::ReadOnly); + + const int rho_init_size = 256; + auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>(file, keys_rho); + + Real passive_scalar = 0.0; // Useless + + std::cout << "Entering initialisation of rho field"; + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + // Case where the box is filled with perturbations of equal mean amplitude + if (full_box){ + + u(IDN, k, j, i) += perturb_amplitude * rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * (u(IDN, k, j, i) / 29.6); + + } + + // Mean amplitude of the perturbations is modulated by the + else { + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); + const Real r_effective = std::max(r,r_smoothing); + + //u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; + + } + + }, + passive_scalar); + + } + + /************************************************************ + * Setting up a perturbed density field (hardcoded version) + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); + const auto background_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); + + if (mu_rho != 0.0) { + + auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); + + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft_rho.SetPhases(pmb.get(), pin); + + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); + + Real v2_sum_rho = 0.0; // used for normalization + + auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + Real perturb = 0.0 ; + + perturb = std::sqrt(SQR(perturb_pack_rho(b, 0, k, j, i)) + SQR(perturb_pack_rho(b, 1, k, j, i)) + SQR(perturb_pack_rho(b, 2, k, j, i))); + u(IDN, k, j, i) = background_rho + mu_rho * perturb; + + }, + v2_sum_rho); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + } + + /************************************************************ + * Set initial velocity perturbations (requires no other velocities for now) + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + + if (sigma_v != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real v2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + auto rho = u(IDN, k, j, i); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, + "Found existing non-zero velocity when setting velocity perturbations."); + + u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); + u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); + u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); + // No need to touch the energy yet as we'll normalize later + + lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * + coords.CellVolume(k, j, i) / SQR(rho); + }, + v2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); + + pmb->par_for( + "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IM1, k, j, i) /= v_norm; + u(IM2, k, j, i) /= v_norm; + u(IM3, k, j, i) /= v_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / + u(IDN, k, j, i); + }); + } + + /************************************************************ + * Set initial magnetic field perturbations (resets magnetic field field) + ************************************************************/ + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + const auto alpha_b = pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0/3.0); + const auto density_scale = pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); + const auto standard_B = pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); + const auto r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); + const auto rho_type = pin->GetOrAddInteger("problem/cluster/init_perturb", "rho_type", 0); // test + + if (sigma_b != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real b2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + // Defining a new table that will contains the values of the perturbed magnetic field, and magnetic energy + parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, + pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + if (standard_B){ + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0; + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * + coords.CellVolume(k, j, i); + }, + b2_sum); + + } else { + + // Idea: separate into two loops, one par_for and one par_reduce + + std::cout << "Setting up perturbed magnetic field." << std::endl; + + for (int b = 0; b < md->NumBlocks(); b++) { + + std::cout << "Treating block number " << b << " out of " << md->NumBlocks() << std::endl; + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // Computation of the P_rho_profile require entire domain of the block (boundary conditions for curl computation) + IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + auto &coords = pmb->coords; + + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::MagneticPerturbations", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + const Real x = coords.Xc<1>(i); + const Real xp = coords.Xc<1>(i+1); + const Real xm = coords.Xc<1>(i-1); + const Real y = coords.Xc<2>(j); + const Real yp = coords.Xc<2>(j+1); + const Real ym = coords.Xc<2>(j-1); + const Real z = coords.Xc<3>(k); + const Real zp = coords.Xc<3>(k+1); + const Real zm = coords.Xc<3>(k-1); + + const Real r = sqrt(SQR(x) + SQR(y) + SQR(z)); + const Real r_ip = sqrt(SQR(xp) + SQR(y) + SQR(z)); + const Real r_im = sqrt(SQR(xm) + SQR(y) + SQR(z)); + const Real r_jp = sqrt(SQR(x) + SQR(yp) + SQR(z)); + const Real r_jm = sqrt(SQR(x) + SQR(ym) + SQR(z)); + const Real r_kp = sqrt(SQR(x) + SQR(y) + SQR(zp)); + const Real r_km = sqrt(SQR(x) + SQR(y) + SQR(zm)); + + const auto rho = P_rho_profile.rho_from_r(r); + const auto rho_ip = P_rho_profile.rho_from_r(r_ip); + const auto rho_im = P_rho_profile.rho_from_r(r_im); + const auto rho_jp = P_rho_profile.rho_from_r(r_jp); + const auto rho_jm = P_rho_profile.rho_from_r(r_jm); + const auto rho_kp = P_rho_profile.rho_from_r(r_kp); + const auto rho_km = P_rho_profile.rho_from_r(r_km); + + // const Real rho_r = P_rho_profile.rho_from_r(r); + + // Normalization condition here, which we want to lift. To do so, we'll need + // to copy the values of the magnetic field into a new array, which we will + // use to store the perturbed magnetic field and then rescale it, until values + // are added to u = cons(b) + + //PARTHENON_REQUIRE( + // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + // "Found existing non-zero B when setting magnetic field perturbations."); + + // First, needs to rescale the perturb_pack, ie. the magnetic field potential + Real perturb2_k_jp_i,perturb2_k_jm_i,perturb1_kp_j_i,perturb1_km_j_i; + Real perturb0_kp_j_i,perturb0_km_j_i,perturb2_k_j_ip,perturb2_k_j_im; + Real perturb1_k_j_ip,perturb1_k_j_im,perturb0_k_jp_i,perturb0_k_jm_i; + + // Rescaling the magnetic potential + + perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i) * std::pow(r_jp, alpha_b); + perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i) * std::pow(r_jm, alpha_b); + perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i) * std::pow(r_kp, alpha_b); + perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i) * std::pow(r_km, alpha_b); + perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i) * std::pow(r_kp, alpha_b); + perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i) * std::pow(r_km, alpha_b); + perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1) * std::pow(r_ip, alpha_b); + perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1) * std::pow(r_im, alpha_b); + perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1) * std::pow(r_ip, alpha_b); + perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1) * std::pow(r_im, alpha_b); + perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i) * std::pow(r_jp, alpha_b); + perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i) * std::pow(r_jm, alpha_b); + + // Then, compute the curl of the magnetic field + + Real curlBx,curlBy,curlBz; + + curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - + (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0 ; + curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - + (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0 ; + curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - + (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0 ; + + dB(b,0, k, j, i) = curlBx; + dB(b,1, k, j, i) = curlBy; + dB(b,2, k, j, i) = curlBz; + + }); + } + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + const auto &coords = cons.GetCoords(b); + + Real curlBx,curlBy,curlBz; + + curlBx = dB(b,0, k, j, i); + curlBy = dB(b,1, k, j, i); + curlBz = dB(b,2, k, j, i); + + lsum += (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); + }, + b2_sum); + } + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); + + if (standard_B){ + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); + }); + + } else { + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + dB(b, 0, k, j, i) /= b_norm; + dB(b, 1, k, j, i) /= b_norm; + dB(b, 2, k, j, i) /= b_norm; + + // Computing energy + dB(b, 3, k, j, i) += + 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + SQR(dB(b, 2, k, j, i))); + + // Updating the MHD vector + + u(IB1, k, j, i) += dB(b, 0, k, j, i); + u(IB2, k, j, i) += dB(b, 1, k, j, i); + u(IB3, k, j, i) += dB(b, 2, k, j, i); + u(IEN, k, j, i) += dB(b, 3, k, j, i); + + }); + } + } +} + +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { + // get hydro + auto pkg = pmb->packages.Get("Hydro"); + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + // get prim vars + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; + + // get derived fields + auto &log10_radius = data->Get("log10_cell_radius").data; + auto &entropy = data->Get("entropy").data; + auto &mach_sonic = data->Get("mach_sonic").data; + auto &temperature = data->Get("temperature").data; + + // for computing temperature from primitives + auto units = pkg->Param("units"); + auto mbar_over_kb = pkg->Param("mbar_over_kb"); + auto mbar = mbar_over_kb * units.k_boltzmann(); + + // fill derived vars (*including ghost cells*) + auto &coords = pmb->coords; + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real v1 = prim(IV1, k, j, i); + const Real v2 = prim(IV2, k, j, i); + const Real v3 = prim(IV3, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute radius + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r2 = SQR(x) + SQR(y) + SQR(z); + log10_radius(k, j, i) = 0.5 * std::log10(r2); + + // compute entropy + const Real K = P / std::pow(rho / mbar, gam); + entropy(k, j, i) = K; + + const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + const Real M_s = v_mag / c_s; + mach_sonic(k, j, i) = M_s; + + // compute temperature + temperature(k, j, i) = mbar_over_kb * P / rho; + }); + + if (pkg->Param("enable_cooling") == Cooling::tabular) { + auto &cooling_time = data->Get("cooling_time").data; + + // get cooling function + const cooling::TabularCooling &tabular_cooling = + pkg->Param("tabular_cooling"); + const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute cooling time + const Real eint = P / (rho * gm1); + const Real edot = cooling_table_obj.DeDt(eint, rho); + cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; + }); + } + + if (pkg->Param("fluid") == Fluid::glmmhd) { + auto &plasma_beta = data->Get("plasma_beta").data; + auto &mach_alfven = data->Get("mach_alfven").data; + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + const Real Bx = prim(IB1, k, j, i); + const Real By = prim(IB2, k, j, i); + const Real Bz = prim(IB3, k, j, i); + const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + + // compute Alfven mach number + const Real v_A = std::sqrt(B2 / rho); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; + + // compute plasma beta + plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; + }); + } +} + +} // namespace cluster diff --git a/src/pgen/old_cluster/cluster_22nov.cpp b/src/pgen/old_cluster/cluster_22nov.cpp new file mode 100644 index 00000000..3aaf94d2 --- /dev/null +++ b/src/pgen/old_cluster/cluster_22nov.cpp @@ -0,0 +1,1306 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster.cpp +// \brief Idealized galaxy cluster problem generator +// +// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in +// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, +// optionally with an initial magnetic tower field. Includes AGN feedback, AGN +// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback +//======================================================================================== + +// C headers + +// C++ headers +#include // min, max +#include // sqrt() +#include // fopen(), fprintf(), freopen() +#include // endl +#include +#include // stringstream +#include // runtime_error +#include // c_str() + +// #include // HDF5 +#include "../../external/HighFive/include/highfive/H5Easy.hpp" + +// Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" +#include +#include + +// AthenaPK headers +#include "../eos/adiabatic_glmmhd.hpp" +#include "../eos/adiabatic_hydro.hpp" +#include "../hydro/hydro.hpp" +#include "../hydro/srcterms/gravitational_field.hpp" +#include "../hydro/srcterms/tabular_cooling.hpp" +#include "../main.hpp" +#include "../utils/few_modes_ft.hpp" +#include "../utils/few_modes_ft_lognormal.hpp" + +// Cluster headers +#include "cluster/agn_feedback.hpp" +#include "cluster/agn_triggering.hpp" +#include "cluster/cluster_clips.hpp" +#include "cluster/cluster_gravity.hpp" +#include "cluster/cluster_reductions.hpp" +#include "cluster/entropy_profiles.hpp" +#include "cluster/hydrostatic_equilibrium_sphere.hpp" +#include "cluster/magnetic_tower.hpp" +#include "cluster/snia_feedback.hpp" +#include "cluster/stellar_feedback.hpp" + +namespace cluster { +using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; +using utils::few_modes_ft::FewModesFT; +using utils::few_modes_ft_log::FewModesFTLog; + +void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); + + if (gravity_srcterm) { + const ClusterGravity &cluster_gravity = + hydro_pkg->Param("cluster_gravity"); + + GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); + } + + const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); + agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); + + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); + + const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); +}; +void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real dt) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); + stellar_feedback.FeedbackSrcTerm(md, dt, tm); + + ApplyClusterClips(md, tm, dt); +} + +Real ClusterEstimateTimestep(MeshData *md) { + Real min_dt = std::numeric_limits::max(); + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // TODO time constraints imposed by thermal AGN feedback, jet velocity, + // magnetic tower + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); + min_dt = std::min(min_dt, agn_triggering_min_dt); + + return min_dt; +} + +//======================================================================================== +//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor +//! *hydro_pkg) \brief Init package data from parameter input +//======================================================================================== + +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { + + /************************************************************ + * Read Uniform Gas + ************************************************************/ + + const bool init_uniform_gas = + pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); + hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); + + if (init_uniform_gas) { + const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); + const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); + const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); + const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); + const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); + + hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); + hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); + hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); + hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); + hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_uniform_b_field = pin->GetOrAddBoolean( + "problem/cluster/uniform_b_field", "init_uniform_b_field", false); + hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); + + if (init_uniform_b_field) { + const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); + const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); + const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); + + hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); + hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); + hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", + "init_dipole_b_field", false); + hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); + + if (init_dipole_b_field) { + const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); + const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); + const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); + + hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); + hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); + hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); + } + + /************************************************************ + * Read Cluster Gravity Parameters + ************************************************************/ + + // Build cluster_gravity object + ClusterGravity cluster_gravity(pin, hydro_pkg); + + // Include gravity as a source term during evolution + const bool gravity_srcterm = + pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); + hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); + + /************************************************************ + * Read Initial Entropy Profile + ************************************************************/ + + // Create hydrostatic sphere with ACCEPT entropy profile + ACCEPTEntropyProfile entropy_profile(pin); + + HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, + entropy_profile); + + // Create hydrostatic sphere with ISOTHERMAL entropy profile + + + + /************************************************************ + * Read Precessing Jet Coordinate system + ************************************************************/ + + JetCoordsFactory jet_coords_factory(pin, hydro_pkg); + + /************************************************************ + * Read AGN Feedback + ************************************************************/ + + AGNFeedback agn_feedback(pin, hydro_pkg); + + /************************************************************ + * Read AGN Triggering + ************************************************************/ + + AGNTriggering agn_triggering(pin, hydro_pkg); + + /************************************************************ + * Read Magnetic Tower + ************************************************************/ + + // Build Magnetic Tower + MagneticTower magnetic_tower(pin, hydro_pkg); + + // Determine if magnetic_tower_power_scaling is needed + // Is AGN Power and Magnetic fraction non-zero? + bool magnetic_tower_power_scaling = + (agn_feedback.magnetic_fraction_ != 0 && + (agn_feedback.fixed_power_ != 0 || + agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); + hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); + + /************************************************************ + * Read SNIA Feedback + ************************************************************/ + + SNIAFeedback snia_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Stellar Feedback + ************************************************************/ + + StellarFeedback stellar_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Clips (ceilings and floors) + ************************************************************/ + + // Disable all clips by default with a negative radius clip + Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); + + // By default disable floors by setting a negative value + Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); + + // By default disable ceilings by setting to infinity + Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", + std::numeric_limits::infinity()); + Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", + std::numeric_limits::infinity()); + Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", + std::numeric_limits::infinity()); + Real eceil = Tceil; + if (eceil < std::numeric_limits::infinity()) { + if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { + PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " + "Either set a 'units' block and the 'hydro/He_mass_fraction' in " + "input file or use a pressure floor " + "(defined code units) instead."); + } + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); + } + hydro_pkg->AddParam("cluster_dfloor", dfloor); + hydro_pkg->AddParam("cluster_eceil", eceil); + hydro_pkg->AddParam("cluster_vceil", vceil); + hydro_pkg->AddParam("cluster_vAceil", vAceil); + hydro_pkg->AddParam("cluster_clip_r", clip_r); + + /************************************************************ + * Start running reductions into history outputs for clips, stellar mass, cold + * gas, and AGN extent + ************************************************************/ + + /* FIXME(forrestglines) This implementation with a reduction into Params might + be broken in several ways. + 1. Each reduction in params is Rank local. Multiple meshblocks packs per + rank adding to these params is not thread-safe + 2. These Params are not carried over between restarts. If a restart dump is + made and a history output is not, then the mass/energy between the last + history output and the restart dump is lost + */ + std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", + "removed_eceil_energy", "removed_vceil_energy", + "added_vAceil_mass"}; + + // Add a param for each reduction, then add it as a summation reduction for + // history outputs + auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); + + for (auto reduction_str : reduction_strs) { + hydro_pkg->AddParam(reduction_str, 0.0, true); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, + [reduction_str](MeshData *md) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const Real reduction = hydro_pkg->Param(reduction_str); + // Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str, 0.0); + return reduction; + }, + reduction_str)); + } + + // Add history reduction for total cold gas using stellar mass threshold + const Real cold_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); + if (cold_thresh > 0) { + hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); + } + const Real agn_tracer_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); + if (agn_tracer_thresh >= 0) { + PARTHENON_REQUIRE( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); + hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); + } + + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); + + /************************************************************ + * Add derived fields + * NOTE: these must be filled in UserWorkBeforeOutput + ************************************************************/ + + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); + + // log10 of cell-centered radius + hydro_pkg->AddField("log10_cell_radius", m); + // entropy + hydro_pkg->AddField("entropy", m); + // sonic Mach number v/c_s + hydro_pkg->AddField("mach_sonic", m); + // temperature + hydro_pkg->AddField("temperature", m); + + if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { + // cooling time + hydro_pkg->AddField("cooling_time", m); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + // alfven Mach number v_A/c_s + hydro_pkg->AddField("mach_alfven", m); + + // plasma beta + hydro_pkg->AddField("plasma_beta", m); + } + + /************************************************************ + * Read Density perturbation + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); // Mean density of perturbations + + if (mu_rho != 0.0) { + + auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", "k_min_rho"); // Minimum wavenumber of perturbation + auto num_modes_rho = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); + auto sol_weight_rho = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); + uint32_t rseed_rho = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); + + // Computing the kmax ie. the Nyquist limit + auto grid_ni = pin->GetOrAddInteger("parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis + auto k_max_rho = grid_ni / 2; + + const auto t_corr_rho = 1e-10; + auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog(num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes + + auto few_modes_ft_rho = FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, + k_vec_rho, k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); + + hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + + } + + /************************************************************ + * Read Velocity perturbation + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + if (sigma_v != 0.0) { + // peak of init vel perturb + auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); + auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); + + PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || + (k_peak_v > 0.0 && l_peak_v <= 0.0), + "Setting initial velocity perturbation requires a single " + "length scale by either setting l_peak_v or k_peak_v."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_v > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_v = Lx / l_peak_v; + } + + auto num_modes_v = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); + auto sol_weight_v = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); + uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); + // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const auto t_corr = 1e-10; + + auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); + + auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, + k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); + hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + + /************************************************************ + * Read Magnetic field perturbation + ************************************************************/ + + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, + "Requested initial magnetic field perturbations but not " + "solving the MHD equations.") + // peak of init magnetic field perturb + auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); + auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); + PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || + (k_peak_b > 0.0 && l_peak_b <= 0.0), + "Setting initial B perturbation requires a single " + "length scale by either setting l_peak_b or k_peak_b."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_b > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_b = Lx / l_peak_b; + } + + auto num_modes_b = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); + uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); + // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const auto t_corr = 1e-10; + // This field should by construction have no compressive modes, so we fix the number. + const auto sol_weight_b = 1.0; + + auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); + + const bool fill_ghosts = true; // as we fill a vector potential to calc B + auto few_modes_ft = + FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, + sol_weight_b, t_corr, rseed_b, fill_ghosts); + hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now). Only add if not already done for the velocity. + if (sigma_v == 0.0) { + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + } + +} + +//======================================================================================== +//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) +//! \brief Generate problem data for all blocks on rank +// +// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that +// reductions work +//======================================================================================== + +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + + // This could be more optimized, but require a refactor of init routines being called. + // However, given that it's just called during initial setup, this should not be a + // performance concern. + + // Defining a table within which the values of the hydrostatic density profile will be stored + auto pmc = md->GetBlockData(0)->GetBlockPointer(); + const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); + + parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho",grid_size + 4,grid_size + 4,grid_size + 4); + + //parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho", md->NumBlocks(), + // pmc->cellbounds.ncellsk(IndexDomain::entire), + // pmc->cellbounds.ncellsj(IndexDomain::entire), + // pmc->cellbounds.ncellsi(IndexDomain::entire)); + + for (int b = 0; b < md->NumBlocks(); b++) { + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + auto &coords = pmb->coords; + + // Get Adiabatic Index + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + /************************************************************ + * Initialize the initial hydro state + ************************************************************/ + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + const auto isothermal_sphere = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); + const auto isothermal_hernquist = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); + + if (init_uniform_gas) { + const Real rho = hydro_pkg->Param("uniform_gas_rho"); + const Real ux = hydro_pkg->Param("uniform_gas_ux"); + const Real uy = hydro_pkg->Param("uniform_gas_uy"); + const Real uz = hydro_pkg->Param("uniform_gas_uz"); + const Real pres = hydro_pkg->Param("uniform_gas_pres"); + + const Real Mx = rho * ux; + const Real My = rho * uy; + const Real Mz = rho * uz; + const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + u(IDN, k, j, i) = rho; + u(IM1, k, j, i) = Mx; + u(IM2, k, j, i) = My; + u(IM3, k, j, i) = Mz; + u(IEN, k, j, i) = E; + }); + + // end if(init_uniform_gas) + } else if (isothermal_sphere) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real mu = hydro_pkg->Param("mu"); + const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); + const Real grav_const = units.gravitational_constant(); + + std::cout << "Entering isothermal sphere generation \n" ; + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real rho_r = prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); + const Real P_r = (prefactor/2) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } else if (isothermal_hernquist) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real m_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); + const Real r_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); + const Real mu = hydro_pkg->Param("mu"); + const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); + const Real grav_const = units.gravitational_constant(); + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real phi_r = - grav_const * m_bcg_s / (r_effective + r_bcg_s); + const Real rho_r = rho_0 * std::exp( - mu * units.mh() / (units.k_boltzmann() * T_bcg_s) * phi_r); + const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } + + else { + + /************************************************************ + * Initialize a HydrostaticEquilibriumSphere + ************************************************************/ + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + // Get pressure and density from generated profile + const Real P_r = P_rho_profile.P_from_r(r); + const Real rho_r = P_rho_profile.rho_from_r(r); + + //u(IDN, k, j, i) = rho_r; + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + // Updating hydrostatic_rho table + hydrostatic_rho(gks + k, gjs + j, gis + i) = rho_r; + + }); + + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + /************************************************************ + * Initialize the initial magnetic field state via a vector potential + ************************************************************/ + parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + /************************************************************ + * Initialize an initial magnetic tower + ************************************************************/ + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + + /************************************************************ + * Add dipole magnetic field to the magnetic potential + ************************************************************/ + const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); + if (init_dipole_b_field) { + const Real mx = hydro_pkg->Param("dipole_b_field_mx"); + const Real my = hydro_pkg->Param("dipole_b_field_my"); + const Real mz = hydro_pkg->Param("dipole_b_field_mz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + + const Real m_cross_r_x = my * z - mz * y; + const Real m_cross_r_y = mz * x - mx * z; + const Real m_cross_r_z = mx * y - mx * y; + + // To check whether there is some component before initiating perturbations + std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; + + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); + A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); + A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + }); + } + + /************************************************************ + * Apply the potential to the conserved variables + ************************************************************/ + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + u(IB1, k, j, i) = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + + SQR(u(IB3, k, j, i))); + }); + + /************************************************************ + * Add uniform magnetic field to the conserved variables + ************************************************************/ + const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); + if (init_uniform_b_field) { + const Real bx = hydro_pkg->Param("uniform_b_field_bx"); + const Real by = hydro_pkg->Param("uniform_b_field_by"); + const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + const Real bx_i = u(IB1, k, j, i); + const Real by_i = u(IB2, k, j, i); + const Real bz_i = u(IB3, k, j, i); + + u(IB1, k, j, i) += bx; + u(IB2, k, j, i) += by; + u(IB3, k, j, i) += bz; + + // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + + // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that + u(IEN, k, j, i) += + bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + }); + // end if(init_uniform_b_field) + } + + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) + } + + /************************************************************ + * Initial parameters + ************************************************************/ + + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto fluid = hydro_pkg->Param("fluid"); + auto const &cons = md->PackVariables(std::vector{"cons"}); + const auto num_blocks = md->NumBlocks(); + + /************************************************************ + * Set initial density perturbations (read from HDF5 file) + ************************************************************/ + + const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); + const bool full_box = pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); + const bool spherical_collapse = pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); + + hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); + + Real passive_scalar = 0.0; // Not useful here + + // Spherical collapse test with an initial overdensity + + if (spherical_collapse == true) { + + // Create an homogeneous sphere of a density superior or inferior to the background density + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius + const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); + const Real background_density = pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); + const Real foreground_density = pin->GetOrAddReal("problem/cluster/init_perturb", "foreground_density", 150); + + // Setting an initial + u(IDN, k, j, i) = background_density; + + // For any cell at a radius less then overdensity radius, set the density to foreground_density + if (r < overdensity_radius){ + u(IDN, k, j, i) = foreground_density; + } + + }, + passive_scalar); + + } + + /* -------------- Setting up a clumpy atmosphere -------------- + + 1) Extract the values of the density from an input hdf5 file using H5Easy + 2) Initiate the associated density field + 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) + 4) Optionnaly, add a central overdensity + + */ + + if (init_perturb_rho == true) { + + auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", "init_perturb_rho_file","none"); + + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real perturb_amplitude = pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); + + std::string keys_rho = "data"; + H5Easy::File file(filename_rho, HighFive::File::ReadOnly); + + const int rho_init_size = 260; + auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>(file, keys_rho); + + Real passive_scalar = 0.0; // Useless + + std::cout << "Entering initialisation of rho field"; + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + // Case where the box is filled with perturbations of equal mean amplitude + if (full_box){ + + u(IDN, k, j, i) += perturb_amplitude * rho_init[gks + k][gjs + j][gis + i] * (u(IDN, k, j, i) / 29.6); + hydrostatic_rho(gks + k, gjs + j, gis + i) += perturb_amplitude * rho_init[gks + k][gjs + j][gis + i] * (u(IDN, k, j, i) / 29.6); + } + + + }, + passive_scalar); + + + } + + /************************************************************ + * Set initial velocity perturbations (requires no other velocities for now) + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + + if (sigma_v != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real v2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + auto rho = u(IDN, k, j, i); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, + "Found existing non-zero velocity when setting velocity perturbations."); + + u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); + u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); + u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); + // No need to touch the energy yet as we'll normalize later + + lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * + coords.CellVolume(k, j, i) / SQR(rho); + }, + v2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); + + pmb->par_for( + "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IM1, k, j, i) /= v_norm; + u(IM2, k, j, i) /= v_norm; + u(IM3, k, j, i) /= v_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / + u(IDN, k, j, i); + }); + } + + /************************************************************ + * Set initial magnetic field perturbations (resets magnetic field field) + ************************************************************/ + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + const auto alpha_b = pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0/3.0); + const auto density_scale = pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); + const auto standard_B = pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); + const auto r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); + + if (sigma_b != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real b2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + // Defining a new table that will contains the values of the perturbed magnetic field, and magnetic energy + parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, + pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + // Modifying the magnetic potential so that it follows the rho profile + pmb->par_for( + "Init sigma_b", 0, num_blocks - 1, kb.s-1, kb.e+1, jb.s-1, jb.e+1, ib.s-1, ib.e+1, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; + + //perturb_pack(b, 0, k, j, i) *= std::pow(u(IDN, k, j, i),alpha_b); + //perturb_pack(b, 1, k, j, i) *= std::pow(u(IDN, k, j, i),alpha_b); + //perturb_pack(b, 2, k, j, i) *= std::pow(u(IDN, k, j, i),alpha_b); + + perturb_pack(b, 0, k, j, i) *= std::pow(hydrostatic_rho(gks + k, gjs + j, gis + i),alpha_b); + perturb_pack(b, 1, k, j, i) *= std::pow(hydrostatic_rho(gks + k, gjs + j, gis + i),alpha_b); + perturb_pack(b, 2, k, j, i) *= std::pow(hydrostatic_rho(gks + k, gjs + j, gis + i),alpha_b); + + //perturb_pack(b, 0, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); + //perturb_pack(b, 1, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); + //perturb_pack(b, 2, k, j, i) *= std::pow(hydrostatic_rho(b, k, j, i),alpha_b); + + }); + + if (standard_B){ + + std::cout << "Entering standard B" << std::endl; + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0; + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * + coords.CellVolume(k, j, i); + }, + b2_sum); + + } else { + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + // First, needs to rescale the perturb_pack, ie. the magnetic field potential + Real perturb2_k_jp_i,perturb2_k_jm_i,perturb1_kp_j_i,perturb1_km_j_i; + Real perturb0_kp_j_i,perturb0_km_j_i,perturb2_k_j_ip,perturb2_k_j_im; + Real perturb1_k_j_ip,perturb1_k_j_im,perturb0_k_jp_i,perturb0_k_jm_i; + + perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i); + perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i); + perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i); + perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i); + perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i); + perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i); + perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1); + perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1); + perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1); + perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1); + perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i); + perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i); + + // Then, compute the curl of the magnetic field + + Real curlBx,curlBy,curlBz; + + curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - + (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0 ; + curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - + (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0 ; + curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - + (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0 ; + + dB(b, 0, k, j, i) = curlBx; + dB(b, 1, k, j, i) = curlBy; + dB(b, 2, k, j, i) = curlBz; + + lsum += (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); + + }, + b2_sum); + } + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); + + if (standard_B){ + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); + }); + + } else { + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + dB(b, 0, k, j, i) /= b_norm; + dB(b, 1, k, j, i) /= b_norm; + dB(b, 2, k, j, i) /= b_norm; + + // Computing energy + dB(b, 3, k, j, i) += + 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + SQR(dB(b, 2, k, j, i))); + + // Updating the MHD vector + + u(IB1, k, j, i) += dB(b, 0, k, j, i); + u(IB2, k, j, i) += dB(b, 1, k, j, i); + u(IB3, k, j, i) += dB(b, 2, k, j, i); + u(IEN, k, j, i) += dB(b, 3, k, j, i); + + }); + } + } +} + +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { + // get hydro + auto pkg = pmb->packages.Get("Hydro"); + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + // get prim vars + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; + + // get derived fields + auto &log10_radius = data->Get("log10_cell_radius").data; + auto &entropy = data->Get("entropy").data; + auto &mach_sonic = data->Get("mach_sonic").data; + auto &temperature = data->Get("temperature").data; + + // for computing temperature from primitives + auto units = pkg->Param("units"); + auto mbar_over_kb = pkg->Param("mbar_over_kb"); + auto mbar = mbar_over_kb * units.k_boltzmann(); + + // fill derived vars (*including ghost cells*) + auto &coords = pmb->coords; + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real v1 = prim(IV1, k, j, i); + const Real v2 = prim(IV2, k, j, i); + const Real v3 = prim(IV3, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute radius + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r2 = SQR(x) + SQR(y) + SQR(z); + log10_radius(k, j, i) = 0.5 * std::log10(r2); + + // compute entropy + const Real K = P / std::pow(rho / mbar, gam); + entropy(k, j, i) = K; + + const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + const Real M_s = v_mag / c_s; + mach_sonic(k, j, i) = M_s; + + // compute temperature + temperature(k, j, i) = mbar_over_kb * P / rho; + }); + + if (pkg->Param("enable_cooling") == Cooling::tabular) { + auto &cooling_time = data->Get("cooling_time").data; + + // get cooling function + const cooling::TabularCooling &tabular_cooling = + pkg->Param("tabular_cooling"); + const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute cooling time + const Real eint = P / (rho * gm1); + const Real edot = cooling_table_obj.DeDt(eint, rho); + cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; + }); + } + + if (pkg->Param("fluid") == Fluid::glmmhd) { + auto &plasma_beta = data->Get("plasma_beta").data; + auto &mach_alfven = data->Get("mach_alfven").data; + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + const Real Bx = prim(IB1, k, j, i); + const Real By = prim(IB2, k, j, i); + const Real Bz = prim(IB3, k, j, i); + const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + + // compute Alfven mach number + const Real v_A = std::sqrt(B2 / rho); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; + + // compute plasma beta + plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; + }); + } +} + +} // namespace cluster diff --git a/src/pgen/cluster_modified.cpp b/src/pgen/old_cluster/cluster_modified.cpp similarity index 100% rename from src/pgen/cluster_modified.cpp rename to src/pgen/old_cluster/cluster_modified.cpp diff --git a/src/pgen/old_cluster/cluster_temp.cpp b/src/pgen/old_cluster/cluster_temp.cpp new file mode 100644 index 00000000..44f1bbfc --- /dev/null +++ b/src/pgen/old_cluster/cluster_temp.cpp @@ -0,0 +1,1404 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster.cpp +// \brief Idealized galaxy cluster problem generator +// +// Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in +// hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, +// optionally with an initial magnetic tower field. Includes AGN feedback, AGN +// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback +//======================================================================================== + +// C headers + +// C++ headers +#include // min, max +#include // sqrt() +#include // fopen(), fprintf(), freopen() +#include // endl +#include +#include // stringstream +#include // runtime_error +#include // c_str() + +// #include // HDF5 +#include "../../external/HighFive/include/highfive/H5Easy.hpp" + +// Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" +#include +#include + +// AthenaPK headers +#include "../eos/adiabatic_glmmhd.hpp" +#include "../eos/adiabatic_hydro.hpp" +#include "../hydro/hydro.hpp" +#include "../hydro/srcterms/gravitational_field.hpp" +#include "../hydro/srcterms/tabular_cooling.hpp" +#include "../main.hpp" +#include "../utils/few_modes_ft.hpp" +#include "../utils/few_modes_ft_lognormal.hpp" + +// Cluster headers +#include "cluster/agn_feedback.hpp" +#include "cluster/agn_triggering.hpp" +#include "cluster/cluster_clips.hpp" +#include "cluster/cluster_gravity.hpp" +#include "cluster/cluster_reductions.hpp" +#include "cluster/entropy_profiles.hpp" +#include "cluster/hydrostatic_equilibrium_sphere.hpp" +#include "cluster/magnetic_tower.hpp" +#include "cluster/snia_feedback.hpp" +#include "cluster/stellar_feedback.hpp" + +namespace cluster { +using namespace parthenon::driver::prelude; +using namespace parthenon::package::prelude; +using utils::few_modes_ft::FewModesFT; +using utils::few_modes_ft_log::FewModesFTLog; + +void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); + + if (gravity_srcterm) { + const ClusterGravity &cluster_gravity = + hydro_pkg->Param("cluster_gravity"); + + GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); + } + + const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); + agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); + + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); + + const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); +}; +void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real dt) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); + stellar_feedback.FeedbackSrcTerm(md, dt, tm); + + ApplyClusterClips(md, tm, dt); +} + +Real ClusterEstimateTimestep(MeshData *md) { + Real min_dt = std::numeric_limits::max(); + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // TODO time constraints imposed by thermal AGN feedback, jet velocity, + // magnetic tower + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); + min_dt = std::min(min_dt, agn_triggering_min_dt); + + return min_dt; +} + +//======================================================================================== +//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor +//! *hydro_pkg) \brief Init package data from parameter input +//======================================================================================== + +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { + + /************************************************************ + * Read Uniform Gas + ************************************************************/ + + const bool init_uniform_gas = + pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); + hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); + + if (init_uniform_gas) { + const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); + const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); + const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); + const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); + const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); + + hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); + hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); + hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); + hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); + hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_uniform_b_field = pin->GetOrAddBoolean( + "problem/cluster/uniform_b_field", "init_uniform_b_field", false); + hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); + + if (init_uniform_b_field) { + const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); + const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); + const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); + + hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); + hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); + hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", + "init_dipole_b_field", false); + hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); + + if (init_dipole_b_field) { + const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); + const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); + const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); + + hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); + hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); + hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); + } + + /************************************************************ + * Read Cluster Gravity Parameters + ************************************************************/ + + // Build cluster_gravity object + ClusterGravity cluster_gravity(pin, hydro_pkg); + + // Include gravity as a source term during evolution + const bool gravity_srcterm = + pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); + hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); + + /************************************************************ + * Read Initial Entropy Profile + ************************************************************/ + + // Create hydrostatic sphere with ACCEPT entropy profile + ACCEPTEntropyProfile entropy_profile(pin); + + HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, + entropy_profile); + + // Create hydrostatic sphere with ISOTHERMAL entropy profile + + + + /************************************************************ + * Read Precessing Jet Coordinate system + ************************************************************/ + + JetCoordsFactory jet_coords_factory(pin, hydro_pkg); + + /************************************************************ + * Read AGN Feedback + ************************************************************/ + + AGNFeedback agn_feedback(pin, hydro_pkg); + + /************************************************************ + * Read AGN Triggering + ************************************************************/ + + AGNTriggering agn_triggering(pin, hydro_pkg); + + /************************************************************ + * Read Magnetic Tower + ************************************************************/ + + // Build Magnetic Tower + MagneticTower magnetic_tower(pin, hydro_pkg); + + // Determine if magnetic_tower_power_scaling is needed + // Is AGN Power and Magnetic fraction non-zero? + bool magnetic_tower_power_scaling = + (agn_feedback.magnetic_fraction_ != 0 && + (agn_feedback.fixed_power_ != 0 || + agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); + hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); + + /************************************************************ + * Read SNIA Feedback + ************************************************************/ + + SNIAFeedback snia_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Stellar Feedback + ************************************************************/ + + StellarFeedback stellar_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Clips (ceilings and floors) + ************************************************************/ + + // Disable all clips by default with a negative radius clip + Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); + + // By default disable floors by setting a negative value + Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); + + // By default disable ceilings by setting to infinity + Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", + std::numeric_limits::infinity()); + Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", + std::numeric_limits::infinity()); + Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", + std::numeric_limits::infinity()); + Real eceil = Tceil; + if (eceil < std::numeric_limits::infinity()) { + if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { + PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " + "Either set a 'units' block and the 'hydro/He_mass_fraction' in " + "input file or use a pressure floor " + "(defined code units) instead."); + } + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); + } + hydro_pkg->AddParam("cluster_dfloor", dfloor); + hydro_pkg->AddParam("cluster_eceil", eceil); + hydro_pkg->AddParam("cluster_vceil", vceil); + hydro_pkg->AddParam("cluster_vAceil", vAceil); + hydro_pkg->AddParam("cluster_clip_r", clip_r); + + /************************************************************ + * Start running reductions into history outputs for clips, stellar mass, cold + * gas, and AGN extent + ************************************************************/ + + /* FIXME(forrestglines) This implementation with a reduction into Params might + be broken in several ways. + 1. Each reduction in params is Rank local. Multiple meshblocks packs per + rank adding to these params is not thread-safe + 2. These Params are not carried over between restarts. If a restart dump is + made and a history output is not, then the mass/energy between the last + history output and the restart dump is lost + */ + std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", + "removed_eceil_energy", "removed_vceil_energy", + "added_vAceil_mass"}; + + // Add a param for each reduction, then add it as a summation reduction for + // history outputs + auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); + + for (auto reduction_str : reduction_strs) { + hydro_pkg->AddParam(reduction_str, 0.0, true); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, + [reduction_str](MeshData *md) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const Real reduction = hydro_pkg->Param(reduction_str); + // Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str, 0.0); + return reduction; + }, + reduction_str)); + } + + // Add history reduction for total cold gas using stellar mass threshold + const Real cold_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); + if (cold_thresh > 0) { + hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); + } + const Real agn_tracer_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); + if (agn_tracer_thresh >= 0) { + PARTHENON_REQUIRE( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); + hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); + } + + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); + + /************************************************************ + * Add derived fields + * NOTE: these must be filled in UserWorkBeforeOutput + ************************************************************/ + + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); + + // log10 of cell-centered radius + hydro_pkg->AddField("log10_cell_radius", m); + // entropy + hydro_pkg->AddField("entropy", m); + // sonic Mach number v/c_s + hydro_pkg->AddField("mach_sonic", m); + // temperature + hydro_pkg->AddField("temperature", m); + + if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { + // cooling time + hydro_pkg->AddField("cooling_time", m); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + // alfven Mach number v_A/c_s + hydro_pkg->AddField("mach_alfven", m); + + // plasma beta + hydro_pkg->AddField("plasma_beta", m); + } + + /************************************************************ + * Read Density perturbation + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); // Mean density of perturbations + + if (mu_rho != 0.0) { + + auto k_min_rho = pin->GetReal("problem/cluster/init_perturb", "k_min_rho"); // Minimum wavenumber of perturbation + auto num_modes_rho = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_rho", 40); + auto sol_weight_rho = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_rho", 1.0); + uint32_t rseed_rho = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_rho", 1); + + // Computing the kmax ie. the Nyquist limit + auto grid_ni = pin->GetOrAddInteger("parthenon/mesh", "nx1", 64); // Assuming cubic grid with equal size in each axis + auto k_max_rho = grid_ni / 2; + + const auto t_corr_rho = 1e-10; + auto k_vec_rho = utils::few_modes_ft_log::MakeRandomModesLog(num_modes_rho, k_min_rho, k_max_rho, rseed_rho); // Generating random modes + + auto few_modes_ft_rho = FewModesFTLog(pin, hydro_pkg, "cluster_perturb_rho", num_modes_rho, + k_vec_rho, k_min_rho, k_max_rho, sol_weight_rho, t_corr_rho, rseed_rho); + + hydro_pkg->AddParam<>("cluster/few_modes_ft_rho", few_modes_ft_rho); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + + } + + /************************************************************ + * Read Velocity perturbation + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + if (sigma_v != 0.0) { + // peak of init vel perturb + auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); + auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); + + PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || + (k_peak_v > 0.0 && l_peak_v <= 0.0), + "Setting initial velocity perturbation requires a single " + "length scale by either setting l_peak_v or k_peak_v."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_v > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_v = Lx / l_peak_v; + } + + auto num_modes_v = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); + auto sol_weight_v = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); + uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); + // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const auto t_corr = 1e-10; + + auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); + + auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, + k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); + hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + + /************************************************************ + * Read Magnetic field perturbation + ************************************************************/ + + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, + "Requested initial magnetic field perturbations but not " + "solving the MHD equations.") + // peak of init magnetic field perturb + auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); + auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); + PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || + (k_peak_b > 0.0 && l_peak_b <= 0.0), + "Setting initial B perturbation requires a single " + "length scale by either setting l_peak_b or k_peak_b."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_b > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_b = Lx / l_peak_b; + } + + auto num_modes_b = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); + uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); + // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const auto t_corr = 1e-10; + // This field should by construction have no compressive modes, so we fix the number. + const auto sol_weight_b = 1.0; + + auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); + + const bool fill_ghosts = true; // as we fill a vector potential to calc B + auto few_modes_ft = + FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, + sol_weight_b, t_corr, rseed_b, fill_ghosts); + hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now). Only add if not already done for the velocity. + if (sigma_v == 0.0) { + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + } + +} + +//======================================================================================== +//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) +//! \brief Generate problem data for all blocks on rank +// +// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that +// reductions work +//======================================================================================== + +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + + // This could be more optimized, but require a refactor of init routines being called. + // However, given that it's just called during initial setup, this should not be a + // performance concern. + + // Defining a table within which the values of the hydrostatic density profile will be stored + auto pmc = md->GetBlockData(0)->GetBlockPointer(); + const auto grid_size = pin->GetOrAddInteger("problem/cluster/mesh", "nx1", 256); + + // Here, the pmc-> contains the number of cells of each MESHBLOCK, including the 2*n_ghost ghost cells + parthenon::ParArray4D hydrostatic_rho("hydrostatic_rho", md->NumBlocks(), + pmc->cellbounds.ncellsk(IndexDomain::entire), + pmc->cellbounds.ncellsj(IndexDomain::entire), + pmc->cellbounds.ncellsi(IndexDomain::entire)); + + for (int b = 0; b < md->NumBlocks(); b++) { + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + + auto &coords = pmb->coords; + + // Get Adiabatic Index + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + /************************************************************ + * Initialize the initial hydro state + ************************************************************/ + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + const auto isothermal_sphere = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_sphere", false); + const auto isothermal_hernquist = pin->GetOrAddBoolean("problem/cluster/gravity", "isothermal_hernquist", false); + + if (init_uniform_gas) { + const Real rho = hydro_pkg->Param("uniform_gas_rho"); + const Real ux = hydro_pkg->Param("uniform_gas_ux"); + const Real uy = hydro_pkg->Param("uniform_gas_uy"); + const Real uz = hydro_pkg->Param("uniform_gas_uz"); + const Real pres = hydro_pkg->Param("uniform_gas_pres"); + + const Real Mx = rho * ux; + const Real My = rho * uy; + const Real Mz = rho * uz; + const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + u(IDN, k, j, i) = rho; + u(IM1, k, j, i) = Mx; + u(IM2, k, j, i) = My; + u(IM3, k, j, i) = Mz; + u(IEN, k, j, i) = E; + }); + + // end if(init_uniform_gas) + } else if (isothermal_sphere) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real mu = hydro_pkg->Param("mu"); + const Real prefactor = 2 * units.k_boltzmann() * T_bcg_s / (mu * units.mh()); + const Real grav_const = units.gravitational_constant(); + + std::cout << "Entering isothermal sphere generation \n" ; + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real rho_r = prefactor * 1 / (4 * M_PI * grav_const * r_effective * r_effective); + const Real P_r = (prefactor/2) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } else if (isothermal_hernquist) { + + const Real T_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "T_bcg_s", 10000); + const Real m_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); + const Real r_bcg_s = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); + const Real mu = hydro_pkg->Param("mu"); + const Real rho_0 = pin->GetOrAddReal("problem/cluster/gravity", "rho_0", 1e3); + const Real grav_const = units.gravitational_constant(); + + // Generating the profile + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::IsothermalSphere", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + const Real r_effective = std::max(r_smoothing,r); + + const Real phi_r = - grav_const * m_bcg_s / (r_effective + r_bcg_s); + const Real rho_r = rho_0 * std::exp( - mu * units.mh() / (units.k_boltzmann() * T_bcg_s) * phi_r); + const Real P_r = units.k_boltzmann() * T_bcg_s / (mu * units.mh()) * rho_r; + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + }); + + } + + else { + /************************************************************ + * Initialize a HydrostaticEquilibriumSphere + ************************************************************/ + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + // Get pressure and density from generated profile + const Real P_r = P_rho_profile.P_from_r(r); + const Real rho_r = P_rho_profile.rho_from_r(r); + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + + // Updating hydrostatic_rho table + hydrostatic_rho(b, k, j, i) = rho_r; + + }); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + /************************************************************ + * Initialize the initial magnetic field state via a vector potential + ************************************************************/ + parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + /************************************************************ + * Initialize an initial magnetic tower + ************************************************************/ + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + + /************************************************************ + * Add dipole magnetic field to the magnetic potential + ************************************************************/ + const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); + if (init_dipole_b_field) { + const Real mx = hydro_pkg->Param("dipole_b_field_mx"); + const Real my = hydro_pkg->Param("dipole_b_field_my"); + const Real mz = hydro_pkg->Param("dipole_b_field_mz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + + const Real m_cross_r_x = my * z - mz * y; + const Real m_cross_r_y = mz * x - mx * z; + const Real m_cross_r_z = mx * y - mx * y; + + // To check whether there is some component before initiating perturbations + std::cout << "A(0, k, j, i)=" << A(0, k, j, i) << std::endl; + + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); + A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); + A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + }); + } + + /************************************************************ + * Apply the potential to the conserved variables + ************************************************************/ + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + u(IB1, k, j, i) = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + + SQR(u(IB3, k, j, i))); + }); + + /************************************************************ + * Add uniform magnetic field to the conserved variables + ************************************************************/ + const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); + if (init_uniform_b_field) { + const Real bx = hydro_pkg->Param("uniform_b_field_bx"); + const Real by = hydro_pkg->Param("uniform_b_field_by"); + const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + const Real bx_i = u(IB1, k, j, i); + const Real by_i = u(IB2, k, j, i); + const Real bz_i = u(IB3, k, j, i); + + u(IB1, k, j, i) += bx; + u(IB2, k, j, i) += by; + u(IB3, k, j, i) += bz; + + // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + + // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that + u(IEN, k, j, i) += + bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + }); + // end if(init_uniform_b_field) + } + + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) + } + + /************************************************************ + * Initial parameters + ************************************************************/ + + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto fluid = hydro_pkg->Param("fluid"); + auto const &cons = md->PackVariables(std::vector{"cons"}); + const auto num_blocks = md->NumBlocks(); + + /************************************************************ + * Set initial density perturbations (read from HDF5 file) + ************************************************************/ + + const bool init_perturb_rho = pin->GetOrAddBoolean("problem/cluster/init_perturb", "init_perturb_rho", false); + const bool full_box = pin->GetOrAddBoolean("problem/cluster/init_perturb", "full_box", true); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.0); + const bool spherical_collapse = pin->GetOrAddBoolean("problem/cluster/init_perturb", "spherical_collapse", false); + + hydro_pkg->AddParam<>("init_perturb_rho", init_perturb_rho); + + Real passive_scalar = 0.0; // Not useful here + + // Spherical collapse test with an initial overdensity + + if (spherical_collapse == true) { + + // Create an homogeneous sphere of a density superior or inferior to the background density + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); // Computing radius + const Real overdensity_radius = pin->GetOrAddReal("problem/cluster/init_perturb", "overdensity_radius", 0.01); + const Real background_density = pin->GetOrAddReal("problem/cluster/init_perturb", "background_density", 3); + const Real foreground_density = pin->GetOrAddReal("problem/cluster/init_perturb", "foreground_density", 150); + + // Setting an initial + u(IDN, k, j, i) = background_density; + + // For any cell at a radius less then overdensity radius, set the density to foreground_density + if (r < overdensity_radius){ + u(IDN, k, j, i) = foreground_density; + } + + }, + passive_scalar); + + } + + /* -------------- Setting up a clumpy atmosphere -------------- + + 1) Extract the values of the density from an input hdf5 file using H5Easy + 2) Initiate the associated density field + 3) Optionnaly, add some overpressure ring to check behavior (overpressure_ring bool) + 4) Optionnaly, add a central overdensity + + */ + + if (init_perturb_rho == true) { + + auto filename_rho = pin->GetOrAddString("problem/cluster/init_perturb", "init_perturb_rho_file","none"); + + const Real r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 1e-6); + const Real perturb_amplitude = pin->GetOrAddReal("problem/cluster/init_perturb", "perturb_amplitude", 1); + const Real thickness_ism = pin->GetOrAddReal("problem/cluster/init_perturb", "thickness_ism", 0.01); + + std::string keys_rho = "data"; + H5Easy::File file(filename_rho, HighFive::File::ReadOnly); + + const int rho_init_size = 256; + auto rho_init = H5Easy::load, rho_init_size>, rho_init_size>>(file, keys_rho); + + Real passive_scalar = 0.0; // Useless + + std::cout << "Entering initialisation of rho field"; + + pmb->par_reduce( + "Init density field", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + auto pmbb = md->GetBlockData(b)->GetBlockPointer(); // Meshblock b + + const auto gis = pmbb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmbb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmbb->loc.lx3() * pmb->block_size.nx3; + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + // Case where the box is filled with perturbations of equal mean amplitude + if (full_box){ + + u(IDN, k, j, i) += perturb_amplitude * rho_init[gks + k - 2][gjs + j - 2][gis + i - 2] * (u(IDN, k, j, i) / 29.6); + + } + + // Mean amplitude of the perturbations is modulated by the + else { + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r = std::sqrt(SQR(x) + SQR(y) + SQR(z)); + const Real r_effective = std::max(r,r_smoothing); + + //u(IDN, k, j, i) += rho_init[gks + k - 2][gjs + j - 2][gis + i - 2]; + + } + + }, + passive_scalar); + + } + + /************************************************************ + * Setting up a perturbed density field (hardcoded version) + ************************************************************/ + + const auto mu_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "mu_rho", 0.0); + const auto background_rho = pin->GetOrAddReal("problem/cluster/init_perturb", "background_rho", 0.0); + + if (mu_rho != 0.0) { + + auto few_modes_ft_rho = hydro_pkg->Param("cluster/few_modes_ft_rho"); + + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft_rho.SetPhases(pmb.get(), pin); + + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft_rho.Generate(md, dt, "tmp_perturb"); + + Real v2_sum_rho = 0.0; // used for normalization + + auto perturb_pack_rho = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + + Real perturb = 0.0 ; + + perturb = std::sqrt(SQR(perturb_pack_rho(b, 0, k, j, i)) + SQR(perturb_pack_rho(b, 1, k, j, i)) + SQR(perturb_pack_rho(b, 2, k, j, i))); + u(IDN, k, j, i) = background_rho + mu_rho * perturb; + + }, + v2_sum_rho); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum_rho, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + } + + /************************************************************ + * Set initial velocity perturbations (requires no other velocities for now) + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + + if (sigma_v != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real v2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + auto rho = u(IDN, k, j, i); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, + "Found existing non-zero velocity when setting velocity perturbations."); + + u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); + u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); + u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); + // No need to touch the energy yet as we'll normalize later + + lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * + coords.CellVolume(k, j, i) / SQR(rho); + }, + v2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); + + pmb->par_for( + "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IM1, k, j, i) /= v_norm; + u(IM2, k, j, i) /= v_norm; + u(IM3, k, j, i) /= v_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / + u(IDN, k, j, i); + }); + } + + /************************************************************ + * Set initial magnetic field perturbations (resets magnetic field field) + ************************************************************/ + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + const auto alpha_b = pin->GetOrAddReal("problem/cluster/init_perturb", "alpha_b", 2.0/3.0); + const auto density_scale = pin->GetOrAddReal("problem/cluster/init_perturb", "density_scale", 29.6); + const auto standard_B = pin->GetOrAddBoolean("problem/cluster/init_perturb", "standard_B", true); + const auto r_smoothing = pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 5e-3); + const auto rho_type = pin->GetOrAddInteger("problem/cluster/init_perturb", "rho_type", 0); // test + + if (sigma_b != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real b2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + // Defining a new table that will contains the values of the perturbed magnetic field, and magnetic energy + parthenon::ParArray5D dB("turbulent magnetic field", num_blocks, 4, + pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + if (standard_B){ + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0; + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * + coords.CellVolume(k, j, i); + }, + b2_sum); + + } else { + + // Idea: separate into two loops, one par_for and one par_reduce + + std::cout << "Setting up perturbed magnetic field." << std::endl; + + for (int b = 0; b < md->NumBlocks(); b++) { + + std::cout << "Treating block number " << b << " out of " << md->NumBlocks() << std::endl; + + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // Computation of the P_rho_profile require entire domain of the block (boundary conditions for curl computation) + IndexRange ib_entire = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb_entire = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb_entire = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + auto &coords = pmb->coords; + + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib_entire, jb_entire, kb_entire, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::MagneticPerturbations", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + + const Real x = coords.Xc<1>(i); + const Real xp = coords.Xc<1>(i+1); + const Real xm = coords.Xc<1>(i-1); + const Real y = coords.Xc<2>(j); + const Real yp = coords.Xc<2>(j+1); + const Real ym = coords.Xc<2>(j-1); + const Real z = coords.Xc<3>(k); + const Real zp = coords.Xc<3>(k+1); + const Real zm = coords.Xc<3>(k-1); + + const Real r = sqrt(SQR(x) + SQR(y) + SQR(z)); + const Real r_ip = sqrt(SQR(xp) + SQR(y) + SQR(z)); + const Real r_im = sqrt(SQR(xm) + SQR(y) + SQR(z)); + const Real r_jp = sqrt(SQR(x) + SQR(yp) + SQR(z)); + const Real r_jm = sqrt(SQR(x) + SQR(ym) + SQR(z)); + const Real r_kp = sqrt(SQR(x) + SQR(y) + SQR(zp)); + const Real r_km = sqrt(SQR(x) + SQR(y) + SQR(zm)); + + const auto rho = P_rho_profile.rho_from_r(r); + const auto rho_ip = P_rho_profile.rho_from_r(r_ip); + const auto rho_im = P_rho_profile.rho_from_r(r_im); + const auto rho_jp = P_rho_profile.rho_from_r(r_jp); + const auto rho_jm = P_rho_profile.rho_from_r(r_jm); + const auto rho_kp = P_rho_profile.rho_from_r(r_kp); + const auto rho_km = P_rho_profile.rho_from_r(r_km); + + // const Real rho_r = P_rho_profile.rho_from_r(r); + + // Normalization condition here, which we want to lift. To do so, we'll need + // to copy the values of the magnetic field into a new array, which we will + // use to store the perturbed magnetic field and then rescale it, until values + // are added to u = cons(b) + + //PARTHENON_REQUIRE( + // u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + // "Found existing non-zero B when setting magnetic field perturbations."); + + // First, needs to rescale the perturb_pack, ie. the magnetic field potential + Real perturb2_k_jp_i,perturb2_k_jm_i,perturb1_kp_j_i,perturb1_km_j_i; + Real perturb0_kp_j_i,perturb0_km_j_i,perturb2_k_j_ip,perturb2_k_j_im; + Real perturb1_k_j_ip,perturb1_k_j_im,perturb0_k_jp_i,perturb0_k_jm_i; + + // Rescaling the magnetic potential + + perturb2_k_jp_i = perturb_pack(b, 2, k, j + 1, i) * std::pow(r_jp, alpha_b); + perturb2_k_jm_i = perturb_pack(b, 2, k, j - 1, i) * std::pow(r_jm, alpha_b); + perturb1_kp_j_i = perturb_pack(b, 1, k + 1, j, i) * std::pow(r_kp, alpha_b); + perturb1_km_j_i = perturb_pack(b, 1, k - 1, j, i) * std::pow(r_km, alpha_b); + perturb0_kp_j_i = perturb_pack(b, 0, k + 1, j, i) * std::pow(r_kp, alpha_b); + perturb0_km_j_i = perturb_pack(b, 0, k - 1, j, i) * std::pow(r_km, alpha_b); + perturb2_k_j_ip = perturb_pack(b, 2, k, j, i + 1) * std::pow(r_ip, alpha_b); + perturb2_k_j_im = perturb_pack(b, 2, k, j, i - 1) * std::pow(r_im, alpha_b); + perturb1_k_j_ip = perturb_pack(b, 1, k, j, i + 1) * std::pow(r_ip, alpha_b); + perturb1_k_j_im = perturb_pack(b, 1, k, j, i - 1) * std::pow(r_im, alpha_b); + perturb0_k_jp_i = perturb_pack(b, 0, k, j + 1, i) * std::pow(r_jp, alpha_b); + perturb0_k_jm_i = perturb_pack(b, 0, k, j - 1, i) * std::pow(r_jm, alpha_b); + + // Then, compute the curl of the magnetic field + + Real curlBx,curlBy,curlBz; + + curlBx = (perturb2_k_jp_i - perturb2_k_jm_i) / coords.Dxc<2>(j) / 2.0 - + (perturb1_kp_j_i - perturb1_km_j_i) / coords.Dxc<3>(k) / 2.0 ; + curlBy = (perturb0_kp_j_i - perturb0_km_j_i) / coords.Dxc<3>(k) / 2.0 - + (perturb2_k_j_ip - perturb2_k_j_im) / coords.Dxc<1>(i) / 2.0 ; + curlBz = (perturb1_k_j_ip - perturb1_k_j_im) / coords.Dxc<1>(i) / 2.0 - + (perturb0_k_jp_i - perturb0_k_jm_i) / coords.Dxc<2>(j) / 2.0 ; + + dB(b,0, k, j, i) = curlBx; + dB(b,1, k, j, i) = curlBy; + dB(b,2, k, j, i) = curlBz; + + }); + } + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + + const auto &coords = cons.GetCoords(b); + + Real curlBx,curlBy,curlBz; + + curlBx = dB(b,0, k, j, i); + curlBy = dB(b,1, k, j, i); + curlBz = dB(b,2, k, j, i); + + lsum += (SQR(curlBx) + SQR(curlBy) + SQR(curlBz)) * coords.CellVolume(k, j, i); + }, + b2_sum); + } + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); + + if (standard_B){ + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); + }); + + } else { + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + dB(b, 0, k, j, i) /= b_norm; + dB(b, 1, k, j, i) /= b_norm; + dB(b, 2, k, j, i) /= b_norm; + + // Computing energy + dB(b, 3, k, j, i) += + 0.5 * (SQR(dB(b, 0, k, j, i)) + SQR(dB(b, 1, k, j, i)) + SQR(dB(b, 2, k, j, i))); + + // Updating the MHD vector + + u(IB1, k, j, i) += dB(b, 0, k, j, i); + u(IB2, k, j, i) += dB(b, 1, k, j, i); + u(IB3, k, j, i) += dB(b, 2, k, j, i); + u(IEN, k, j, i) += dB(b, 3, k, j, i); + + }); + } + } +} + +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { + // get hydro + auto pkg = pmb->packages.Get("Hydro"); + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); + + // get prim vars + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; + + // get derived fields + auto &log10_radius = data->Get("log10_cell_radius").data; + auto &entropy = data->Get("entropy").data; + auto &mach_sonic = data->Get("mach_sonic").data; + auto &temperature = data->Get("temperature").data; + + // for computing temperature from primitives + auto units = pkg->Param("units"); + auto mbar_over_kb = pkg->Param("mbar_over_kb"); + auto mbar = mbar_over_kb * units.k_boltzmann(); + + // fill derived vars (*including ghost cells*) + auto &coords = pmb->coords; + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real v1 = prim(IV1, k, j, i); + const Real v2 = prim(IV2, k, j, i); + const Real v3 = prim(IV3, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute radius + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r2 = SQR(x) + SQR(y) + SQR(z); + log10_radius(k, j, i) = 0.5 * std::log10(r2); + + // compute entropy + const Real K = P / std::pow(rho / mbar, gam); + entropy(k, j, i) = K; + + const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + const Real M_s = v_mag / c_s; + mach_sonic(k, j, i) = M_s; + + // compute temperature + temperature(k, j, i) = mbar_over_kb * P / rho; + }); + + if (pkg->Param("enable_cooling") == Cooling::tabular) { + auto &cooling_time = data->Get("cooling_time").data; + + // get cooling function + const cooling::TabularCooling &tabular_cooling = + pkg->Param("tabular_cooling"); + const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute cooling time + const Real eint = P / (rho * gm1); + const Real edot = cooling_table_obj.DeDt(eint, rho); + cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; + }); + } + + if (pkg->Param("fluid") == Fluid::glmmhd) { + auto &plasma_beta = data->Get("plasma_beta").data; + auto &mach_alfven = data->Get("mach_alfven").data; + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + const Real Bx = prim(IB1, k, j, i); + const Real By = prim(IB2, k, j, i); + const Real Bz = prim(IB3, k, j, i); + const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + + // compute Alfven mach number + const Real v_A = std::sqrt(B2 / rho); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; + + // compute plasma beta + plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; + }); + } +} + +} // namespace cluster diff --git a/src/pgen/cluster_working.cpp b/src/pgen/old_cluster/cluster_working.cpp similarity index 100% rename from src/pgen/cluster_working.cpp rename to src/pgen/old_cluster/cluster_working.cpp diff --git a/src/utils/few_modes_ft_lognormal.cpp b/src/utils/few_modes_ft_lognormal.cpp index 9a6b2ae8..f04d0fe3 100644 --- a/src/utils/few_modes_ft_lognormal.cpp +++ b/src/utils/few_modes_ft_lognormal.cpp @@ -382,6 +382,18 @@ ParArray2D MakeRandomModesLog(const int num_modes, const Real k_min, const while (n_mode < num_modes && n_attempt < max_attempts) { n_attempt += 1; + // ============================= + // Uniformly taken in lin scaled + // ============================= + + //kx1 = dist(rng); + //kx2 = dist(rng); + //kx3 = dist(rng); + + // ============================= + // Uniformly taken in log scaled + // ============================= + skx1 = dist(rng); skx2 = dist(rng); skx3 = dist(rng); @@ -397,16 +409,12 @@ ParArray2D MakeRandomModesLog(const int num_modes, const Real k_min, const kx1 = skx1 * std::floor(std::pow(10,lkx1)); kx2 = skx2 * std::floor(std::pow(10,lkx2)); kx3 = skx3 * std::floor(std::pow(10,lkx3)); - - //kx1 = std::floor(std::pow(10,lkx1)); - //kx2 = std::floor(std::pow(10,lkx2)); - //kx3 = std::floor(std::pow(10,lkx3)); k_mag = std::sqrt(SQR(kx1) + SQR(kx2) + SQR(kx3)); - + // Expected amplitude of the spectral function. If this is changed, it also needs to // be changed in the FMFT class (or abstracted). - ampl = std::pow((k_mag / k_low), -5./3.); + ampl = std::pow((k_mag / k_high), -5./3.); // Check is mode was already picked by chance mode_exists = false;