diff --git a/.gitignore b/.gitignore index a325e21d..5dfd5c88 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,7 @@ compile_commands.json # CMake Test files Testing + +# generate_cluster_input.ipynb files +inputs/cluster/generate_cluster_input.ipynb +inputs/cluster/my_cluster.input diff --git a/README.md b/README.md index a3207114..04881cd0 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ AthenaPK: a performance portable version based on [Athena++](https://github.com/PrincetonUniversity/athena), [Parthenon](https://github.com/parthenon-hpc-lab/parthenon) and [Kokkos](https://github.com/kokkos/kokkos). -## Current state of the code +## Overview -For this reason, it is highly recommended to only use AthenaPK with the Kokkos and Parthenon versions that are provided by the submodules (see [building](#building)) and to build everything (AthenaPK, Parthenon, and Kokkos) together from source. +It is highly recommended to only use AthenaPK with the Kokkos and Parthenon versions that are provided by the submodules (see [building](#building)) and to build everything (AthenaPK, Parthenon, and Kokkos) together from source. Neither other versions or nor using preinstalled Parthenon/Kokkos libraries have been tested. Current features include @@ -76,19 +76,22 @@ Obtain all (AthenaPK, Parthenon, and Kokkos) sources Most of the general build instructions and options for Parthenon (see [here](https://parthenon-hpc-lab.github.io/parthenon/develop/src/building.html)) also apply to AthenaPK. The following examples are a few standard cases. -Most simple configuration (only CPU, no MPI, no HDF5). +Most simple configuration (only CPU, no MPI). The `Kokkos_ARCH_...` parameter should be adjusted to match the target machine where AthenaPK will be executed. A full list of architecture keywords is available on the [Kokkos wiki](https://kokkos.github.io/kokkos-core-wiki/keywords.html#architecture-keywords). - - # configure with enabling Broadwell architecture (AVX2) instructions - cmake -S. -Bbuild-host -DKokkos_ARCH_BDW=ON -DPARTHENON_DISABLE_MPI=ON -DPARTHENON_DISABLE_HDF5=ON + # configure with enabling Intel Broadwell or similar architecture (AVX2) instructions + cmake -S. -Bbuild-host -DKokkos_ARCH_BDW=ON -DPARTHENON_DISABLE_MPI=ON # now build with cd build-host && make # or alternatively cmake --build build-host -An Intel Skylake system (AVX512 instructions) with NVidia Volta V100 GPUs and with MPI and HDF5 enabled (the latter is the default option, so they don't need to be specified) +If `cmake` has troubling finding the HDF5 library (which is required for writing analysis outputs or +restartings simulation) an additional hint to the location of the library can be provided via +`-DHDF5_ROOT=/path/to/local/hdf5` on the first `cmake` command for configuration. + +An Intel Skylake system (AVX512 instructions) with NVidia Volta V100 GPUs and with MPI enabled (the latter is the default option, so they don't need to be specified) cmake -S. -Bbuild-gpu -DKokkos_ARCH_SKX=ON -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_VOLTA70=ON # now build with diff --git a/docs/cluster.md b/docs/cluster.md new file mode 100644 index 00000000..b4d83983 --- /dev/null +++ b/docs/cluster.md @@ -0,0 +1,565 @@ +# Galaxy Cluster and Cluster-like Problem Setup + +This problem generator initializes an isolated ideal galaxy cluster (or a +galaxy-cluster-like object). Simulations begin as a spherically symmetric set up of +gas in hydrostatic equilibrium with a fixed defined gravitational potential and +an ACCEPT-like entropy profile. An initial magnetic tower can also be included. + +In addition to the fixed gravitational potential source term, the problem +generator also includes AGN feedback via any combination of the injection of a +magnetic tower, kinetic jet, and a flat volumetric thermal dump around the +fixed AGN. Feedback via the magnetic tower and jet can be set to precess around +the z-axis. The AGN feedback power can be set to a fixed power or triggered via +Boosted Bondi accretion, Bondi-Schaye accretion, or cold gas near the AGN. + +## Units + +All parameters in `` are defined in code units. Code units can +be defined under ` +#Units parameters +code_length_cgs = 3.085677580962325e+24 # in cm +code_mass_cgs = 1.98841586e+47 # in g +code_time_cgs = 3.15576e+16 # in s +``` +will set the code length-unit to 1 Mpc, the code mass to 10^14 solar masses, +and the code time to 1 Gyr. + + +## Fixed Gravitational Profile + +A gravitational profile can be defined including components from an NFW dark +matter halo, a brightest cluster galaxy (BCG), and a point-source central +supermassive black hole. This gravitational potential is used to determine +initial conditions in hydrostatic equilbrium and by default as a source term +during evolution. Parameters for the gravitationl profile are placed into ``. + + +The toggles to include different components are as follows: +``` + +include_nfw_g = True +which_bcg_g = HERNQUIST #or NONE +include_smbh_g = True +``` +Where `include_nfw_g` for the NFW dark-matter halo ([Navarro +1997](doi.org/10.1086/304888)) is boolean; `which_bcg_g` for the BCG can be +`NONE` for no BCG, `HERNQUIST` for Hernquist profile ([Hernquist +1990](doi.org/10.1086/168845)), and `include_smbh_g` for the SMBH is +boolean. + +Parameters for the NFW profile are +``` + +c_nfw = 6.0 # Unitless +m_nfw_200 = 10.0 # in code_mass +``` +which adds a gravitational acceleration defined by + +$$ +g_{\text{NFW}}(r) = + \frac{G}{r^2} + \frac{M_{NFW} \left [ \ln{\left(1 + \frac{r}{R_{NFW}} \right )} - \frac{r}{r+R_{NFW}} \right ]} + { \ln{\left(1 + c_{NFW}\right)} - \frac{ c_{NFW}}{1 + c_{NFW}} } +$$ +The scale radius $R_{NFW}$ for the NFW profile is computed from +$$ +R_{NFW} = \left ( \frac{M_{NFW}}{ 4 \pi \rho_{NFW} \left [ \ln{\left ( 1 + c_{NFW} \right )} - c_{NFW}/\left(1 + c_{NFW} \right ) \right ] }\right )^{1/3} +$$ +where the scale density $\rho_{NFW}$ is computed from +$$ +\rho_{NFW} = \frac{200}{3} \rho_{crit} \frac{c_{NFW}^3}{\ln{\left ( 1 + c_{NFW} \right )} - c_{NFW}/\left(1 + c_{NFW} \right )}. +$$ +The critical density $\rho_{crit}$ is computed from +$$ + \frac{3 H_0^2}{8 \pi G}. +$$ + +Parameters for the HERNQUIST BCG are controlled via: +``` + + +m_bcg_s = 0.001 # in code_mass +r_bcg_s = 0.004 # in code_length +``` +where a HERNQUIST profile adds a gravitational acceleration defined by + +$$ + g_{BCG}(r) = G \frac{ M_{BCG} }{R_{BCG}^2} \frac{1}{\left( 1 + \frac{r}{R_{BCG}}\right)^2} +$$ + +Gravitational acceleration from the SMBH is inserted as a point source defined solely by its mass +``` + +m_smbh = 1.0e-06 # in code_mass +``` + +Some acceleration profiles may be ill-defined at the origin. For this, we provide a smoothing length parameter, +``` + +g_smoothing_radius = 0.0 # in code_length +``` +which works as a minimum r when the gravitational potential is applied. It +effectively modifies the gravitation acceleration to work as + +$$ +\tilde{g} (r) = g( max( r, r_{smooth})) +$$ + +By default, the gravitational profile used to create the initial conditions is +also used as an accelerating source term during evolution. This source term can +be turned off, letting this gravitational profile to only apply to +initialization, with the following parameter in ``. +``` + +gravity_srcterm = False +``` + +## Entropy Profile + +The `cluster` problem generator initializes a galaxy-cluster-like system with an entropy profile following the ACCEPT profile + +$$ + K(r) = K_{0} + K_{100} \left ( r/ 100 \text{ kpc} \right )^{\alpha_K} + $$ + +where we are using the entropy $K$ is defined as + +$$ + K \equiv \frac{ k_bT}{n_e^{2/3} } + $$ + +This profile is determined by these parameters +``` + +k_0 = 8.851337676479303e-121 # in code_length**4*code_mass/code_time**2 +k_100 = 1.3277006514718954e-119 # in code_length**4*code_mass/code_time**2 +r_k = 0.1 # in code_length +alpha_k = 1.1 # unitless +``` + +## Defining Initial Hydrostatic Equilibrium + +With a gravitational profile and initial entropy profile defined, the system of +equations for initial hydrostatic equilibrium are still not closed. In order to +close them, we fix the density of the cluster to a defined value at a defined +radius. This radius and density is set by the parameters +``` + +r_fix = 2.0 # in code_length +rho_fix = 0.01477557589278723 # in code_mass/code_length**3 +``` +In each meshblock the equations for hydrostatic equilbirium and the entropy +profile are integrated to obtain density and pressure profiles inward and +outward from `r_fix` as needed to cover the meshblock. The parameter +`r_sampling` controls the resolution of the profiles created, where higher +values give higher resolution. +``` + +r_sampling = 4.0 +``` +Specifically, the resolution of the 1D profile for each meshblock is either +`min(dx,dy,dz)/r_sampling` or `r_k/r_sampling`, whichever is smaller. + +## Initial perturbations + +Initial perturbations for both the velocity field and the magnetic field are +supported. + +*Note* that these intial perturbation are currently incompatible with +other initial conditions that modify the velocity or magnetic field, e.g., +an initial magnetic dipole or a uniform field. +This restriction could be lifted if required/desired but the normalization +of the fields would need to be adjusted. + +In general, the perturbations will be seeded by an inverse parabolic +spectral profile centered on a peak wavenumber (in normalized units, i.e., +`k_peak = 2` mean half the box size) with a finite number of modes (default 40) +randomly chosen between `k_peak/2` and `2*k_peak`. + +Pertubations are controlled by the following parameters: +``` + +# for the velocity field +sigma_v = 0.0 # volume weighted RMS |v| in code velocity; default: 0.0 (disabled) +l_peak_v = ??? # lengthscale (in code length units) where the velocity spectrum peaks. No default value. +k_peak_v = ??? # (exclusive alternative to l_peak_v): wavenumber in normalized units where the velocity spectrum peaks. No default value. +num_modes_v = 40 # (optional) number of wavemodes in spectral space; default: 40 +sol_weight_v = 1.0 # (optional) power in solenoidal (rotational) modes of the perturbation. Range between 0 (fully compressive) and 1.0 (default, fully solenoidal). +rseed_v = 1 # (optional) integer seed for RNG for wavenumbers and amplitudes + +# for the magnetic field +sigma_b = 0.0 # volume weighted RMS |B| in code magnetic; default: 0.0 (disabled) +l_peak_b = ??? # lengthscale (in code length units) where the magnetic field spectrum peaks. No default value. +k_peak_b = ??? # (exclusive alternative to l_peak_b): wavenumber in normalized units where the magnetic field spectrum peaks. No default value. +num_modes_b = 40 # (optional) number of wavemodes in spectral space; default: 40 +rseed_b = 2 # (optional) integer seed for RNG for wavenumbers and amplitudes +``` + +## AGN Triggering + +If AGN triggering is enabled, at the end of each time step, a mass accretion +rate `mdot` is determined from conditions around the AGN according to the +different triggering prescriptions. The accreted mass is removed from the gas +around the AGN, with details depending on each prescription explained below, +and is used as input for AGN feedback power. + +The triggering prescriptions currently implemented are "boosted Bondi +accretion" ([Bondi 1952](doi.org/10.1093/mnras/112.2.195), [Meece +2017](doi.org/10.3847/1538-4357/aa6fb1)), "Bondi-Schaye accretion" ([Bondi and +Schaye 2009](doi.org/10.1111/j.1365-2966.2009.15043.x)), and "cold gas" +([Meece 2017](doi.org/10.3847/1538-4357/aa6fb1)). These modes can be chosen via +``` + +triggering_mode = COLD_GAS # or NONE, BOOSTED_BONDI, BONDI_SCHAYE +``` +where `triggering_mode=NONE` will disable AGN triggering. + +With BOOSTED_BONDI accretion, the mass rate of accretion follows + +$$ +\dot{M} = \alpha \frac { 2 \pi G^2 M^2_{SMBH} \hat {\rho} } { +\left ( \hat{v}^2 + \hat{c}_s^2 \right ) ^{3/2} } +$$ + +where $\hat{rho}$, $\hat{v}$, and $\hat{c}_s$ are respectively the mass weighted density, +velocity, and sound speed within the accretion region. The mass of the SMBH, +the radius of the sphere of accretion around the AGN, and the $\alpha$ parameter +can be set with +``` + +m_smbh = 1.0e-06 # in code_mass + + +accretion_radius = 0.001 # in code_length +bondi_alpha= 100.0 # unitless +``` +With BONDI_SCHAYE accretion, the `$\alpha$` used for BOOSTED_BONDI accretion is modified to depend on the number density following: + +$$ +\alpha = + \begin{cases} +1 & n \leq n_0 \\\\ + ( n/n_0 ) ^\beta & n > n_0\\\\ +\end{cases} +$$ + +where `n` is the mass weighted mean density within the accretion region and the parameter `n_0` and `beta` can be set with +``` + +bondi_n0= 2.9379989445851786e+72 # in 1/code_length**3 +bondi_beta= 2.0 # unitless +``` + +With both BOOSTED_BONDI and BONDI_SCHAYE accretion, mass is removed from each +cell within the accretion zone at a mass weighted rate. E.g. the mass in each +cell within the accretion region changes by +``` +new_cell_mass = cell_mass - cell_mass/total_mass*mdot*dt; +``` +where `total_mass` is the total mass within the accretion zone. The accreted +mass is removed from the gas which momentum density and energy density +unchanged. Thus velocities and temperatures will increase where mass is +removed. + + +With COLD_GAS accretion, the accretion rate becomes the total mass within the accretion zone equal to or +below a defined cold temperature threshold divided by a defined accretion +timescale. The temperature threshold and accretion timescale are defined by +``` + +cold_temp_thresh= 100000.0 +cold_t_acc= 0.1 +``` +Mass is removed from each cell in the accretion zone on the accretion +timescale. E.g. for each cell in the accretion zone with cold gas +``` +new_cell_mass = cell_mass - cell_mass/cold_t_acc*dt; +``` +As with the Bondi-like accretion prescriptions, this mass is removed such that +the momentum and energy densities are unchanged. + + +## AGN Feedback + +AGN feedback can be both triggered via the mechanisms in the section above and with a fixed power. +``` + +fixed_power = 0.0 +efficiency = 0.001 +``` +Where and `mdot` calculated from AGN triggering will lead to an an AGN feedback +power of `agn_power = efficiency*mdot*c**2`. The parameter `efficiency` is +specifically the AGN's effiency converting in-falling mass into energy in the +jet. The fixed power and triggered power are not mutually exclusive; if both +`fixed_power` is defined and triggering is enabled with a non-zero +`efficiency`, then the `fixed_power` will be added to the triggered AGN power. + + +AGN feedback can be injected via any combination of an injected magnetic tower, +a thermal dump around the AGN, and a kinetic jet. The fraction deposited into +each mechansim can be controlled via +``` + +magnetic_fraction = 0.3333 +thermal_fraction = 0.3333 +kinetic_fraction = 0.3333 +``` +These values are automatically normalized to sum to 1.0 at run time. + +### Thermal feedback + +Thermal feedback is deposited at a flat power density within a sphere of defined radius +``` + +thermal_radius = 0.0005 +``` +Mass is also injected into the sphere at a flat density rate with the existing +velocity and temperature to match the accreted mass proportioned to thermal +feedback, e.g. +``` +thermal_injected_mass = mdot * (1 - efficiency) * normalized_thermal_fraction; +``` + +### Kinetic feedback + +Kinetic feedback is deposited into two disks along the axis of the jet within a +defined radius, thickness of each disk, and an offset above and below the plane +of the AGN disk where each disk begins. +``` + +kinetic_jet_radius = 0.0005 +kinetic_jet_thickness = 0.0005 +kinetic_jet_offset = 0.0005 +``` +Along the axis of the jet, kinetic energy will be deposited as far away as +`kinetic_jet_offset+kinetic_jet_thickness` in either direction. With a z-axis +aligned jet, `kinetic_jet_thickness` should be a multiple of the deposition +zone grid size, otherwise feedback will be lost due to systematic integration +error. + +The axis of the jet can be set to precess with +``` + +jet_theta= 0.15 # in radians +jet_phi0= 0.2 # in radians +jet_phi_dot= 628.3185307179587 # in radians/code_time +``` +at defined precession angle off of the z-axis (`jet_theta`), an initial +azimuthal angle (`jet_phi0`), and an rate of azimuthal precession +(`jet_phi_dot`). + +Kinetic jet feedback is injected is injected as if disk of fixed temperature +and velocity and changing density to match the AGN triggering rate were added +to the existing ambient gas. Either or both the jet temperature $T_{jet}$ and +velocity $v_{jet}$ can be set via +``` + +#kinetic_jet_velocity = 13.695710297774411 # code_length/code_time +kinetic_jet_temperature = 1e7 # K +``` +However, $T_{jet}$ and $v_{jet}$ must be non-negative and fulfill +$$ +v_{jet} = \sqrt{ 2 \left ( \epsilon c^2 - (1 - \epsilon) \frac{k_B T_{jet}}{ \mu m_h \left( \gamma - 1 \right} \right ) } +$$ +to ensure that the sum of rest mass energy, thermal energy, and kinetic energy of the new gas sums to $\dot{M} c^2$. Note that these equations places limits on $T_{jet}$ and $v_{jet}$, specifically +$$ +v_{jet} \leq c \sqrt{ 2 \epsilon } \qquad \text{and} \qquad \frac{k_B T_{jet}}{ \mu m_h \left( \gamma - 1 \right} \leq c^2 \frac{ \epsilon}{1 - \epsilon} +$$ +If the above equations are not satified then an exception will be thrown at +initialization. If neither $T_{jet}$ nor $v_{jet}$ are specified, then +$v_{jet}$ will be computed assuming $T_{jet}=0$ and a warning will be given +that the temperature of the jet is assumed to be 0 K. + +The total mass injected with kinetic jet feedback at each time step is +``` +kinetic_injected_mass = mdot * (1 - efficiency) * normalized_kinetic_fraction; +``` +In each cell the added density, momentum, and energy are +``` +kinetic_injected_density = kinetic_injected_mass/(2*kinetic_jet_thickness*pi*kinetic_jet_radius**2) +kinetic_injected_momentum_density = kinetc_injected_density*kinetic_jet_velocity**2 +kinetic_injected_energy_density = mdot*efficiency*normalized_kinetic_fraction/(2*kinetic_jet_thickness*pi*kinetic_jet_radius**2 +``` +Note that this only guarentees a fixed change in momentum density and total +energy density; changes in kinetic energy density will depend on the velocity +of the ambient gas. Temperature will also change but should always increase +with kinetic jet feedback. + +#### Tracing jet material + +Material launched by the kinetic jet component can be traced by setting +``` + +enable_tracer = true # disabled by default +``` + +At the moment, this also requires to enable a single passive scalar by setting +``` + +nscalars = 1 +``` +This may change in the future as the current implementation restricts the +passive scalar use to tracing jet material. +This is a design decision motivated by simplicity and not for technical +reasons (KISS!). + +Note that all material launched from within the jet region is traced, i.e., +passive scalar concentration does not differentiate between original cell +material and mass added through the kinetic jet feedback mechanism. + + +### Magnetic feedback + +Multiple options exists to inject magnetic fields: +- a simple field loop (donut) configuration (better suited for kinetically dominated jets at larger scales) +- a more complex pinching tower model (particularly suited for magnetically dominated jets at small scales) + +The option is controlled by the following parameter +``` + +potential_type = li # alternative: "donut" +``` + +#### Pinching magnetic tower + +Magnetic feedback is injected following ([Li 2006](doi.org/10.1086/501499)) +where the injected magnetic field follows + +$$ +\begin{align} +\mathcal{B}_r &=\mathcal{B}_0 2 \frac{h r}{\ell^2} \exp{ \left ( \frac{-r^2 - h^2}{\ell^2} \right )} \\\\ +\mathcal{B}_\theta &=\mathcal{B}_0 \alpha \frac{r}{\ell} \exp{ \left ( \frac{-r^2 - h^2}{\ell^2} \right ) } \\\\ +\mathcal{B}_h &=\mathcal{B}_0 2 \left( 1 - \frac{r^2}{\ell^2} \right ) \exp{ \left ( \frac{-r^2 - h^2}{\ell^2} \right )} \\\\ +\end{align} +$$ + +which has the corresponding vector potential field + +$$ +\begin{align} +\mathcal{A}_r &= 0 \\\\ +\mathcal{A}_{\theta} &= \mathcal{B}_0 \ell \frac{r}{\ell} \exp{ \left ( \frac{-r^2 - h^2}{\ell^2} \right )} \\\\ +\mathcal{A}_h &= \mathcal{B}_0 \ell \frac{\alpha}{2}\exp{ \left ( \frac{-r^2 - h^2}{\ell^2} \right )} +\end{align} +$$ + +The parameters $\alpha$ and $\ell$ can be changed with +``` + +potential_type = li +li_alpha = 20 +l_scale = 0.001 +``` +When injected as a fraction of + +Mass is also injected along with the magnetic field following + +$$ +\dot{\rho} = \dot{\rho}_B * \exp{ \frac{ -r^2 + -h^2}{\ell^2} } +$$ + +where $\dot{\rho}_B$ is set to + +$$ +\dot{\rho}_B = \frac{3 \pi}{2} \frac{\dot{M} \left ( 1 - \epsilon \right ) f_{magnetic}}{\ell^3} +$$ + +so that the total mass injected matches the accreted mass propotioned to magnetic feedback. + +``` + +l_mass_scale = 0.001 +``` + +Mass injection by the tower is enabled by default. +It can be disabled by setting +``` + +enable_magnetic_tower_mass_injection = false +``` +In this case, the injected mass through kinetic and thermal feedback +according to their ratio. + +#### Simple field loop (donut) feedback + +Magnetic energy is injected according to the following simple potential +$$ +A_h(r, \theta, h) = B_0 L \exp^\left ( -r^2/L^2 \right)$ for $h_\mathrm{offset} \leq |h| \leq h_\mathrm{offset} + h_\mathrm{thickness} +$$ +resultig in a magnetic field configuration of +$$ +B_\theta(r, \theta, h) = 2 B_0 r /L \exp^\left ( -r^2/L^2 \right)$ for $h_\mathrm{offset} \leq |h| \leq h_\mathrm{offset} + h_\mathrm{thickness} +$$ +with all other components being zero. + + +``` + +potential_type = donut +l_scale = 0.0005 # in code length +donut_offset = 0.001 # in code length +donut_thickness = 0.0001 # in code length +``` + +It is recommended to match the donut thickness to the thickness of the kinetic jet launching region +and to choose a lengthscale that is half the jet launching radius. +This results in almost all injected fields being confined to the launching region (and thus being +carried along with a dominant jet). + +Mass feedback for the donut model is currently identical to the Li tower above +(and, thus, the parameters above pertaining to the mass injection equally apply here). + +#### Fixed magnetic feedback + +Magnetic feedback can also be inserted at runtime and injected at a fixed +increase in magnetic field, and additional mass can be injected at a fixed +rate. +This works for all magnetic vector potentials. +``` + +initial_field = 0.12431560000204142 +fixed_field_rate = 1.0 +fixed_mass_rate = 1.0 +``` + +## SNIA Feedback + +Following [Prasad 2020](doi.org/10.1093/mnras/112.2.195), AthenaPK can inject +mass and energy from type Ia supernovae following the mass profile of the BCG. +This SNIA feedback can be configured with +``` + +power_per_bcg_mass = 0.0015780504379367209 # in units code_length**2/code_time**3 +mass_rate_per_bcg_mass = 0.00315576 # in units 1/code_time +disabled = False +``` +where `power_per_bcg_mass` and `mass_rate_per_bcg_mass` is the power and mass +per time respectively injected per BCG mass at a given radius. This SNIA +feedback is otherwise fixed in time, spherically symmetric, and dependant on +the BCG specified in ``. + +## Stellar feedback + +Cold, dense, potentially magnetically supported gas may accumulate in a small +disk-like structure around the AGN depending on the specific setup. +In reality, this gas would form stars. + +In absence of star particles (or similar) in the current setup, we use a simple +formulation to convert some fraction of this gas to thermal energy. +More specifically, gas within a given radius, above a critical number density +and below a temperature threshold, will be converted to thermal energy with a +given efficiency. +The parameters can be set as follows (numerical values here just for illustration purpose -- by default they are all 0, i.e., stellar feedback is disabled). + +``` + +stellar_radius = 0.025 # in code length +efficiency = 5e-6 +number_density_threshold = 2.93799894e+74 # in code_length**-3 +temperature_threshold = 2e4 # in K +``` + +Note that all parameters need to be specified explicitly for the feedback to work +(i.e., no hidden default values). \ No newline at end of file diff --git a/docs/pgen.md b/docs/pgen.md index f2ff8f53..0b58b9a2 100644 --- a/docs/pgen.md +++ b/docs/pgen.md @@ -1,11 +1,73 @@ -# Problem specific callbacks +# Problem generators -## Source functions +Different simulation setups in AthenaPK are controlled via the so-called problem generators. +New problem generators can easily be added and we are happy to accept and merge contibuted problem +generators by any user via pull requests. + +## Addding a new problem generator + +In general, four small steps are required: + +### 1. Add a new source file to the `src/pgen` folder + +The file shoud includes at least the `void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md)` +function, which is used to initialize the data at the beginning of the simulation. +Alternatively, the `MeshBlock` version (`void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin)`) can be used that +operates on a single block at a time rather than on a collection of blocks -- though this is not recommended from a performance point of view. + +The function (and all other functions in that file) should be encapsulated in their own namespace (that, ideally, is named +identical to the file itself) so that the functions name are unique across different problem generators. + +Tip: Do not write the problem generator file from scratch but mix and match from existing ones, +e.g., start with a simple one like [orszag_tang.cpp](../src/pgen/orszag_tang.cpp). + +### 2. Add new function(s) to `pgen.hpp` + +All callback functions, i.e., at least the `ProblemGenerator` plus additional optional ones (see step 5 below), +need to be added to [pgen.hpp](../src/pgen/pgen.hpp). +Again, just follow the existing pattern in the file and add the new function declarations with the appropriate namespace. + +### 3. Add callbacks to `main.cpp` + +All problem specific callback functions need to be enrolled in the [`main.cpp`](../src/main.cpp) file. +The selection (via the input file) is controlled by the `problem_id` string in the `` block. +Again, for consistency it is recommended to pick a string that matches the namespace and problem generator file. + +### 4. Ensure new problem generator is compiled + +Add the new source file to [src/pgen/CMakeLists.txt](../src/pgen/CMakeLists.txt) so that it will be compiled +along all other problem generators. + +### 5. (Optional) Add more additional callback + +In addition to the `ProblemGenerator` that initializes the data, other callback functions exists +that allow to modify the behavior of AthenaPK on a problem specific basis. +See [Callback functions](#Callback-functions)] below for available options. + +### 6. (Optional but most likely required) Write an input file + +In theory, one can hardcode all paramters in the source file (like in the +[orszag_tang.cpp](../src/pgen/orszag_tang.cpp) problem generator) but it +prevents modification of the problem setup once the binary is compiled. + +The more common usecase is to create an input file that contains a problem specific +input block. +The convention here is to have a block named `` where `NAME` is the name +of the problem generator (or namespace used). +For example, the Sod shock tube problem generator processes the input file with lines like +``` + Real rho_l = pin->GetOrAddReal("problem/sod", "rho_l", 1.0); +``` +to set the density on the left hand side of the domain. + +## Callback functions + +### Source functions Additional (physical) source terms (e.g., the ones typically on the right hand side of system of equations) can be added in various ways from an algorithmic point of view. -### Unsplit +#### Unsplit Unsplit sources are added at each stage of the (multi-stage) integration after the (conserved) fluxes are calculated. @@ -23,7 +85,7 @@ by implementing a function with that signature and assigning it to the Note, there is no requirement to call the function `MyUnsplitSource`. -### Split first order (generally not recommended) +#### Split first order (generally not recommended) If for special circumstances sources need to be added in a fully operator split way, i.e., the source function is only called *after* the full hydro (or MHD) integration, @@ -38,12 +100,27 @@ Note, as the name suggests, this method is only first order accurate (while ther is no requirement to call the function `MyFirstOrderSource`). -### Split second order (Strang splitting) +#### Split second order (Strang splitting) -Not implemented yet. +Strang splitting achieves second order by interleaving the main hydro/MHD update +with a source update. +In practice, AthenaPK calls Strang split source with `0.5*dt` before the first stage of +each integrator and with `0.5*dt` after the last stage of the integrator. +Note that for consistency, a Strang split source terms should update both conserved +and primitive variables in all zones (i.e., also in the ghost zones as those +are currently not updated after calling a source function). + +Users can enroll a custom source function +```c++ +void MyStrangSplitSource(MeshData *md, const Real beta_dt); +``` +by implementing a function with that signature and assigning it to the +`Hydro::ProblemSourceStrangSplit` callback (currently in `main.cpp`). + +Note, there is no requirement to call the function `MyStrangSplitSource`. -## Timestep restrictions +### Timestep restrictions If additional problem specific physics are implemented (e.g., through the source functions above) that require a custom timestep, it can be added via the @@ -55,4 +132,60 @@ Real ProblemEstimateTimestep(MeshData *md); The return value is expected to be the minimum value over all blocks in the contained in the `MeshData` container, cf., the hydro/mhd `EstimateTimestep` function. Note, that the hyperbolic CFL value is currently applied after the function call, i.e., -it is also applied to the problem specific function. \ No newline at end of file +it is also applied to the problem specific function. + +### Additional initialization on startup (adding variables/fields, custom history output, ...) + +Sometimes a problem generator requires a more general processing of the input file +and/or needs to make certain global adjustments (e.g., adding a custom callback function +for the history output or adding a new field). +This can be achieved via modifying the AthenaPK "package" (currently called +`parthenon::StateDescriptor`) through the following function. +```c++ +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg) +``` + +For example, the [src/pgen/turbulence.cpp](../[src/pgen/turbulence.cpp]) problem generator +add an additional field (here for an acceleration field) by calling +```c++ + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + pkg->AddField("acc", m); +``` +in the `ProblemInitPackageData`. + +### Additional initialization on mesh creation/remeshing/load balancing + +For some problem generators it is required to initialize data on "new" blocks. +These new blocks can, for example, be created during mesh refinement +(or derefinement) and this data is typically not active data (like +conserved or primitive variables as those are handled automatically) +but more general data. +Once example is the phase information in the turbulence driver that +does not vary over time but spatially (and therefore at a per block level). + +The appropriate callback to enroll is +```c++ +void InitMeshBlockUserData(MeshBlock *pmb, ParameterInput *pin) +``` + +### UserWorkAfterLoop + +If additional work is required once the main loop finishes (i.e., once the +simulation reaches its final time) computations can be done inside the +```c++ +void UserWorkAfterLoop(Mesh *mesh, ParameterInput *pin, parthenon::SimTime &tm) +``` +callback function. +This is, for example, done in the linear wave problem generator to calculate the +error norms for convergence tests. + +### MeshBlockUserWorkBeforeOutput + +Sometimes it is desirable to further process data before an output file is written. +For this the +```c++ +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) +``` +callback is available, that is called once every time right before a data output +(hdf5 or restart) is being written. diff --git a/external/Kokkos b/external/Kokkos index af5caae7..62d2b6c8 160000 --- a/external/Kokkos +++ b/external/Kokkos @@ -1 +1 @@ -Subproject commit af5caae75ef07e93177dcc23b180ec6983942696 +Subproject commit 62d2b6c879b74b6ae7bd06eb3e5e80139c4708e6 diff --git a/external/parthenon b/external/parthenon index 7da5c6a2..5b6bb619 160000 --- a/external/parthenon +++ b/external/parthenon @@ -1 +1 @@ -Subproject commit 7da5c6a29792b47cff2411390d91ef2c8f386304 +Subproject commit 5b6bb61906f7c278f9724ee9f38e79dee8707098 diff --git a/inputs/blast_3d_amr.in b/inputs/blast_3d_amr.in index 70b6a695..93a3c2a3 100644 --- a/inputs/blast_3d_amr.in +++ b/inputs/blast_3d_amr.in @@ -55,7 +55,7 @@ scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM type = pressure_gradient threshold_pressure_gradient = 0.1 - + pressure_ambient = 0.001 # ambient pressure pressure_ratio = 1.6e8 # Pressure ratio initially radius_outer = 0.03125 # Radius of the outer sphere diff --git a/inputs/cluster/agn_triggering.in b/inputs/cluster/agn_triggering.in new file mode 100644 index 00000000..1ba04ec3 --- /dev/null +++ b/inputs/cluster/agn_triggering.in @@ -0,0 +1,117 @@ +################################################################################ +# Input file for testing AGN triggering without fluid +# evolution +################################################################################ + +problem = AGN Triggering Test + + +problem_id = cluster # problem ID: basename of output filenames + + +file_type = hst # History data dump +dt = 1e-4 # time increment between outputs + + +file_type = hdf5 # HDF5 data dump +variables = cons,prim # Variables to be output +dt = 0.1 # Time increment between outputs +id = vars # Name to append to output + + +cfl_number = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number +nlim = -1 # cycle limit +tlim = 0.1 # time limit +integrator = vl2 # time integration algorithm + + + +refinement = static +nghost = 2 + +nx1 = 64 # Number of zones in X1-direction +x1min =-0.1 # minimum value of X1 +x1max = 0.1 # maximum value of X1 +ix1_bc = outflow # inner-X1 boundary flag +ox1_bc = outflow # outer-X1 boundary flag + +nx2 = 64 # Number of zones in X2-direction +x2min =-0.1 # minimum value of X2 +x2max = 0.1 # maximum value of X2 +ix2_bc = outflow # inner-X2 boundary flag +ox2_bc = outflow # outer-X2 boundary flag + +nx3 = 64 # Number of zones in X3-direction +x3min =-0.1 # minimum value of X3 +x3max = 0.1 # maximum value of X3 +ix3_bc = outflow # inner-X3 boundary flag +ox3_bc = outflow # outer-X3 boundary flag + + +x1min = -0.025 +x1max = 0.025 +x2min = -0.025 +x2max = 0.025 +x3min = -0.025 +x3max = 0.025 +level = 2 + + + +nx1 = 8 # Number of zones in X1-direction +nx2 = 8 # Number of zones in X2-direction +nx3 = 8 # Number of zones in X3-direction + + +fluid = euler +gamma = 1.6666666666666667 # gamma = C_p/C_v +eos = adiabatic +riemann = none +reconstruction = dc +calc_dt_hyp = true +scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM + +He_mass_fraction = 0.25 + + +#Units parameters +#Note: All other parameters for the cluster are in terms of these units +code_length_cgs = 3.085677580962325e+24 # 1 Mpc in cm +code_mass_cgs = 1.98841586e+47 # 1e14 Msun in g +code_time_cgs = 3.15576e+16 # 1 Gyr in s + + + + +#Define SMBH for Bondi accretion +m_smbh = 1.0e-06 + +#Disable gravity as a source term +gravity_srcterm = false + + +#Initialize with a uniform gas +init_uniform_gas = true +rho = 14775.575892787232 +ux = 0.0006136272991326239 +uy = 0.0004090848660884159 +uz =-0.0005113560826105199 +pres = 1.5454368403867562 + + +triggering_mode = COLD_GAS +accretion_radius = 0.02 +cold_temp_thresh = 7198.523584993224 +cold_t_acc = 0.1 +bondi_alpha = 100 +bondi_beta = 2 +bondi_n0 = 1.4928506511614283e+74 +write_to_file=false +triggering_filename= agn_triggering.dat + + +#Don't do any feedback with the triggering +disabled = true + + +disabled = True diff --git a/inputs/cluster/cluster.in b/inputs/cluster/cluster.in new file mode 100644 index 00000000..e3192524 --- /dev/null +++ b/inputs/cluster/cluster.in @@ -0,0 +1,190 @@ + + +problem = Isolated galaxy cluster + + +problem_id = cluster # problem ID: basename of output filenames + + +file_type = hst # History data dump +dt = 1e-3 # time increment between outputs (1 Myr) + + +file_type = hdf5 # HDF5 data dump +variables = prim # Variables to be output +dt = 1.e-2 # Time increment between outputs (10 Myr) +id = prim # Name to append to output + + +cfl = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number +nlim = -1 # cycle limit +tlim = 1e-1 # time limit (100 Myr) +integrator = vl2 # time integration algorithm + + + +refinement = static +nghost = 2 + +nx1 = 128 # Number of zones in X1-direction +x1min =-1.6 # minimum value of X1 +x1max = 1.6 # maximum value of X1 +ix1_bc = outflow # inner-X1 boundary flag +ox1_bc = outflow # outer-X1 boundary flag + +nx2 = 128 # Number of zones in X2-direction +x2min =-1.6 # minimum value of X2 +x2max = 1.6 # maximum value of X2 +ix2_bc = outflow # inner-X2 boundary flag +ox2_bc = outflow # outer-X2 boundary flag + +nx3 = 128 # Number of zones in X3-direction +x3min =-1.6 # minimum value of X3 +x3max = 1.6 # maximum value of X3 +ix3_bc = outflow # inner-X3 boundary flag +ox3_bc = outflow # outer-X3 boundary flag + + +x1min = -0.4 +x1max = 0.4 +x2min = -0.4 +x2max = 0.4 +x3min = -0.4 +x3max = 0.4 +level = 1 + + +x1min = -0.025 +x1max = 0.025 +x2min = -0.025 +x2max = 0.025 +x3min = -0.025 +x3max = 0.025 +level = 3 + + +nx1 = 32 # Number of zones in X1-direction +nx2 = 32 # Number of zones in X2-direction +nx3 = 32 # Number of zones in X3-direction + + +fluid = glmmhd +gamma = 1.6666666666666667 # gamma = C_p/C_v +eos = adiabatic +riemann = hlld +reconstruction = plm +scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM + +He_mass_fraction = 0.25 + + +#Units parameters +#Note: All other parameters for the cluster are in terms of these units +code_length_cgs = 3.085677580962325e+24 # 1 Mpc in cm +code_mass_cgs = 1.98841586e+47 # 1e14 Msun in g +code_time_cgs = 3.15576e+16 # 1 Gyr in s + + +enable_cooling=tabular +table_filename=schure.cooling +log_temp_col=0 +log_lambda_col=1 +lambda_units_cgs=1.0 + +integrator=rk45 +cfl=0.1 +max_iter=100 +d_e_tol=1e-08 +d_log_temp_tol=1e-08 + + +hubble_parameter = 0.0715898515654728 + + +#Include gravity as a source term +gravity_srcterm = true +#NOTE: Use this line instead to disable gravity source term +#gravity_srcterm = false + +#Which gravitational fields to include +include_nfw_g = True +which_bcg_g = HERNQUIST +include_smbh_g = True + +#NFW parameters +c_nfw = 6.0 +m_nfw_200 = 10.0 + +#BCG parameters +m_bcg_s = 0.001 +r_bcg_s = 0.004 + +#SMBH parameters +m_smbh = 1.0e-06 + +#Smooth gravity at origin, for numerical reasons +g_smoothing_radius = 0.0 + +#NOTE: Uncomment these lines to use a uniform initial gas instead of hydrostatic equilbrium +# +##Initialize with a uniform gas +#init_uniform_gas = true +#rho = 147.7557589278723 +#ux = 0 +#uy = 0 +#uz = 0 +#pres = 1.5454368403867562 + + +#Entropy profile parameters +k_0 = 8.851337676479303e-121 +k_100 = 1.3277006514718954e-119 +r_k = 0.1 +alpha_k = 1.1 + + +#Fix density at radius to close system of equations +r_fix = 2.0 +rho_fix = 0.01477557589278723 + +#Building the radii at which to sample initial rho,P +r_sampling = 4.0 + + +triggering_mode = COLD_GAS +#NOTE: Change to this line to disable AGN triggering +#triggering_mode = NONE +accretion_radius = 0.0005 +cold_temp_thresh= 10000.0 +cold_t_acc= 0.1 +bondi_alpha= 100.0 +bondi_beta= 2.0 +bondi_n0= 2.9379989445851786e+72 + + +jet_phi= 0.15 +jet_theta_dot= 628.3185307179587 + + +efficiency = 0.001 +magnetic_fraction = 0.4 +thermal_fraction = 0.3 +kinetic_fraction = 0.3 +#NOTE: Change to these lines to disable magnetic AGN feedback +#magnetic_fraction = 0.0 +#thermal_fraction = 0.5 +#kinetic_fraction = 0.5 + +thermal_radius = 0.1 +kinetic_jet_radius = 0.1 +kinetic_jet_thickness = 0.05 +kinetic_jet_offset = 0.05 + + +potential_type = li +li_alpha = 20 +l_scale = 0.001 +initial_field = 0.12431560000204142 +#NOTE: Change to this line to disable initial magnetic tower +#initial_field = 0. +l_mass_scale = 0.001 diff --git a/inputs/cluster/cooling.in b/inputs/cluster/cooling.in index c56a2e8b..63f8dd20 100644 --- a/inputs/cluster/cooling.in +++ b/inputs/cluster/cooling.in @@ -1,6 +1,8 @@ - +################################################################################ +# Input file for testing tabular cooling +################################################################################ -problem = Isolated galaxy cluster +problem = Cooling Test problem_id = cluster # problem ID: basename of output filenames @@ -20,7 +22,6 @@ cfl = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number nlim = -1 # cycle limit tlim = 1.0 # time limit integrator = vl2 # time integration algorithm -perf_cycle_offset = 10 # interval for stdout summary info @@ -56,36 +57,23 @@ gamma = 1.6666666666666667 # gamma = C_p/C_v eos = adiabatic riemann = hlle reconstruction = plm -use_scratch = false scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM He_mass_fraction = 0.25 #Units parameters -code_length_cgs = 3.085677580962325e+24 -code_mass_cgs = 1.98841586e+47 -code_time_cgs = 3.15576e+16 - - - -#Disable gravity as a source term -gravity_srcterm = false - -#Initialize with a uniform gas -init_uniform_gas = true -uniform_gas_rho = 147.7557589278723 -uniform_gas_ux = 0 -uniform_gas_uy = 0 -uniform_gas_uz = 0 -uniform_gas_pres = 1.5454368403867562 +#Note: All other parameters for the cluster are in terms of these units +code_length_cgs = 3.085677580962325e+24 # 1 Mpc in cm +code_mass_cgs = 1.98841586e+47 # 1e14 Msun in g +code_time_cgs = 3.15576e+16 # 1 Gyr in s enable_cooling = tabular table_filename = schure.cooling log_temp_col = 0 log_lambda_col = 1 -lambda_units_cgs = 1 +lambda_units_cgs = 1 #erg cm^3/s in cgs, as used in schure.cooling integrator = rk12 max_iter = 100 @@ -93,3 +81,20 @@ cfl = 0.1 d_log_temp_tol = 1e-8 d_e_tol = 1e-8 + + + +#Disable gravity as a source term +gravity_srcterm = false + + +#Initialize with a uniform gas +init_uniform_gas = true +rho = 147.7557589278723 +ux = 0 +uy = 0 +uz = 0 +pres = 1.5454368403867562 + + +disabled = True diff --git a/inputs/cluster/generate_cluster_input.ipynb b/inputs/cluster/generate_cluster_input.ipynb new file mode 100644 index 00000000..449283f1 --- /dev/null +++ b/inputs/cluster/generate_cluster_input.ipynb @@ -0,0 +1,576 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "3f50554f-678e-4a0c-a1a6-cc2a634393b6", + "metadata": {}, + "source": [ + "# Generate AthenaPK inputs for Cluster-like Objects\n", + "\n", + "Notebook to help with generating AthenaPK input files for running cluster-like simulations with the `cluster` problem generator, including AGN feedback and triggering. Check `docs/cluster.md` for more details on the components and parameters of the `cluster` problem generator. Every section marked `CHANGEME` is intended to be modified to change the initial setup.\n", + "\n", + "The `cluster` problem generator uses code units for parameter definitions. This notebook manages the conversion from astronomical units to code units.\n", + "\n", + "Required Python libraries:\n", + "\n", + "- [`unyt`](https://unyt.readthedocs.io/en/stable/), tested with `unyt v2.9.2`\n", + "- [`numpy`](https://numpy.org/), tested with `numpy 1.23.1`\n", + "\n", + "Tested with Python 3.9" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40314e1e", + "metadata": {}, + "outputs": [], + "source": [ + "import unyt\n", + "import numpy as np\n", + "import copy\n", + "import itertools\n", + "import os" + ] + }, + { + "cell_type": "markdown", + "id": "9b1c6149-a856-4dd9-ad87-22f3ed10ac44", + "metadata": { + "tags": [] + }, + "source": [ + "## CHANGEME: `filename` to write input file to\n", + "\n", + "Make sure the path containing the filename exists" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "963663a7-2148-4163-b97f-57af8bb33aaa", + "metadata": {}, + "outputs": [], + "source": [ + "filename = \"my_cluster.input\"" + ] + }, + { + "cell_type": "markdown", + "id": "0a2dd78b", + "metadata": {}, + "source": [ + "## CHANGEME: Define the code units to use throughout the file\n", + "\n", + "Note that you need to reload the notebook if you change these" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5f30640c", + "metadata": {}, + "outputs": [], + "source": [ + "# Use MPC, 1e14 Msun, and 1 Gyr for code units\n", + "unyt.define_unit(\"code_length\",(1,unyt.Mpc))\n", + "unyt.define_unit(\"code_mass\",(1e14,unyt.Msun))\n", + "unyt.define_unit(\"code_time\",(1,unyt.Gyr))" + ] + }, + { + "cell_type": "markdown", + "id": "93908a05", + "metadata": {}, + "source": [ + "## CHANGEME: Define AthenaPK parameters for the different general and cluster modules\n", + "\n", + "Read `docs/cluster.md` for more detailed descriptions " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "779e5a45", + "metadata": {}, + "outputs": [], + "source": [ + "params_text = f\"\"\"\n", + "\n", + "fluid = glmmhd\n", + "gamma = 5./3. # gamma = C_p/C_v\n", + "eos = adiabatic\n", + "riemann = hlld\n", + "reconstruction = plm\n", + "scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM\n", + "Tfloor = {unyt.unyt_quantity(1e4,\"K\").v}\n", + "\n", + "first_order_flux_correct = True\n", + "\n", + "He_mass_fraction = 0.25\n", + "\n", + "\n", + "#Units parameters\n", + "code_length_cgs = {unyt.unyt_quantity(1,\"code_length\").in_units(\"cm\").v}\n", + "code_mass_cgs = {unyt.unyt_quantity(1,\"code_mass\").in_units(\"g\").v}\n", + "code_time_cgs = {unyt.unyt_quantity(1,\"code_time\").in_units(\"s\").v}\n", + "\n", + "\n", + "enable_cooling = tabular\n", + "table_filename = schure.cooling\n", + "log_temp_col = 0 # Column to read temperature in cooling table\n", + "log_lambda_col = 1 # Column to read lambda in cooling table\n", + "lambda_units_cgs = {unyt.unyt_quantity(1,\"erg*cm**3/s\").v}\n", + "\n", + "integrator = townsend\n", + "cfl = 0.1 # Restricts hydro step based on fraction of minimum cooling time\n", + "min_timestep = {unyt.unyt_quantity(1,\"Gyr\").in_units(\"code_time\").v}\n", + "d_e_tol = 1e-8\n", + "d_log_temp_tol = 1e-8\n", + "\n", + "\n", + "hubble_parameter = {unyt.unyt_quantity(70,\"km*s**-1*Mpc**-1\").in_units(\"1/code_time\").v}\n", + "\n", + "\n", + "#Include gravity as a source term\n", + "gravity_srcterm = True\n", + "\n", + "#Which gravitational fields to include\n", + "include_nfw_g = True\n", + "which_bcg_g = HERNQUIST\n", + "include_smbh_g = True\n", + "\n", + "#NFW parameters\n", + "c_nfw = 6.0\n", + "m_nfw_200 = {unyt.unyt_quantity(1e15,\"Msun\").in_units(\"code_mass\").v}\n", + "\n", + "#BCG parameters\n", + "m_bcg_s = {unyt.unyt_quantity(1e11,\"Msun\").in_units(\"code_mass\").v}\n", + "r_bcg_s = {unyt.unyt_quantity(4,\"kpc\").in_units(\"code_length\").v}\n", + "\n", + "#SMBH parameters\n", + "m_smbh = {unyt.unyt_quantity(1e8,\"Msun\").in_units(\"code_mass\").v}\n", + "\n", + "#Smooth gravity at origin, for numerical reasons\n", + "g_smoothing_radius = {unyt.unyt_quantity(0,\"code_length\").v}\n", + "\n", + "\n", + "#Entropy profile parameters\n", + "k_0 = {unyt.unyt_quantity(10,\"keV*cm**2\").in_units(\"code_length**4*code_mass/code_time**2\").v}\n", + "k_100 = {unyt.unyt_quantity(150,\"keV*cm**2\").in_units(\"code_length**4*code_mass/code_time**2\").v}\n", + "r_k = {unyt.unyt_quantity(100,\"kpc\").in_units(\"code_length\").v}\n", + "alpha_k = 1.1\n", + "\n", + "\n", + "#Fix density at radius to close system of equations\n", + "r_fix = {unyt.unyt_quantity(2e3,\"kpc\").in_units(\"code_length\").v}\n", + "rho_fix = {unyt.unyt_quantity(1e-28,\"g*cm**-3\").in_units(\"code_mass/code_length**3\").v}\n", + "\n", + "#Building the radii at which to sample initial rho,P\n", + "r_sampling = 4.0\n", + "\n", + "\n", + "#Which triggering mode (BOOSTED_BONDI, BOOTH_SCHAYE, COLD_GAS, NONE)\n", + "triggering_mode = COLD_GAS\n", + "\n", + "#Radius of accretion for triggering\n", + "accretion_radius = {unyt.unyt_quantity(1,\"kpc\").in_units(\"code_length\").v}\n", + "\n", + "#BOOSTED_BONDI and BOOTH_SCHAYE Parameters\n", + "bondi_alpha = 100.0\n", + "bondi_beta = 2.0\n", + "bondi_n0 = {unyt.unyt_quantity(0.1,\"cm**-3\").in_units(\"code_length**-3\").v}\n", + "\n", + "#COLD_GAS Parameters\n", + "cold_temp_thresh = {unyt.unyt_quantity(1e5,\"K\").in_units(\"K\").v}\n", + "cold_t_acc = {unyt.unyt_quantity(100,\"Myr\").in_units(\"code_time\").v}\n", + "\n", + "write_to_file = True\n", + "\n", + "\n", + "jet_theta = 0.15\n", + "jet_phi_dot = {(2*np.pi/unyt.unyt_quantity(10,\"Myr\")).in_units(\"code_time**-1\").v}\n", + "jet_phi0 = 0.2\n", + "\n", + "\n", + "# Fixed power, added on top of triggered feedback\n", + "fixed_power = {unyt.unyt_quantity(0,\"erg/s\").in_units(\"code_length**2*code_mass/code_time**3\").v}\n", + "\n", + "# Efficieny in conversion of AGN accreted mass to AGN feedback energy\n", + "efficiency = 1e-3\n", + "\n", + "# Fraction allocated to different mechanisms\n", + "magnetic_fraction = 0.333\n", + "thermal_fraction = 0.333\n", + "kinetic_fraction = 0.333\n", + "\n", + "# Thermal feedback parameters\n", + "thermal_radius = {unyt.unyt_quantity(0.5,\"kpc\").in_units(\"code_length\").v}\n", + "\n", + "# Kinetic jet feedback parameters\n", + "kinetic_jet_radius = {unyt.unyt_quantity(0.5,\"kpc\").in_units(\"code_length\").v}\n", + "kinetic_jet_thickness = {unyt.unyt_quantity(0.5,\"kpc\").in_units(\"code_length\").v}\n", + "kinetic_jet_offset = {unyt.unyt_quantity(0.5,\"kpc\").in_units(\"code_length\").v}\n", + "kinetic_jet_temperature = {unyt.unyt_quantity(1e6,\"K\").in_units(\"K\").v}\n", + "\n", + "\n", + "alpha = 20\n", + "l_scale = {unyt.unyt_quantity(1,\"kpc\").in_units(\"code_length\").v}\n", + "initial_field = {unyt.unyt_quantity(1e-6,\"G\").in_units(\"code_mass**(1/2)*code_length**(-1/2)*code_time**-1\").v}\n", + "l_mass_scale = {unyt.unyt_quantity(1,\"kpc\").in_units(\"code_length\").v}\n", + "\n", + "\n", + "\n", + "power_per_bcg_mass = {unyt.unyt_quantity(1e51*3e-14,\"erg/yr/Msun\").in_units(\"code_length**2/code_time**3\").v}\n", + "mass_rate_per_bcg_mass = {unyt.unyt_quantity(1e-19,\"1/s\").in_units(\"1/code_time\").v}\n", + "disabled = False\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "636d1d12-9b0c-4a21-bded-5dc5e9feb40d", + "metadata": {}, + "source": [ + "## CHANGEME: Define the data output for the simulation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43effc3a-425b-4568-bb5c-534034afd2e5", + "metadata": {}, + "outputs": [], + "source": [ + "output_text = f\"\"\"\n", + "\n", + "file_type = hst # History data dump\n", + "dt = {unyt.unyt_quantity(0.1,\"Myr\").in_units(\"code_time\").v} # time increment between outputs\n", + "\n", + "\n", + "file_type = rst # restart data dump\n", + "dt = {unyt.unyt_quantity(1.0,\"Myr\").in_units(\"code_time\").v} # Time increment between outputs\n", + "id = restart\n", + "\n", + "# hdf5_compression_level = 0\n", + "use_final_label = false\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "2f219a5f-8f79-4035-a023-03a4a09a2b9e", + "metadata": {}, + "source": [ + "## CHANGEME: Define the time constraints for the simulation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11db692a-a428-42f6-9e90-bec04cf7584d", + "metadata": {}, + "outputs": [], + "source": [ + "time_text=f\"\"\"\n", + "\n", + "cfl = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number\n", + "tlim = {unyt.unyt_quantity(10,\"Myr\").in_units(\"code_time\").v} # time limit\n", + "integrator = vl2 # time integration algorithm\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c988b2d0-6ef6-43a9-9d94-e1faa5d1ab3c", + "metadata": {}, + "outputs": [], + "source": [ + "## CHANGEME: Define static mesh refinement levels. Used below by `smr_generator` to make the mesh input" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3bbd06c1-3390-40f0-83d3-d16fc6d1441b", + "metadata": {}, + "outputs": [], + "source": [ + "# Number of cells on each side in base mesh\n", + "base_nx = 64\n", + "# List of levels of refinement for SMR regions\n", + "base_width = unyt.unyt_quantity(200,\"kpc\")\n", + "\n", + "#List of levels of refinement for SMR regions\n", + "smr_levels = [2,]\n", + "#List of widths (in code length units) of SMR regions\n", + "smr_widths = unyt.unyt_array([25,],\"kpc\")\n", + "\n", + "# Number of cells on each side of meshblocks\n", + "mb_nx=32\n" + ] + }, + { + "cell_type": "markdown", + "id": "798f7b28", + "metadata": {}, + "source": [ + "## Define different mesh sizes/hierarchies\n", + "\n", + "Define an SMR mesh for the simulation. We provide an automatically generated SMR mesh with `smr_generator`, or you can craft your SMR mesh by hand." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0c61c53-3959-4208-9f80-07d2c6255ebf", + "metadata": {}, + "outputs": [], + "source": [ + "def smr_generator(base_nx, base_width,\n", + " smr_levels,smr_widths,\n", + " mb_nx=32,quiet=False,\n", + " mem_per_device=40e9):\n", + " \"\"\"\n", + " Helper function to quickly define static-mesh refinement meshes for AthenaPK.\n", + " By default, prints out information like smallest cell size, total number of\n", + " cells, estimated data outputs, and estimated NVIDIA A100s needed to run the\n", + " simulation.\n", + " \n", + " Parameters:\n", + " base_nx : Number of cells on each side in base mesh\n", + " base_width : Width of base mesh (in code length units)\n", + "\n", + " smr_levels : List of levels of refinement for SMR regions\n", + " smr_widths : List of widths (in code length units) of SMR regions\n", + "\n", + " mb_nx=32 : Number of cells on each side of meshblocks\n", + " quiet=False : Silence printing of SMR information\n", + " \n", + " Returns: mesh_text, info\n", + " mesh_text: \n", + " \"\"\"\n", + " base_width = base_width.in_units(\"code_length\").v\n", + " smr_widths = smr_widths.in_units(\"code_length\").v\n", + " \n", + " base_dx = base_width/base_nx\n", + " \n", + " specified_widths = {0:base_width}\n", + " for level,width in zip(smr_levels,smr_widths):\n", + " specified_widths[level] = width\n", + " \n", + " #Setup each of the SMR levels to determine the true necessary widths\n", + " levels = np.arange(np.max(smr_levels,0)+1,dtype=int)\n", + " \n", + " meshes = {level:{\"dx\":(base_dx/(2.**level))} for level in levels}\n", + " \n", + " #Assume even number of mesh blocks, using this function\n", + " def ceil_even(x):\n", + " return int(np.ceil(x/2.)*2)\n", + " \n", + " #Create levels for static refinement, starting from highest level\n", + " level = levels[-1]\n", + " #Full number of meshblocks to cover the level along a side\n", + " meshes[level][\"full_nx_mb\"] = ceil_even( specified_widths[level]/(meshes[level][\"dx\"]*mb_nx))\n", + " #Full number of cells to cover level\n", + " meshes[level][\"full_nx\"] = meshes[level][\"full_nx_mb\"]*mb_nx\n", + " #Actual number of meshblocks in this level\n", + " meshes[level][\"n_mb\"] = meshes[level][\"full_nx_mb\"]**3\n", + " \n", + " meshes[level][\"width\"] = meshes[level][\"full_nx\"]*meshes[level][\"dx\"]\n", + " \n", + " #Compute widths of lower levels, extrapolating from highest level\n", + " for level,finer_level in reversed(list(zip(levels[:-1],levels[1:]))):\n", + " dx = meshes[level][\"dx\"]\n", + " \n", + " #This level's width is the max of the specified level width, expanded to fit with \n", + " #mesh block sizes, or the higher SMR level with 2 buffering mesh blocks on this level\n", + " if level in specified_widths.keys():\n", + " mb_specified_width = ceil_even( specified_widths[level]/(dx*mb_nx))*mb_nx*dx\n", + " else:\n", + " mb_specified_width = 0\n", + " meshes[level][\"width\"] = np.max([\n", + " mb_specified_width,\n", + " meshes[finer_level][\"width\"] + 2*mb_nx*dx])\n", + " \n", + " #Calculate number of cells to cover full length of level\n", + " meshes[level][\"full_nx\"] = int(meshes[level][\"width\"]/dx)\n", + " #Calculate number of meshblocks along a side to cover full level\n", + " meshes[level][\"full_nx_mb\"] = int(meshes[level][\"full_nx\"]/mb_nx)\n", + " #Calculate total number of meshblocks in this level, subtracting \n", + " #the blocks already covered in a higher level\n", + " meshes[level][\"n_mb\"] = int( meshes[level][\"full_nx_mb\"]**3 \n", + " - (meshes[finer_level][\"width\"]/(dx*mb_nx))**3)\n", + " \n", + " \n", + " #Flesh out details of all levels\n", + " for level in levels:\n", + " \n", + " meshes[level][\"xmax\"] = meshes[level][\"width\"]/2. ##Needed for creating the input file\n", + " \n", + " if level in specified_widths.keys():\n", + " meshes[level][\"specified_width_used\"] = ( meshes[level][\"width\"] == specified_widths[level])\n", + " else:\n", + " meshes[level][\"specified_width_used\"] = True\n", + " \n", + " meshes[level][\"total_cells\"] = meshes[level][\"n_mb\"]*mb_nx**3\n", + " \n", + " info = {}\n", + " info[\"all_sane\"] = np.all( [mesh[\"specified_width_used\"] for mesh in meshes.values()] )\n", + " info[\"total_cells\"] = np.sum([mesh[\"total_cells\"] for mesh in meshes.values()])\n", + " info[\"total_n_mb\"] = np.sum([mesh[\"n_mb\"] for mesh in meshes.values()])\n", + "\n", + " bytes_per_real = 8\n", + "\n", + " \n", + " reals_output_per_cell = 9\n", + " reals_used_per_cell = reals_output_per_cell*13\n", + "\n", + " info[\"total_used_memory\"] = info[\"total_cells\"]*bytes_per_real*reals_used_per_cell\n", + " info[\"total_output_memory\"] = info[\"total_cells\"]*bytes_per_real*reals_output_per_cell\n", + " \n", + " if not quiet:\n", + " \n", + " finest_dx = unyt.unyt_quantity(meshes[levels[-1]][\"dx\"],\"code_length\")\n", + " print(f\"Finest level covered by { finest_dx } , { finest_dx.in_units('pc') } cells\" )\n", + " \n", + " print(\"Do level widths match specified widths: \", info[\"all_sane\"])\n", + " print(\"\\t Widths: \",[ mesh[\"width\"] for mesh in meshes.values()])\n", + " print(\"\\t NX: \",[ mesh[\"full_nx\"] for mesh in meshes.values()])\n", + " print(\"\\t NX Meshblocks: \",[ mesh[\"full_nx_mb\"] for mesh in meshes.values()])\n", + " print(\"\\t N Meshblocks: \",[ mesh[\"n_mb\"] for mesh in meshes.values()])\n", + " \n", + " print(f\"Total cells: {info['total_cells']} or aprox. {np.cbrt(info['total_cells']):.1f}**3\")\n", + " print(f\"Total meshblocks: {info['total_n_mb']}\" )\n", + " print(f\"Total memory needed: {info['total_used_memory']/1e9} GB\")\n", + " print(f\"Total memory per output: {info['total_output_memory']/1e9} GB\")\n", + " print(f\"Devices needed with {mem_per_device/1e9:.2e} GB per deivce: {info['total_used_memory']/mem_per_device:.2e} \")\n", + " \n", + " print()\n", + "\n", + " #Base mesh text\n", + " base_xmax = base_width/2.\n", + " base_mesh_text = f\"\"\"\n", + "\n", + "refinement = static\n", + "nghost = 2\n", + "\n", + "nx1 = {base_nx} # Number of zones in X1-direction\n", + "x1min =-{base_xmax} # minimum value of X1\n", + "x1max = {base_xmax} # maximum value of X1\n", + "ix1_bc = outflow # inner-X1 boundary flag\n", + "ox1_bc = outflow # outer-X1 boundary flag\n", + "\n", + "nx2 = {base_nx} # Number of zones in X2-direction\n", + "x2min =-{base_xmax} # minimum value of X2\n", + "x2max = {base_xmax} # maximum value of X2\n", + "ix2_bc = outflow # inner-X2 boundary flag\n", + "ox2_bc = outflow # outer-X2 boundary flag\n", + "\n", + "nx3 = {base_nx} # Number of zones in X3-direction\n", + "x3min =-{base_xmax} # minimum value of X3\n", + "x3max = {base_xmax} # maximum value of X3\n", + "ix3_bc = outflow # inner-X3 boundary flag\n", + "ox3_bc = outflow # outer-X3 boundary flag\n", + "\n", + "\n", + "nx1 = {mb_nx} # Number of zones in X1-direction\n", + "nx2 = {mb_nx} # Number of zones in X2-direction\n", + "nx3 = {mb_nx} # Number of zones in X3-direction\n", + "\n", + "\"\"\"\n", + " \n", + " #\n", + " smr_texts = []\n", + " for level in smr_levels:\n", + " smr_texts.append(\n", + "f\"\"\"\n", + "\n", + "x1min = -{meshes[level][\"xmax\"]} \n", + "x1max = {meshes[level][\"xmax\"]}\n", + "x2min = -{meshes[level][\"xmax\"]}\n", + "x2max = {meshes[level][\"xmax\"]}\n", + "x3min = -{meshes[level][\"xmax\"]}\n", + "x3max = {meshes[level][\"xmax\"]}\n", + "level = {level}\n", + "\n", + "\"\"\")\n", + " return base_mesh_text + \"\".join(smr_texts),info" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "54cafbc6-e20d-49f1-8a57-10e5bbec52ab", + "metadata": {}, + "outputs": [], + "source": [ + "mesh_text,mesh_info = smr_generator( base_nx, base_width,\n", + " smr_levels, smr_widths,\n", + " mb_nx, quiet=False,\n", + " mem_per_device=40e9) #Report devices needed using memory of NVidia A100\n", + "# print(mesh_text)" + ] + }, + { + "cell_type": "markdown", + "id": "e8c95676-83e2-4347-8750-cf06aa1e293d", + "metadata": {}, + "source": [ + "## Write input file to `filename`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc6b5970", + "metadata": {}, + "outputs": [], + "source": [ + "input_text = f\"\"\" \n", + "# File autogenerated with Python script\n", + "# Changes might be overwritten!\n", + "\n", + "problem = Isolated galaxy cluster\n", + "\n", + "\n", + "problem_id = cluster # problem ID: basename of output filenames\n", + "\n", + "{output_text}\n", + "\n", + "{time_text}\n", + "\n", + "{mesh_text}\n", + "\n", + "{params_text}\n", + "\n", + "\"\"\"\n", + "\n", + "with open(filename,\"w\") as f:\n", + " f.write(input_text)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:.conda-py39]", + "language": "python", + "name": "conda-env-.conda-py39-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/inputs/cluster/hse.in b/inputs/cluster/hse.in index 5d99d003..7bb5080d 100644 --- a/inputs/cluster/hse.in +++ b/inputs/cluster/hse.in @@ -1,6 +1,9 @@ - +################################################################################ +# Input file for testing hydrostatic equilbrium setup for galaxy cluster-like +# objects +################################################################################ -problem = Isolated galaxy cluster +problem = Galaxy Cluster Hydrostatic Equilibrium Test problem_id = cluster # problem ID: basename of output filenames @@ -20,7 +23,6 @@ cfl = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number nlim = -1 # cycle limit tlim = 1e-3 # time limit integrator = vl2 # time integration algorithm -perf_cycle_offset = 10 # interval for stdout summary info @@ -66,20 +68,22 @@ gamma = 1.6666666666666667 # gamma = C_p/C_v eos = adiabatic riemann = hlle reconstruction = plm -use_scratch = false scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM He_mass_fraction = 0.25 #Units parameters -code_length_cgs = 3.085677580962325e+24 -code_mass_cgs = 1.98841586e+47 -code_time_cgs = 3.15576e+16 +#Note: All other parameters for the cluster are in terms of these units +code_length_cgs = 3.085677580962325e+24 # 1 Mpc in cm +code_mass_cgs = 1.98841586e+47 # 1e14 Msun in g +code_time_cgs = 3.15576e+16 # 1 Gyr in s hubble_parameter = 0.0715898515654728 + + #Which gravitational fields to include include_nfw_g = True which_bcg_g = HERNQUIST @@ -87,14 +91,14 @@ include_smbh_g = True #NFW parameters c_nfw = 6.0 -M_nfw_200 = 10.000000000000002 +m_nfw_200 = 10.0 #BCG parameters -M_bcg_s = 0.0010000000000000002 -R_bcg_s = 0.004 +m_bcg_s = 0.001 +r_bcg_s = 0.004 #SMBH parameters -M_smbh = 1.0000000000000002e-06 +m_smbh = 1.0e-06 #Smooth gravity at origin, for numerical reasons g_smoothing_radius = 1e-6 @@ -102,18 +106,24 @@ g_smoothing_radius = 1e-6 #Include gravity as a source term gravity_srcterm = true + + #Entropy profile parameters -K_0 = 8.851337676479303e-121 -K_100 = 1.3277006514718954e-119 -R_K = 0.1 -alpha_K = 1.1 +k_0 = 8.851337676479303e-121 +k_100 = 1.3277006514718954e-119 +r_k = 0.1 +alpha_k = 1.1 + + #Fix density at radius to close system of equations -R_fix = 2.0 +r_fix = 2.0 rho_fix = 0.01477557589278723 #Building the radii at which to sample initial rho,P -R_sampling = 4.0 -max_dR = 0.001 +r_sampling = 4.0 test_he_sphere = true + + +disabled = True diff --git a/inputs/cluster/hydro_agn_feedback.in b/inputs/cluster/hydro_agn_feedback.in new file mode 100644 index 00000000..a909819a --- /dev/null +++ b/inputs/cluster/hydro_agn_feedback.in @@ -0,0 +1,120 @@ +################################################################################ +# Input file for testing kinetic and thermal AGN feedback without fluid +# evolution +################################################################################ + +problem = Hydro AGN Feedback Test + + +problem_id = cluster # problem ID: basename of output filenames + + +file_type = hst # History data dump +dt = 1e-4 # time increment between outputs + + +file_type = hdf5 # HDF5 data dump +variables = cons,prim # Variables to be output +dt = 0.01 # Time increment between outputs +id = vars # Name to append to output + + +cfl = 1e1 # The Courant, Friedrichs, & Lewy (CFL) Number +nlim = -1 # cycle limit +tlim = 0.1 # time limit +integrator = vl2 # time integration algorithm + + + +refinement = static +nghost = 2 + +nx1 = 64 # Number of zones in X1-direction +x1min =-0.1 # minimum value of X1 +x1max = 0.1 # maximum value of X1 +ix1_bc = outflow # inner-X1 boundary flag +ox1_bc = outflow # outer-X1 boundary flag + +nx2 = 64 # Number of zones in X2-direction +x2min =-0.1 # minimum value of X2 +x2max = 0.1 # maximum value of X2 +ix2_bc = outflow # inner-X2 boundary flag +ox2_bc = outflow # outer-X2 boundary flag + +nx3 = 64 # Number of zones in X3-direction +x3min =-0.1 # minimum value of X3 +x3max = 0.1 # maximum value of X3 +ix3_bc = outflow # inner-X3 boundary flag +ox3_bc = outflow # outer-X3 boundary flag + + +x1min = -0.025 +x1max = 0.025 +x2min = -0.025 +x2max = 0.025 +x3min = -0.025 +x3max = 0.025 +level = 2 + + + +nx1 = 8 # Number of zones in X1-direction +nx2 = 8 # Number of zones in X2-direction +nx3 = 8 # Number of zones in X3-direction + + +fluid = euler +gamma = 1.6666666666666667 # gamma = C_p/C_v +eos = adiabatic +riemann = none +reconstruction = dc +calc_dt_hyp = true +scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM + +He_mass_fraction = 0.25 + + +#Units parameters +#Note: All other parameters for the cluster are in terms of these units +code_length_cgs = 3.085677580962325e+24 # 1 Mpc in cm +code_mass_cgs = 1.98841586e+47 # 1e14 Msun in g +code_time_cgs = 3.15576e+16 # 1 Gyr in s + + + + +#Disable gravity as a source term +gravity_srcterm = false + + +#Initialize with a uniform gas +init_uniform_gas = true +rho = 147.7557589278723 +ux = 0 +uy = 0 +uz = 0 +pres = 1.5454368403867562 + + +#Define a precessing jet +jet_phi0 = 1.2 +jet_phi_dot = 1 +jet_theta = 0.4 + + +fixed_power = 1.65998282e-04 +efficiency = 1e-3 + +magnetic_fraction = = 0.0 +thermal_fraction = 0.5 +kinetic_fraction = 0.5 + +thermal_radius = 0.0125 + +kinetic_jet_radius = 0.02 +kinetic_jet_thickness = 0.01 +kinetic_jet_offset = 0.01 +kinetic_jet_temperature = 1e7 + + +disabled = True diff --git a/inputs/cluster/magnetic_tower.in b/inputs/cluster/magnetic_tower.in new file mode 100644 index 00000000..44ba58a0 --- /dev/null +++ b/inputs/cluster/magnetic_tower.in @@ -0,0 +1,121 @@ +################################################################################ +# Input file for testing magnetic tower AGN feedback without fluid +# evolution +################################################################################ + +problem = Magnetic Tower Test + + +problem_id = cluster # problem ID: basename of output filenames + + +file_type = hst # History data dump +dt = 1e-4 # time increment between outputs + + +file_type = hdf5 # HDF5 data dump +variables = cons,prim # Variables to be output +dt = 0.01 # Time increment between outputs +id = vars # Name to append to output + + +cfl_number = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number +nlim = -1 # cycle limit +tlim = 0.01 # time limit +integrator = vl2 # time integration algorithm + + + +refinement = static +nghost = 2 + +nx1 = 64 # Number of zones in X1-direction +x1min =-0.1 # minimum value of X1 +x1max = 0.1 # maximum value of X1 +ix1_bc = outflow # inner-X1 boundary flag +ox1_bc = outflow # outer-X1 boundary flag + +nx2 = 64 # Number of zones in X2-direction +x2min =-0.1 # minimum value of X2 +x2max = 0.1 # maximum value of X2 +ix2_bc = outflow # inner-X2 boundary flag +ox2_bc = outflow # outer-X2 boundary flag + +nx3 = 64 # Number of zones in X3-direction +x3min =-0.1 # minimum value of X3 +x3max = 0.1 # maximum value of X3 +ix3_bc = outflow # inner-X3 boundary flag +ox3_bc = outflow # outer-X3 boundary flag + + +x1min = -0.025 +x1max = 0.025 +x2min = -0.025 +x2max = 0.025 +x3min = -0.025 +x3max = 0.025 +level = 2 + + + +nx1 = 8 # Number of zones in X1-direction +nx2 = 8 # Number of zones in X2-direction +nx3 = 8 # Number of zones in X3-direction + + +fluid = glmmhd +gamma = 1.6666666666666667 # gamma = C_p/C_v +eos = adiabatic +riemann = none +reconstruction = dc +calc_dt_hyp = true +scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM + +He_mass_fraction = 0.25 + + +#Units parameters +#Note: All other parameters for the cluster are in terms of these units +code_length_cgs = 3.085677580962325e+24 # 1 Mpc in cm +code_mass_cgs = 1.98841586e+47 # 1e14 Msun in g +code_time_cgs = 3.15576e+16 # 1 Gyr in s + + + + +#Disable gravity as a source term +gravity_srcterm = false + + +#Initialize with a uniform gas +init_uniform_gas = true +rho = 147.7557589278723 +ux = 0 +uy = 0 +uz = 0 +pres = 1.5454368403867562 + + +#Define a precessing jet +jet_theta = 0.2 +jet_phi_dot = 0 +jet_phi0 = 1 + + +fixed_power = 0.0016599828166743962 +efficiency = 1e-3 +magnetic_fraction = 1 +kinetic_fraction = 0 +thermal_fraction = 0 + + +potential_type = li +li_alpha = 20 +l_scale = 0.01 +initial_field = 0.12431560000204142 +fixed_field_rate = 12.431560000204144 +fixed_mass_rate = 1.7658562333594375e-05 +l_mass_scale = 0.005 + + +disabled = True diff --git a/inputs/cluster/my_cluster.input b/inputs/cluster/my_cluster.input new file mode 100644 index 00000000..3823f7b8 --- /dev/null +++ b/inputs/cluster/my_cluster.input @@ -0,0 +1,201 @@ + +# File autogenerated with Python script +# Changes might be overwritten! + +problem = Isolated galaxy cluster + + +problem_id = cluster # problem ID: basename of output filenames + + + +file_type = hst # History data dump +dt = 0.0001 # time increment between outputs + + +file_type = rst # restart data dump +dt = 0.001 # Time increment between outputs +id = restart + +# hdf5_compression_level = 0 +use_final_label = false + + + + +cfl = 0.3 # The Courant, Friedrichs, & Lewy (CFL) Number +tlim = 0.01 # time limit +integrator = vl2 # time integration algorithm + + + + +refinement = static +nghost = 2 + +nx1 = 64 # Number of zones in X1-direction +x1min =-0.1 # minimum value of X1 +x1max = 0.1 # maximum value of X1 +ix1_bc = outflow # inner-X1 boundary flag +ox1_bc = outflow # outer-X1 boundary flag + +nx2 = 64 # Number of zones in X2-direction +x2min =-0.1 # minimum value of X2 +x2max = 0.1 # maximum value of X2 +ix2_bc = outflow # inner-X2 boundary flag +ox2_bc = outflow # outer-X2 boundary flag + +nx3 = 64 # Number of zones in X3-direction +x3min =-0.1 # minimum value of X3 +x3max = 0.1 # maximum value of X3 +ix3_bc = outflow # inner-X3 boundary flag +ox3_bc = outflow # outer-X3 boundary flag + + +nx1 = 32 # Number of zones in X1-direction +nx2 = 32 # Number of zones in X2-direction +nx3 = 32 # Number of zones in X3-direction + + + +x1min = -0.025 +x1max = 0.025 +x2min = -0.025 +x2max = 0.025 +x3min = -0.025 +x3max = 0.025 +level = 2 + + + + + +fluid = glmmhd +gamma = 5./3. # gamma = C_p/C_v +eos = adiabatic +riemann = hlld +reconstruction = plm +scratch_level = 0 # 0 is actual scratch (tiny); 1 is HBM +Tfloor = 10000.0 + +first_order_flux_correct = True + +He_mass_fraction = 0.25 + + +#Units parameters +code_length_cgs = 3.085677580962325e+24 +code_mass_cgs = 1.98841586e+47 +code_time_cgs = 3.15576e+16 + + +enable_cooling = tabular +table_filename = schure.cooling +log_temp_col = 0 # Column to read temperature in cooling table +log_lambda_col = 1 # Column to read lambda in cooling table +lambda_units_cgs = 1 + +integrator = townsend +cfl = 0.1 # Restricts hydro step based on fraction of minimum cooling time +min_timestep = 1.0 +d_e_tol = 1e-8 +d_log_temp_tol = 1e-8 + + +hubble_parameter = 0.0715898515654728 + + +#Include gravity as a source term +gravity_srcterm = True + +#Which gravitational fields to include +include_nfw_g = True +which_bcg_g = HERNQUIST +include_smbh_g = True + +#NFW parameters +c_nfw = 6.0 +m_nfw_200 = 10.000000000000002 + +#BCG parameters +m_bcg_s = 0.0010000000000000002 +r_bcg_s = 0.004 + +#SMBH parameters +m_smbh = 1.0000000000000002e-06 + +#Smooth gravity at origin, for numerical reasons +g_smoothing_radius = 0 + + +#Entropy profile parameters +k_0 = 8.851337676479303e-121 +k_100 = 1.3277006514718954e-119 +r_k = 0.1 +alpha_k = 1.1 + + +#Fix density at radius to close system of equations +r_fix = 2.0 +rho_fix = 0.01477557589278723 + +#Building the radii at which to sample initial rho,P +r_sampling = 4.0 + + +#Which triggering mode (BOOSTED_BONDI, BOOTH_SCHAYE, COLD_GAS, NONE) +triggering_mode = COLD_GAS + +#Radius of accretion for triggering +accretion_radius = 0.001 + +#BOOSTED_BONDI and BOOTH_SCHAYE Parameters +bondi_alpha = 100.0 +bondi_beta = 2.0 +bondi_n0 = 2.9379989445851786e+72 + +#COLD_GAS Parameters +cold_temp_thresh = 100000.0 +cold_t_acc = 0.1 + +write_to_file = True + + +jet_theta = 0.15 +jet_phi_dot = 628.3185307179587 +jet_phi0 = 0.2 + + +# Fixed power, added on top of triggered feedback +fixed_power = 0.0 + +# Efficieny in conversion of AGN accreted mass to AGN feedback energy +efficiency = 1e-3 + +# Fraction allocated to different mechanisms +magnetic_fraction = 0.333 +thermal_fraction = 0.333 +kinetic_fraction = 0.333 + +# Thermal feedback parameters +thermal_radius = 0.0005 + +# Kinetic jet feedback parameters +kinetic_jet_radius = 0.0005 +kinetic_jet_thickness = 0.0005 +kinetic_jet_offset = 0.0005 +kinetic_jet_temperature = 1000000.0 + + +alpha = 20 +l_scale = 0.001 +initial_field = 0.12431560000204142 +l_mass_scale = 0.001 + + + +power_per_bcg_mass = 0.0015780504379367209 +mass_rate_per_bcg_mass = 0.00315576 +disabled = False + + diff --git a/inputs/cluster/gnat-sternberg.cooling b/inputs/cooling_tables/gnat-sternberg.cooling similarity index 100% rename from inputs/cluster/gnat-sternberg.cooling rename to inputs/cooling_tables/gnat-sternberg.cooling diff --git a/inputs/cluster/schure.cooling b/inputs/cooling_tables/schure.cooling similarity index 100% rename from inputs/cluster/schure.cooling rename to inputs/cooling_tables/schure.cooling diff --git a/inputs/cooling_tables/sutherland_dopita.cooling b/inputs/cooling_tables/sutherland_dopita.cooling new file mode 100644 index 00000000..5e329103 --- /dev/null +++ b/inputs/cooling_tables/sutherland_dopita.cooling @@ -0,0 +1,782 @@ +# Cooling table for solar metallicity, 1/3 solar metallicity +# (adapted from PLUTO code by Deovrat Prasad) +# temperature log(K), cooling rate/ne^3 (erg cm^3/s) +# This is the cooling table based on Sutherland & Dopita (1993), ApJ, 88, 253 +# logT 1/3 solar 1 solar +3.0064233e+00 -2.4478679e+01 -2.4024876e+01 +3.0154017e+00 -2.4475474e+01 -2.4022199e+01 +3.0244036e+00 -2.4472293e+01 -2.4019538e+01 +3.0333835e+00 -2.4469109e+01 -2.4016884e+01 +3.0424180e+00 -2.4465936e+01 -2.4014241e+01 +3.0513841e+00 -2.4462760e+01 -2.4011606e+01 +3.0603956e+00 -2.4459595e+01 -2.4008978e+01 +3.0694091e+00 -2.4456416e+01 -2.4006361e+01 +3.0783843e+00 -2.4453248e+01 -2.4003747e+01 +3.0873909e+00 -2.4450078e+01 -2.4001139e+01 +3.0963885e+00 -2.4446918e+01 -2.3998526e+01 +3.1053739e+00 -2.4443746e+01 -2.3995937e+01 +3.1143774e+00 -2.4440572e+01 -2.3993363e+01 +3.1233616e+00 -2.4437398e+01 -2.3990762e+01 +3.1323878e+00 -2.4434223e+01 -2.3988176e+01 +3.1413557e+00 -2.4431036e+01 -2.3985563e+01 +3.1503573e+00 -2.4427861e+01 -2.3983008e+01 +3.1593566e+00 -2.4424662e+01 -2.3980427e+01 +3.1683501e+00 -2.4421464e+01 -2.3977819e+01 +3.1773633e+00 -2.4418266e+01 -2.3975227e+01 +3.1863629e+00 -2.4415036e+01 -2.3972610e+01 +3.1953461e+00 -2.4411818e+01 -2.3970008e+01 +3.2043642e+00 -2.4408568e+01 -2.3967381e+01 +3.2133584e+00 -2.4405320e+01 -2.3964770e+01 +3.2223522e+00 -2.4402053e+01 -2.3962135e+01 +3.2313421e+00 -2.4398766e+01 -2.3959516e+01 +3.2403495e+00 -2.4395472e+01 -2.3956834e+01 +3.2493451e+00 -2.4392395e+01 -2.3954403e+01 +3.2583259e+00 -2.4388999e+01 -2.3951636e+01 +3.2673360e+00 -2.4385662e+01 -2.3948963e+01 +3.2763239e+00 -2.4382266e+01 -2.3946192e+01 +3.2853322e+00 -2.4378876e+01 -2.3943476e+01 +3.2943339e+00 -2.4375439e+01 -2.3940664e+01 +3.3033257e+00 -2.4371989e+01 -2.3937869e+01 +3.3123255e+00 -2.4368516e+01 -2.3935056e+01 +3.3213084e+00 -2.4365009e+01 -2.3932185e+01 +3.3303123e+00 -2.4361481e+01 -2.3929297e+01 +3.3393123e+00 -2.4357921e+01 -2.3926355e+01 +3.3483049e+00 -2.4354332e+01 -2.3923396e+01 +3.3573058e+00 -2.4350714e+01 -2.3920385e+01 +3.3663109e+00 -2.4347038e+01 -2.3917286e+01 +3.3752977e+00 -2.4343308e+01 -2.3914174e+01 +3.3842996e+00 -2.4339533e+01 -2.3910978e+01 +3.3932944e+00 -2.4335697e+01 -2.3907771e+01 +3.4022958e+00 -2.4331810e+01 -2.3904447e+01 +3.4112998e+00 -2.4327856e+01 -2.3901114e+01 +3.4202859e+00 -2.4323846e+01 -2.3897669e+01 +3.4292838e+00 -2.4319764e+01 -2.3894183e+01 +3.4382891e+00 -2.4315621e+01 -2.3890590e+01 +3.4472821e+00 -2.4311402e+01 -2.3886926e+01 +3.4562749e+00 -2.4307100e+01 -2.3883193e+01 +3.4652787e+00 -2.4302727e+01 -2.3879327e+01 +3.4742746e+00 -2.4298268e+01 -2.3875398e+01 +3.4832734e+00 -2.4293726e+01 -2.3871375e+01 +3.4922714e+00 -2.4289088e+01 -2.3867228e+01 +3.5012647e+00 -2.4284364e+01 -2.3862963e+01 +3.5102634e+00 -2.4279535e+01 -2.3858582e+01 +3.5192634e+00 -2.4274595e+01 -2.3854089e+01 +3.5282609e+00 -2.4269557e+01 -2.3849458e+01 +3.5372523e+00 -2.4264401e+01 -2.3844694e+01 +3.5462465e+00 -2.4259124e+01 -2.3839802e+01 +3.5552517e+00 -2.4253732e+01 -2.3834756e+01 +3.5642517e+00 -2.4248221e+01 -2.3829533e+01 +3.5732430e+00 -2.4242574e+01 -2.3824169e+01 +3.5822452e+00 -2.4236774e+01 -2.3818642e+01 +3.5912427e+00 -2.4230837e+01 -2.3812902e+01 +3.6002321e+00 -2.4224746e+01 -2.3807015e+01 +3.6092315e+00 -2.4218503e+01 -2.3800931e+01 +3.6182260e+00 -2.4212100e+01 -2.3794633e+01 +3.6272327e+00 -2.4205533e+01 -2.3788132e+01 +3.6362270e+00 -2.4198795e+01 -2.3781438e+01 +3.6452257e+00 -2.4190723e+01 -2.3773117e+01 +3.6542247e+00 -2.4183447e+01 -2.3765787e+01 +3.6632202e+00 -2.4175972e+01 -2.3758180e+01 +3.6722180e+00 -2.4168277e+01 -2.3750313e+01 +3.6812141e+00 -2.4160359e+01 -2.3742153e+01 +3.6902049e+00 -2.4152205e+01 -2.3733745e+01 +3.6992045e+00 -2.4143815e+01 -2.3725034e+01 +3.7081999e+00 -2.4135175e+01 -2.3716021e+01 +3.7172043e+00 -2.4126273e+01 -2.3706682e+01 +3.7261973e+00 -2.4117100e+01 -2.3697042e+01 +3.7351995e+00 -2.4107638e+01 -2.3687040e+01 +3.7441912e+00 -2.4097872e+01 -2.3676686e+01 +3.7531922e+00 -2.4087778e+01 -2.3665949e+01 +3.7621907e+00 -2.4077342e+01 -2.3654803e+01 +3.7711831e+00 -2.4066538e+01 -2.3643248e+01 +3.7801804e+00 -2.4055340e+01 -2.3631248e+01 +3.7891787e+00 -2.4043711e+01 -2.3618777e+01 +3.7981809e+00 -2.4031629e+01 -2.3605794e+01 +3.8071764e+00 -2.4019047e+01 -2.3592286e+01 +3.8161750e+00 -2.4005943e+01 -2.3578248e+01 +3.8251729e+00 -2.3992252e+01 -2.3563631e+01 +3.8341663e+00 -2.3977984e+01 -2.3548428e+01 +3.8431642e+00 -2.3963012e+01 -2.3532614e+01 +3.8521627e+00 -2.3947421e+01 -2.3516284e+01 +3.8611579e+00 -2.3930924e+01 -2.3499160e+01 +3.8701580e+00 -2.3913533e+01 -2.3481302e+01 +3.8791532e+00 -2.3895103e+01 -2.3462697e+01 +3.8881514e+00 -2.3875561e+01 -2.3443275e+01 +3.8971486e+00 -2.3854773e+01 -2.3423014e+01 +3.9061464e+00 -2.3832535e+01 -2.3401844e+01 +3.9151412e+00 -2.3808773e+01 -2.3379729e+01 +3.9241397e+00 -2.3783254e+01 -2.3356616e+01 +3.9331379e+00 -2.3755896e+01 -2.3332482e+01 +3.9421371e+00 -2.3726466e+01 -2.3307285e+01 +3.9511334e+00 -2.3694864e+01 -2.3280984e+01 +3.9601282e+00 -2.3660946e+01 -2.3253560e+01 +3.9691269e+00 -2.3626886e+01 -2.3227869e+01 +3.9781257e+00 -2.3587976e+01 -2.3198123e+01 +3.9871208e+00 -2.3546269e+01 -2.3166898e+01 +3.9961175e+00 -2.3501855e+01 -2.3134280e+01 +4.0051376e+00 -2.3455015e+01 -2.3100163e+01 +4.0141003e+00 -2.3405099e+01 -2.3064503e+01 +4.0231289e+00 -2.3352686e+01 -2.3027274e+01 +4.0320947e+00 -2.3297880e+01 -2.2988430e+01 +4.0411162e+00 -2.3240884e+01 -2.2947999e+01 +4.0501090e+00 -2.3181926e+01 -2.2905878e+01 +4.0591088e+00 -2.3121266e+01 -2.2862203e+01 +4.0681116e+00 -2.3059200e+01 -2.2816987e+01 +4.0771134e+00 -2.2996066e+01 -2.2770446e+01 +4.0861106e+00 -2.2932111e+01 -2.2722437e+01 +4.0950996e+00 -2.2867676e+01 -2.2673234e+01 +4.1040772e+00 -2.2803078e+01 -2.2623022e+01 +4.1130739e+00 -2.2738642e+01 -2.2572076e+01 +4.1220848e+00 -2.2674649e+01 -2.2520338e+01 +4.1310730e+00 -2.2611686e+01 -2.2468726e+01 +4.1400679e+00 -2.2550013e+01 -2.2417221e+01 +4.1490651e+00 -2.2490233e+01 -2.2366521e+01 +4.1580608e+00 -2.2432797e+01 -2.2317025e+01 +4.1670809e+00 -2.2378325e+01 -2.2269347e+01 +4.1760623e+00 -2.2327505e+01 -2.2224026e+01 +4.1850603e+00 -2.2281067e+01 -2.2182091e+01 +4.1940701e+00 -2.2239615e+01 -2.2144202e+01 +4.2030601e+00 -2.2167274e+01 -2.2083372e+01 +4.2120544e+00 -2.2127989e+01 -2.2046956e+01 +4.2210489e+00 -2.2094112e+01 -2.2015099e+01 +4.2300400e+00 -2.2065896e+01 -2.1988134e+01 +4.2390491e+00 -2.2043452e+01 -2.1966134e+01 +4.2480469e+00 -2.2026484e+01 -2.1948616e+01 +4.2570543e+00 -2.2015104e+01 -2.1936291e+01 +4.2660434e+00 -2.2008663e+01 -2.1928228e+01 +4.2750348e+00 -2.2006595e+01 -2.1923906e+01 +4.2840245e+00 -2.2008278e+01 -2.1922741e+01 +4.2930309e+00 -2.2013094e+01 -2.1924088e+01 +4.3020277e+00 -2.2020433e+01 -2.1927383e+01 +4.3110330e+00 -2.2029751e+01 -2.1932111e+01 +4.3200216e+00 -2.2040544e+01 -2.1937719e+01 +4.3290316e+00 -2.2052380e+01 -2.1943820e+01 +4.3380180e+00 -2.2064886e+01 -2.1950085e+01 +4.3470176e+00 -2.2077737e+01 -2.1956206e+01 +4.3560067e+00 -2.2090658e+01 -2.1961976e+01 +4.3650197e+00 -2.2103413e+01 -2.1967140e+01 +4.3740147e+00 -2.2115805e+01 -2.1971591e+01 +4.3830070e+00 -2.2127646e+01 -2.1975145e+01 +4.3920107e+00 -2.2138812e+01 -2.1977819e+01 +4.4010040e+00 -2.2149188e+01 -2.1979473e+01 +4.4100007e+00 -2.2158641e+01 -2.1980053e+01 +4.4189969e+00 -2.2167095e+01 -2.1979556e+01 +4.4279889e+00 -2.2174515e+01 -2.1977984e+01 +4.4369891e+00 -2.2180693e+01 -2.1974981e+01 +4.4459932e+00 -2.2185926e+01 -2.1971225e+01 +4.4549820e+00 -2.2190265e+01 -2.1966616e+01 +4.4639825e+00 -2.2193298e+01 -2.1961141e+01 +4.4729757e+00 -2.2194723e+01 -2.1954325e+01 +4.4819726e+00 -2.2195200e+01 -2.1946614e+01 +4.4909693e+00 -2.2195186e+01 -2.1938246e+01 +4.4999756e+00 -2.2194812e+01 -2.1929408e+01 +4.5089739e+00 -2.2193698e+01 -2.1919987e+01 +4.5179608e+00 -2.2192025e+01 -2.1910024e+01 +4.5269593e+00 -2.2190427e+01 -2.1899871e+01 +4.5359647e+00 -2.2188217e+01 -2.1889208e+01 +4.5449605e+00 -2.2185593e+01 -2.1878210e+01 +4.5539558e+00 -2.2182494e+01 -2.1866749e+01 +4.5629587e+00 -2.2179615e+01 -2.1855332e+01 +4.5719533e+00 -2.2175451e+01 -2.1843178e+01 +4.5809478e+00 -2.2171585e+01 -2.1831120e+01 +4.5899496e+00 -2.2166929e+01 -2.1818614e+01 +4.5989436e+00 -2.2161132e+01 -2.1805541e+01 +4.6079373e+00 -2.2155510e+01 -2.1792527e+01 +4.6169374e+00 -2.2148925e+01 -2.1778977e+01 +4.6259295e+00 -2.2141120e+01 -2.1764876e+01 +4.6349305e+00 -2.2133217e+01 -2.1750655e+01 +4.6439262e+00 -2.2123996e+01 -2.1735914e+01 +4.6529229e+00 -2.2114147e+01 -2.1720858e+01 +4.6619262e+00 -2.2103518e+01 -2.1705468e+01 +4.6709228e+00 -2.2092030e+01 -2.1689732e+01 +4.6799182e+00 -2.2079835e+01 -2.1673849e+01 +4.6889179e+00 -2.2066619e+01 -2.1657459e+01 +4.6979090e+00 -2.2052522e+01 -2.1640753e+01 +4.7069140e+00 -2.2034963e+01 -2.1622767e+01 +4.7159115e+00 -2.2018172e+01 -2.1605128e+01 +4.7249064e+00 -2.2001183e+01 -2.1587472e+01 +4.7339031e+00 -2.1983301e+01 -2.1569538e+01 +4.7428979e+00 -2.1964530e+01 -2.1551340e+01 +4.7518947e+00 -2.1944966e+01 -2.1532940e+01 +4.7608972e+00 -2.1924599e+01 -2.1514293e+01 +4.7698940e+00 -2.1903368e+01 -2.1495407e+01 +4.7788889e+00 -2.1881636e+01 -2.1476423e+01 +4.7878854e+00 -2.1859461e+01 -2.1457336e+01 +4.7968864e+00 -2.1836928e+01 -2.1438231e+01 +4.8058813e+00 -2.1814005e+01 -2.1419440e+01 +4.8148799e+00 -2.1791021e+01 -2.1400455e+01 +4.8238783e+00 -2.1768097e+01 -2.1381638e+01 +4.8328728e+00 -2.1745404e+01 -2.1363071e+01 +4.8418723e+00 -2.1723492e+01 -2.1345025e+01 +4.8508667e+00 -2.1702480e+01 -2.1327560e+01 +4.8598645e+00 -2.1682564e+01 -2.1310789e+01 +4.8688618e+00 -2.1664301e+01 -2.1294975e+01 +4.8778607e+00 -2.1646892e+01 -2.1279708e+01 +4.8868572e+00 -2.1630933e+01 -2.1265296e+01 +4.8958533e+00 -2.1616400e+01 -2.1251711e+01 +4.9048507e+00 -2.1603260e+01 -2.1238922e+01 +4.9138509e+00 -2.1591573e+01 -2.1226923e+01 +4.9228448e+00 -2.1581053e+01 -2.1215682e+01 +4.9318442e+00 -2.1571703e+01 -2.1205205e+01 +4.9408401e+00 -2.1563456e+01 -2.1195499e+01 +4.9498387e+00 -2.1556236e+01 -2.1186613e+01 +4.9588361e+00 -2.1550013e+01 -2.1178585e+01 +4.9678334e+00 -2.1544759e+01 -2.1171495e+01 +4.9768312e+00 -2.1540487e+01 -2.1165427e+01 +4.9858305e+00 -2.1537153e+01 -2.1160434e+01 +4.9948273e+00 -2.1534781e+01 -2.1156593e+01 +5.0038051e+00 -2.1533607e+01 -2.1154226e+01 +5.0128372e+00 -2.1533088e+01 -2.1152717e+01 +5.0218093e+00 -2.1534558e+01 -2.1152748e+01 +5.0308020e+00 -2.1535898e+01 -2.1153484e+01 +5.0398106e+00 -2.1537452e+01 -2.1154747e+01 +5.0487913e+00 -2.1539343e+01 -2.1156512e+01 +5.0578182e+00 -2.1541241e+01 -2.1158515e+01 +5.0668103e+00 -2.1543163e+01 -2.1160491e+01 +5.0758024e+00 -2.1544820e+01 -2.1162191e+01 +5.0847907e+00 -2.1546070e+01 -2.1163423e+01 +5.0938068e+00 -2.1546789e+01 -2.1164031e+01 +5.1028109e+00 -2.1546911e+01 -2.1163929e+01 +5.1118000e+00 -2.1546422e+01 -2.1163075e+01 +5.1208042e+00 -2.1545308e+01 -2.1161472e+01 +5.1297865e+00 -2.1543619e+01 -2.1159167e+01 +5.1387762e+00 -2.1541407e+01 -2.1156232e+01 +5.1477690e+00 -2.1538757e+01 -2.1152761e+01 +5.1567914e+00 -2.1535734e+01 -2.1148840e+01 +5.1657783e+00 -2.1532436e+01 -2.1144596e+01 +5.1747864e+00 -2.1528958e+01 -2.1140123e+01 +5.1837822e+00 -2.1525361e+01 -2.1135530e+01 +5.1927625e+00 -2.1521765e+01 -2.1130933e+01 +5.2017521e+00 -2.1518214e+01 -2.1126412e+01 +5.2107732e+00 -2.1514804e+01 -2.1122059e+01 +5.2197678e+00 -2.1511576e+01 -2.1117937e+01 +5.2287596e+00 -2.1508582e+01 -2.1114091e+01 +5.2377448e+00 -2.1505818e+01 -2.1110536e+01 +5.2467447e+00 -2.1503306e+01 -2.1107266e+01 +5.2557548e+00 -2.1501001e+01 -2.1104241e+01 +5.2647470e+00 -2.1498872e+01 -2.1101412e+01 +5.2737418e+00 -2.1496863e+01 -2.1098705e+01 +5.2827354e+00 -2.1498982e+01 -2.1097671e+01 +5.2917461e+00 -2.1496809e+01 -2.1094911e+01 +5.3007259e+00 -2.1494606e+01 -2.1092116e+01 +5.3097366e+00 -2.1492414e+01 -2.1089296e+01 +5.3187310e+00 -2.1490273e+01 -2.1086483e+01 +5.3277267e+00 -2.1488237e+01 -2.1083751e+01 +5.3367198e+00 -2.1486370e+01 -2.1081205e+01 +5.3457265e+00 -2.1484802e+01 -2.1078969e+01 +5.3547229e+00 -2.1483663e+01 -2.1077223e+01 +5.3637248e+00 -2.1483108e+01 -2.1076160e+01 +5.3727095e+00 -2.1483332e+01 -2.1076016e+01 +5.3817106e+00 -2.1484550e+01 -2.1077057e+01 +5.3907055e+00 -2.1486982e+01 -2.1079558e+01 +5.3997083e+00 -2.1490878e+01 -2.1083825e+01 +5.4086978e+00 -2.1496482e+01 -2.1090145e+01 +5.4177041e+00 -2.1503970e+01 -2.1098787e+01 +5.4267064e+00 -2.1513499e+01 -2.1109954e+01 +5.4357011e+00 -2.1525158e+01 -2.1123759e+01 +5.4447004e+00 -2.1538907e+01 -2.1140219e+01 +5.4536852e+00 -2.1554614e+01 -2.1159198e+01 +5.4626824e+00 -2.1572011e+01 -2.1180423e+01 +5.4716877e+00 -2.1590743e+01 -2.1203502e+01 +5.4806823e+00 -2.1610409e+01 -2.1227935e+01 +5.4896773e+00 -2.1630580e+01 -2.1253187e+01 +5.4986826e+00 -2.1650781e+01 -2.1278692e+01 +5.5076805e+00 -2.1670643e+01 -2.1303932e+01 +5.5166676e+00 -2.1689838e+01 -2.1328438e+01 +5.5256666e+00 -2.1708121e+01 -2.1351845e+01 +5.5346733e+00 -2.1725311e+01 -2.1373855e+01 +5.5436708e+00 -2.1741315e+01 -2.1394286e+01 +5.5526682e+00 -2.1756218e+01 -2.1413087e+01 +5.5616618e+00 -2.1769628e+01 -2.1429994e+01 +5.5706597e+00 -2.1782095e+01 -2.1445293e+01 +5.5796579e+00 -2.1793147e+01 -2.1458745e+01 +5.5886526e+00 -2.1803382e+01 -2.1470685e+01 +5.5976513e+00 -2.1812479e+01 -2.1481013e+01 +5.6066500e+00 -2.1820879e+01 -2.1490018e+01 +5.6156450e+00 -2.1828303e+01 -2.1497641e+01 +5.6246430e+00 -2.1835053e+01 -2.1504151e+01 +5.6336401e+00 -2.1841065e+01 -2.1509592e+01 +5.6426327e+00 -2.1846551e+01 -2.1514165e+01 +5.6516365e+00 -2.1851613e+01 -2.1518013e+01 +5.6606284e+00 -2.1856298e+01 -2.1521289e+01 +5.6696237e+00 -2.1860751e+01 -2.1524155e+01 +5.6786276e+00 -2.1865090e+01 -2.1526747e+01 +5.6876181e+00 -2.1869377e+01 -2.1529251e+01 +5.6966185e+00 -2.1873739e+01 -2.1531830e+01 +5.7056157e+00 -2.1879294e+01 -2.1535108e+01 +5.7146147e+00 -2.1885389e+01 -2.1538862e+01 +5.7236116e+00 -2.1891841e+01 -2.1543179e+01 +5.7326109e+00 -2.1898769e+01 -2.1548214e+01 +5.7416085e+00 -2.1906333e+01 -2.1554085e+01 +5.7506010e+00 -2.1914531e+01 -2.1560888e+01 +5.7595999e+00 -2.1923396e+01 -2.1568652e+01 +5.7686011e+00 -2.1932929e+01 -2.1577312e+01 +5.7775935e+00 -2.1943019e+01 -2.1586751e+01 +5.7865953e+00 -2.1953544e+01 -2.1596742e+01 +5.7955881e+00 -2.1964250e+01 -2.1607004e+01 +5.8045892e+00 -2.1974899e+01 -2.1617191e+01 +5.8135877e+00 -2.1985311e+01 -2.1627033e+01 +5.8225799e+00 -2.1995249e+01 -2.1636256e+01 +5.8315819e+00 -2.2004571e+01 -2.1644625e+01 +5.8405765e+00 -2.2013197e+01 -2.1652007e+01 +5.8495730e+00 -2.2021062e+01 -2.1658368e+01 +5.8585733e+00 -2.2028172e+01 -2.1663700e+01 +5.8675677e+00 -2.2034573e+01 -2.1668107e+01 +5.8765642e+00 -2.2040324e+01 -2.1671661e+01 +5.8855647e+00 -2.2045526e+01 -2.1674525e+01 +5.8945597e+00 -2.2050278e+01 -2.1676851e+01 +5.9035566e+00 -2.2054689e+01 -2.1678754e+01 +5.9125568e+00 -2.2058861e+01 -2.1680436e+01 +5.9215512e+00 -2.2062909e+01 -2.1681999e+01 +5.9305517e+00 -2.2066923e+01 -2.1683589e+01 +5.9395492e+00 -2.2071005e+01 -2.1685332e+01 +5.9485450e+00 -2.2075230e+01 -2.1687336e+01 +5.9575450e+00 -2.2079673e+01 -2.1689689e+01 +5.9665406e+00 -2.2084379e+01 -2.1692440e+01 +5.9755375e+00 -2.2089386e+01 -2.1695617e+01 +5.9845363e+00 -2.2094701e+01 -2.1699252e+01 +5.9935332e+00 -2.2100305e+01 -2.1703313e+01 +6.0025116e+00 -2.2106177e+01 -2.1707744e+01 +6.0115282e+00 -2.2112265e+01 -2.1712489e+01 +6.0205270e+00 -2.2118518e+01 -2.1717469e+01 +6.0295055e+00 -2.2124158e+01 -2.1722368e+01 +6.0385009e+00 -2.2131009e+01 -2.1727787e+01 +6.0475085e+00 -2.2137773e+01 -2.1733228e+01 +6.0565237e+00 -2.2144463e+01 -2.1738666e+01 +6.0655050e+00 -2.2151054e+01 -2.1744028e+01 +6.0745239e+00 -2.2157541e+01 -2.1749336e+01 +6.0835026e+00 -2.2163936e+01 -2.1754586e+01 +6.0925101e+00 -2.2170253e+01 -2.1759825e+01 +6.1015065e+00 -2.2176513e+01 -2.1765027e+01 +6.1104887e+00 -2.2182745e+01 -2.1770267e+01 +6.1194868e+00 -2.2188995e+01 -2.1775571e+01 +6.1284962e+00 -2.2195309e+01 -2.1781018e+01 +6.1374807e+00 -2.2201736e+01 -2.1786668e+01 +6.1465001e+00 -2.2208344e+01 -2.1792554e+01 +6.1554879e+00 -2.2215190e+01 -2.1798794e+01 +6.1644718e+00 -2.2222334e+01 -2.1805458e+01 +6.1734776e+00 -2.2229848e+01 -2.1812592e+01 +6.1824717e+00 -2.2237802e+01 -2.1820305e+01 +6.1914790e+00 -2.2246241e+01 -2.1828654e+01 +6.2004674e+00 -2.2255230e+01 -2.1837704e+01 +6.2094614e+00 -2.2264824e+01 -2.1847498e+01 +6.2184567e+00 -2.2275061e+01 -2.1858143e+01 +6.2274753e+00 -2.2285972e+01 -2.1869666e+01 +6.2364617e+00 -2.2297595e+01 -2.1882099e+01 +6.2454633e+00 -2.2309955e+01 -2.1895479e+01 +6.2544514e+00 -2.2323050e+01 -2.1909777e+01 +6.2634467e+00 -2.2336884e+01 -2.1925110e+01 +6.2724450e+00 -2.2351425e+01 -2.1941346e+01 +6.2814425e+00 -2.2366622e+01 -2.1958489e+01 +6.2904353e+00 -2.2382413e+01 -2.1976459e+01 +6.2994419e+00 -2.2398712e+01 -2.1995163e+01 +6.3084363e+00 -2.2415409e+01 -2.2014520e+01 +6.3174365e+00 -2.2432409e+01 -2.2034361e+01 +6.3264383e+00 -2.2449576e+01 -2.2054571e+01 +6.3354378e+00 -2.2466813e+01 -2.2075023e+01 +6.3444316e+00 -2.2484020e+01 -2.2095609e+01 +6.3534353e+00 -2.2501097e+01 -2.2116219e+01 +6.3624259e+00 -2.2518070e+01 -2.2136820e+01 +6.3714189e+00 -2.2534573e+01 -2.2157197e+01 +6.3804102e+00 -2.2551186e+01 -2.2177551e+01 +6.3894142e+00 -2.2567223e+01 -2.2197548e+01 +6.3984088e+00 -2.2582944e+01 -2.2217226e+01 +6.4074079e+00 -2.2598117e+01 -2.2236467e+01 +6.4164075e+00 -2.2612806e+01 -2.2255191e+01 +6.4254038e+00 -2.2626922e+01 -2.2273322e+01 +6.4344092e+00 -2.2640430e+01 -2.2290798e+01 +6.4434038e+00 -2.2653315e+01 -2.2307567e+01 +6.4523998e+00 -2.2665566e+01 -2.2323608e+01 +6.4613935e+00 -2.2677161e+01 -2.2338898e+01 +6.4703958e+00 -2.2688077e+01 -2.2353390e+01 +6.4793881e+00 -2.2698406e+01 -2.2367168e+01 +6.4883815e+00 -2.2708121e+01 -2.2380218e+01 +6.4973858e+00 -2.2717242e+01 -2.2392524e+01 +6.5063833e+00 -2.2725796e+01 -2.2404129e+01 +6.5153837e+00 -2.2733792e+01 -2.2415047e+01 +6.5243831e+00 -2.2741291e+01 -2.2425309e+01 +6.5333780e+00 -2.2748240e+01 -2.2434884e+01 +6.5423772e+00 -2.2754685e+01 -2.2443806e+01 +6.5513646e+00 -2.2760625e+01 -2.2452102e+01 +6.5603610e+00 -2.2766091e+01 -2.2459733e+01 +6.5693622e+00 -2.2770958e+01 -2.2466686e+01 +6.5783641e+00 -2.2775622e+01 -2.2473170e+01 +6.5873629e+00 -2.2779604e+01 -2.2478940e+01 +6.5963551e+00 -2.2783412e+01 -2.2484232e+01 +6.6053482e+00 -2.2786535e+01 -2.2488839e+01 +6.6143487e+00 -2.2789521e+01 -2.2493049e+01 +6.6233527e+00 -2.2791908e+01 -2.2496632e+01 +6.6323459e+00 -2.2794173e+01 -2.2499805e+01 +6.6413452e+00 -2.2795826e+01 -2.2502448e+01 +6.6503367e+00 -2.2797430e+01 -2.2504789e+01 +6.6593361e+00 -2.2798521e+01 -2.2506626e+01 +6.6683300e+00 -2.2799560e+01 -2.2508190e+01 +6.6773332e+00 -2.2800135e+01 -2.2509311e+01 +6.6863323e+00 -2.2800656e+01 -2.2510224e+01 +6.6953240e+00 -2.2800739e+01 -2.2510717e+01 +6.7043221e+00 -2.2800848e+01 -2.2511055e+01 +6.7133225e+00 -2.2800547e+01 -2.2511055e+01 +6.7223212e+00 -2.2800245e+01 -2.2510900e+01 +6.7313147e+00 -2.2799587e+01 -2.2510463e+01 +6.7403153e+00 -2.2798958e+01 -2.2509901e+01 +6.7493111e+00 -2.2798057e+01 -2.2509143e+01 +6.7583062e+00 -2.2797212e+01 -2.2508372e+01 +6.7673043e+00 -2.2796097e+01 -2.2507407e+01 +6.7763016e+00 -2.2795039e+01 -2.2506430e+01 +6.7853014e+00 -2.2793795e+01 -2.2505345e+01 +6.7942998e+00 -2.2792635e+01 -2.2504289e+01 +6.8032932e+00 -2.2791290e+01 -2.2503167e+01 +6.8122915e+00 -2.2790083e+01 -2.2502158e+01 +6.8212908e+00 -2.2788719e+01 -2.2501097e+01 +6.8302871e+00 -2.2787546e+01 -2.2500230e+01 +6.8392832e+00 -2.2786217e+01 -2.2499297e+01 +6.8482815e+00 -2.2785024e+01 -2.2498544e+01 +6.8572782e+00 -2.2783755e+01 -2.2497819e+01 +6.8662755e+00 -2.2782648e+01 -2.2497259e+01 +6.8752755e+00 -2.2781464e+01 -2.2496754e+01 +6.8842685e+00 -2.2780442e+01 -2.2496454e+01 +6.8932678e+00 -2.2779343e+01 -2.2496209e+01 +6.9022640e+00 -2.2778429e+01 -2.2496168e+01 +6.9112642e+00 -2.2777466e+01 -2.2496209e+01 +6.9202590e+00 -2.2776660e+01 -2.2496468e+01 +6.9292554e+00 -2.2775830e+01 -2.2496795e+01 +6.9382545e+00 -2.2775105e+01 -2.2497341e+01 +6.9472523e+00 -2.2774536e+01 -2.2498092e+01 +6.9562501e+00 -2.2773942e+01 -2.2498982e+01 +6.9652488e+00 -2.2773503e+01 -2.2500107e+01 +6.9742445e+00 -2.2773220e+01 -2.2501496e+01 +6.9832428e+00 -2.2772936e+01 -2.2503070e+01 +6.9922397e+00 -2.2772833e+01 -2.2504955e+01 +7.0012576e+00 -2.2772911e+01 -2.2507128e+01 +7.0102151e+00 -2.2773065e+01 -2.2509578e+01 +7.0192410e+00 -2.2773400e+01 -2.2512381e+01 +7.0282458e+00 -2.2773942e+01 -2.2515586e+01 +7.0372272e+00 -2.2774613e+01 -2.2519117e+01 +7.0462219e+00 -2.2775467e+01 -2.2523024e+01 +7.0552254e+00 -2.2776504e+01 -2.2527331e+01 +7.0642333e+00 -2.2777700e+01 -2.2532022e+01 +7.0732050e+00 -2.2779056e+01 -2.2537093e+01 +7.0822107e+00 -2.2780546e+01 -2.2542527e+01 +7.0912096e+00 -2.2782200e+01 -2.2548352e+01 +7.1001982e+00 -2.2783940e+01 -2.2554458e+01 +7.1092072e+00 -2.2785739e+01 -2.2560825e+01 +7.1181986e+00 -2.2787653e+01 -2.2567480e+01 +7.1272020e+00 -2.2789521e+01 -2.2574254e+01 +7.1361813e+00 -2.2791397e+01 -2.2581169e+01 +7.1451964e+00 -2.2793255e+01 -2.2588162e+01 +7.1541804e+00 -2.2795012e+01 -2.2595115e+01 +7.1631912e+00 -2.2796695e+01 -2.2602043e+01 +7.1721941e+00 -2.2798193e+01 -2.2608835e+01 +7.1811859e+00 -2.2799587e+01 -2.2615503e+01 +7.1901916e+00 -2.2800821e+01 -2.2621929e+01 +7.1991790e+00 -2.2801838e+01 -2.2628120e+01 +7.2081725e+00 -2.2802664e+01 -2.2634044e+01 +7.2171680e+00 -2.2803299e+01 -2.2639653e+01 +7.2261615e+00 -2.2803713e+01 -2.2644932e+01 +7.2351748e+00 -2.2803934e+01 -2.2649868e+01 +7.2441534e+00 -2.2803934e+01 -2.2654430e+01 +7.2531683e+00 -2.2803741e+01 -2.2658684e+01 +7.2621662e+00 -2.2803354e+01 -2.2662581e+01 +7.2711676e+00 -2.2802775e+01 -2.2666150e+01 +7.2801457e+00 -2.2802031e+01 -2.2669404e+01 +7.2891428e+00 -2.2800986e+01 -2.2672008e+01 +7.2981542e+00 -2.2799861e+01 -2.2674566e+01 +7.3071536e+00 -2.2798603e+01 -2.2676830e+01 +7.3161382e+00 -2.2796885e+01 -2.2678029e+01 +7.3251461e+00 -2.2795283e+01 -2.2679604e+01 +7.3341319e+00 -2.2793552e+01 -2.2680915e+01 +7.3431328e+00 -2.2791693e+01 -2.2681958e+01 +7.3521246e+00 -2.2789708e+01 -2.2682752e+01 +7.3611232e+00 -2.2787626e+01 -2.2683317e+01 +7.3701243e+00 -2.2785421e+01 -2.2683631e+01 +7.3791241e+00 -2.2783096e+01 -2.2683757e+01 +7.3881190e+00 -2.2780704e+01 -2.2683652e+01 +7.3971228e+00 -2.2778221e+01 -2.2683380e+01 +7.4061142e+00 -2.2775622e+01 -2.2682898e+01 +7.4151070e+00 -2.2772936e+01 -2.2682166e+01 +7.4241136e+00 -2.2770190e+01 -2.2681332e+01 +7.4331135e+00 -2.2767385e+01 -2.2680353e+01 +7.4421033e+00 -2.2764497e+01 -2.2679210e+01 +7.4511107e+00 -2.2761552e+01 -2.2677925e+01 +7.4601007e+00 -2.2758553e+01 -2.2676500e+01 +7.4691000e+00 -2.2755451e+01 -2.2674957e+01 +7.4780901e+00 -2.2752346e+01 -2.2673275e+01 +7.4870959e+00 -2.2749165e+01 -2.2671478e+01 +7.4960851e+00 -2.2745960e+01 -2.2669566e+01 +7.5050821e+00 -2.2742681e+01 -2.2667562e+01 +7.5140827e+00 -2.2739356e+01 -2.2665446e+01 +7.5230828e+00 -2.2736009e+01 -2.2663220e+01 +7.5320789e+00 -2.2732594e+01 -2.2660926e+01 +7.5410798e+00 -2.2729181e+01 -2.2658526e+01 +7.5500815e+00 -2.2725680e+01 -2.2656060e+01 +7.5590683e+00 -2.2722207e+01 -2.2653510e+01 +7.5680726e+00 -2.2718648e+01 -2.2650878e+01 +7.5770665e+00 -2.2715096e+01 -2.2648165e+01 +7.5860694e+00 -2.2711482e+01 -2.2645411e+01 +7.5950661e+00 -2.2707855e+01 -2.2642580e+01 +7.6040640e+00 -2.2704191e+01 -2.2639691e+01 +7.6130592e+00 -2.2700536e+01 -2.2636745e+01 +7.6220585e+00 -2.2696804e+01 -2.2633745e+01 +7.6310479e+00 -2.2693103e+01 -2.2630691e+01 +7.6400539e+00 -2.2689349e+01 -2.2627604e+01 +7.6490426e+00 -2.2685585e+01 -2.2624464e+01 +7.6580400e+00 -2.2681791e+01 -2.2621275e+01 +7.6670418e+00 -2.2678008e+01 -2.2618037e+01 +7.6760348e+00 -2.2674156e+01 -2.2614769e+01 +7.6850338e+00 -2.2670338e+01 -2.2611473e+01 +7.6940345e+00 -2.2666472e+01 -2.2608130e+01 +7.7030333e+00 -2.2662621e+01 -2.2604778e+01 +7.7120264e+00 -2.2658724e+01 -2.2601366e+01 +7.7210270e+00 -2.2654842e+01 -2.2597962e+01 +7.7300228e+00 -2.2650936e+01 -2.2594500e+01 +7.7390182e+00 -2.2647027e+01 -2.2591048e+01 +7.7480174e+00 -2.2643095e+01 -2.2587556e+01 +7.7570162e+00 -2.2639179e+01 -2.2584059e+01 +7.7660111e+00 -2.2635224e+01 -2.2580523e+01 +7.7750130e+00 -2.2631267e+01 -2.2576984e+01 +7.7840036e+00 -2.2627309e+01 -2.2573424e+01 +7.7930007e+00 -2.2623350e+01 -2.2569877e+01 +7.8020002e+00 -2.2619373e+01 -2.2566294e+01 +7.8109982e+00 -2.2615396e+01 -2.2562709e+01 +7.8199977e+00 -2.2611402e+01 -2.2559107e+01 +7.8289948e+00 -2.2607426e+01 -2.2555502e+01 +7.8379922e+00 -2.2603417e+01 -2.2551881e+01 +7.8469862e+00 -2.2599427e+01 -2.2548275e+01 +7.8559853e+00 -2.2595423e+01 -2.2544637e+01 +7.8649855e+00 -2.2591421e+01 -2.2541015e+01 +7.8739829e+00 -2.2587388e+01 -2.2537362e+01 +7.8829797e+00 -2.2583376e+01 -2.2533726e+01 +7.8919777e+00 -2.2579351e+01 -2.2530075e+01 +7.9009731e+00 -2.2575347e+01 -2.2526425e+01 +7.9099677e+00 -2.2571315e+01 -2.2522777e+01 +7.9189682e+00 -2.2567287e+01 -2.2519117e+01 +7.9279654e+00 -2.2563249e+01 -2.2515444e+01 +7.9369609e+00 -2.2559217e+01 -2.2511788e+01 +7.9459607e+00 -2.2555159e+01 -2.2508120e+01 +7.9549561e+00 -2.2551124e+01 -2.2504456e+01 +7.9639530e+00 -2.2547079e+01 -2.2500780e+01 +7.9729523e+00 -2.2543027e+01 -2.2497109e+01 +7.9819499e+00 -2.2538967e+01 -2.2493427e+01 +7.9909468e+00 -2.2534930e+01 -2.2489737e+01 +7.9999435e+00 -2.2530856e+01 -2.2486050e+01 +8.0089407e+00 -2.2526805e+01 -2.2482356e+01 +8.0179511e+00 -2.2522734e+01 -2.2478653e+01 +8.0269416e+00 -2.2518658e+01 -2.2474955e+01 +8.0359498e+00 -2.2514577e+01 -2.2471238e+01 +8.0449315e+00 -2.2510506e+01 -2.2467526e+01 +8.0539232e+00 -2.2506416e+01 -2.2463795e+01 +8.0629203e+00 -2.2502324e+01 -2.2460071e+01 +8.0719188e+00 -2.2498243e+01 -2.2456329e+01 +8.0809150e+00 -2.2494145e+01 -2.2452582e+01 +8.0899051e+00 -2.2490045e+01 -2.2448831e+01 +8.0989205e+00 -2.2485931e+01 -2.2445063e+01 +8.1079219e+00 -2.2481828e+01 -2.2441291e+01 +8.1169065e+00 -2.2477712e+01 -2.2437517e+01 +8.1259040e+00 -2.2473583e+01 -2.2433716e+01 +8.1349099e+00 -2.2469467e+01 -2.2429924e+01 +8.1438888e+00 -2.2465327e+01 -2.2426108e+01 +8.1528996e+00 -2.2461188e+01 -2.2422290e+01 +8.1619068e+00 -2.2457050e+01 -2.2418460e+01 +8.1709068e+00 -2.2452902e+01 -2.2414618e+01 +8.1798963e+00 -2.2448745e+01 -2.2410766e+01 +8.1889004e+00 -2.2444579e+01 -2.2406902e+01 +8.1978868e+00 -2.2440417e+01 -2.2403029e+01 +8.2068798e+00 -2.2436246e+01 -2.2399136e+01 +8.2158754e+00 -2.2432068e+01 -2.2395245e+01 +8.2248696e+00 -2.2427884e+01 -2.2391335e+01 +8.2338841e+00 -2.2423693e+01 -2.2387407e+01 +8.2428643e+00 -2.2419497e+01 -2.2383482e+01 +8.2518571e+00 -2.2415296e+01 -2.2379531e+01 +8.2608581e+00 -2.2411090e+01 -2.2375574e+01 +8.2698630e+00 -2.2406869e+01 -2.2371611e+01 +8.2788679e+00 -2.2402645e+01 -2.2367624e+01 +8.2878689e+00 -2.2398418e+01 -2.2363632e+01 +8.2968626e+00 -2.2394189e+01 -2.2359628e+01 +8.3058456e+00 -2.2389947e+01 -2.2355621e+01 +8.3148570e+00 -2.2385704e+01 -2.2351591e+01 +8.3238500e+00 -2.2381450e+01 -2.2347551e+01 +8.3328423e+00 -2.2377196e+01 -2.2343499e+01 +8.3418498e+00 -2.2372931e+01 -2.2339438e+01 +8.3508486e+00 -2.2368658e+01 -2.2335367e+01 +8.3598355e+00 -2.2364386e+01 -2.2331288e+01 +8.3688259e+00 -2.2360105e+01 -2.2327191e+01 +8.3778342e+00 -2.2355808e+01 -2.2323078e+01 +8.3868377e+00 -2.2351513e+01 -2.2318967e+01 +8.3958329e+00 -2.2347212e+01 -2.2314832e+01 +8.4048166e+00 -2.2342906e+01 -2.2310700e+01 +8.4138193e+00 -2.2338585e+01 -2.2306546e+01 +8.4228196e+00 -2.2334269e+01 -2.2302387e+01 +8.4318139e+00 -2.2329940e+01 -2.2298208e+01 +8.4408147e+00 -2.2325607e+01 -2.2294025e+01 +8.4498177e+00 -2.2321263e+01 -2.2289832e+01 +8.4588040e+00 -2.2316917e+01 -2.2285628e+01 +8.4678004e+00 -2.2312560e+01 -2.2281415e+01 +8.4768027e+00 -2.2308203e+01 -2.2277185e+01 +8.4858066e+00 -2.2303836e+01 -2.2272955e+01 +8.4947944e+00 -2.2299461e+01 -2.2268709e+01 +8.5037907e+00 -2.2295078e+01 -2.2264449e+01 +8.5127911e+00 -2.2290688e+01 -2.2260182e+01 +8.5217916e+00 -2.2286291e+01 -2.2255903e+01 +8.5307886e+00 -2.2281889e+01 -2.2251618e+01 +8.5397784e+00 -2.2277481e+01 -2.2247314e+01 +8.5487824e+00 -2.2273061e+01 -2.2243007e+01 +8.5577838e+00 -2.2268637e+01 -2.2238689e+01 +8.5667791e+00 -2.2264202e+01 -2.2234361e+01 +8.5757765e+00 -2.2259764e+01 -2.2230017e+01 +8.5847721e+00 -2.2255316e+01 -2.2225673e+01 +8.5937733e+00 -2.2250859e+01 -2.2221313e+01 +8.6027651e+00 -2.2246394e+01 -2.2216933e+01 +8.6117658e+00 -2.2241929e+01 -2.2212554e+01 +8.6207605e+00 -2.2237449e+01 -2.2208169e+01 +8.6297561e+00 -2.2232963e+01 -2.2203766e+01 +8.6387587e+00 -2.2228464e+01 -2.2199359e+01 +8.6477545e+00 -2.2223960e+01 -2.2194934e+01 +8.6567496e+00 -2.2219445e+01 -2.2190508e+01 +8.6657498e+00 -2.2214927e+01 -2.2186059e+01 +8.6747418e+00 -2.2210391e+01 -2.2181609e+01 +8.6837403e+00 -2.2205854e+01 -2.2177139e+01 +8.6927412e+00 -2.2201301e+01 -2.2172663e+01 +8.7017406e+00 -2.2196741e+01 -2.2168175e+01 +8.7107349e+00 -2.2192167e+01 -2.2163670e+01 +8.7197289e+00 -2.2187588e+01 -2.2159160e+01 +8.7287270e+00 -2.2182997e+01 -2.2154635e+01 +8.7377252e+00 -2.2178395e+01 -2.2150096e+01 +8.7467276e+00 -2.2173783e+01 -2.2145548e+01 +8.7557224e+00 -2.2169155e+01 -2.2140988e+01 +8.7647215e+00 -2.2164519e+01 -2.2136415e+01 +8.7737133e+00 -2.2159875e+01 -2.2131826e+01 +8.7827162e+00 -2.2155212e+01 -2.2127226e+01 +8.7917117e+00 -2.2150544e+01 -2.2122617e+01 +8.8007102e+00 -2.2145858e+01 -2.2117988e+01 +8.8097078e+00 -2.2141162e+01 -2.2113351e+01 +8.8187008e+00 -2.2136451e+01 -2.2108697e+01 +8.8276987e+00 -2.2131732e+01 -2.2104025e+01 +8.8366975e+00 -2.2126993e+01 -2.2099343e+01 +8.8456932e+00 -2.2122243e+01 -2.2094647e+01 +8.8546946e+00 -2.2117475e+01 -2.2089936e+01 +8.8636916e+00 -2.2112698e+01 -2.2085207e+01 +8.8726864e+00 -2.2107900e+01 -2.2080462e+01 +8.8816870e+00 -2.2103088e+01 -2.2075705e+01 +8.8906836e+00 -2.2098258e+01 -2.2070929e+01 +8.8996783e+00 -2.2093417e+01 -2.2066133e+01 +8.9086780e+00 -2.2088555e+01 -2.2061325e+01 +8.9176735e+00 -2.2083678e+01 -2.2056496e+01 +8.9266716e+00 -2.2078782e+01 -2.2051651e+01 +8.9356685e+00 -2.2073869e+01 -2.2046787e+01 +8.9446652e+00 -2.2068934e+01 -2.2041905e+01 +8.9536631e+00 -2.2063979e+01 -2.2037006e+01 +8.9626581e+00 -2.2059011e+01 -2.2032082e+01 +8.9716562e+00 -2.2054015e+01 -2.2027140e+01 +8.9806532e+00 -2.2049003e+01 -2.2022180e+01 +8.9896545e+00 -2.2043966e+01 -2.2017195e+01 +8.9986516e+00 -2.2038911e+01 -2.2012191e+01 +9.0076624e+00 -2.2033835e+01 -2.2007159e+01 +9.0166573e+00 -2.2028729e+01 -2.2002111e+01 +9.0256335e+00 -2.2023604e+01 -2.1997057e+01 +9.0346285e+00 -2.2018453e+01 -2.1991954e+01 +9.0436373e+00 -2.2013278e+01 -2.1986826e+01 +9.0526170e+00 -2.2008079e+01 -2.1981674e+01 +9.0616409e+00 -2.2002850e+01 -2.1976501e+01 +9.0706288e+00 -2.1997575e+01 -2.1971307e+01 +9.0796153e+00 -2.1992295e+01 -2.1966054e+01 +9.0886321e+00 -2.1986994e+01 -2.1960824e+01 +9.0976043e+00 -2.1981674e+01 -2.1955539e+01 +9.1066328e+00 -2.1976295e+01 -2.1950201e+01 +9.1156105e+00 -2.1970900e+01 -2.1944889e+01 +9.1246020e+00 -2.1965452e+01 -2.1939491e+01 +9.1336028e+00 -2.1959991e+01 -2.1934084e+01 +9.1426084e+00 -2.1954481e+01 -2.1928670e+01 +9.1516150e+00 -2.1948963e+01 -2.1923178e+01 +9.1605886e+00 -2.1943400e+01 -2.1917681e+01 +9.1695863e+00 -2.1937794e+01 -2.1912148e+01 +9.1786029e+00 -2.1932148e+01 -2.1906578e+01 +9.1876053e+00 -2.1926465e+01 -2.1900976e+01 +9.1965907e+00 -2.1920746e+01 -2.1895308e+01 +9.2055833e+00 -2.1914995e+01 -2.1889612e+01 +9.2145790e+00 -2.1909213e+01 -2.1883891e+01 +9.2235738e+00 -2.1903368e+01 -2.1878145e+01 +9.2325896e+00 -2.1897498e+01 -2.1872312e+01 +9.2415714e+00 -2.1891570e+01 -2.1866493e+01 +9.2505664e+00 -2.1885589e+01 -2.1860593e+01 +9.2595700e+00 -2.1879591e+01 -2.1854648e+01 +9.2685780e+00 -2.1873544e+01 -2.1848661e+01 +9.2775634e+00 -2.1867420e+01 -2.1842634e+01 +9.2865687e+00 -2.1861255e+01 -2.1836570e+01 +9.2955671e+00 -2.1855052e+01 -2.1830443e+01 +9.3045552e+00 -2.1848814e+01 -2.1824256e+01 +9.3135509e+00 -2.1842483e+01 -2.1818042e+01 +9.3225501e+00 -2.1836123e+01 -2.1811775e+01 +9.3315488e+00 -2.1829709e+01 -2.1805430e+01 +9.3405433e+00 -2.1823243e+01 -2.1799040e+01 +9.3495495e+00 -2.1816702e+01 -2.1792608e+01 +9.3585440e+00 -2.1810117e+01 -2.1786111e+01 +9.3675423e+00 -2.1803465e+01 -2.1779578e+01 +9.3765405e+00 -2.1796777e+01 -2.1772962e+01 +9.3855348e+00 -2.1790003e+01 -2.1766293e+01 +9.3945392e+00 -2.1783148e+01 -2.1759551e+01 +9.4035323e+00 -2.1776270e+01 -2.1752763e+01 +9.4125277e+00 -2.1769296e+01 -2.1745887e+01 +9.4215217e+00 -2.1762255e+01 -2.1738975e+01 +9.4305265e+00 -2.1755179e+01 -2.1731984e+01 +9.4395222e+00 -2.1747997e+01 -2.1724919e+01 +9.4485208e+00 -2.1740765e+01 -2.1717809e+01 +9.4575186e+00 -2.1733439e+01 -2.1710612e+01 +9.4665117e+00 -2.1726073e+01 -2.1703335e+01 +9.4755114e+00 -2.1718603e+01 -2.1695984e+01 +9.4845134e+00 -2.1711080e+01 -2.1688585e+01 +9.4934999e+00 -2.1703466e+01 -2.1681081e+01 +9.5024954e+00 -2.1695768e+01 -2.1673521e+01 +9.5114957e+00 -2.1687992e+01 -2.1665868e+01 +9.5204966e+00 -2.1680145e+01 -2.1658150e+01 +9.5294945e+00 -2.1672212e+01 -2.1650353e+01 +9.5384858e+00 -2.1664201e+01 -2.1642465e+01 +9.5474916e+00 -2.1656099e+01 -2.1634493e+01 +9.5564834e+00 -2.1647914e+01 -2.1626444e+01 +9.5654818e+00 -2.1639634e+01 -2.1618307e+01 +9.5744827e+00 -2.1631267e+01 -2.1610090e+01 +9.5834822e+00 -2.1622821e+01 -2.1601782e+01 +9.5924766e+00 -2.1614287e+01 -2.1593375e+01 +9.6014733e+00 -2.1605653e+01 -2.1584893e+01 +9.6104685e+00 -2.1596931e+01 -2.1576312e+01 +9.6194690e+00 -2.1588111e+01 -2.1567640e+01 +9.6284605e+00 -2.1579186e+01 -2.1558871e+01 +9.6374597e+00 -2.1570183e+01 -2.1550013e+01 +9.6464625e+00 -2.1561078e+01 -2.1541060e+01 +9.6554554e+00 -2.1551881e+01 -2.1532007e+01 +9.6644539e+00 -2.1542572e+01 -2.1522864e+01 +9.6734541e+00 -2.1533177e+01 -2.1513612e+01 +9.6824520e+00 -2.1523676e+01 -2.1504275e+01 +9.6914440e+00 -2.1514066e+01 -2.1494823e+01 +9.7004441e+00 -2.1504372e+01 -2.1485280e+01 +9.7094396e+00 -2.1494565e+01 -2.1475643e+01 +9.7184353e+00 -2.1484656e+01 -2.1465898e+01 +9.7274355e+00 -2.1474644e+01 -2.1456044e+01 +9.7364363e+00 -2.1464529e+01 -2.1446093e+01 +9.7454340e+00 -2.1454309e+01 -2.1436045e+01 +9.7544248e+00 -2.1443987e+01 -2.1425876e+01 +9.7634280e+00 -2.1433563e+01 -2.1415612e+01 +9.7724244e+00 -2.1423026e+01 -2.1405254e+01 +9.7814179e+00 -2.1412390e+01 -2.1394770e+01 +9.7904189e+00 -2.1401636e+01 -2.1384197e+01 +9.7994164e+00 -2.1390790e+01 -2.1373506e+01 +9.8084136e+00 -2.1379822e+01 -2.1362720e+01 +9.8174067e+00 -2.1368759e+01 -2.1351816e+01 +9.8264053e+00 -2.1357585e+01 -2.1340807e+01 +9.8354052e+00 -2.1346305e+01 -2.1329698e+01 +9.8444026e+00 -2.1334916e+01 -2.1318478e+01 +9.8534001e+00 -2.1323425e+01 -2.1307153e+01 +9.8623938e+00 -2.1311829e+01 -2.1295721e+01 +9.8713919e+00 -2.1300119e+01 -2.1284180e+01 +9.8803905e+00 -2.1288311e+01 -2.1272540e+01 +9.8893858e+00 -2.1276388e+01 -2.1260784e+01 +9.8983851e+00 -2.1264369e+01 -2.1248929e+01 +9.9073845e+00 -2.1252239e+01 -2.1236969e+01 +9.9163802e+00 -2.1240007e+01 -2.1224900e+01 +9.9253791e+00 -2.1227671e+01 -2.1212724e+01 +9.9343772e+00 -2.1215233e+01 -2.1200453e+01 +9.9433708e+00 -2.1202691e+01 -2.1188070e+01 +9.9523710e+00 -2.1190050e+01 -2.1175588e+01 +9.9613689e+00 -2.1177302e+01 -2.1162999e+01 +9.9703655e+00 -2.1164455e+01 -2.1150310e+01 +9.9793617e+00 -2.1151509e+01 -2.1137523e+01 +9.9883583e+00 -2.1138466e+01 -2.1124632e+01 diff --git a/inputs/orszag_tang.in b/inputs/orszag_tang.in new file mode 100644 index 00000000..b6eab07e --- /dev/null +++ b/inputs/orszag_tang.in @@ -0,0 +1,61 @@ +# AthenaPK - a performance portable block structured AMR MHD code +# Copyright (c) 2023, Athena Parthenon Collaboration. All rights reserved. +# Licensed under the BSD 3-Clause License (the "LICENSE"); + + +problem = Orszag-Tang vortex # Orszag,S. & Tang,W., J. Fluid Mech., 90, 129 (1998) + + +problem_id = orszag_tang + + +refinement = none +nghost = 3 + +nx1 = 256 +x1min = -0.5 +x1max = 0.5 +ix1_bc = periodic +ox1_bc = periodic + +nx2 = 256 +x2min = -0.5 +x2max = 0.5 +ix2_bc = periodic +ox2_bc = periodic + +nx3 = 1 +x3min = -0.5 +x3max = 0.5 +ix3_bc = periodic +ox3_bc = periodic + + +nx1 = 64 +nx2 = 256 +nx3 = 1 + + +integrator = vl2 +cfl = 0.4 +tlim = 1.0 +nlim = -1 +perf_cycle_offset = 2 # number of inital cycles not to be included in perf calc + + +fluid = glmmhd +eos = adiabatic +riemann = hlld +reconstruction = ppm +gamma = 1.666666666666667 # gamma = C_p/C_v +first_order_flux_correct = true + + +file_type = hdf5 +dt = 0.01 +id = prim +variables = prim + + +file_type = hst +dt = 0.1 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 7f60ecac..ab8556f7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -13,11 +13,13 @@ add_executable( hydro/hydro_driver.cpp hydro/hydro.cpp hydro/glmmhd/dedner_source.cpp + hydro/prolongation/custom_ops.hpp hydro/srcterms/gravitational_field.hpp hydro/srcterms/tabular_cooling.hpp hydro/srcterms/tabular_cooling.cpp refinement/gradient.cpp refinement/other.cpp + utils/few_modes_ft.cpp ) add_subdirectory(pgen) diff --git a/src/eos/adiabatic_glmmhd.cpp b/src/eos/adiabatic_glmmhd.cpp index 10e93877..24b8a155 100644 --- a/src/eos/adiabatic_glmmhd.cpp +++ b/src/eos/adiabatic_glmmhd.cpp @@ -37,16 +37,12 @@ void AdiabaticGLMMHDEOS::ConservedToPrimitive(MeshData *md) const { auto jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); auto kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - auto gam = GetGamma(); - auto gm1 = gam - 1.0; - auto density_floor_ = GetDensityFloor(); - auto pressure_floor_ = GetPressureFloor(); - auto e_floor_ = GetInternalEFloor(); - auto pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const auto nhydro = pkg->Param("nhydro"); const auto nscalars = pkg->Param("nscalars"); + auto this_on_device = (*this); + parthenon::par_for( DEFAULT_LOOP_PATTERN, "ConservedToPrimitive", parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, @@ -55,74 +51,6 @@ void AdiabaticGLMMHDEOS::ConservedToPrimitive(MeshData *md) const { auto &prim = prim_pack(b); // auto &nu = entropy_pack(b); - Real &u_d = cons(IDN, k, j, i); - Real &u_m1 = cons(IM1, k, j, i); - Real &u_m2 = cons(IM2, k, j, i); - Real &u_m3 = cons(IM3, k, j, i); - Real &u_e = cons(IEN, k, j, i); - Real &u_b1 = cons(IB1, k, j, i); - Real &u_b2 = cons(IB2, k, j, i); - Real &u_b3 = cons(IB3, k, j, i); - Real &u_psi = cons(IPS, k, j, i); - - Real &w_d = prim(IDN, k, j, i); - Real &w_vx = prim(IV1, k, j, i); - Real &w_vy = prim(IV2, k, j, i); - Real &w_vz = prim(IV3, k, j, i); - Real &w_p = prim(IPR, k, j, i); - Real &w_Bx = prim(IB1, k, j, i); - Real &w_By = prim(IB2, k, j, i); - Real &w_Bz = prim(IB3, k, j, i); - Real &w_psi = prim(IPS, k, j, i); - - // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) - // and the code will fail if a negative density is encountered. - PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, - "Got negative density. Consider enabling first-order flux " - "correction or setting a reasonble density floor."); - // apply density floor, without changing momentum or energy - u_d = (u_d > density_floor_) ? u_d : density_floor_; - w_d = u_d; - - Real di = 1.0 / u_d; - w_vx = u_m1 * di; - w_vy = u_m2 * di; - w_vz = u_m3 * di; - - w_Bx = u_b1; - w_By = u_b2; - w_Bz = u_b3; - w_psi = u_psi; - - Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); - Real e_B = 0.5 * (SQR(u_b1) + SQR(u_b2) + SQR(u_b3)); - w_p = gm1 * (u_e - e_k - e_B); - - // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) - // and the code will fail if a negative pressure is encountered. - PARTHENON_REQUIRE( - w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, - "Got negative pressure. Consider enabling first-order flux " - "correction or setting a reasonble pressure or temperature floor."); - - // Pressure floor (if present) takes precedence over temperature floor - if ((pressure_floor_ > 0.0) && (w_p < pressure_floor_)) { - // apply pressure floor, correct total energy - u_e = (pressure_floor_ / gm1) + e_k + e_B; - w_p = pressure_floor_; - } - - // temperature (internal energy) based pressure floor - const Real eff_pressure_floor = gm1 * u_d * e_floor_; - if (w_p < eff_pressure_floor) { - // apply temperature floor, correct total energy - u_e = (u_d * e_floor_) + e_k + e_B; - w_p = eff_pressure_floor; - } - - // Convert passive scalars - for (auto n = nhydro; n < nhydro + nscalars; ++n) { - prim(n, k, j, i) = cons(n, k, j, i) * di; - } + return this_on_device.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); }); } diff --git a/src/eos/adiabatic_glmmhd.hpp b/src/eos/adiabatic_glmmhd.hpp index 9add7160..baebc681 100644 --- a/src/eos/adiabatic_glmmhd.hpp +++ b/src/eos/adiabatic_glmmhd.hpp @@ -24,8 +24,10 @@ using parthenon::Real; class AdiabaticGLMMHDEOS : public EquationOfState { public: AdiabaticGLMMHDEOS(Real pressure_floor, Real density_floor, Real internal_e_floor, - Real gamma) - : EquationOfState(pressure_floor, density_floor, internal_e_floor), gamma_{gamma} {} + Real velocity_ceiling, Real internal_e_ceiling, Real gamma) + : EquationOfState(pressure_floor, density_floor, internal_e_floor, velocity_ceiling, + internal_e_ceiling), + gamma_{gamma} {} void ConservedToPrimitive(MeshData *md) const override; @@ -52,8 +54,120 @@ class AdiabaticGLMMHDEOS : public EquationOfState { } // + //---------------------------------------------------------------------------------------- + // \!fn Real EquationOfState::ConsToPrim(View4D cons, View4D prim, const int& k, const + // int& j, const int& i) \brief Fills an array of primitives given an array of + // conserveds, potentially updating the conserved with floors + template + KOKKOS_INLINE_FUNCTION void ConsToPrim(View4D cons, View4D prim, const int &nhydro, + const int &nscalars, const int &k, const int &j, + const int &i) const { + auto gam = GetGamma(); + auto gm1 = gam - 1.0; + auto density_floor_ = GetDensityFloor(); + auto pressure_floor_ = GetPressureFloor(); + auto e_floor_ = GetInternalEFloor(); + + auto velocity_ceiling_ = GetVelocityCeiling(); + auto e_ceiling_ = GetInternalECeiling(); + + Real &u_d = cons(IDN, k, j, i); + Real &u_m1 = cons(IM1, k, j, i); + Real &u_m2 = cons(IM2, k, j, i); + Real &u_m3 = cons(IM3, k, j, i); + Real &u_e = cons(IEN, k, j, i); + Real &u_b1 = cons(IB1, k, j, i); + Real &u_b2 = cons(IB2, k, j, i); + Real &u_b3 = cons(IB3, k, j, i); + Real &u_psi = cons(IPS, k, j, i); + + Real &w_d = prim(IDN, k, j, i); + Real &w_vx = prim(IV1, k, j, i); + Real &w_vy = prim(IV2, k, j, i); + Real &w_vz = prim(IV3, k, j, i); + Real &w_p = prim(IPR, k, j, i); + Real &w_Bx = prim(IB1, k, j, i); + Real &w_By = prim(IB2, k, j, i); + Real &w_Bz = prim(IB3, k, j, i); + Real &w_psi = prim(IPS, k, j, i); + + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) + // and the code will fail if a negative density is encountered. + PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, + "Got negative density. Consider enabling first-order flux " + "correction or setting a reasonble density floor."); + // apply density floor, without changing momentum or energy + u_d = (u_d > density_floor_) ? u_d : density_floor_; + w_d = u_d; + + Real di = 1.0 / u_d; + w_vx = u_m1 * di; + w_vy = u_m2 * di; + w_vz = u_m3 * di; + + w_Bx = u_b1; + w_By = u_b2; + w_Bz = u_b3; + w_psi = u_psi; + + Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); + Real e_B = 0.5 * (SQR(u_b1) + SQR(u_b2) + SQR(u_b3)); + w_p = gm1 * (u_e - e_k - e_B); + + // apply velocity ceiling. By default ceiling is std::numeric_limits::infinity() + const Real w_v2 = SQR(w_vx) + SQR(w_vy) + SQR(w_vz); + if (w_v2 > SQR(velocity_ceiling_)) { + const Real w_v = sqrt(w_v2); + w_vx *= velocity_ceiling_ / w_v; + w_vy *= velocity_ceiling_ / w_v; + w_vz *= velocity_ceiling_ / w_v; + + u_m1 *= velocity_ceiling_ / w_v; + u_m2 *= velocity_ceiling_ / w_v; + u_m3 *= velocity_ceiling_ / w_v; + + Real e_k_new = 0.5 * u_d * SQR(velocity_ceiling_); + u_e -= e_k - e_k_new; + e_k = e_k_new; + } + + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) + // and the code will fail if a negative pressure is encountered. + PARTHENON_REQUIRE(w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, + "Got negative pressure. Consider enabling first-order flux " + "correction or setting a reasonble pressure or temperature floor."); + + // Pressure floor (if present) takes precedence over temperature floor + if ((pressure_floor_ > 0.0) && (w_p < pressure_floor_)) { + // apply pressure floor, correct total energy + u_e = (pressure_floor_ / gm1) + e_k + e_B; + w_p = pressure_floor_; + } + + // temperature (internal energy) based pressure floor + const Real eff_pressure_floor = gm1 * u_d * e_floor_; + if (w_p < eff_pressure_floor) { + // apply temperature floor, correct total energy + u_e = (u_d * e_floor_) + e_k + e_B; + w_p = eff_pressure_floor; + } + + // temperature (internal energy) based pressure ceiling + const Real eff_pressure_ceiling = gm1 * u_d * e_ceiling_; + if (w_p > eff_pressure_ceiling) { + // apply temperature ceiling, correct total energy + u_e = (u_d * e_ceiling_) + e_k + e_B; + w_p = eff_pressure_ceiling; + } + + // Convert passive scalars + for (auto n = nhydro; n < nhydro + nscalars; ++n) { + prim(n, k, j, i) = cons(n, k, j, i) * di; + } + } + private: Real gamma_; // ratio of specific heats }; -#endif // EOS_ADIABATIC_GLMMHD_HPP_ \ No newline at end of file +#endif // EOS_ADIABATIC_GLMMHD_HPP_ diff --git a/src/eos/adiabatic_hydro.cpp b/src/eos/adiabatic_hydro.cpp index 91032d5d..20e53a75 100644 --- a/src/eos/adiabatic_hydro.cpp +++ b/src/eos/adiabatic_hydro.cpp @@ -36,75 +36,20 @@ void AdiabaticHydroEOS::ConservedToPrimitive(MeshData *md) const { auto ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); auto jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); auto kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); - Real gm1 = GetGamma() - 1.0; - auto density_floor_ = GetDensityFloor(); - auto pressure_floor_ = GetPressureFloor(); - auto e_floor_ = GetInternalEFloor(); auto pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const auto nhydro = pkg->Param("nhydro"); const auto nscalars = pkg->Param("nscalars"); + auto this_on_device = (*this); + parthenon::par_for( DEFAULT_LOOP_PATTERN, "ConservedToPrimitive", parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const auto &cons = cons_pack(b); auto &prim = prim_pack(b); - Real &u_d = cons(IDN, k, j, i); - Real &u_m1 = cons(IM1, k, j, i); - Real &u_m2 = cons(IM2, k, j, i); - Real &u_m3 = cons(IM3, k, j, i); - Real &u_e = cons(IEN, k, j, i); - - Real &w_d = prim(IDN, k, j, i); - Real &w_vx = prim(IV1, k, j, i); - Real &w_vy = prim(IV2, k, j, i); - Real &w_vz = prim(IV3, k, j, i); - Real &w_p = prim(IPR, k, j, i); - - // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) - // and the code will fail if a negative density is encountered. - PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, - "Got negative density. Consider enabling first-order flux " - "correction or setting a reasonble density floor."); - // apply density floor, without changing momentum or energy - u_d = (u_d > density_floor_) ? u_d : density_floor_; - w_d = u_d; - - Real di = 1.0 / u_d; - w_vx = u_m1 * di; - w_vy = u_m2 * di; - w_vz = u_m3 * di; - - Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); - w_p = gm1 * (u_e - e_k); - - // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) - // and the code will fail if a negative pressure is encountered. - PARTHENON_REQUIRE( - w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, - "Got negative pressure. Consider enabling first-order flux " - "correction or setting a reasonble pressure or temperature floor."); - - // Pressure floor (if present) takes precedence over temperature floor - if ((pressure_floor_ > 0.0) && (w_p < pressure_floor_)) { - // apply pressure floor, correct total energy - u_e = (pressure_floor_ / gm1) + e_k; - w_p = pressure_floor_; - } - - // temperature (internal energy) based pressure floor - const Real eff_pressure_floor = gm1 * u_d * e_floor_; - if (w_p < eff_pressure_floor) { - // apply temperature floor, correct total energy - u_e = (u_d * e_floor_) + e_k; - w_p = eff_pressure_floor; - } - // Convert passive scalars - for (auto n = nhydro; n < nhydro + nscalars; ++n) { - prim(n, k, j, i) = cons(n, k, j, i) * di; - } + return this_on_device.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); }); } diff --git a/src/eos/adiabatic_hydro.hpp b/src/eos/adiabatic_hydro.hpp index 75c2744f..bf317923 100644 --- a/src/eos/adiabatic_hydro.hpp +++ b/src/eos/adiabatic_hydro.hpp @@ -7,6 +7,7 @@ // C headers // C++ headers +#include #include // std::numeric_limits // Parthenon headers @@ -24,8 +25,10 @@ using parthenon::Real; class AdiabaticHydroEOS : public EquationOfState { public: AdiabaticHydroEOS(Real pressure_floor, Real density_floor, Real internal_e_floor, - Real gamma) - : EquationOfState(pressure_floor, density_floor, internal_e_floor), gamma_{gamma} {} + Real velocity_ceiling, Real internal_e_ceiling, Real gamma) + : EquationOfState(pressure_floor, density_floor, internal_e_floor, velocity_ceiling, + internal_e_ceiling), + gamma_{gamma} {} void ConservedToPrimitive(MeshData *md) const override; @@ -41,8 +44,105 @@ class AdiabaticHydroEOS : public EquationOfState { return std::sqrt(gamma_ * prim[IPR] / prim[IDN]); } + //---------------------------------------------------------------------------------------- + // \!fn Real EquationOfState::ConsToPrim(View4D cons, View4D prim, const int& k, const + // int& j, const int& i) \brief Fills an array of primitives given an array of + // conserveds, potentially updating the conserved with floors + template + KOKKOS_INLINE_FUNCTION void ConsToPrim(View4D cons, View4D prim, const int &nhydro, + const int &nscalars, const int &k, const int &j, + const int &i) const { + Real gm1 = GetGamma() - 1.0; + auto density_floor_ = GetDensityFloor(); + auto pressure_floor_ = GetPressureFloor(); + auto e_floor_ = GetInternalEFloor(); + + auto velocity_ceiling_ = GetVelocityCeiling(); + auto e_ceiling_ = GetInternalECeiling(); + + Real &u_d = cons(IDN, k, j, i); + Real &u_m1 = cons(IM1, k, j, i); + Real &u_m2 = cons(IM2, k, j, i); + Real &u_m3 = cons(IM3, k, j, i); + Real &u_e = cons(IEN, k, j, i); + + Real &w_d = prim(IDN, k, j, i); + Real &w_vx = prim(IV1, k, j, i); + Real &w_vy = prim(IV2, k, j, i); + Real &w_vz = prim(IV3, k, j, i); + Real &w_p = prim(IPR, k, j, i); + + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) + // and the code will fail if a negative density is encountered. + PARTHENON_REQUIRE(u_d > 0.0 || density_floor_ > 0.0, + "Got negative density. Consider enabling first-order flux " + "correction or setting a reasonble density floor."); + // apply density floor, without changing momentum or energy + u_d = (u_d > density_floor_) ? u_d : density_floor_; + w_d = u_d; + + Real di = 1.0 / u_d; + w_vx = u_m1 * di; + w_vy = u_m2 * di; + w_vz = u_m3 * di; + + Real e_k = 0.5 * di * (SQR(u_m1) + SQR(u_m2) + SQR(u_m3)); + w_p = gm1 * (u_e - e_k); + + // apply velocity ceiling. By default ceiling is std::numeric_limits::infinity() + const Real w_v2 = SQR(w_vx) + SQR(w_vy) + SQR(w_vz); + if (w_v2 > SQR(velocity_ceiling_)) { + const Real w_v = sqrt(w_v2); + w_vx *= velocity_ceiling_ / w_v; + w_vy *= velocity_ceiling_ / w_v; + w_vz *= velocity_ceiling_ / w_v; + + u_m1 *= velocity_ceiling_ / w_v; + u_m2 *= velocity_ceiling_ / w_v; + u_m3 *= velocity_ceiling_ / w_v; + + Real e_k_new = 0.5 * u_d * SQR(velocity_ceiling_); + u_e -= e_k - e_k_new; + e_k = e_k_new; + } + + // Let's apply floors explicitly, i.e., by default floor will be disabled (<=0) + // and the code will fail if a negative pressure is encountered. + PARTHENON_REQUIRE(w_p > 0.0 || pressure_floor_ > 0.0 || e_floor_ > 0.0, + "Got negative pressure. Consider enabling first-order flux " + "correction or setting a reasonble pressure or temperature floor."); + + // Pressure floor (if present) takes precedence over temperature floor + if ((pressure_floor_ > 0.0) && (w_p < pressure_floor_)) { + // apply pressure floor, correct total energy + u_e = (pressure_floor_ / gm1) + e_k; + w_p = pressure_floor_; + } + + // temperature (internal energy) based pressure floor + const Real eff_pressure_floor = gm1 * u_d * e_floor_; + if (w_p < eff_pressure_floor) { + // apply temperature floor, correct total energy + u_e = (u_d * e_floor_) + e_k; + w_p = eff_pressure_floor; + } + + // temperature (internal energy) based pressure ceiling + const Real eff_pressure_ceiling = gm1 * u_d * e_ceiling_; + if (w_p > eff_pressure_ceiling) { + // apply temperature ceiling, correct total energy + u_e = (u_d * e_ceiling_) + e_k; + w_p = eff_pressure_ceiling; + } + + // Convert passive scalars + for (auto n = nhydro; n < nhydro + nscalars; ++n) { + prim(n, k, j, i) = cons(n, k, j, i) * di; + } + } + private: Real gamma_; // ratio of specific heats }; -#endif // EOS_ADIABATIC_HYDRO_HPP_ \ No newline at end of file +#endif // EOS_ADIABATIC_HYDRO_HPP_ diff --git a/src/eos/eos.hpp b/src/eos/eos.hpp index b6986d7e..b75eedce 100644 --- a/src/eos/eos.hpp +++ b/src/eos/eos.hpp @@ -32,9 +32,11 @@ using parthenon::Real; class EquationOfState { public: - EquationOfState(Real pressure_floor, Real density_floor, Real internal_e_floor) + EquationOfState(Real pressure_floor, Real density_floor, Real internal_e_floor, + Real velocity_ceiling, Real internal_e_ceiling) : pressure_floor_(pressure_floor), density_floor_(density_floor), - internal_e_floor_(internal_e_floor) {} + internal_e_floor_(internal_e_floor), velocity_ceiling_(velocity_ceiling), + internal_e_ceiling_(internal_e_ceiling) {} virtual void ConservedToPrimitive(MeshData *md) const = 0; KOKKOS_INLINE_FUNCTION @@ -47,8 +49,15 @@ class EquationOfState { KOKKOS_INLINE_FUNCTION Real GetInternalEFloor() const { return internal_e_floor_; } + KOKKOS_INLINE_FUNCTION + Real GetVelocityCeiling() const { return velocity_ceiling_; } + + KOKKOS_INLINE_FUNCTION + Real GetInternalECeiling() const { return internal_e_ceiling_; } + private: Real pressure_floor_, density_floor_, internal_e_floor_; + Real velocity_ceiling_, internal_e_ceiling_; }; #endif // EOS_EOS_HPP_ diff --git a/src/hydro/hydro.cpp b/src/hydro/hydro.cpp index 6b6c5ee0..6d0e087c 100644 --- a/src/hydro/hydro.cpp +++ b/src/hydro/hydro.cpp @@ -30,6 +30,7 @@ #include "glmmhd/glmmhd.hpp" #include "hydro.hpp" #include "outputs/outputs.hpp" +#include "prolongation/custom_ops.hpp" #include "rsolvers/rsolvers.hpp" #include "srcterms/tabular_cooling.hpp" #include "utils/error_checking.hpp" @@ -102,13 +103,16 @@ Real HydroHst(MeshData *md) { divb += (cons(IB3, k + 1, j, i) - cons(IB3, k - 1, j, i)) / coords.Dxc<3>(k, j, i); } - lsum += 0.5 * - (std::sqrt(SQR(coords.Dxc<1>(k, j, i)) + SQR(coords.Dxc<2>(k, j, i)) + - SQR(coords.Dxc<3>(k, j, i)))) * - std::abs(divb) / - std::sqrt(SQR(cons(IB1, k, j, i)) + SQR(cons(IB2, k, j, i)) + - SQR(cons(IB3, k, j, i))) * - coords.CellVolume(k, j, i); + + Real abs_b = std::sqrt(SQR(cons(IB1, k, j, i)) + SQR(cons(IB2, k, j, i)) + + SQR(cons(IB3, k, j, i))); + + lsum += (abs_b != 0) ? 0.5 * + (std::sqrt(SQR(coords.Dxc<1>(k, j, i)) + + SQR(coords.Dxc<2>(k, j, i)) + + SQR(coords.Dxc<3>(k, j, i)))) * + std::abs(divb) / abs_b * coords.CellVolume(k, j, i) + : 0; // Add zero when abs_b ==0 } }, sum); @@ -139,6 +143,14 @@ TaskStatus AddUnsplitSources(MeshData *md, const SimTime &tm, const Real b if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { hydro_pkg->Param("glmmhd_source")(md, beta_dt); } + const auto &enable_cooling = hydro_pkg->Param("enable_cooling"); + + if (enable_cooling == Cooling::tabular) { + const TabularCooling &tabular_cooling = + hydro_pkg->Param("tabular_cooling"); + + tabular_cooling.SrcTerm(md, beta_dt); + } if (ProblemSourceUnsplit != nullptr) { ProblemSourceUnsplit(md, tm, beta_dt); } @@ -149,14 +161,6 @@ TaskStatus AddUnsplitSources(MeshData *md, const SimTime &tm, const Real b TaskStatus AddSplitSourcesFirstOrder(MeshData *md, const SimTime &tm) { auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - const auto &enable_cooling = hydro_pkg->Param("enable_cooling"); - - if (enable_cooling == Cooling::tabular) { - const TabularCooling &tabular_cooling = - hydro_pkg->Param("tabular_cooling"); - - tabular_cooling.SrcTerm(md, tm.dt); - } if (ProblemSourceFirstOrder != nullptr) { ProblemSourceFirstOrder(md, tm, tm.dt); } @@ -164,17 +168,6 @@ TaskStatus AddSplitSourcesFirstOrder(MeshData *md, const SimTime &tm) { } TaskStatus AddSplitSourcesStrang(MeshData *md, const SimTime &tm) { - // auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); - - // const auto &enable_cooling = hydro_pkg->Param("enable_cooling"); - - // if (enable_cooling == Cooling::tabular) { - // const TabularCooling &tabular_cooling = - // hydro_pkg->Param("tabular_cooling"); - - // tabular_cooling.SrcTerm(md, 0.5 * tm.dt); - // } - if (ProblemSourceStrangSplit != nullptr) { ProblemSourceStrangSplit(md, tm, tm.dt); } @@ -382,10 +375,6 @@ std::shared_ptr Initialize(ParameterInput *pin) { auto first_order_flux_correct = pin->GetOrAddBoolean("hydro", "first_order_flux_correct", false); - if (first_order_flux_correct && integrator != Integrator::vl2) { - PARTHENON_FAIL("Please use 'vl2' integrator with first order flux correction. Other " - "integrators have not been tested.") - } pkg->AddParam<>("first_order_flux_correct", first_order_flux_correct); if (first_order_flux_correct) { if (fluid == Fluid::euler) { @@ -411,7 +400,9 @@ std::shared_ptr Initialize(ParameterInput *pin) { auto units = pkg->Param("units"); const auto He_mass_fraction = pin->GetReal("hydro", "He_mass_fraction"); const auto mu = 1 / (He_mass_fraction * 3. / 4. + (1 - He_mass_fraction) * 2); + const auto mu_e = 1 / (He_mass_fraction * 2. / 4. + (1 - He_mass_fraction)); pkg->AddParam<>("mu", mu); + pkg->AddParam<>("mu_e", mu_e); pkg->AddParam<>("He_mass_fraction", He_mass_fraction); // Following convention in the astro community, we're using mh as unit for the mean // molecular weight @@ -434,6 +425,23 @@ std::shared_ptr Initialize(ParameterInput *pin) { efloor = Tfloor / mbar_over_kb / (gamma - 1.0); } + // By default disable ceilings by setting to infinity + Real vceil = + pin->GetOrAddReal("hydro", "vceil", std::numeric_limits::infinity()); + Real Tceil = + pin->GetOrAddReal("hydro", "Tceil", std::numeric_limits::infinity()); + Real eceil = Tceil; + if (eceil < std::numeric_limits::infinity()) { + if (!pkg->AllParams().hasKey("mbar_over_kb")) { + PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " + "Either set a 'units' block and the 'hydro/He_mass_fraction' in " + "input file or use a pressure floor " + "(defined code units) instead."); + } + auto mbar_over_kb = pkg->Param("mbar_over_kb"); + eceil = Tceil / mbar_over_kb / (gamma - 1.0); + } + auto conduction = Conduction::none; auto conduction_str = pin->GetOrAddString("diffusion", "conduction", "none"); if (conduction_str == "spitzer") { @@ -467,12 +475,12 @@ std::shared_ptr Initialize(ParameterInput *pin) { pkg->AddParam<>("conduction", conduction); if (fluid == Fluid::euler) { - AdiabaticHydroEOS eos(pfloor, dfloor, efloor, gamma); + AdiabaticHydroEOS eos(pfloor, dfloor, efloor, vceil, eceil, gamma); pkg->AddParam<>("eos", eos); pkg->FillDerivedMesh = ConsToPrim; pkg->EstimateTimestepMesh = EstimateTimestep; } else if (fluid == Fluid::glmmhd) { - AdiabaticGLMMHDEOS eos(pfloor, dfloor, efloor, gamma); + AdiabaticGLMMHDEOS eos(pfloor, dfloor, efloor, vceil, eceil, gamma); pkg->AddParam<>("eos", eos); pkg->FillDerivedMesh = ConsToPrim; pkg->EstimateTimestepMesh = EstimateTimestep; @@ -498,7 +506,7 @@ std::shared_ptr Initialize(ParameterInput *pin) { pkg->AddParam<>("enable_cooling", cooling); if (cooling == Cooling::tabular) { - TabularCooling tabular_cooling(pin); + TabularCooling tabular_cooling(pin, pkg); pkg->AddParam<>("tabular_cooling", tabular_cooling); } @@ -542,6 +550,8 @@ std::shared_ptr Initialize(ParameterInput *pin) { Metadata m( {Metadata::Cell, Metadata::Independent, Metadata::FillGhost, Metadata::WithFluxes}, std::vector({nhydro + nscalars}), cons_labels); + m.RegisterRefinementOps(); pkg->AddField("cons", m); m = Metadata({Metadata::Cell, Metadata::Derived}, std::vector({nhydro + nscalars}), @@ -956,7 +966,13 @@ TaskStatus CalculateFluxes(std::shared_ptr> &md) { // (multiple calls) versus extra memory usage is. template TaskStatus FirstOrderFluxCorrect(MeshData *u0_data, MeshData *u1_data, - const Real gam0, const Real gam1, const Real beta_dt) { + const Real gam0_, const Real gam1_, + const Real beta_dt_) { + // Work around for CUDA <=11.6 + const Real gam0 = gam0_; + const Real gam1 = gam1_; + const Real beta_dt = beta_dt_; + auto pmb = u0_data->GetBlockData(0)->GetBlockPointer(); IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); @@ -964,6 +980,7 @@ TaskStatus FirstOrderFluxCorrect(MeshData *u0_data, MeshData *u1_dat std::vector flags_ind({Metadata::Independent}); auto u0_cons_pack = u0_data->PackVariablesAndFluxes(flags_ind); + auto const &u0_prim_pack = u0_data->PackVariables(std::vector{"prim"}); auto u1_cons_pack = u1_data->PackVariablesAndFluxes(flags_ind); auto pkg = pmb->packages.Get("Hydro"); @@ -976,14 +993,6 @@ TaskStatus FirstOrderFluxCorrect(MeshData *u0_data, MeshData *u1_dat if (fluid == Fluid::glmmhd) { c_h = pkg->Param("c_h"); } - // Using "u1_prim" as "u0_prim" here because all current integrators start with copying - // the initial state to the "u0" register, see conditional for `stage == 1` in the - // hydro_driver where normally only "cons" is copied but in case for flux correction - // "prim", too. This means both during stage 1 and during stage 2 `u1` holds the - // original data at the beginning of the timestep. For flux correction we want to make a - // full (dt) low order update using the original data and thus use the "prim" data from - // u1 here. - auto const &u0_prim_pack = u1_data->PackVariables(std::vector{"prim"}); const int ndim = pmb->pmy_mesh->ndim; diff --git a/src/hydro/hydro_driver.cpp b/src/hydro/hydro_driver.cpp index 73a2e8e4..e33b1b0d 100644 --- a/src/hydro/hydro_driver.cpp +++ b/src/hydro/hydro_driver.cpp @@ -12,11 +12,13 @@ // Parthenon headers #include "amr_criteria/refinement_package.hpp" -#include "bvals/cc/bvals_cc_in_one.hpp" +#include "bvals/comms/bvals_in_one.hpp" #include "prolong_restrict/prolong_restrict.hpp" #include // AthenaPK headers #include "../eos/adiabatic_hydro.hpp" +#include "../pgen/cluster/agn_triggering.hpp" +#include "../pgen/cluster/magnetic_tower.hpp" #include "glmmhd/glmmhd.hpp" #include "hydro.hpp" #include "hydro_driver.hpp" @@ -86,6 +88,44 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { // required to be 1 or blocks.size() but could also only apply to a subset of blocks. auto num_task_lists_executed_independently = blocks.size(); + const int num_partitions = pmesh->DefaultNumPartitions(); + + // calculate agn triggering accretion rate + if ((stage == 1) && + hydro_pkg->AllParams().hasKey("agn_triggering_reduce_accretion_rate") && + hydro_pkg->Param("agn_triggering_reduce_accretion_rate")) { + + // need to make sure that there's only one region in order to MPI_reduce to work + TaskRegion &single_task_region = tc.AddRegion(1); + auto &tl = single_task_region[0]; + // First globally reset triggering quantities + auto prev_task = + tl.AddTask(none, cluster::AGNTriggeringResetTriggering, hydro_pkg.get()); + + // Adding one task for each partition. Given that they're all in one task list + // they'll be executed sequentially. Given that a par_reduce to a host var is + // blocking it's also save to store the variable in the Params for now. + for (int i = 0; i < num_partitions; i++) { + auto &mu0 = pmesh->mesh_data.GetOrAdd("base", i); + auto new_agn_triggering = + tl.AddTask(prev_task, cluster::AGNTriggeringReduceTriggering, mu0.get(), tm.dt); + prev_task = new_agn_triggering; + } +#ifdef MPI_PARALLEL + auto reduce_agn_triggering = + tl.AddTask(prev_task, cluster::AGNTriggeringMPIReduceTriggering, hydro_pkg.get()); + prev_task = reduce_agn_triggering; +#endif + + // Remove accreted gas + for (int i = 0; i < num_partitions; i++) { + auto &mu0 = pmesh->mesh_data.GetOrAdd("base", i); + auto new_remove_accreted_gas = + tl.AddTask(prev_task, cluster::AGNTriggeringFinalizeTriggering, mu0.get(), tm); + prev_task = new_remove_accreted_gas; + } + } + for (int i = 0; i < blocks.size(); i++) { auto &pmb = blocks[i]; // Using "base" as u0, which already exists (and returned by using plain Get()) @@ -99,8 +139,6 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { } } - const int num_partitions = pmesh->DefaultNumPartitions(); - // Calculate hyperbolic divergence cleaning speed // TODO(pgrete) Calculating mindx is only required after remeshing. Need to find a clean // solution for this one-off global reduction. @@ -150,6 +188,48 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { hydro_pkg.get()); } + // calculate magnetic tower scaling + if ((stage == 1) && hydro_pkg->AllParams().hasKey("magnetic_tower_power_scaling") && + hydro_pkg->Param("magnetic_tower_power_scaling")) { + const auto &magnetic_tower = + hydro_pkg->Param("magnetic_tower"); + + // need to make sure that there's only one region in order to MPI_reduce to work + TaskRegion &single_task_region = tc.AddRegion(1); + auto &tl = single_task_region[0]; + // First globally reset magnetic_tower_linear_contrib and + // magnetic_tower_quadratic_contrib + auto prev_task = + tl.AddTask(none, cluster::MagneticTowerResetPowerContribs, hydro_pkg.get()); + + // Adding one task for each partition. Given that they're all in one task list + // they'll be executed sequentially. Given that a par_reduce to a host var is + // blocking it's also save to store the variable in the Params for now. + for (int i = 0; i < num_partitions; i++) { + auto &mu0 = pmesh->mesh_data.GetOrAdd("base", i); + auto new_magnetic_tower_power_contrib = + tl.AddTask(prev_task, cluster::MagneticTowerReducePowerContribs, mu0.get(), tm); + prev_task = new_magnetic_tower_power_contrib; + } +#ifdef MPI_PARALLEL + auto reduce_magnetic_tower_power_contrib = tl.AddTask( + prev_task, + [](StateDescriptor *hydro_pkg) { + Real magnetic_tower_contribs[] = { + hydro_pkg->Param("magnetic_tower_linear_contrib"), + hydro_pkg->Param("magnetic_tower_quadratic_contrib")}; + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, magnetic_tower_contribs, 2, + MPI_PARTHENON_REAL, MPI_SUM, MPI_COMM_WORLD)); + hydro_pkg->UpdateParam("magnetic_tower_linear_contrib", + magnetic_tower_contribs[0]); + hydro_pkg->UpdateParam("magnetic_tower_quadratic_contrib", + magnetic_tower_contribs[1]); + return TaskStatus::complete; + }, + hydro_pkg.get()); +#endif + } + // First add split sources before the main time integration if (stage == 1) { TaskRegion &strang_init_region = tc.AddRegion(num_partitions); @@ -197,7 +277,7 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { auto &tl = single_tasklist_per_pack_region[i]; auto &mu0 = pmesh->mesh_data.GetOrAdd("base", i); auto &mu1 = pmesh->mesh_data.GetOrAdd("u1", i); - tl.AddTask(none, parthenon::cell_centered_bvars::StartReceiveFluxCorrections, mu0); + tl.AddTask(none, parthenon::StartReceiveFluxCorrections, mu0); const auto flux_str = (stage == 1) ? "flux_first_stage" : "flux_other_stage"; FluxFun_t *calc_flux_fun = hydro_pkg->Param(flux_str); @@ -217,13 +297,10 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { } auto send_flx = - tl.AddTask(first_order_flux_correct, - parthenon::cell_centered_bvars::LoadAndSendFluxCorrections, mu0); + tl.AddTask(first_order_flux_correct, parthenon::LoadAndSendFluxCorrections, mu0); auto recv_flx = - tl.AddTask(first_order_flux_correct, - parthenon::cell_centered_bvars::ReceiveFluxCorrections, mu0); - auto set_flx = - tl.AddTask(recv_flx, parthenon::cell_centered_bvars::SetFluxCorrections, mu0); + tl.AddTask(first_order_flux_correct, parthenon::ReceiveFluxCorrections, mu0); + auto set_flx = tl.AddTask(recv_flx, parthenon::SetFluxCorrections, mu0); // compute the divergence of fluxes of conserved variables auto update = tl.AddTask( @@ -259,8 +336,8 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { // TODO(someone) experiment with split (local/nonlocal) comms with respect to // performance for various tests (static, amr, block sizes) and then decide on the // best impl. Go with default call (split local/nonlocal) for now. - parthenon::cell_centered_bvars::AddBoundaryExchangeTasks(source_split_first_order, tl, - mu0, pmesh->multilevel); + parthenon::AddBoundaryExchangeTasks(source_split_first_order, tl, mu0, + pmesh->multilevel); } TaskRegion &async_region_3 = tc.AddRegion(num_task_lists_executed_independently); @@ -268,9 +345,11 @@ TaskCollection HydroDriver::MakeTaskCollection(BlockList_t &blocks, int stage) { auto &tl = async_region_3[i]; auto &u0 = blocks[i]->meshblock_data.Get("base"); auto prolongBound = none; - if (pmesh->multilevel) { - prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, u0); - } + // Currently taken care of by AddBoundaryExchangeTasks above. + // Needs to be reintroduced once we reintroduce split (local/nonlocal) communication. + // if (pmesh->multilevel) { + // prolongBound = tl.AddTask(none, parthenon::ProlongateBoundaries, u0); + //} // set physical boundaries auto set_bc = tl.AddTask(prolongBound, parthenon::ApplyBoundaryConditions, u0); diff --git a/src/hydro/prolongation/custom_ops.hpp b/src/hydro/prolongation/custom_ops.hpp new file mode 100644 index 00000000..02d2b15f --- /dev/null +++ b/src/hydro/prolongation/custom_ops.hpp @@ -0,0 +1,184 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2022, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the BSD 3-Clause License (the "LICENSE"). +//======================================================================================== +// Parthenon performance portable AMR framework +// Copyright(C) 2020-2022 The Parthenon collaboration +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +// (C) (or copyright) 2021. Triad National Security, LLC. All rights reserved. +// +// This program was produced under U.S. Government contract 89233218CNA000001 +// for Los Alamos National Laboratory (LANL), which is operated by Triad +// National Security, LLC for the U.S. Department of Energy/National Nuclear +// Security Administration. All rights in the program are reserved by Triad +// National Security, LLC, and the U.S. Department of Energy/National Nuclear +// Security Administration. The Government is granted for itself and others +// acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +// in this material to reproduce, prepare derivative works, distribute copies to +// the public, perform publicly and display publicly, and to permit others to do +// so. +//======================================================================================== + +#ifndef HYDRO_PROLONGATION_CUSTOM_OPS_HPP_ +#define HYDRO_PROLONGATION_CUSTOM_OPS_HPP_ + +#include +#include + +#include "basic_types.hpp" +#include "coordinates/coordinates.hpp" // for coordinates +#include "kokkos_abstraction.hpp" // ParArray +#include "mesh/domain.hpp" // for IndesShape +#include "prolong_restrict/pr_ops.hpp" + +namespace Hydro { +namespace refinement_ops { + +using parthenon::Coordinates_t; +using parthenon::ParArray6D; +using parthenon::TE; +using parthenon::TopologicalElement; + +// Multi-dimensional, limited prolongation: +// Multi-dim stencil corresponds to eq (5) in Stone et al. (2020). +// Limiting based on implementation in AMReX (see +// https://github.com/AMReX-Codes/amrex/blob/735c3513153f1d06f783e64f455816be85fb3602/Src/AmrCore/AMReX_MFInterp_3D_C.H#L89) +// to preserve extrema. +struct ProlongateCellMinModMultiD { + static constexpr bool OperationRequired(TopologicalElement fel, + TopologicalElement cel) { + return fel == cel; + } + template + KOKKOS_FORCEINLINE_FUNCTION static void + Do(const int l, const int m, const int n, const int k, const int j, const int i, + const IndexRange &ckb, const IndexRange &cjb, const IndexRange &cib, + const IndexRange &kb, const IndexRange &jb, const IndexRange &ib, + const Coordinates_t &coords, const Coordinates_t &coarse_coords, + const ParArrayND *pcoarse, + const ParArrayND *pfine) { + using namespace parthenon::refinement_ops::util; + auto &coarse = *pcoarse; + auto &fine = *pfine; + + constexpr int element_idx = static_cast(el) % 3; + + const int fi = (DIM > 0) ? (i - cib.s) * 2 + ib.s : ib.s; + const int fj = (DIM > 1) ? (j - cjb.s) * 2 + jb.s : jb.s; + const int fk = (DIM > 2) ? (k - ckb.s) * 2 + kb.s : kb.s; + + constexpr bool INCLUDE_X1 = + (DIM > 0) && (el == TE::CC || el == TE::F2 || el == TE::F3 || el == TE::E1); + constexpr bool INCLUDE_X2 = + (DIM > 1) && (el == TE::CC || el == TE::F3 || el == TE::F1 || el == TE::E2); + constexpr bool INCLUDE_X3 = + (DIM > 2) && (el == TE::CC || el == TE::F1 || el == TE::F2 || el == TE::E3); + + const Real fc = coarse(element_idx, l, m, n, k, j, i); + + Real dx1fm = 0; + Real dx1fp = 0; + Real gx1c = 0; + if constexpr (INCLUDE_X1) { + Real dx1m, dx1p; + GetGridSpacings<1, el>(coords, coarse_coords, cib, ib, i, fi, &dx1m, &dx1p, &dx1fm, + &dx1fp); + gx1c = GradMinMod(fc, coarse(element_idx, l, m, n, k, j, i - 1), + coarse(element_idx, l, m, n, k, j, i + 1), dx1m, dx1p); + } + + Real dx2fm = 0; + Real dx2fp = 0; + Real gx2c = 0; + if constexpr (INCLUDE_X2) { + Real dx2m, dx2p; + GetGridSpacings<2, el>(coords, coarse_coords, cjb, jb, j, fj, &dx2m, &dx2p, &dx2fm, + &dx2fp); + gx2c = GradMinMod(fc, coarse(element_idx, l, m, n, k, j - 1, i), + coarse(element_idx, l, m, n, k, j + 1, i), dx2m, dx2p); + } + Real dx3fm = 0; + Real dx3fp = 0; + Real gx3c = 0; + if constexpr (INCLUDE_X3) { + Real dx3m, dx3p; + GetGridSpacings<3, el>(coords, coarse_coords, ckb, kb, k, fk, &dx3m, &dx3p, &dx3fm, + &dx3fp); + gx3c = GradMinMod(fc, coarse(element_idx, l, m, n, k - 1, j, i), + coarse(element_idx, l, m, n, k + 1, j, i), dx3m, dx3p); + } + + // Max. expected total difference. (dx#fm/p are positive by construction) + Real dqmax = std::abs(gx1c) * std::max(dx1fm, dx1fp); + int jlim = 0; + int klim = 0; + if constexpr (DIM > 1) { + dqmax += std::abs(gx2c) * std::max(dx2fm, dx2fp); + jlim = 1; + } + if constexpr (DIM > 2) { + dqmax += std::abs(gx3c) * std::max(dx3fm, dx3fp); + klim = 1; + } + // Min/max values of all coarse cells used here + Real qmin = fc; + Real qmax = fc; + for (int koff = -klim; koff <= klim; koff++) { + for (int joff = -jlim; joff <= jlim; joff++) { + for (int ioff = -1; ioff <= 1; ioff++) { + qmin = + std::min(qmin, coarse(element_idx, l, m, n, k + koff, j + joff, i + ioff)); + qmax = + std::max(qmax, coarse(element_idx, l, m, n, k + koff, j + joff, i + ioff)); + } + } + } + + // Scaling factor to limit all slopes simultaneously + Real alpha = 1.0; + if (dqmax * alpha > (qmax - fc)) { + alpha = (qmax - fc) / dqmax; + } + if (dqmax * alpha > (fc - qmin)) { + alpha = (fc - qmin) / dqmax; + } + + // Ensure no new extrema are introduced in multi-D + gx1c *= alpha; + gx2c *= alpha; + gx3c *= alpha; + + // KGF: add the off-centered quantities first to preserve FP symmetry + // JMM: Extraneous quantities are zero + fine(element_idx, l, m, n, fk, fj, fi) = + fc - (gx1c * dx1fm + gx2c * dx2fm + gx3c * dx3fm); + if constexpr (INCLUDE_X1) + fine(element_idx, l, m, n, fk, fj, fi + 1) = + fc + (gx1c * dx1fp - gx2c * dx2fm - gx3c * dx3fm); + if constexpr (INCLUDE_X2) + fine(element_idx, l, m, n, fk, fj + 1, fi) = + fc - (gx1c * dx1fm - gx2c * dx2fp + gx3c * dx3fm); + if constexpr (INCLUDE_X2 && INCLUDE_X1) + fine(element_idx, l, m, n, fk, fj + 1, fi + 1) = + fc + (gx1c * dx1fp + gx2c * dx2fp - gx3c * dx3fm); + if constexpr (INCLUDE_X3) + fine(element_idx, l, m, n, fk + 1, fj, fi) = + fc - (gx1c * dx1fm + gx2c * dx2fm - gx3c * dx3fp); + if constexpr (INCLUDE_X3 && INCLUDE_X1) + fine(element_idx, l, m, n, fk + 1, fj, fi + 1) = + fc + (gx1c * dx1fp - gx2c * dx2fm + gx3c * dx3fp); + if constexpr (INCLUDE_X3 && INCLUDE_X2) + fine(element_idx, l, m, n, fk + 1, fj + 1, fi) = + fc - (gx1c * dx1fm - gx2c * dx2fp - gx3c * dx3fp); + if constexpr (INCLUDE_X3 && INCLUDE_X2 && INCLUDE_X1) + fine(element_idx, l, m, n, fk + 1, fj + 1, fi + 1) = + fc + (gx1c * dx1fp + gx2c * dx2fp + gx3c * dx3fp); + } +}; +} // namespace refinement_ops +} // namespace Hydro + +#endif // HYDRO_PROLONGATION_CUSTOM_OPS_HPP_ diff --git a/src/hydro/srcterms/gravitational_field.hpp b/src/hydro/srcterms/gravitational_field.hpp index 4e0edcdf..31feb1fe 100644 --- a/src/hydro/srcterms/gravitational_field.hpp +++ b/src/hydro/srcterms/gravitational_field.hpp @@ -1,6 +1,6 @@ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file gravitational_field.hpp @@ -53,13 +53,10 @@ void GravitationalFieldSrcTerm(parthenon::MeshData *md, // Apply g_r as a source term const Real den = prim(IDN, k, j, i); - const Real src = - (r == 0) ? 0 - : beta_dt * den * g_r / r; // FIXME watch out for previous /r errors + const Real src = (r == 0) ? 0 : beta_dt * den * g_r / r; cons(IM1, k, j, i) -= src * coords.Xc<1>(i); cons(IM2, k, j, i) -= src * coords.Xc<2>(j); cons(IM3, k, j, i) -= src * coords.Xc<3>(k); - // FIXME Double check this cons(IEN, k, j, i) -= src * (coords.Xc<1>(i) * prim(IV1, k, j, i) + coords.Xc<2>(j) * prim(IV2, k, j, i) + coords.Xc<3>(k) * prim(IV3, k, j, i)); diff --git a/src/hydro/srcterms/tabular_cooling.cpp b/src/hydro/srcterms/tabular_cooling.cpp index bbdc6c30..9164f4cc 100644 --- a/src/hydro/srcterms/tabular_cooling.cpp +++ b/src/hydro/srcterms/tabular_cooling.cpp @@ -9,6 +9,7 @@ //======================================================================================== // C++ headers +#include #include #include @@ -26,17 +27,9 @@ namespace cooling { using namespace parthenon; -TabularCooling::TabularCooling(ParameterInput *pin) { - Units units(pin); - - const Real He_mass_fraction = pin->GetReal("hydro", "He_mass_fraction"); - const Real H_mass_fraction = 1.0 - He_mass_fraction; - const Real mu = 1 / (He_mass_fraction * 3. / 4. + (1 - He_mass_fraction) * 2); - - gm1_ = pin->GetReal("hydro", "gamma") - 1.0; - - mu_mh_gm1_by_k_B_ = mu * units.mh() * gm1_ / units.k_boltzmann(); - X_by_mh_ = H_mass_fraction / units.mh(); +TabularCooling::TabularCooling(ParameterInput *pin, + std::shared_ptr hydro_pkg) { + auto units = hydro_pkg->Param("units"); const std::string table_filename = pin->GetString("cooling", "table_filename"); @@ -196,7 +189,10 @@ TabularCooling::TabularCooling(ParameterInput *pin) { lambda_final_ = std::pow(10.0, log_lambdas[n_temp_ - 1]); // Setup log_lambdas_ used in Dedt() - if ((integrator_ != CoolIntegrator::townsend) || (cooling_time_cfl_ > 0.0)) { + { + // log_lambdas is used if the integrator isn't Townsend, if the cooling CFL + // is set, or if cooling time is a extra derived field. Since we don't have + // a good way to check the last condition we always initialize log_lambdas_ log_lambdas_ = ParArray1D("log_lambdas_", n_temp_); // Read log_lambdas in host_log_lambdas, changing to code units along the way @@ -259,6 +255,15 @@ TabularCooling::TabularCooling(ParameterInput *pin) { Kokkos::deep_copy(townsend_alpha_k_, host_townsend_alpha_k); Kokkos::deep_copy(townsend_Y_k_, host_townsend_Y_k); } + + // Create a lightweight object for computing cooling rates within kernels + const auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + const auto adiabatic_index = hydro_pkg->Param("AdiabaticIndex"); + const auto He_mass_fraction = hydro_pkg->Param("He_mass_fraction"); + + cooling_table_obj_ = CoolingTableObj(log_lambdas_, log_temp_start_, log_temp_final_, + d_log_temp_, n_temp_, mbar_over_kb, + adiabatic_index, 1.0 - He_mass_fraction, units); } void TabularCooling::SrcTerm(MeshData *md, const Real dt) const { @@ -282,16 +287,10 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt const bool mhd_enabled = hydro_pkg->Param("fluid") == Fluid::glmmhd; // Grab member variables for compiler - // Everything needed by DeDt - const Real mu_mh_gm1_by_k_B = mu_mh_gm1_by_k_B_; - const Real X_by_mh = X_by_mh_; - const Real log_temp_start = log_temp_start_; - const Real log_temp_final = log_temp_final_; - const Real d_log_temp = d_log_temp_; - const unsigned int n_temp = n_temp_; - const auto log_lambdas = log_lambdas_; - - const Real gm1 = gm1_; + const CoolingTableObj cooling_table_obj = cooling_table_obj_; + const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; + const unsigned int max_iter = max_iter_; const Real min_sub_dt = dt / max_iter; @@ -303,7 +302,7 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - const Real internal_e_floor = temp_floor / mu_mh_gm1_by_k_B; // specific internal en. + const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. // Grab some necessary variables const auto &prim_pack = md->PackVariables(std::vector{"prim"}); @@ -336,14 +335,11 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt internal_e /= rho; const Real internal_e_initial = internal_e; - const Real n_h2_by_rho = rho * X_by_mh * X_by_mh; - bool dedt_valid = true; // Wrap DeDt into a functor for the RKStepper auto DeDt_wrapper = [&](const Real t, const Real e, bool &valid) { - return DeDt(e, mu_mh_gm1_by_k_B, n_h2_by_rho, log_temp_start, log_temp_final, - d_log_temp, n_temp, log_lambdas, valid); + return cooling_table_obj.DeDt(e, rho, valid); }; Real sub_t = 0; // current subcycle time @@ -394,11 +390,16 @@ void TabularCooling::SubcyclingFixedIntSrcTerm(MeshData *md, const Real dt if (!dedt_valid) { if (sub_dt == min_sub_dt) { - PARTHENON_FAIL("FATAL ERROR in [TabularCooling::SubcyclingSplitSrcTerm]: " - "Minumum sub_dt leads to negative internal energy"); + // Cooling is so fast that even the minimum subcycle dt would lead to + // negative internal energy -- so just cool to the floor of the cooling + // table + sub_dt = (dt - sub_t); + internal_e_next_h = internal_e_floor; + reattempt_sub = false; + } else { + reattempt_sub = true; + sub_dt = min_sub_dt; } - reattempt_sub = true; - sub_dt = min_sub_dt; } else { // Compute error @@ -483,17 +484,19 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, // Grab member variables for compiler const auto dt = dt_; // HACK capturing parameters still broken with Cuda 11.6 ... - const auto mu_mh_gm1_by_k_B = mu_mh_gm1_by_k_B_; - const auto X_by_mh = X_by_mh_; + + const auto units = hydro_pkg->Param("units"); + const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; + const Real X_by_mh2 = + std::pow((1 - hydro_pkg->Param("He_mass_fraction")) / units.mh(), 2); const auto lambdas = lambdas_; const auto temps = temps_; const auto alpha_k = townsend_alpha_k_; const auto Y_k = townsend_Y_k_; - const auto gm1 = gm1_; - - const auto internal_e_floor = T_floor_ / mu_mh_gm1_by_k_B; + const auto internal_e_floor = T_floor_ / mbar_gm1_over_kb; const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table // Grab some necessary variables @@ -542,13 +545,13 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, return; } - auto temp = mu_mh_gm1_by_k_B * internal_e; + auto temp = mbar_gm1_over_kb * internal_e; // Temperature is above floor (see conditional above) but below cooling table: // -> no cooling if (temp < temp_cool_floor) { return; } - const Real n_h2_by_rho = rho * X_by_mh * X_by_mh; + const Real n_h2_by_rho = rho * X_by_mh2; // Get the index of the right temperature bin // TODO(?) this could be optimized for using a binary search @@ -565,7 +568,7 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, // Compute the adjusted TEF for new timestep (Eqn. 26) (term in brackets) const auto tef_adj = - tef + lambda_final * dt / temp_final * mu_mh_gm1_by_k_B * n_h2_by_rho; + tef + lambda_final * dt / temp_final * mbar_gm1_over_kb * n_h2_by_rho; // TEF is a strictly decreasing function and new_tef > tef // Check if the new TEF falls into a lower bin, i.e., find the right bin for A7 @@ -582,8 +585,8 @@ void TabularCooling::TownsendSrcTerm(parthenon::MeshData *md, 1.0 / (1.0 - alpha_k(idx))); // Set new temp (at the lowest to the lower end of the cooling table) const auto internal_e_new = temp_new > temp_cool_floor - ? temp_new / mu_mh_gm1_by_k_B - : temp_cool_floor / mu_mh_gm1_by_k_B; + ? temp_new / mbar_gm1_over_kb + : temp_cool_floor / mbar_gm1_over_kb; cons(IEN, k, j, i) += rho * (internal_e_new - internal_e); // Latter technically not required if no other tasks follows before // ConservedToPrim conversion, but keeping it for now (better safe than sorry). @@ -595,25 +598,23 @@ Real TabularCooling::EstimateTimeStep(MeshData *md) const { if (cooling_time_cfl_ <= 0.0) { return std::numeric_limits::max(); } - // Grab member variables for compiler - // Everything needed by DeDt - const Real mu_mh_gm1_by_k_B = mu_mh_gm1_by_k_B_; - const Real X_by_mh = X_by_mh_; - const Real log_temp_start = log_temp_start_; - const Real log_temp_final = log_temp_final_; - const Real d_log_temp = d_log_temp_; - const unsigned int n_temp = n_temp_; - const auto log_lambdas = log_lambdas_; + if (isnan(cooling_time_cfl_) || isinf(cooling_time_cfl_)) { + return std::numeric_limits::infinity(); + } - const Real gm1 = gm1_; + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const CoolingTableObj cooling_table_obj = cooling_table_obj_; + const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + const auto mbar_gm1_over_kb = hydro_pkg->Param("mbar_over_kb") * gm1; // Determine the cooling floor, whichever is higher of the cooling table floor // or fluid solver floor const auto temp_cool_floor = std::pow(10.0, log_temp_start_); // low end of cool table const Real temp_floor = (T_floor_ > temp_cool_floor) ? T_floor_ : temp_cool_floor; - const Real internal_e_floor = temp_floor / mu_mh_gm1_by_k_B; // specific internal en. + const Real internal_e_floor = temp_floor / mbar_gm1_over_kb; // specific internal en. // Grab some necessary variables const auto &prim_pack = md->PackVariables(std::vector{"prim"}); @@ -635,15 +636,10 @@ Real TabularCooling::EstimateTimeStep(MeshData *md) const { const Real rho = prim(IDN, k, j, i); const Real pres = prim(IPR, k, j, i); - const Real n_h2_by_rho = rho * X_by_mh * X_by_mh; const Real internal_e = pres / (rho * gm1); - bool dedt_valid = true; - - const Real de_dt = - DeDt(internal_e, mu_mh_gm1_by_k_B, n_h2_by_rho, log_temp_start, - log_temp_final, d_log_temp, n_temp, log_lambdas, dedt_valid); + const Real de_dt = cooling_table_obj.DeDt(internal_e, rho); // Compute cooling time // If de_dt is zero (temperature is smaller than lower end of cooling table) or @@ -674,15 +670,8 @@ void TabularCooling::TestCoolingTable(ParameterInput *pin) const { // Grab member variables for compiler // Everything needed by DeDt - const auto mu_mh_gm1_by_k_B = mu_mh_gm1_by_k_B_; - const auto X_by_mh = X_by_mh_; - const auto log_temp_start = log_temp_start_; - const auto log_temp_final = log_temp_final_; - const auto d_log_temp = d_log_temp_; - const unsigned int n_temp = n_temp_; - const auto log_lambdas = log_lambdas_; - - const Real gm1 = gm1_; + const CoolingTableObj cooling_table_obj = cooling_table_obj_; + const auto gm1 = pin->GetReal("hydro", "gamma") - 1.0; // Make some device arrays to store the test data ParArray2D d_rho("d_rho", n_rho, n_pres), d_pres("d_pres", n_rho, n_pres), @@ -697,17 +686,11 @@ void TabularCooling::TestCoolingTable(ParameterInput *pin) const { d_rho(j, i) = rho; d_pres(j, i) = pres; - const Real n_h2_by_rho = rho * X_by_mh * X_by_mh; - const Real internal_e = pres / (rho * gm1); d_internal_e(j, i) = internal_e; - bool dedt_valid = true; - - const Real de_dt = - DeDt(internal_e, mu_mh_gm1_by_k_B, n_h2_by_rho, log_temp_start, - log_temp_final, d_log_temp, n_temp, log_lambdas, dedt_valid); + const Real de_dt = cooling_table_obj.DeDt(internal_e, rho); d_de_dt(j, i) = de_dt; }); diff --git a/src/hydro/srcterms/tabular_cooling.hpp b/src/hydro/srcterms/tabular_cooling.hpp index dd0efeef..c1ffa8b2 100644 --- a/src/hydro/srcterms/tabular_cooling.hpp +++ b/src/hydro/srcterms/tabular_cooling.hpp @@ -25,6 +25,7 @@ // AthenaPK headers #include "../../main.hpp" +#include "../../units.hpp" #ifdef MPI_PARALLEL #include @@ -86,6 +87,98 @@ struct RK45Stepper { enum class CoolIntegrator { undefined, rk12, rk45, townsend }; +class CoolingTableObj { + /************************************************************ + * Cooling Table Object, for interpolating a cooling rate out of a cooling + * table. Currently assumes evenly space log_temperatures in cooling table + * + * Lightweight object intended for inlined computation within kernels + ************************************************************/ + private: + // Log cooling rate/ne^3 + parthenon::ParArray1D log_lambdas_; + + // Spacing of cooling table + // TODO: assumes evenly spaced cooling table + parthenon::Real log_temp_start_, log_temp_final_, d_log_temp_; + unsigned int n_temp_; + + // Mean molecular mass * ( adiabatic_index -1) / boltzmann_constant + parthenon::Real mbar_gm1_over_k_B_; + + // (Hydrogen mass fraction / hydrogen atomic mass)^2 + parthenon::Real x_H_over_m_h2_; + + public: + CoolingTableObj() + : log_lambdas_(), log_temp_start_(NAN), log_temp_final_(NAN), d_log_temp_(NAN), + n_temp_(0), mbar_gm1_over_k_B_(NAN), x_H_over_m_h2_(NAN) {} + CoolingTableObj(const parthenon::ParArray1D log_lambdas, + const parthenon::Real log_temp_start, + const parthenon::Real log_temp_final, const parthenon::Real d_log_temp, + const unsigned int n_temp, const parthenon::Real mbar_over_kb, + const parthenon::Real adiabatic_index, const parthenon::Real x_H, + const Units units) + : log_lambdas_(log_lambdas), log_temp_start_(log_temp_start), + log_temp_final_(log_temp_final), d_log_temp_(d_log_temp), n_temp_(n_temp), + mbar_gm1_over_k_B_(mbar_over_kb * (adiabatic_index - 1)), + x_H_over_m_h2_(SQR(x_H / units.mh())) {} + + // Interpolate a cooling rate from the table + // from internal energy density and density + KOKKOS_INLINE_FUNCTION parthenon::Real + DeDt(const parthenon::Real &e, const parthenon::Real &rho, bool &is_valid) const { + using namespace parthenon; + + if (e < 0 || std::isnan(e)) { + is_valid = false; + return 0; + } + + const Real temp = mbar_gm1_over_k_B_ * e; + const Real log_temp = log10(temp); + Real log_lambda; + if (log_temp < log_temp_start_) { + return 0; + } else if (log_temp > log_temp_final_) { + // Above table + // Return de/dt + // TODO(forrestglines):Currently free-free cooling is used for + // temperatures above the table. This behavior could be generalized via + // templates + log_lambda = 0.5 * log_temp - 0.5 * log_temp_final_ + log_lambdas_(n_temp_ - 1); + } else { + // Inside table, interpolate assuming log spaced temperatures + + // Determine where temp is in the table + const unsigned int i_temp = + static_cast((log_temp - log_temp_start_) / d_log_temp_); + const Real log_temp_i = log_temp_start_ + d_log_temp_ * i_temp; + + // log_temp should be between log_temps[i_temp] and log_temps[i_temp+1] + PARTHENON_REQUIRE(log_temp >= log_temp_i && log_temp <= log_temp_i + d_log_temp_, + "FATAL ERROR in [CoolingTable::DeDt]: Failed to find log_temp"); + + const Real log_lambda_i = log_lambdas_(i_temp); + const Real log_lambda_ip1 = log_lambdas_(i_temp + 1); + + // Linearly interpolate lambda at log_temp + log_lambda = log_lambda_i + (log_temp - log_temp_i) * + (log_lambda_ip1 - log_lambda_i) / d_log_temp_; + } + // Return de/dt + const Real lambda = pow(10., log_lambda); + const Real de_dt = -lambda * x_H_over_m_h2_ * rho; + return de_dt; + } + + KOKKOS_INLINE_FUNCTION parthenon::Real DeDt(const parthenon::Real &e, + const parthenon::Real &rho) const { + bool is_valid = true; + return DeDt(e, rho, is_valid); + } +}; + class TabularCooling { private: // Defines uniformly spaced log temperature range of the table @@ -105,17 +198,12 @@ class TabularCooling { // Townsend cooling power law indices parthenon::ParArray1D townsend_alpha_k_; - // Some constants - // mean_molecular_mass*mh*(adiabatic_index-1)/k_B - parthenon::Real mu_mh_gm1_by_k_B_; - // H_mass_fraction/mh - parthenon::Real X_by_mh_; - // adiabatic_index -1 - parthenon::Real gm1_; - CoolIntegrator integrator_; - // Temperature floor of the fluid solver (assumed in Kelvin) + // Temperature floor (assumed in Kelvin and only used in cooling function) + // This is either the temperature floor used by the hydro method or the + // lowest temperature in the cooling table (assuming zero cooling below the + // table), whichever temperature is higher parthenon::Real T_floor_; // Maximum number of iterations/subcycles @@ -124,68 +212,21 @@ class TabularCooling { // Cooling CFL parthenon::Real cooling_time_cfl_; + // Minimum timestep that the cooling may limit the simulation timestep + // Use nonpositive values to disable + parthenon::Real min_cooling_timestep_; + // Tolerances parthenon::Real d_log_temp_tol_, d_e_tol_; // Used for roundoff as subcycle approaches end of timestep static constexpr parthenon::Real KEpsilon_ = 1e-12; - // Interpolate a cooling rate from the table - static KOKKOS_INLINE_FUNCTION parthenon::Real - DeDt(const parthenon::Real &e, const parthenon::Real &mu_mh_gm1_by_k_B, - const parthenon::Real &n_h2_by_rho, const parthenon::Real &log_temp_start, - const parthenon::Real &log_temp_final, const parthenon::Real &d_log_temp, - const unsigned int n_temp, - const parthenon::ParArray1D &log_lambdas, bool &valid) { - using namespace parthenon; - - if (e < 0 || std::isnan(e)) { - valid = false; - return 0; - } - - const Real temp = mu_mh_gm1_by_k_B * e; - const Real log_temp = log10(temp); - Real log_lambda; - if (log_temp < log_temp_start) { - // Below table - return 0 or first entry? - // log_lambda = log_lambdas(0); - // TODO(forrestglines):Currently, no cooling is implemented above the - // table. This behavior could be generalized via templates - return 0; - } else if (log_temp > log_temp_final) { - // Above table - // Return de/dt - // TODO(forrestglines):Currently free-free cooling is implemented above the - // table. This behavior could be generalized via templates - log_lambda = 0.5 * log_temp - 0.5 * log_temp_final + log_lambdas(n_temp - 1); - } else { - // Inside table - - // Determine where temp is in the table - const unsigned int i_temp = - static_cast((log_temp - log_temp_start) / d_log_temp); - const Real log_temp_i = log_temp_start + d_log_temp * i_temp; - - // log_temp should be between log_temps[i_temp] and log_temps[i_temp+1] - if (log_temp < log_temp_i || log_temp > log_temp_i + d_log_temp) { - // FIXME exception - } - - const Real log_lambda_i = log_lambdas(i_temp); - const Real log_lambda_ip1 = log_lambdas(i_temp + 1); - - // Linearly interpolate lambda at log_temp - log_lambda = log_lambda_i + - (log_temp - log_temp_i) * (log_lambda_ip1 - log_lambda_i) / d_log_temp; - } - // Return de/dt - const Real lambda = pow(10., log_lambda); - return -lambda * n_h2_by_rho; - } + CoolingTableObj cooling_table_obj_; public: - TabularCooling(parthenon::ParameterInput *pin); + TabularCooling(parthenon::ParameterInput *pin, + std::shared_ptr hydro_pkg); void SrcTerm(parthenon::MeshData *md, const parthenon::Real dt) const; @@ -201,6 +242,9 @@ class TabularCooling { parthenon::Real EstimateTimeStep(parthenon::MeshData *md) const; + // Get a lightweight object for computing cooling rate from the cooling table + const CoolingTableObj GetCoolingTableObj() const { return cooling_table_obj_; } + void TestCoolingTable(parthenon::ParameterInput *pin) const; }; diff --git a/src/main.cpp b/src/main.cpp index 8cadf7c0..87d82021 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -85,8 +85,12 @@ int main(int argc, char *argv[]) { Hydro::ProblemInitPackageData = rand_blast::ProblemInitPackageData; Hydro::ProblemSourceFirstOrder = rand_blast::RandomBlasts; } else if (problem == "cluster") { - pman.app_input->ProblemGenerator = cluster::ProblemGenerator; - Hydro::ProblemSourceUnsplit = cluster::ClusterSrcTerm; + pman.app_input->MeshProblemGenerator = cluster::ProblemGenerator; + pman.app_input->MeshBlockUserWorkBeforeOutput = cluster::UserWorkBeforeOutput; + Hydro::ProblemInitPackageData = cluster::ProblemInitPackageData; + Hydro::ProblemSourceUnsplit = cluster::ClusterUnsplitSrcTerm; + Hydro::ProblemSourceFirstOrder = cluster::ClusterSplitSrcTerm; + Hydro::ProblemEstimateTimestep = cluster::ClusterEstimateTimestep; } else if (problem == "sod") { pman.app_input->ProblemGenerator = sod::ProblemGenerator; } else if (problem == "turbulence") { @@ -116,10 +120,6 @@ int main(int argc, char *argv[]) { // This line actually runs the simulation driver.Execute(); } - // very ugly cleanup... - if (problem == "turbulence") { - turbulence::Cleanup(); - } // call MPI_Finalize and Kokkos::finalize if necessary pman.ParthenonFinalize(); diff --git a/src/main.hpp b/src/main.hpp index b73efbf8..bbcd0786 100644 --- a/src/main.hpp +++ b/src/main.hpp @@ -39,6 +39,8 @@ enum class Conduction { none, spitzer, thermal_diff }; enum class Hst { idx, ekin, emag, divb }; +enum class CartesianDir { x, y, z }; + constexpr parthenon::Real float_min{std::numeric_limits::min()}; #endif // MAIN_HPP_ diff --git a/src/pgen/CMakeLists.txt b/src/pgen/CMakeLists.txt index 9bca96a8..382b5b7b 100644 --- a/src/pgen/CMakeLists.txt +++ b/src/pgen/CMakeLists.txt @@ -6,7 +6,14 @@ target_sources(athenaPK PRIVATE blast.cpp cloud.cpp cluster.cpp + cluster/agn_feedback.cpp + cluster/agn_triggering.cpp + cluster/cluster_clips.cpp + cluster/cluster_reductions.cpp cluster/hydrostatic_equilibrium_sphere.cpp + cluster/magnetic_tower.cpp + cluster/snia_feedback.cpp + cluster/stellar_feedback.cpp cpaw.cpp diffusion.cpp field_loop.cpp diff --git a/src/pgen/cloud.cpp b/src/pgen/cloud.cpp index ed39b2c2..bc9fed25 100644 --- a/src/pgen/cloud.cpp +++ b/src/pgen/cloud.cpp @@ -14,6 +14,7 @@ // Parthenon headers #include "mesh/mesh.hpp" +#include #include #include #include @@ -222,8 +223,8 @@ void InflowWindX2(std::shared_ptr> &mbd, bool coarse) { const auto Bx_ = Bx; const auto By_ = By; pmb->par_for_bndry( - "InflowWindX2", nb, IndexDomain::inner_x2, coarse, - KOKKOS_LAMBDA(const int, const int &k, const int &j, const int &i) { + "InflowWindX2", nb, IndexDomain::inner_x2, parthenon::TopologicalElement::CC, + coarse, KOKKOS_LAMBDA(const int, const int &k, const int &j, const int &i) { cons(IDN, k, j, i) = rho_wind_; cons(IM2, k, j, i) = mom_wind_; cons(IEN, k, j, i) = rhoe_wind_ + 0.5 * mom_wind_ * mom_wind_ / rho_wind_; diff --git a/src/pgen/cluster.cpp b/src/pgen/cluster.cpp index 5102e32b..a2b10f7e 100644 --- a/src/pgen/cluster.cpp +++ b/src/pgen/cluster.cpp @@ -1,6 +1,6 @@ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file cluster.cpp @@ -8,8 +8,8 @@ // // Setups up an idealized galaxy cluster with an ACCEPT-like entropy profile in // hydrostatic equilbrium with an NFW+BCG+SMBH gravitational profile, -// optionally with an initial magnetic tower field. Includes tabular cooling, -// AGN feedback, AGN triggering via cold gas, simple SNIA Feedback +// optionally with an initial magnetic tower field. Includes AGN feedback, AGN +// triggering via cold gas, simple SNIA Feedback, and simple stellar feedback //======================================================================================== // C headers @@ -25,25 +25,42 @@ #include // c_str() // Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" #include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" #include #include // AthenaPK headers +#include "../eos/adiabatic_glmmhd.hpp" +#include "../eos/adiabatic_hydro.hpp" #include "../hydro/hydro.hpp" #include "../hydro/srcterms/gravitational_field.hpp" +#include "../hydro/srcterms/tabular_cooling.hpp" #include "../main.hpp" +#include "../utils/few_modes_ft.hpp" // Cluster headers +#include "cluster/agn_feedback.hpp" +#include "cluster/agn_triggering.hpp" +#include "cluster/cluster_clips.hpp" #include "cluster/cluster_gravity.hpp" +#include "cluster/cluster_reductions.hpp" #include "cluster/entropy_profiles.hpp" #include "cluster/hydrostatic_equilibrium_sphere.hpp" +#include "cluster/magnetic_tower.hpp" +#include "cluster/snia_feedback.hpp" +#include "cluster/stellar_feedback.hpp" namespace cluster { using namespace parthenon::driver::prelude; using namespace parthenon::package::prelude; +using utils::few_modes_ft::FewModesFT; -void ClusterSrcTerm(MeshData *md, const parthenon::SimTime, const Real beta_dt) { +void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt) { auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); const bool &gravity_srcterm = hydro_pkg->Param("gravity_srcterm"); @@ -54,149 +71,840 @@ void ClusterSrcTerm(MeshData *md, const parthenon::SimTime, const Real bet GravitationalFieldSrcTerm(md, beta_dt, cluster_gravity); } + + const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); + agn_feedback.FeedbackSrcTerm(md, beta_dt, tm); + + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + magnetic_tower.FixedFieldSrcTerm(md, beta_dt, tm); + + const auto &snia_feedback = hydro_pkg->Param("snia_feedback"); + snia_feedback.FeedbackSrcTerm(md, beta_dt, tm); +}; +void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real dt) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &stellar_feedback = hydro_pkg->Param("stellar_feedback"); + stellar_feedback.FeedbackSrcTerm(md, dt, tm); + + ApplyClusterClips(md, tm, dt); +} + +Real ClusterEstimateTimestep(MeshData *md) { + Real min_dt = std::numeric_limits::max(); + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // TODO time constraints imposed by thermal AGN feedback, jet velocity, + // magnetic tower + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + const Real agn_triggering_min_dt = agn_triggering.EstimateTimeStep(md); + min_dt = std::min(min_dt, agn_triggering_min_dt); + + return min_dt; } //======================================================================================== -//! \fn void InitUserMeshData(Mesh *mesh, ParameterInput *pin) -// \brief Function to initialize problem-specific data in mesh class. Can also be used -// to initialize variables which are global to (and therefore can be passed to) other -// functions in this file. Called in Mesh constructor. +//! \fn void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor +//! *hydro_pkg) \brief Init package data from parameter input //======================================================================================== -void ProblemGenerator(MeshBlock *pmb, parthenon::ParameterInput *pin) { - auto hydro_pkg = pmb->packages.Get("Hydro"); - if (pmb->lid == 0) { - /************************************************************ - * Read Uniform Gas - ************************************************************/ +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) { - const bool init_uniform_gas = - pin->GetOrAddBoolean("problem/cluster", "init_uniform_gas", false); - hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); + /************************************************************ + * Read Uniform Gas + ************************************************************/ - if (init_uniform_gas) { - const Real uniform_gas_rho = pin->GetReal("problem/cluster", "uniform_gas_rho"); - const Real uniform_gas_ux = pin->GetReal("problem/cluster", "uniform_gas_ux"); - const Real uniform_gas_uy = pin->GetReal("problem/cluster", "uniform_gas_uy"); - const Real uniform_gas_uz = pin->GetReal("problem/cluster", "uniform_gas_uz"); - const Real uniform_gas_pres = pin->GetReal("problem/cluster", "uniform_gas_pres"); - - hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); - hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); - hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); - hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); - hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); + const bool init_uniform_gas = + pin->GetOrAddBoolean("problem/cluster/uniform_gas", "init_uniform_gas", false); + hydro_pkg->AddParam<>("init_uniform_gas", init_uniform_gas); + + if (init_uniform_gas) { + const Real uniform_gas_rho = pin->GetReal("problem/cluster/uniform_gas", "rho"); + const Real uniform_gas_ux = pin->GetReal("problem/cluster/uniform_gas", "ux"); + const Real uniform_gas_uy = pin->GetReal("problem/cluster/uniform_gas", "uy"); + const Real uniform_gas_uz = pin->GetReal("problem/cluster/uniform_gas", "uz"); + const Real uniform_gas_pres = pin->GetReal("problem/cluster/uniform_gas", "pres"); + + hydro_pkg->AddParam<>("uniform_gas_rho", uniform_gas_rho); + hydro_pkg->AddParam<>("uniform_gas_ux", uniform_gas_ux); + hydro_pkg->AddParam<>("uniform_gas_uy", uniform_gas_uy); + hydro_pkg->AddParam<>("uniform_gas_uz", uniform_gas_uz); + hydro_pkg->AddParam<>("uniform_gas_pres", uniform_gas_pres); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_uniform_b_field = pin->GetOrAddBoolean( + "problem/cluster/uniform_b_field", "init_uniform_b_field", false); + hydro_pkg->AddParam<>("init_uniform_b_field", init_uniform_b_field); + + if (init_uniform_b_field) { + const Real uniform_b_field_bx = pin->GetReal("problem/cluster/uniform_b_field", "bx"); + const Real uniform_b_field_by = pin->GetReal("problem/cluster/uniform_b_field", "by"); + const Real uniform_b_field_bz = pin->GetReal("problem/cluster/uniform_b_field", "bz"); + + hydro_pkg->AddParam<>("uniform_b_field_bx", uniform_b_field_bx); + hydro_pkg->AddParam<>("uniform_b_field_by", uniform_b_field_by); + hydro_pkg->AddParam<>("uniform_b_field_bz", uniform_b_field_bz); + } + + /************************************************************ + * Read Uniform Magnetic Field + ************************************************************/ + + const bool init_dipole_b_field = pin->GetOrAddBoolean("problem/cluster/dipole_b_field", + "init_dipole_b_field", false); + hydro_pkg->AddParam<>("init_dipole_b_field", init_dipole_b_field); + + if (init_dipole_b_field) { + const Real dipole_b_field_mx = pin->GetReal("problem/cluster/dipole_b_field", "mx"); + const Real dipole_b_field_my = pin->GetReal("problem/cluster/dipole_b_field", "my"); + const Real dipole_b_field_mz = pin->GetReal("problem/cluster/dipole_b_field", "mz"); + + hydro_pkg->AddParam<>("dipole_b_field_mx", dipole_b_field_mx); + hydro_pkg->AddParam<>("dipole_b_field_my", dipole_b_field_my); + hydro_pkg->AddParam<>("dipole_b_field_mz", dipole_b_field_mz); + } + + /************************************************************ + * Read Cluster Gravity Parameters + ************************************************************/ + + // Build cluster_gravity object + ClusterGravity cluster_gravity(pin, hydro_pkg); + // hydro_pkg->AddParam<>("cluster_gravity", cluster_gravity); + + // Include gravity as a source term during evolution + const bool gravity_srcterm = + pin->GetBoolean("problem/cluster/gravity", "gravity_srcterm"); + hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); + + /************************************************************ + * Read Initial Entropy Profile + ************************************************************/ + + // Build entropy_profile object + ACCEPTEntropyProfile entropy_profile(pin); + + /************************************************************ + * Build Hydrostatic Equilibrium Sphere + ************************************************************/ + + HydrostaticEquilibriumSphere hse_sphere(pin, hydro_pkg, cluster_gravity, + entropy_profile); + + /************************************************************ + * Read Precessing Jet Coordinate system + ************************************************************/ + + JetCoordsFactory jet_coords_factory(pin, hydro_pkg); + + /************************************************************ + * Read AGN Feedback + ************************************************************/ + + AGNFeedback agn_feedback(pin, hydro_pkg); + + /************************************************************ + * Read AGN Triggering + ************************************************************/ + AGNTriggering agn_triggering(pin, hydro_pkg); + + /************************************************************ + * Read Magnetic Tower + ************************************************************/ + + // Build Magnetic Tower + MagneticTower magnetic_tower(pin, hydro_pkg); + + // Determine if magnetic_tower_power_scaling is needed + // Is AGN Power and Magnetic fraction non-zero? + bool magnetic_tower_power_scaling = + (agn_feedback.magnetic_fraction_ != 0 && + (agn_feedback.fixed_power_ != 0 || + agn_triggering.triggering_mode_ != AGNTriggeringMode::NONE)); + hydro_pkg->AddParam("magnetic_tower_power_scaling", magnetic_tower_power_scaling); + + /************************************************************ + * Read SNIA Feedback + ************************************************************/ + + SNIAFeedback snia_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Stellar Feedback + ************************************************************/ + + StellarFeedback stellar_feedback(pin, hydro_pkg); + + /************************************************************ + * Read Clips (ceilings and floors) + ************************************************************/ + + // Disable all clips by default with a negative radius clip + Real clip_r = pin->GetOrAddReal("problem/cluster/clips", "clip_r", -1.0); + + // By default disable floors by setting a negative value + Real dfloor = pin->GetOrAddReal("problem/cluster/clips", "dfloor", -1.0); + + // By default disable ceilings by setting to infinity + Real vceil = pin->GetOrAddReal("problem/cluster/clips", "vceil", + std::numeric_limits::infinity()); + Real vAceil = pin->GetOrAddReal("problem/cluster/clips", "vAceil", + std::numeric_limits::infinity()); + Real Tceil = pin->GetOrAddReal("problem/cluster/clips", "Tceil", + std::numeric_limits::infinity()); + Real eceil = Tceil; + if (eceil < std::numeric_limits::infinity()) { + if (!hydro_pkg->AllParams().hasKey("mbar_over_kb")) { + PARTHENON_FAIL("Temperature ceiling requires units and gas composition. " + "Either set a 'units' block and the 'hydro/He_mass_fraction' in " + "input file or use a pressure floor " + "(defined code units) instead."); } + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + eceil = Tceil / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); + } + hydro_pkg->AddParam("cluster_dfloor", dfloor); + hydro_pkg->AddParam("cluster_eceil", eceil); + hydro_pkg->AddParam("cluster_vceil", vceil); + hydro_pkg->AddParam("cluster_vAceil", vAceil); + hydro_pkg->AddParam("cluster_clip_r", clip_r); + + /************************************************************ + * Start running reductions into history outputs for clips, stellar mass, cold + * gas, and AGN extent + ************************************************************/ + + /* FIXME(forrestglines) This implementation with a reduction into Params might + be broken in several ways. + 1. Each reduction in params is Rank local. Multiple meshblocks packs per + rank adding to these params is not thread-safe + 2. These Params are not carried over between restarts. If a restart dump is + made and a history output is not, then the mass/energy between the last + history output and the restart dump is lost + */ + std::string reduction_strs[] = {"stellar_mass", "added_dfloor_mass", + "removed_eceil_energy", "removed_vceil_energy", + "added_vAceil_mass"}; + + // Add a param for each reduction, then add it as a summation reduction for + // history outputs + auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); + + for (auto reduction_str : reduction_strs) { + hydro_pkg->AddParam(reduction_str, 0.0, true); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, + [reduction_str](MeshData *md) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const Real reduction = hydro_pkg->Param(reduction_str); + // Reset the running count for this reduction between history outputs + hydro_pkg->UpdateParam(reduction_str, 0.0); + return reduction; + }, + reduction_str)); + } - /************************************************************ - * Read Cluster Gravity Parameters - ************************************************************/ + // Add history reduction for total cold gas using stellar mass threshold + const Real cold_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "cold_temp_thresh", 0.0); + if (cold_thresh > 0) { + hydro_pkg->AddParam("reduction_cold_threshold", cold_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::sum, LocalReduceColdGas, "cold_mass")); + } + const Real agn_tracer_thresh = + pin->GetOrAddReal("problem/cluster/reductions", "agn_tracer_thresh", -1.0); + if (agn_tracer_thresh >= 0) { + PARTHENON_REQUIRE( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false), + "AGN Tracer must be enabled to reduce AGN tracer extent"); + hydro_pkg->AddParam("reduction_agn_tracer_threshold", agn_tracer_thresh); + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::max, LocalReduceAGNExtent, "agn_extent")); + } - // Build cluster_gravity object - ClusterGravity cluster_gravity(pin); - hydro_pkg->AddParam<>("cluster_gravity", cluster_gravity); + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); - // Include gravity as a source term during evolution - const bool gravity_srcterm = pin->GetBoolean("problem/cluster", "gravity_srcterm"); - hydro_pkg->AddParam<>("gravity_srcterm", gravity_srcterm); + /************************************************************ + * Add derived fields + * NOTE: these must be filled in UserWorkBeforeOutput + ************************************************************/ - /************************************************************ - * Read Initial Entropy Profile - ************************************************************/ + auto m = Metadata({Metadata::Cell, Metadata::OneCopy}, std::vector({1})); - // Build entropy_profile object - ACCEPTEntropyProfile entropy_profile(pin); + // log10 of cell-centered radius + hydro_pkg->AddField("log10_cell_radius", m); + // entropy + hydro_pkg->AddField("entropy", m); + // sonic Mach number v/c_s + hydro_pkg->AddField("mach_sonic", m); + // temperature + hydro_pkg->AddField("temperature", m); + + if (hydro_pkg->Param("enable_cooling") == Cooling::tabular) { + // cooling time + hydro_pkg->AddField("cooling_time", m); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + // alfven Mach number v_A/c_s + hydro_pkg->AddField("mach_alfven", m); + + // plasma beta + hydro_pkg->AddField("plasma_beta", m); + } + + /************************************************************ + * Add infrastructure for initial pertubations + ************************************************************/ + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + if (sigma_v != 0.0) { + // peak of init vel perturb + auto l_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_v", -1.0); + auto k_peak_v = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_v", -1.0); + + PARTHENON_REQUIRE_THROWS((l_peak_v > 0.0 && k_peak_v <= 0.0) || + (k_peak_v > 0.0 && l_peak_v <= 0.0), + "Setting initial velocity perturbation requires a single " + "length scale by either setting l_peak_v or k_peak_v."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_v > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_v = Lx / l_peak_v; + } + auto num_modes_v = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_v", 40); + auto sol_weight_v = + pin->GetOrAddReal("problem/cluster/init_perturb", "sol_weight_v", 1.0); + uint32_t rseed_v = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_v", 1); + // In principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const auto t_corr = 1e-10; + + auto k_vec_v = utils::few_modes_ft::MakeRandomModes(num_modes_v, k_peak_v, rseed_v); + + auto few_modes_ft = FewModesFT(pin, hydro_pkg, "cluster_perturb_v", num_modes_v, + k_vec_v, k_peak_v, sol_weight_v, t_corr, rseed_v); + hydro_pkg->AddParam<>("cluster/few_modes_ft_v", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now) + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + PARTHENON_REQUIRE_THROWS(hydro_pkg->Param("fluid") == Fluid::glmmhd, + "Requested initial magnetic field perturbations but not " + "solving the MHD equations.") + // peak of init magnetic field perturb + auto l_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "l_peak_b", -1.0); + auto k_peak_b = pin->GetOrAddReal("problem/cluster/init_perturb", "k_peak_b", -1.0); + PARTHENON_REQUIRE_THROWS((l_peak_b > 0.0 && k_peak_b <= 0.0) || + (k_peak_b > 0.0 && l_peak_b <= 0.0), + "Setting initial B perturbation requires a single " + "length scale by either setting l_peak_b or k_peak_b."); + // Set peak wavemode as required by few_modes_fft when not directly given + if (l_peak_b > 0) { + const auto Lx = pin->GetReal("parthenon/mesh", "x1max") - + pin->GetReal("parthenon/mesh", "x1min"); + // Note that this assumes a cubic box + k_peak_b = Lx / l_peak_b; + } + auto num_modes_b = + pin->GetOrAddInteger("problem/cluster/init_perturb", "num_modes_b", 40); + uint32_t rseed_b = pin->GetOrAddInteger("problem/cluster/init_perturb", "rseed_b", 2); + // In principle arbitrary because the inital A_hat is 0 and the A_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const auto t_corr = 1e-10; + // This field should by construction have no compressive modes, so we fix the number. + const auto sol_weight_b = 1.0; + + auto k_vec_b = utils::few_modes_ft::MakeRandomModes(num_modes_b, k_peak_b, rseed_b); + + const bool fill_ghosts = true; // as we fill a vector potential to calc B + auto few_modes_ft = + FewModesFT(pin, hydro_pkg, "cluster_perturb_b", num_modes_b, k_vec_b, k_peak_b, + sol_weight_b, t_corr, rseed_b, fill_ghosts); + hydro_pkg->AddParam<>("cluster/few_modes_ft_b", few_modes_ft); + + // Add field for initial perturation (must not need to be consistent but defining it + // this way is easier for now). Only add if not already done for the velocity. + if (sigma_v == 0.0) { + Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("tmp_perturb", m); + } + } +} + +//======================================================================================== +//! \fn void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) +//! \brief Generate problem data for all blocks on rank +// +// Note, this requires that parthenon/mesh/pack_size=-1 during initialization so that +// reductions work +//======================================================================================== + +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { + // This could be more optimized, but require a refactor of init routines being called. + // However, given that it's just called during initial setup, this should not be a + // performance concern. + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + + // Initialize the conserved variables + auto &u = pmb->meshblock_data.Get()->Get("cons").data; + + auto &coords = pmb->coords; + + // Get Adiabatic Index + const Real gam = pin->GetReal("hydro", "gamma"); + const Real gm1 = (gam - 1.0); /************************************************************ - * Build Hydrostatic Equilibrium Sphere + * Initialize the initial hydro state ************************************************************/ + const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); + if (init_uniform_gas) { + const Real rho = hydro_pkg->Param("uniform_gas_rho"); + const Real ux = hydro_pkg->Param("uniform_gas_ux"); + const Real uy = hydro_pkg->Param("uniform_gas_uy"); + const Real uz = hydro_pkg->Param("uniform_gas_uz"); + const Real pres = hydro_pkg->Param("uniform_gas_pres"); + + const Real Mx = rho * ux; + const Real My = rho * uy; + const Real Mz = rho * uz; + const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "Cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + u(IDN, k, j, i) = rho; + u(IM1, k, j, i) = Mx; + u(IM2, k, j, i) = My; + u(IM3, k, j, i) = Mz; + u(IEN, k, j, i) = E; + }); + + // end if(init_uniform_gas) + } else { + /************************************************************ + * Initialize a HydrostaticEquilibriumSphere + ************************************************************/ + const auto &he_sphere = + hydro_pkg + ->Param>( + "hydrostatic_equilibirum_sphere"); + + const auto P_rho_profile = he_sphere.generate_P_rho_profile(ib, jb, kb, coords); + + // initialize conserved variables + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::UniformGas", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Calculate radius + const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + + coords.Xc<2>(j) * coords.Xc<2>(j) + + coords.Xc<3>(k) * coords.Xc<3>(k)); + + // Get pressure and density from generated profile + const Real P_r = P_rho_profile.P_from_r(r); + const Real rho_r = P_rho_profile.rho_from_r(r); + + // Fill conserved states, 0 initial velocity + u(IDN, k, j, i) = rho_r; + u(IM1, k, j, i) = 0.0; + u(IM2, k, j, i) = 0.0; + u(IM3, k, j, i) = 0.0; + u(IEN, k, j, i) = P_r / gm1; + }); + } + + if (hydro_pkg->Param("fluid") == Fluid::glmmhd) { + /************************************************************ + * Initialize the initial magnetic field state via a vector potential + ************************************************************/ + parthenon::ParArray4D A("A", 3, pmb->cellbounds.ncellsk(IndexDomain::entire), + pmb->cellbounds.ncellsj(IndexDomain::entire), + pmb->cellbounds.ncellsi(IndexDomain::entire)); + + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + /************************************************************ + * Initialize an initial magnetic tower + ************************************************************/ + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + magnetic_tower.AddInitialFieldToPotential(pmb.get(), a_kb, a_jb, a_ib, A); + + /************************************************************ + * Add dipole magnetic field to the magnetic potential + ************************************************************/ + const auto &init_dipole_b_field = hydro_pkg->Param("init_dipole_b_field"); + if (init_dipole_b_field) { + const Real mx = hydro_pkg->Param("dipole_b_field_mx"); + const Real my = hydro_pkg->Param("dipole_b_field_my"); + const Real mz = hydro_pkg->Param("dipole_b_field_mz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), a_kb.s, a_kb.e, a_jb.s, a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r3 = pow(SQR(x) + SQR(y) + SQR(z), 3. / 2); + + const Real m_cross_r_x = my * z - mz * y; + const Real m_cross_r_y = mz * x - mx * z; + const Real m_cross_r_z = mx * y - mx * y; + + A(0, k, j, i) += m_cross_r_x / (4 * M_PI * r3); + A(1, k, j, i) += m_cross_r_y / (4 * M_PI * r3); + A(2, k, j, i) += m_cross_r_z / (4 * M_PI * r3); + }); + } + + /************************************************************ + * Apply the potential to the conserved variables + ************************************************************/ + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyMagneticPotential", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + u(IB1, k, j, i) = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + u(IEN, k, j, i) += 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + + SQR(u(IB3, k, j, i))); + }); + + /************************************************************ + * Add uniform magnetic field to the conserved variables + ************************************************************/ + const auto &init_uniform_b_field = hydro_pkg->Param("init_uniform_b_field"); + if (init_uniform_b_field) { + const Real bx = hydro_pkg->Param("uniform_b_field_bx"); + const Real by = hydro_pkg->Param("uniform_b_field_by"); + const Real bz = hydro_pkg->Param("uniform_b_field_bz"); + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "cluster::ProblemGenerator::ApplyUniformBField", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + const Real bx_i = u(IB1, k, j, i); + const Real by_i = u(IB2, k, j, i); + const Real bz_i = u(IB3, k, j, i); + + u(IB1, k, j, i) += bx; + u(IB2, k, j, i) += by; + u(IB3, k, j, i) += bz; + + // Old magnetic energy is b_i^2, new Magnetic energy should be 0.5*(b_i + + // b)^2, add b_i*b + 0.5b^2 to old energy to accomplish that + u(IEN, k, j, i) += + bx_i * bx + by_i * by + bz_i * bz + 0.5 * (SQR(bx) + SQR(by) + SQR(bz)); + }); + // end if(init_uniform_b_field) + } - HydrostaticEquilibriumSphere hse_sphere(pin, cluster_gravity, entropy_profile); - hydro_pkg->AddParam<>("hydrostatic_equilibirum_sphere", hse_sphere); + } // END if(hydro_pkg->Param("fluid") == Fluid::glmmhd) } + /************************************************************ + * Set initial velocity perturbations (requires no other velocities for now) + ************************************************************/ + + auto pmb = md->GetBlockData(0)->GetBlockPointer(); IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::interior); IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::interior); IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::interior); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto fluid = hydro_pkg->Param("fluid"); + auto const &cons = md->PackVariables(std::vector{"cons"}); + const auto num_blocks = md->NumBlocks(); + + const auto sigma_v = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_v", 0.0); + + if (sigma_v != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_v"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital v_hat is 0 and the v_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_v) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real v2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + auto rho = u(IDN, k, j, i); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IM1, k, j, i) == 0.0 && u(IM2, k, j, i) == 0.0 && u(IM3, k, j, i) == 0.0, + "Found existing non-zero velocity when setting velocity perturbations."); + + u(IM1, k, j, i) = rho * perturb_pack(b, 0, k, j, i); + u(IM2, k, j, i) = rho * perturb_pack(b, 1, k, j, i); + u(IM3, k, j, i) = rho * perturb_pack(b, 2, k, j, i); + // No need to touch the energy yet as we'll normalize later + + lsum += (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) * + coords.CellVolume(k, j, i) / SQR(rho); + }, + v2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &v2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto v_norm = std::sqrt(v2_sum / (Lx * Ly * Lz) / (SQR(sigma_v))); + + pmb->par_for( + "Norm sigma_v", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IM1, k, j, i) /= v_norm; + u(IM2, k, j, i) /= v_norm; + u(IM3, k, j, i) /= v_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IM1, k, j, i)) + SQR(u(IM2, k, j, i)) + SQR(u(IM3, k, j, i))) / + u(IDN, k, j, i); + }); + } - // initialize conserved variables - auto &rc = pmb->meshblock_data.Get(); - auto &u_dev = rc->Get("cons").data; - auto &coords = pmb->coords; - - // Initialize the conserved variables - auto u = u_dev.GetHostMirrorAndCopy(); + /************************************************************ + * Set initial magnetic field perturbations (resets magnetic field field) + ************************************************************/ + const auto sigma_b = pin->GetOrAddReal("problem/cluster/init_perturb", "sigma_b", 0.0); + if (sigma_b != 0.0) { + auto few_modes_ft = hydro_pkg->Param("cluster/few_modes_ft_b"); + // Init phases on all blocks + for (int b = 0; b < md->NumBlocks(); b++) { + auto pmb = md->GetBlockData(b)->GetBlockPointer(); + few_modes_ft.SetPhases(pmb.get(), pin); + } + // As for t_corr in few_modes_ft, the choice for dt is + // in principle arbitrary because the inital b_hat is 0 and the b_hat_new will contain + // the perturbation (and is normalized in the following to get the desired sigma_b) + const Real dt = 1.0; + few_modes_ft.Generate(md, dt, "tmp_perturb"); + + Real b2_sum = 0.0; // used for normalization + + auto perturb_pack = md->PackVariables(std::vector{"tmp_perturb"}); + + pmb->par_reduce( + "Init sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, Real &lsum) { + const auto &coords = cons.GetCoords(b); + const auto &u = cons(b); + // The following restriction could be lifted, but requires refactoring of the + // logic for the normalization/reduction below + PARTHENON_REQUIRE( + u(IB1, k, j, i) == 0.0 && u(IB2, k, j, i) == 0.0 && u(IB3, k, j, i) == 0.0, + "Found existing non-zero B when setting magnetic field perturbations."); + u(IB1, k, j, i) = + (perturb_pack(b, 2, k, j + 1, i) - perturb_pack(b, 2, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0 - + (perturb_pack(b, 1, k + 1, j, i) - perturb_pack(b, 1, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0; + u(IB2, k, j, i) = + (perturb_pack(b, 0, k + 1, j, i) - perturb_pack(b, 0, k - 1, j, i)) / + coords.Dxc<3>(k) / 2.0 - + (perturb_pack(b, 2, k, j, i + 1) - perturb_pack(b, 2, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0; + u(IB3, k, j, i) = + (perturb_pack(b, 1, k, j, i + 1) - perturb_pack(b, 1, k, j, i - 1)) / + coords.Dxc<1>(i) / 2.0 - + (perturb_pack(b, 0, k, j + 1, i) - perturb_pack(b, 0, k, j - 1, i)) / + coords.Dxc<2>(j) / 2.0; + + // No need to touch the energy yet as we'll normalize later + lsum += (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))) * + coords.CellVolume(k, j, i); + }, + b2_sum); + +#ifdef MPI_PARALLEL + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &b2_sum, 1, MPI_PARTHENON_REAL, + MPI_SUM, MPI_COMM_WORLD)); +#endif // MPI_PARALLEL + + const auto Lx = pmesh->mesh_size.x1max - pmesh->mesh_size.x1min; + const auto Ly = pmesh->mesh_size.x2max - pmesh->mesh_size.x2min; + const auto Lz = pmesh->mesh_size.x3max - pmesh->mesh_size.x3min; + auto b_norm = std::sqrt(b2_sum / (Lx * Ly * Lz) / (SQR(sigma_b))); + + pmb->par_for( + "Norm sigma_b", 0, num_blocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const auto &u = cons(b); + + u(IB1, k, j, i) /= b_norm; + u(IB2, k, j, i) /= b_norm; + u(IB3, k, j, i) /= b_norm; + + u(IEN, k, j, i) += + 0.5 * (SQR(u(IB1, k, j, i)) + SQR(u(IB2, k, j, i)) + SQR(u(IB3, k, j, i))); + }); + } +} - // Get Adiabatic Index +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { + // get hydro + auto pkg = pmb->packages.Get("Hydro"); const Real gam = pin->GetReal("hydro", "gamma"); const Real gm1 = (gam - 1.0); - const auto &init_uniform_gas = hydro_pkg->Param("init_uniform_gas"); - if (init_uniform_gas) { - const Real rho = hydro_pkg->Param("uniform_gas_rho"); - const Real ux = hydro_pkg->Param("uniform_gas_ux"); - const Real uy = hydro_pkg->Param("uniform_gas_uy"); - const Real uz = hydro_pkg->Param("uniform_gas_uz"); - const Real pres = hydro_pkg->Param("uniform_gas_pres"); - - const Real Mx = rho * ux; - const Real My = rho * uy; - const Real Mz = rho * uz; - const Real E = rho * (0.5 * (ux * ux + uy * uy + uz * uz) + pres / (gm1 * rho)); - - for (int k = kb.s; k <= kb.e; k++) { - for (int j = jb.s; j <= jb.e; j++) { - for (int i = ib.s; i <= ib.e; i++) { - - u(IDN, k, j, i) = rho; - u(IM1, k, j, i) = Mx; - u(IM2, k, j, i) = My; - u(IM3, k, j, i) = Mz; - u(IEN, k, j, i) = E; - } - } - } + // get prim vars + auto &data = pmb->meshblock_data.Get(); + auto const &prim = data->Get("prim").data; - } else { - /************************************************************ - * Initialize a HydrostaticEquilibriumSphere - ************************************************************/ - const auto &he_sphere = - hydro_pkg - ->Param>( - "hydrostatic_equilibirum_sphere"); - - const auto P_rho_profile = he_sphere.generate_P_rho_profile>(ib, jb, kb, - coords); - - // initialize conserved variables - for (int k = kb.s; k <= kb.e; k++) { - for (int j = jb.s; j <= jb.e; j++) { - for (int i = ib.s; i <= ib.e; i++) { - - // Calculate radius - const Real r = - sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + coords.Xc<2>(j) * coords.Xc<2>(j) + - coords.Xc<3>(k) * coords.Xc<3>(k)); - - // Get pressure and density from generated profile - const Real P_r = P_rho_profile.P_from_r(r); - const Real rho_r = P_rho_profile.rho_from_r(r); - - // Fill conserved states, 0 initial velocity - u(IDN, k, j, i) = rho_r; - u(IM1, k, j, i) = 0.0; - u(IM2, k, j, i) = 0.0; - u(IM3, k, j, i) = 0.0; - u(IEN, k, j, i) = P_r / gm1; - } - } - } + // get derived fields + auto &log10_radius = data->Get("log10_cell_radius").data; + auto &entropy = data->Get("entropy").data; + auto &mach_sonic = data->Get("mach_sonic").data; + auto &temperature = data->Get("temperature").data; + + // for computing temperature from primitives + auto units = pkg->Param("units"); + auto mbar_over_kb = pkg->Param("mbar_over_kb"); + auto mbar = mbar_over_kb * units.k_boltzmann(); + + // fill derived vars (*including ghost cells*) + auto &coords = pmb->coords; + IndexRange ib = pmb->cellbounds.GetBoundsI(IndexDomain::entire); + IndexRange jb = pmb->cellbounds.GetBoundsJ(IndexDomain::entire); + IndexRange kb = pmb->cellbounds.GetBoundsK(IndexDomain::entire); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real v1 = prim(IV1, k, j, i); + const Real v2 = prim(IV2, k, j, i); + const Real v3 = prim(IV3, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute radius + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + const Real r2 = SQR(x) + SQR(y) + SQR(z); + log10_radius(k, j, i) = 0.5 * std::log10(r2); + + // compute entropy + const Real K = P / std::pow(rho / mbar, gam); + entropy(k, j, i) = K; + + const Real v_mag = std::sqrt(SQR(v1) + SQR(v2) + SQR(v3)); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + const Real M_s = v_mag / c_s; + mach_sonic(k, j, i) = M_s; + + // compute temperature + temperature(k, j, i) = mbar_over_kb * P / rho; + }); + if (pkg->Param("enable_cooling") == Cooling::tabular) { + auto &cooling_time = data->Get("cooling_time").data; + + // get cooling function + const cooling::TabularCooling &tabular_cooling = + pkg->Param("tabular_cooling"); + const auto cooling_table_obj = tabular_cooling.GetCoolingTableObj(); + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::CoolingTime", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + + // compute cooling time + const Real eint = P / (rho * gm1); + const Real edot = cooling_table_obj.DeDt(eint, rho); + cooling_time(k, j, i) = (edot != 0) ? -eint / edot : NAN; + }); } - // copy initialized cons to device - u_dev.DeepCopy(u); + if (pkg->Param("fluid") == Fluid::glmmhd) { + auto &plasma_beta = data->Get("plasma_beta").data; + auto &mach_alfven = data->Get("mach_alfven").data; + + pmb->par_for( + "Cluster::UserWorkBeforeOutput::MHD", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int k, const int j, const int i) { + // get gas properties + const Real rho = prim(IDN, k, j, i); + const Real P = prim(IPR, k, j, i); + const Real Bx = prim(IB1, k, j, i); + const Real By = prim(IB2, k, j, i); + const Real Bz = prim(IB3, k, j, i); + const Real B2 = (SQR(Bx) + SQR(By) + SQR(Bz)); + + // compute Alfven mach number + const Real v_A = std::sqrt(B2 / rho); + const Real c_s = std::sqrt(gam * P / rho); // ideal gas EOS + mach_alfven(k, j, i) = mach_sonic(k, j, i) * c_s / v_A; + + // compute plasma beta + plasma_beta(k, j, i) = (B2 != 0) ? P / (0.5 * B2) : NAN; + }); + } } } // namespace cluster diff --git a/src/pgen/cluster/JetCoordsMath.ipynb b/src/pgen/cluster/JetCoordsMath.ipynb new file mode 100644 index 00000000..1c2e9c4b --- /dev/null +++ b/src/pgen/cluster/JetCoordsMath.ipynb @@ -0,0 +1,406 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "3d49a71c", + "metadata": {}, + "outputs": [], + "source": [ + "import sympy as sy\n", + "import sympy.physics.vector\n", + "import sympy.vector\n", + "from sympy.codegen.ast import Assignment\n", + "import numpy as np" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2762bbb0", + "metadata": {}, + "outputs": [], + "source": [ + "#Define the simulation cartesian coordinate frame\n", + "S_cart = sympy.vector.CoordSys3D(\"S_{cart}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "6c8a2a1b", + "metadata": {}, + "outputs": [], + "source": [ + "#Azimuthal and inclination angle of the jet\n", + "phi_jet, theta_jet = sy.symbols(\"phi_jet theta_jet\")\n", + "\n", + "#Define the cartesian coordinate system of the jet\n", + "J_cart = S_cart.orient_new_space(\"J_{cart}\", theta_jet, phi_jet, 0, \"YZX\")\n", + "\n", + "\n", + "#Define the cylindrical coordinate system of the jet\n", + "J_cyl = J_cart.create_new(\"J_{cyl}\",transformation=\"cylindrical\")" + ] + }, + { + "cell_type": "markdown", + "id": "324f0c8d", + "metadata": {}, + "source": [ + "# Equations and code to convert simulation cartesian coordinates to jet cylindrical coordinates" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "1ce2d3b4", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Equation for jet cartesian coords from simulation cartesian coords\n" + ] + }, + { + "data": { + "text/latex": [ + "$\\displaystyle x_{jet} = x_{sim} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - z_{sim} \\sin{\\left(\\theta_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(x_jet, x_sim*cos(phi_jet)*cos(theta_jet) + y_sim*sin(phi_jet)*cos(theta_jet) - z_sim*sin(theta_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/latex": [ + "$\\displaystyle y_{jet} = - x_{sim} \\sin{\\left(\\phi_{jet} \\right)} + y_{sim} \\cos{\\left(\\phi_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(y_jet, -x_sim*sin(phi_jet) + y_sim*cos(phi_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/latex": [ + "$\\displaystyle z_{jet} = x_{sim} \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)} + z_{sim} \\cos{\\left(\\theta_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(z_jet, x_sim*sin(theta_jet)*cos(phi_jet) + y_sim*sin(phi_jet)*sin(theta_jet) + z_sim*cos(theta_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Latex for jet cartesian coords from simulation cartesian coords\n", + "x_{jet} = x_{sim} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - z_{sim} \\sin{\\left(\\theta_{jet} \\right)}\n", + "y_{jet} = - x_{sim} \\sin{\\left(\\phi_{jet} \\right)} + y_{sim} \\cos{\\left(\\phi_{jet} \\right)}\n", + "z_{jet} = x_{sim} \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)} + z_{sim} \\cos{\\left(\\theta_{jet} \\right)}\n", + "\n", + "Code for jet_cartesian vector as a sim cartesian_vector\n", + "x_jet = x_sim*cos(phi_jet)*cos(theta_jet) + y_sim*sin(phi_jet)*cos(theta_jet) - z_sim*sin(theta_jet);\n", + "y_jet = -x_sim*sin(phi_jet) + y_sim*cos(phi_jet);\n", + "z_jet = x_sim*sin(theta_jet)*cos(phi_jet) + y_sim*sin(phi_jet)*sin(theta_jet) + z_sim*cos(theta_jet);\n", + "\n", + "Equation for jet cylindrical coords from simulation cartesian coords\n" + ] + }, + { + "data": { + "text/latex": [ + "$\\displaystyle pos_{r} = \\sqrt{\\left(- x_{sim} \\sin{\\left(\\phi_{jet} \\right)} + y_{sim} \\cos{\\left(\\phi_{jet} \\right)}\\right)^{2} + \\left(x_{sim} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - z_{sim} \\sin{\\left(\\theta_{jet} \\right)}\\right)^{2}}$" + ], + "text/plain": [ + "Eq(pos_r, sqrt((-x_sim*sin(phi_jet) + y_sim*cos(phi_jet))**2 + (x_sim*cos(phi_jet)*cos(theta_jet) + y_sim*sin(phi_jet)*cos(theta_jet) - z_sim*sin(theta_jet))**2))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/latex": [ + "$\\displaystyle pos_{\\theta} = \\operatorname{atan}_{2}{\\left(- x_{sim} \\sin{\\left(\\phi_{jet} \\right)} + y_{sim} \\cos{\\left(\\phi_{jet} \\right)},x_{sim} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - z_{sim} \\sin{\\left(\\theta_{jet} \\right)} \\right)}$" + ], + "text/plain": [ + "Eq(pos_theta, atan2(-x_sim*sin(phi_jet) + y_sim*cos(phi_jet), x_sim*cos(phi_jet)*cos(theta_jet) + y_sim*sin(phi_jet)*cos(theta_jet) - z_sim*sin(theta_jet)))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/latex": [ + "$\\displaystyle pos_{h} = x_{sim} \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)} + z_{sim} \\cos{\\left(\\theta_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(pos_h, x_sim*sin(theta_jet)*cos(phi_jet) + y_sim*sin(phi_jet)*sin(theta_jet) + z_sim*cos(theta_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Latex for jet cylindrical coords from simulation cartesian coords\n", + "pos_{r} = \\sqrt{\\left(- x_{sim} \\sin{\\left(\\phi_{jet} \\right)} + y_{sim} \\cos{\\left(\\phi_{jet} \\right)}\\right)^{2} + \\left(x_{sim} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - z_{sim} \\sin{\\left(\\theta_{jet} \\right)}\\right)^{2}}\n", + "pos_{\\theta} = \\operatorname{atan}_{2}{\\left(- x_{sim} \\sin{\\left(\\phi_{jet} \\right)} + y_{sim} \\cos{\\left(\\phi_{jet} \\right)},x_{sim} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - z_{sim} \\sin{\\left(\\theta_{jet} \\right)} \\right)}\n", + "pos_{h} = x_{sim} \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)} + y_{sim} \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)} + z_{sim} \\cos{\\left(\\theta_{jet} \\right)}\n", + "Equation for jet cylindrical coords from simulation cartesian coords\n", + "pos_r = sqrt(pow(-x_sim*sin(phi_jet) + y_sim*cos(phi_jet), 2) + pow(x_sim*cos(phi_jet)*cos(theta_jet) + y_sim*sin(phi_jet)*cos(theta_jet) - z_sim*sin(theta_jet), 2));\n", + "pos_theta = atan2(-x_sim*sin(phi_jet) + y_sim*cos(phi_jet), x_sim*cos(phi_jet)*cos(theta_jet) + y_sim*sin(phi_jet)*cos(theta_jet) - z_sim*sin(theta_jet));\n", + "pos_h = x_sim*sin(theta_jet)*cos(phi_jet) + y_sim*sin(phi_jet)*sin(theta_jet) + z_sim*cos(theta_jet);\n" + ] + } + ], + "source": [ + "#Define a position in simulation-cartesian\n", + "x_sim,y_sim,z_sim = sy.symbols(\"x_sim y_sim z_sim\")\n", + "\n", + "pos_sim = S_cart.origin.locate_new(\"p_sim\",x_sim*S_cart.i + y_sim*S_cart.j + z_sim*S_cart.k)\n", + "\n", + "#Express that simulation-cartesian position in jet-cartesian\n", + "pos_jet = pos_sim.express_coordinates(J_cart)\n", + "\n", + "print(\"Equation for jet cartesian coords from simulation cartesian coords\")\n", + "for i,pos_i in enumerate(pos_jet):\n", + " display(sy.Eq(sy.symbols(f\"{'xyz'[i]}_jet\"),pos_i,evaluate=False))\n", + "print()\n", + "\n", + "print(\"Latex for jet cartesian coords from simulation cartesian coords\")\n", + "for i,pos_i in enumerate(pos_jet):\n", + " print(sy.latex(sy.Eq(sy.symbols(f\"{'xyz'[i]}_jet\"),pos_i,evaluate=False)))\n", + "print()\n", + "\n", + "\n", + "print(\"Code for jet_cartesian vector as a sim cartesian_vector\")\n", + "for i,pos_i in enumerate(pos_jet):\n", + " print(sy.ccode(Assignment(sy.symbols(f\"{'xyz'[i]}_jet\"),pos_i)))\n", + "print()\n", + "\n", + "#Express the simulation-cartesian position in jet-cylindrical\n", + "pos_r = sy.sqrt(pos_jet[0]**2 + pos_jet[1]**2)\n", + "pos_theta = sy.atan2(pos_jet[1],pos_jet[0])\n", + "pos_h = pos_jet[2]\n", + "\n", + "print(\"Equation for jet cylindrical coords from simulation cartesian coords\")\n", + "for symbol,var in zip(sy.symbols(\"pos_r pos_theta pos_h\"),(pos_r,pos_theta,pos_h)):\n", + " display(sy.Eq(symbol,var,evaluate=False))\n", + "\n", + "print(\"Latex for jet cylindrical coords from simulation cartesian coords\")\n", + "for symbol,var in zip(sy.symbols(\"pos_r pos_theta pos_h\"),(pos_r,pos_theta,pos_h)):\n", + " print(sy.latex(sy.Eq(symbol,var,evaluate=False)))\n", + " \n", + "print(\"Equation for jet cylindrical coords from simulation cartesian coords\")\n", + "for symbol,var in zip(sy.symbols(\"pos_r pos_theta pos_h\"),(pos_r,pos_theta,pos_h)):\n", + " print(sy.ccode(Assignment(symbol,var)))\n" + ] + }, + { + "cell_type": "markdown", + "id": "7df8918b", + "metadata": {}, + "source": [ + "# Equations and code to convert jet cylindrical vectors to simulation cartesian vectors" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "66b32b15", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Equation for DCM matrix for jet cartesian to sim cartesian\n" + ] + }, + { + "data": { + "text/latex": [ + "$\\displaystyle \\left[\\begin{matrix}\\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} & - \\sin{\\left(\\phi_{jet} \\right)} & \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)}\\\\\\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} & \\cos{\\left(\\phi_{jet} \\right)} & \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)}\\\\- \\sin{\\left(\\theta_{jet} \\right)} & 0 & \\cos{\\left(\\theta_{jet} \\right)}\\end{matrix}\\right]$" + ], + "text/plain": [ + "Matrix([\n", + "[cos(phi_jet)*cos(theta_jet), -sin(phi_jet), sin(theta_jet)*cos(phi_jet)],\n", + "[sin(phi_jet)*cos(theta_jet), cos(phi_jet), sin(phi_jet)*sin(theta_jet)],\n", + "[ -sin(theta_jet), 0, cos(theta_jet)]])" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Latex for DCM matrix for jet cartesian to sim cartesian\n", + "\\left[\\begin{matrix}\\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} & - \\sin{\\left(\\phi_{jet} \\right)} & \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)}\\\\\\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} & \\cos{\\left(\\phi_{jet} \\right)} & \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)}\\\\- \\sin{\\left(\\theta_{jet} \\right)} & 0 & \\cos{\\left(\\theta_{jet} \\right)}\\end{matrix}\\right]\n", + "\n", + "Equations for jet_cartesian vector as a sim cartesian_vector\n" + ] + }, + { + "data": { + "text/latex": [ + "$\\displaystyle v_{xsim} = v_{xjet} \\cos{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} - v_{yjet} \\sin{\\left(\\phi_{jet} \\right)} + v_{zjet} \\sin{\\left(\\theta_{jet} \\right)} \\cos{\\left(\\phi_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(v_xsim, v_xjet*cos(phi_jet)*cos(theta_jet) - v_yjet*sin(phi_jet) + v_zjet*sin(theta_jet)*cos(phi_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/latex": [ + "$\\displaystyle v_{ysim} = v_{xjet} \\sin{\\left(\\phi_{jet} \\right)} \\cos{\\left(\\theta_{jet} \\right)} + v_{yjet} \\cos{\\left(\\phi_{jet} \\right)} + v_{zjet} \\sin{\\left(\\phi_{jet} \\right)} \\sin{\\left(\\theta_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(v_ysim, v_xjet*sin(phi_jet)*cos(theta_jet) + v_yjet*cos(phi_jet) + v_zjet*sin(phi_jet)*sin(theta_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/latex": [ + "$\\displaystyle v_{zsim} = - v_{xjet} \\sin{\\left(\\theta_{jet} \\right)} + v_{zjet} \\cos{\\left(\\theta_{jet} \\right)}$" + ], + "text/plain": [ + "Eq(v_zsim, -v_xjet*sin(theta_jet) + v_zjet*cos(theta_jet))" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Code for jet_cartesian vector as a sim cartesian_vector\n", + "v_xsim = v_xjet*cos(phi_jet)*cos(theta_jet) - v_yjet*sin(phi_jet) + v_zjet*sin(theta_jet)*cos(phi_jet);\n", + "v_ysim = v_xjet*sin(phi_jet)*cos(theta_jet) + v_yjet*cos(phi_jet) + v_zjet*sin(phi_jet)*sin(theta_jet);\n", + "v_zsim = -v_xjet*sin(theta_jet) + v_zjet*cos(theta_jet);\n" + ] + } + ], + "source": [ + "#Get a rotation matrix for vectors from jet-cartesian to simulation-cartesian\n", + "DCM_jet_to_sim = S_cart.rotation_matrix(J_cart)\n", + "\n", + "print(\"Equation for DCM matrix for jet cartesian to sim cartesian\")\n", + "display(DCM_jet_to_sim)\n", + "print()\n", + "\n", + "print(\"Latex for DCM matrix for jet cartesian to sim cartesian\")\n", + "print(sy.latex(DCM_jet_to_sim))\n", + "print()\n", + "\n", + "#Express the equation for jet-cylindrical vectors to simulation-cartesian vectors\n", + "v_x, v_y, v_z = sy.symbols(\"v_xjet v_yjet v_zjet\")\n", + "v_jet = v_x*J_cart.i + v_y*J_cart.j + v_z*J_cart.k\n", + "\n", + "print(\"Equations for jet_cartesian vector as a sim cartesian_vector\")\n", + "for i,unit in enumerate((S_cart.i, S_cart.j, S_cart.k)):\n", + " out = sy.symbols(f\"v_{'xyz'[i]}sim\")\n", + " display(sy.Eq(out,unit.dot(v_jet),evaluate=False))\n", + "print()\n", + "\n", + "print(\"Code for jet_cartesian vector as a sim cartesian_vector\")\n", + "for i,unit in enumerate((S_cart.i, S_cart.j, S_cart.k)):\n", + " out = sy.symbols(f\"v_{'xyz'[i]}sim\")\n", + " print(sy.ccode(Assignment(out,unit.dot(v_jet))))" + ] + }, + { + "cell_type": "markdown", + "id": "0d18aa6e", + "metadata": {}, + "source": [ + "# Verification" + ] + }, + { + "cell_type": "markdown", + "id": "fd3ab37d", + "metadata": {}, + "source": [ + " Verify that a vector along the jet axis points down $(1,\\theta_{jet},\\phi_{jet})$" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "3ea1df6b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Vector along jet axis is consistent\n" + ] + } + ], + "source": [ + "v_jet = J_cyl.k\n", + "v_jet_cart = sy.sin(theta_jet)*sy.cos(phi_jet)*S_cart.i + \\\n", + " sy.sin(theta_jet)*sy.sin(phi_jet)*S_cart.j + \\\n", + " sy.cos(theta_jet)*S_cart.k\n", + "\n", + "if v_jet_cart == sy.vector.express(v_jet,S_cart):\n", + " print(\"Vector along jet axis is consistent\")\n", + "else:\n", + " print(\"FAIL: Vector along jet axis is NOT consistent\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/pgen/cluster/agn_feedback.cpp b/src/pgen/cluster/agn_feedback.cpp new file mode 100644 index 00000000..7523f663 --- /dev/null +++ b/src/pgen/cluster/agn_feedback.cpp @@ -0,0 +1,420 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file agn_feedback.cpp +// \brief Class for injecting AGN feedback via thermal dump, kinetic jet, and magnetic +// tower + +#include + +// Parthenon headers +#include +#include +#include +#include +#include +#include + +// AthenaPK headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" +#include "../../main.hpp" +#include "../../units.hpp" +#include "agn_feedback.hpp" +#include "agn_triggering.hpp" +#include "cluster_utils.hpp" +#include "magnetic_tower.hpp" +#include "utils/error_checking.hpp" + +namespace cluster { +using namespace parthenon; + +AGNFeedback::AGNFeedback(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg) + : fixed_power_(pin->GetOrAddReal("problem/cluster/agn_feedback", "fixed_power", 0.0)), + vceil_(pin->GetOrAddReal("problem/cluster/agn_feedback", "vceil", + std::numeric_limits::infinity())), + efficiency_(pin->GetOrAddReal("problem/cluster/agn_feedback", "efficiency", 1e-3)), + thermal_fraction_( + pin->GetOrAddReal("problem/cluster/agn_feedback", "thermal_fraction", 0.0)), + kinetic_fraction_( + pin->GetOrAddReal("problem/cluster/agn_feedback", "kinetic_fraction", 0.0)), + magnetic_fraction_( + pin->GetOrAddReal("problem/cluster/agn_feedback", "magnetic_fraction", 0.0)), + thermal_radius_( + pin->GetOrAddReal("problem/cluster/agn_feedback", "thermal_radius", 0.01)), + kinetic_jet_radius_( + pin->GetOrAddReal("problem/cluster/agn_feedback", "kinetic_jet_radius", 0.01)), + kinetic_jet_thickness_(pin->GetOrAddReal("problem/cluster/agn_feedback", + "kinetic_jet_thickness", 0.02)), + kinetic_jet_offset_( + pin->GetOrAddReal("problem/cluster/agn_feedback", "kinetic_jet_offset", 0.02)), + enable_tracer_( + pin->GetOrAddBoolean("problem/cluster/agn_feedback", "enable_tracer", false)), + disabled_(pin->GetOrAddBoolean("problem/cluster/agn_feedback", "disabled", false)), + enable_magnetic_tower_mass_injection_(pin->GetOrAddBoolean( + "problem/cluster/agn_feedback", "enable_magnetic_tower_mass_injection", true)) { + + // Normalize the thermal, kinetic, and magnetic fractions to sum to 1.0 + const Real total_frac = thermal_fraction_ + kinetic_fraction_ + magnetic_fraction_; + if (total_frac > 0) { + thermal_fraction_ = thermal_fraction_ / total_frac; + kinetic_fraction_ = kinetic_fraction_ / total_frac; + magnetic_fraction_ = magnetic_fraction_ / total_frac; + } + + PARTHENON_REQUIRE(thermal_fraction_ >= 0 && kinetic_fraction_ >= 0 && + magnetic_fraction_ >= 0, + "AGN feedback energy fractions must be non-negative."); + + // Normalize the thermal, kinetic, and magnetic mass fractions to sum to 1.0 + if (enable_magnetic_tower_mass_injection_) { + thermal_mass_fraction_ = thermal_fraction_; + kinetic_mass_fraction_ = kinetic_fraction_; + magnetic_mass_fraction_ = magnetic_fraction_; + } else { + const auto total_mass_frac = thermal_fraction_ + kinetic_fraction_; + thermal_mass_fraction_ = thermal_fraction_ / total_mass_frac; + kinetic_mass_fraction_ = kinetic_fraction_ / total_mass_frac; + magnetic_mass_fraction_ = 0.0; + } + + ///////////////////////////////////////////////////// + // Read in or calculate jet velocity and temperature. Either and or both can + // be defined but they must satify + // + // v_jet = sqrt( 2*(eps*c^2 - (1-eps)*e_jet) ) + // + // With real, non-negative values for v_jet and e_jet + ///////////////////////////////////////////////////// + + kinetic_jet_velocity_ = NAN; + kinetic_jet_temperature_ = NAN; + kinetic_jet_e_ = NAN; + + const auto units = hydro_pkg->Param("units"); + auto mbar_gm1_over_kb = + hydro_pkg->Param("mbar_over_kb") * (pin->GetReal("hydro", "gamma") - 1); + + // Get jet velocity and temperature/internal_e if in the sim parameters. These are NAN + // otherwise + if (pin->DoesParameterExist("problem/cluster/agn_feedback", "kinetic_jet_velocity")) { + kinetic_jet_velocity_ = + pin->GetReal("problem/cluster/agn_feedback", "kinetic_jet_velocity"); + } + if (pin->DoesParameterExist("problem/cluster/agn_feedback", + "kinetic_jet_temperature")) { + kinetic_jet_temperature_ = + pin->GetReal("problem/cluster/agn_feedback", "kinetic_jet_temperature"); + + kinetic_jet_e_ = kinetic_jet_temperature_ / mbar_gm1_over_kb; + } + + if (std::isnan(kinetic_jet_velocity_) && std::isnan(kinetic_jet_temperature_)) { + // Both velocity and temperature are missing, assume 0K temperature + kinetic_jet_velocity_ = units.speed_of_light() * sqrt(2 * (efficiency_)); + kinetic_jet_temperature_ = 0; + kinetic_jet_e_ = 0; + std::cout << "### WARNING Kinetic jet velocity nor temperature not specified. " + "Assuming 0K temperature jet" + << std::endl; + } else if (std::isnan(kinetic_jet_velocity_)) { + // Velocity is missing, compute it from e_jet + kinetic_jet_velocity_ = sqrt(2 * (efficiency_ * SQR(units.speed_of_light()) - + (1.0 - efficiency_) * kinetic_jet_e_)); + } else if (std::isnan(kinetic_jet_temperature_)) { + // Temperature is missing, compute e_jet and T_jet from v_jet + kinetic_jet_e_ = + (efficiency_ * SQR(units.speed_of_light()) - 0.5 * SQR(kinetic_jet_velocity_)) / + (1 - efficiency_); + kinetic_jet_temperature_ = mbar_gm1_over_kb * kinetic_jet_e_; + } + + // Verify all equations are satified. NAN's here should give failures + PARTHENON_REQUIRE( + fabs(kinetic_jet_velocity_ - sqrt(2 * (efficiency_ * SQR(units.speed_of_light()) - + (1 - efficiency_) * kinetic_jet_e_))) < + 10 * std::numeric_limits::epsilon(), + "Specified kinetic jet velocity and temperature are incompatible with mass to " + "energy conversion efficiency. Either the specified velocity, temperature, or " + "efficiency are incompatible"); + + PARTHENON_REQUIRE(kinetic_jet_velocity_ <= + units.speed_of_light() * sqrt(2 * efficiency_), + "Kinetic jet velocity implies negative temperature of the jet"); + + PARTHENON_REQUIRE(kinetic_jet_e_ <= + SQR(units.speed_of_light()) * efficiency_ / (1 - efficiency_), + "Kinetic jet temperature implies negative kinetic energy of the jet"); + + PARTHENON_REQUIRE(kinetic_jet_velocity_ >= 0, + "Kinetic jet velocity must be non-negative"); + PARTHENON_REQUIRE(kinetic_jet_temperature_ >= 0, + "Kinetic jet temperature must be non-negative"); + + // Compute the internal energy ceiling from the temperature ceiling + const Real tceil = pin->GetOrAddReal("problem/cluster/agn_feedback", "Tceil", + std::numeric_limits::infinity()); + eceil_ = tceil / mbar_gm1_over_kb; + + // Add user history output variable for AGN power + auto hst_vars = hydro_pkg->Param(parthenon::hist_param_key); + if (!disabled_) { + // HACK (forrestglines): The operations should be a + // parthenon::UserHistoryOperation::no_reduce, which is as of writing + // unimplemented + hst_vars.emplace_back(parthenon::HistoryOutputVar( + parthenon::UserHistoryOperation::max, + [this](MeshData *md) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto &agn_feedback = hydro_pkg->Param("agn_feedback"); + return agn_feedback.GetFeedbackPower(hydro_pkg.get()); + }, + "agn_feedback_power")); + } + hydro_pkg->UpdateParam(parthenon::hist_param_key, hst_vars); + + // Double check that tracers are also enabled in fluid solver + PARTHENON_REQUIRE_THROWS(!enable_tracer_ || hydro_pkg->Param("nscalars") == 1, + "Enabling tracer for AGN feedback requires hydro/nscalars=1"); + + hydro_pkg->AddParam<>("agn_feedback", *this); +} + +parthenon::Real AGNFeedback::GetFeedbackPower(StateDescriptor *hydro_pkg) const { + auto units = hydro_pkg->Param("units"); + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + + const Real accretion_rate = agn_triggering.GetAccretionRate(hydro_pkg); + const Real power = + fixed_power_ + accretion_rate * efficiency_ * pow(units.speed_of_light(), 2); + + return power; +} +parthenon::Real AGNFeedback::GetFeedbackMassRate(StateDescriptor *hydro_pkg) const { + auto units = hydro_pkg->Param("units"); + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + + const Real accretion_rate = agn_triggering.GetAccretionRate(hydro_pkg); + + // Return a mass_rate equal to the accretion_rate minus energy-mass conversion + // to feedback energy. We could divert mass to increase the SMBH/leave out + // from mass injection + // + // Also add fixed_power/(efficiency_*c**2) when fixed_power is enabled + const Real mass_rate = accretion_rate * (1 - efficiency_) + + fixed_power_ / (efficiency_ * pow(units.speed_of_light(), 2)); + + return mass_rate; +} + +void AGNFeedback::FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm) const { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + auto fluid = hydro_pkg->Param("fluid"); + if (fluid == Fluid::euler) { + FeedbackSrcTerm(md, beta_dt, tm, hydro_pkg->Param("eos")); + } else if (fluid == Fluid::glmmhd) { + FeedbackSrcTerm(md, beta_dt, tm, hydro_pkg->Param("eos")); + } else { + PARTHENON_FAIL("AGNFeedback::FeedbackSrcTerm: Unknown EOS"); + } +} +template +void AGNFeedback::FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm, const EOS &eos) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + auto units = hydro_pkg->Param("units"); + + const Real power = GetFeedbackPower(hydro_pkg.get()); + const Real mass_rate = GetFeedbackMassRate(hydro_pkg.get()); + + if (power == 0 || disabled_) { + // No AGN feedback, return + return; + } + + PARTHENON_REQUIRE(magnetic_fraction_ != 0 || thermal_fraction_ != 0 || + kinetic_fraction_ != 0, + "AGNFeedback::FeedbackSrcTerm Magnetic, Thermal, and Kinetic " + "fractions are all zero"); + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + const auto nscalars = hydro_pkg->Param("nscalars"); + + //////////////////////////////////////////////////////////////////////////////// + // Thermal quantities + //////////////////////////////////////////////////////////////////////////////// + const Real thermal_radius2 = thermal_radius_ * thermal_radius_; + const Real thermal_scaling_factor = 1 / (4. / 3. * M_PI * pow(thermal_radius_, 3)); + + // Amount of energy/volume to dump in each cell + const Real thermal_feedback = + thermal_fraction_ * power * thermal_scaling_factor * beta_dt; + // Amount of density to dump in each cell + const Real thermal_density = + thermal_mass_fraction_ * mass_rate * thermal_scaling_factor * beta_dt; + + //////////////////////////////////////////////////////////////////////////////// + // Kinetic Jet Quantities + //////////////////////////////////////////////////////////////////////////////// + const Real kinetic_scaling_factor = + 1 / (2 * kinetic_jet_thickness_ * M_PI * pow(kinetic_jet_radius_, 2)); + + const Real kinetic_jet_radius = kinetic_jet_radius_; + const Real kinetic_jet_thickness = kinetic_jet_thickness_; + const Real kinetic_jet_offset = kinetic_jet_offset_; + + // Matches 1/2.*jet_density*jet_velocity*jet_velocity*beta_dt; + // const Real kinetic_feedback = + // kinetic_fraction_ * power * kinetic_scaling_factor * beta_dt; // energy/volume + + // Amount of density to dump in each cell + const Real jet_density = + kinetic_mass_fraction_ * mass_rate * kinetic_scaling_factor * beta_dt; + + // Velocity of added gas + const Real jet_velocity = kinetic_jet_velocity_; + const Real jet_specific_internal_e = kinetic_jet_e_; + + // Amount of momentum density ( density * velocity) to dump in each cell + const Real jet_momentum = jet_density * jet_velocity; + + // Amount of total energy to dump in each cell + const Real jet_feedback = kinetic_fraction_ * power * kinetic_scaling_factor * beta_dt; + + const Real vceil = vceil_; + const Real vceil2 = SQR(vceil); + const Real eceil = eceil_; + const Real gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + const auto enable_tracer = enable_tracer_; + //////////////////////////////////////////////////////////////////////////////// + + const parthenon::Real time = tm.time; + + const auto &jet_coords_factory = + hydro_pkg->Param("jet_coords_factory"); + const JetCoords jet_coords = jet_coords_factory.CreateJetCoords(time); + + // Appy kinietic jet and thermal feedback + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "HydroAGNFeedback::FeedbackSrcTerm", + parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, + ib.e, KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + // Thermal Feedback + if (thermal_feedback > 0 || thermal_density > 0) { + const Real r2 = x * x + y * y + z * z; + // Determine if point is in sphere r<=thermal_radius + if (r2 <= thermal_radius2) { + // Then apply heating + if (thermal_feedback > 0) cons(IEN, k, j, i) += thermal_feedback; + // Add density at constant velocity + if (thermal_density > 0) + AddDensityToConsAtFixedVel(thermal_density, cons, prim, k, j, i); + } + } + + // Kinetic Jet Feedback + if (jet_density > 0) { + // Get position in jet cylindrical coords + Real r, cos_theta, sin_theta, h; + jet_coords.SimCartToJetCylCoords(x, y, z, r, cos_theta, sin_theta, h); + + if (r < kinetic_jet_radius && fabs(h) >= kinetic_jet_offset && + fabs(h) <= kinetic_jet_offset + kinetic_jet_thickness) { + // Cell falls inside jet deposition volume + + // Get the vector of the jet axis + Real jet_axis_x, jet_axis_y, jet_axis_z; + jet_coords.JetCylToSimCartVector(cos_theta, sin_theta, 0, 0, 1, jet_axis_x, + jet_axis_y, jet_axis_z); + + const Real sign_jet = (h > 0) ? 1 : -1; // Above or below jet-disk + + /////////////////////////////////////////////////////////////////// + // We add the kinetic jet with a fixed jet velocity and specific + // internal energy/temperature of the added gas. The density, + // momentum, and total energy added depend on the triggered power. + /////////////////////////////////////////////////////////////////// + + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + + cons(IDN, k, j, i) += jet_density; + cons(IM1, k, j, i) += jet_momentum * sign_jet * jet_axis_x; + cons(IM2, k, j, i) += jet_momentum * sign_jet * jet_axis_y; + cons(IM3, k, j, i) += jet_momentum * sign_jet * jet_axis_z; + cons(IEN, k, j, i) += jet_feedback; + + // Reset tracer to one for the entire material in the jet launching region as + // we cannot distinguish between original material in a cell and new jet + // material in the evolution of the jet. Eventually, we're just interested in + // stuff that came from here. + if (enable_tracer) { + cons(nhydro, k, j, i) = 1.0 * cons(IDN, k, j, i); + } + + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + } + + // Apply velocity ceiling + const Real v2 = + SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + SQR(prim(IV3, k, j, i)); + if (vceil2 > 0 && v2 > vceil2) { + // Fix the velocity to the velocity ceiling + const Real v = sqrt(v2); + cons(IM1, k, j, i) *= vceil / v; + cons(IM2, k, j, i) *= vceil / v; + cons(IM3, k, j, i) *= vceil / v; + prim(IV1, k, j, i) *= vceil / v; + prim(IV2, k, j, i) *= vceil / v; + prim(IV3, k, j, i) *= vceil / v; + + // Remove kinetic energy + cons(IEN, k, j, i) -= 0.5 * prim(IDN, k, j, i) * (v2 - vceil2); + } + + // Apply internal energy ceiling as a pressure ceiling + const Real internal_e = prim(IPR, k, j, i) / (gm1 * prim(IDN, k, j, i)); + if (eceil > 0 && internal_e > eceil) { + cons(IEN, k, j, i) -= prim(IDN, k, j, i) * (internal_e - eceil); + prim(IPR, k, j, i) = gm1 * prim(IDN, k, j, i) * eceil; + } + } + + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + PARTHENON_REQUIRE(prim(IPR, k, j, i) > 0, + "Kinetic injection leads to negative pressure"); + }); + + // Apply magnetic tower feedback + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + const Real magnetic_power = power * magnetic_fraction_; + const Real magnetic_mass_rate = mass_rate * magnetic_mass_fraction_; + magnetic_tower.PowerSrcTerm(magnetic_power, magnetic_mass_rate, md, beta_dt, tm); +} + +} // namespace cluster diff --git a/src/pgen/cluster/agn_feedback.hpp b/src/pgen/cluster/agn_feedback.hpp new file mode 100644 index 00000000..3bec1609 --- /dev/null +++ b/src/pgen/cluster/agn_feedback.hpp @@ -0,0 +1,69 @@ +#ifndef CLUSTER_AGN_FEEDBACK_HPP_ +#define CLUSTER_AGN_FEEDBACK_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file agn_feedback.hpp +// \brief Class for injecting AGN feedback via thermal dump, kinetic jet, and magnetic +// tower + +// parthenon headers +#include +#include +#include +#include +#include + +#include "jet_coords.hpp" + +namespace cluster { + +/************************************************************ + * AGNFeedback + ************************************************************/ +class AGNFeedback { + public: + const parthenon::Real fixed_power_; + parthenon::Real thermal_fraction_, kinetic_fraction_, magnetic_fraction_; + parthenon::Real thermal_mass_fraction_, kinetic_mass_fraction_, magnetic_mass_fraction_; + + // Efficiency converting mass to energy + const parthenon::Real efficiency_; + + // Velocity and temperature ceilings + parthenon::Real vceil_, eceil_; + + // Thermal Heating Parameters + const parthenon::Real thermal_radius_; + + // Kinetic Feedback Parameters + const parthenon::Real kinetic_jet_radius_, kinetic_jet_thickness_, kinetic_jet_offset_; + parthenon::Real kinetic_jet_velocity_, kinetic_jet_temperature_, kinetic_jet_e_; + + // enable passive scalar to trace AGN material + const bool enable_tracer_; + + const bool disabled_; + + const bool enable_magnetic_tower_mass_injection_; + + AGNFeedback(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg); + + parthenon::Real GetFeedbackPower(parthenon::StateDescriptor *hydro_pkg) const; + parthenon::Real GetFeedbackMassRate(parthenon::StateDescriptor *hydro_pkg) const; + + void FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm) const; + + // Apply the feedback from hydrodynamic AGN feedback (kinetic jets and thermal feedback) + template + void FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm, + const EOS &eos) const; +}; + +} // namespace cluster + +#endif // CLUSTER_AGN_FEEDBACK_HPP_ diff --git a/src/pgen/cluster/agn_triggering.cpp b/src/pgen/cluster/agn_triggering.cpp new file mode 100644 index 00000000..242de4fa --- /dev/null +++ b/src/pgen/cluster/agn_triggering.cpp @@ -0,0 +1,586 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file agn_triggering.cpp +// \brief Class for computing AGN triggering from Bondi-like and cold gas accretion + +#include +#include // for ofstream +#include + +// Parthenon headers +#include +#include +#include +#include +#include +#include +#include + +// Athena headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" +#include "../../main.hpp" +#include "../../units.hpp" +#include "agn_feedback.hpp" +#include "agn_triggering.hpp" +#include "cluster_utils.hpp" + +namespace cluster { +using namespace parthenon; + +AGNTriggeringMode ParseAGNTriggeringMode(const std::string &mode_str) { + + if (mode_str == "COLD_GAS") { + return AGNTriggeringMode::COLD_GAS; + } else if (mode_str == "BOOSTED_BONDI") { + return AGNTriggeringMode::BOOSTED_BONDI; + } else if (mode_str == "BOOTH_SCHAYE") { + return AGNTriggeringMode::BOOTH_SCHAYE; + } else if (mode_str == "NONE") { + return AGNTriggeringMode::NONE; + } else { + std::stringstream msg; + msg << "### FATAL ERROR in function [ParseAGNTriggeringMode]" << std::endl + << "Unrecognized AGNTriggeringMode: \"" << mode_str << "\"" << std::endl; + PARTHENON_FAIL(msg); + } + return AGNTriggeringMode::NONE; +} + +AGNTriggering::AGNTriggering(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg, + const std::string &block) + : gamma_(pin->GetReal("hydro", "gamma")), + triggering_mode_( + ParseAGNTriggeringMode(pin->GetOrAddString(block, "triggering_mode", "NONE"))), + accretion_radius_(pin->GetOrAddReal(block, "accretion_radius", 0)), + cold_temp_thresh_(pin->GetOrAddReal(block, "cold_temp_thresh", 0)), + cold_t_acc_(pin->GetOrAddReal(block, "cold_t_acc", 0)), + bondi_alpha_(pin->GetOrAddReal(block, "bondi_alpha", 0)), + bondi_M_smbh_(pin->GetOrAddReal("problem/cluster/gravity", "m_smbh", 0)), + bondi_n0_(pin->GetOrAddReal(block, "bondi_n0", 0)), + bondi_beta_(pin->GetOrAddReal(block, "bondi_beta", 0)), + accretion_cfl_(pin->GetOrAddReal(block, "accretion_cfl", 1e-1)), + remove_accreted_mass_(pin->GetOrAddBoolean(block, "removed_accreted_mass", true)), + write_to_file_(pin->GetOrAddBoolean(block, "write_to_file", false)), + triggering_filename_( + pin->GetOrAddString(block, "triggering_filename", "agn_triggering.dat")) { + + const auto units = hydro_pkg->Param("units"); + const parthenon::Real He_mass_fraction = pin->GetReal("hydro", "He_mass_fraction"); + const parthenon::Real H_mass_fraction = 1.0 - He_mass_fraction; + const parthenon::Real mu = + 1 / (He_mass_fraction * 3. / 4. + (1 - He_mass_fraction) * 2); + + mean_molecular_mass_ = mu * units.atomic_mass_unit(); + + if (triggering_mode_ == AGNTriggeringMode::NONE) { + hydro_pkg->AddParam("agn_triggering_reduce_accretion_rate", false); + } else { + hydro_pkg->AddParam("agn_triggering_reduce_accretion_rate", true); + } + switch (triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + hydro_pkg->AddParam("agn_triggering_cold_mass", 0, Params::Mutability::Restart); + break; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + hydro_pkg->AddParam("agn_triggering_total_mass", 0, + Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_density", 0, + Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_velocity", 0, + Params::Mutability::Restart); + hydro_pkg->AddParam("agn_triggering_mass_weighted_cs", 0, + Params::Mutability::Restart); + break; + } + case AGNTriggeringMode::NONE: { + break; + } + } + + // Set up writing the triggering to file, used for debugging and regression + // testing. Note that this is written every timestep, which is more + // frequently than history outputs. It is also not reduced across ranks and + // so is only valid without MPI + if (write_to_file_ && parthenon::Globals::my_rank == 0) { + // Clear the triggering_file + std::ofstream triggering_file; + triggering_file.open(triggering_filename_, std::ofstream::out | std::ofstream::trunc); + triggering_file.close(); + } + + hydro_pkg->AddParam("agn_triggering", *this); +} + +// Compute Cold gas accretion rate within the accretion radius for cold gas triggering +// and simultaneously remove cold gas (updating conserveds and primitives) +template +void AGNTriggering::ReduceColdMass(parthenon::Real &cold_mass, + parthenon::MeshData *md, + const parthenon::Real dt, const EOS eos) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); + IndexRange int_ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange int_jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange int_kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + const auto nscalars = hydro_pkg->Param("nscalars"); + + const Real accretion_radius2 = pow(accretion_radius_, 2); + + // Reduce just the cold gas + const auto units = hydro_pkg->Param("units"); + const Real mean_molecular_mass_by_kb = mean_molecular_mass_ / units.k_boltzmann(); + + const Real cold_temp_thresh = cold_temp_thresh_; + const Real cold_t_acc = cold_t_acc_; + + const bool remove_accreted_mass = remove_accreted_mass_; + + Real md_cold_mass = 0; + + parthenon::par_reduce( + parthenon::loop_pattern_mdrange_tag, "AGNTriggering::ReduceColdGas", + parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, + ib.e, + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real &team_cold_mass) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const parthenon::Real r2 = + pow(coords.Xc<1>(i), 2) + pow(coords.Xc<2>(j), 2) + pow(coords.Xc<3>(k), 2); + if (r2 < accretion_radius2) { + + const Real temp = + mean_molecular_mass_by_kb * prim(IPR, k, j, i) / prim(IDN, k, j, i); + + if (temp <= cold_temp_thresh) { + + const Real cell_cold_mass = prim(IDN, k, j, i) * coords.CellVolume(k, j, i); + + if (k >= int_kb.s && k <= int_kb.e && j >= int_jb.s && j <= int_jb.e && + i >= int_ib.s && i <= int_ib.e) { + // Only reduce the cold gas that exists on the interior grid + team_cold_mass += cell_cold_mass; + } + + const Real cell_delta_rho = -prim(IDN, k, j, i) / cold_t_acc * dt; + + if (remove_accreted_mass) { + AddDensityToConsAtFixedVelTemp(cell_delta_rho, cons, prim, eos.GetGamma(), + k, j, i); + // Update the Primitives + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + } + } + } + }, + Kokkos::Sum(md_cold_mass)); + cold_mass += md_cold_mass; +} + +// Compute Mass-weighted total density, velocity, and sound speed and total mass +// for Bondi accretion +void AGNTriggering::ReduceBondiTriggeringQuantities( + parthenon::Real &total_mass, parthenon::Real &mass_weighted_density, + parthenon::Real &mass_weighted_velocity, parthenon::Real &mass_weighted_cs, + parthenon::MeshData *md) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + + const Real accretion_radius2 = pow(accretion_radius_, 2); + + // Reduce Mass-weighted total density, velocity, and sound speed and total + // mass (in that order). Will need to divide the three latter quantities by + // total mass in order to get their mass-weighted averaged values + Real total_mass_red, mass_weighted_density_red, mass_weighted_velocity_red, + mass_weighted_cs_red; + + const parthenon::Real gamma = gamma_; + + Kokkos::parallel_reduce( + "AGNTriggering::ReduceBondi", + Kokkos::MDRangePolicy>( + DevExecSpace(), {0, kb.s, jb.s, ib.s}, + {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real <otal_mass_red, Real &lmass_weighted_density_red, + Real &lmass_weighted_velocity_red, Real &lmass_weighted_cs_red) { + auto &prim = prim_pack(b); + const auto &coords = prim_pack.GetCoords(b); + const parthenon::Real r2 = + pow(coords.Xc<1>(i), 2) + pow(coords.Xc<2>(j), 2) + pow(coords.Xc<3>(k), 2); + if (r2 < accretion_radius2) { + const Real cell_mass = prim(IDN, k, j, i) * coords.CellVolume(k, j, i); + + const Real cell_mass_weighted_density = cell_mass * prim(IDN, k, j, i); + const Real cell_mass_weighted_velocity = + cell_mass * sqrt(pow(prim(IV1, k, j, i), 2) + pow(prim(IV2, k, j, i), 2) + + pow(prim(IV3, k, j, i), 2)); + const Real cell_mass_weighted_cs = + cell_mass * sqrt(gamma * prim(IPR, k, j, i) / prim(IDN, k, j, i)); + + ltotal_mass_red += cell_mass; + lmass_weighted_density_red += cell_mass_weighted_density; + lmass_weighted_velocity_red += cell_mass_weighted_velocity; + lmass_weighted_cs_red += cell_mass_weighted_cs; + } + }, + total_mass_red, mass_weighted_density_red, mass_weighted_velocity_red, + mass_weighted_cs_red); + // Save the reduction results to triggering_quantities + total_mass += total_mass_red; + mass_weighted_density += mass_weighted_density_red; + mass_weighted_velocity += mass_weighted_velocity_red; + mass_weighted_cs += mass_weighted_cs_red; +} + +// Remove gas consistent with Bondi accretion +/// i.e. proportional to the accretion rate, weighted by the local gas mass +template +void AGNTriggering::RemoveBondiAccretedGas(parthenon::MeshData *md, + const parthenon::Real dt, + const EOS eos) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // Grab some necessary variables + // FIXME(forrestglines) When reductions are called, is `prim` up to date? + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::entire); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::entire); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::entire); + const auto nhydro = hydro_pkg->Param("nhydro"); + const auto nscalars = hydro_pkg->Param("nscalars"); + + const Real accretion_radius2 = pow(accretion_radius_, 2); + + const Real accretion_rate = GetAccretionRate(hydro_pkg.get()); + const Real total_mass = hydro_pkg->Param("agn_triggering_total_mass"); + + parthenon::par_for( + parthenon::loop_pattern_mdrange_tag, "AGNTriggering::RemoveBondiAccretedGas", + parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, + ib.e, KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const parthenon::Real r2 = + pow(coords.Xc<1>(i), 2) + pow(coords.Xc<2>(j), 2) + pow(coords.Xc<3>(k), 2); + if (r2 < accretion_radius2) { + + const Real cell_delta_rho = + -prim(IDN, k, j, i) / total_mass * accretion_rate * dt; + + AddDensityToConsAtFixedVelTemp(cell_delta_rho, cons, prim, eos.GetGamma(), k, j, + i); + + // Update the Primitives + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + } + }); +} + +// Compute and return the current AGN accretion rate from already globally +// reduced quantities +parthenon::Real +AGNTriggering::GetAccretionRate(parthenon::StateDescriptor *hydro_pkg) const { + switch (triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + // Test the Cold-Gas-like triggering methods + const parthenon::Real cold_mass = hydro_pkg->Param("agn_triggering_cold_mass"); + + return cold_mass / cold_t_acc_; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + // Test the Bondi-like triggering methods + auto units = hydro_pkg->Param("units"); + const Real total_mass = hydro_pkg->Param("agn_triggering_total_mass"); + const Real mean_mass_weighted_density = + hydro_pkg->Param("agn_triggering_mass_weighted_density") / total_mass; + const Real mean_mass_weighted_velocity = + hydro_pkg->Param("agn_triggering_mass_weighted_velocity") / total_mass; + const Real mean_mass_weighted_cs = + hydro_pkg->Param("agn_triggering_mass_weighted_cs") / total_mass; + + Real alpha = 0; + if (triggering_mode_ == AGNTriggeringMode::BOOSTED_BONDI) { + alpha = bondi_alpha_; + } else if (triggering_mode_ == AGNTriggeringMode::BOOTH_SCHAYE) { + const Real mean_mass_weighted_n = mean_mass_weighted_density / mean_molecular_mass_; + alpha = (mean_mass_weighted_n <= bondi_n0_) + ? 1 + : pow(mean_mass_weighted_n / bondi_n0_, bondi_beta_); + } else { + PARTHENON_FAIL("### FATAL ERROR in AGNTriggering::AccretionRate unsupported " + "Bondi-like triggering"); + } + const Real mdot = + alpha * 2 * M_PI * pow(units.gravitational_constant(), 2) * + pow(bondi_M_smbh_, 2) * mean_mass_weighted_density / + (pow(pow(mean_mass_weighted_velocity, 2) + pow(mean_mass_weighted_cs, 2), + 3. / 2.)); + + return mdot; + } + case AGNTriggeringMode::NONE: { + return 0; + } + } + return 0; +} + +parthenon::TaskStatus +AGNTriggeringResetTriggering(parthenon::StateDescriptor *hydro_pkg) { + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + + switch (agn_triggering.triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + hydro_pkg->UpdateParam("agn_triggering_cold_mass", 0); + break; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + hydro_pkg->UpdateParam("agn_triggering_total_mass", 0); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_density", 0); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_velocity", 0); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_cs", 0); + break; + } + case AGNTriggeringMode::NONE: { + break; + } + } + return TaskStatus::complete; +} + +parthenon::TaskStatus +AGNTriggeringReduceTriggering(parthenon::MeshData *md, + const parthenon::Real dt) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + + switch (agn_triggering.triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + Real cold_mass = hydro_pkg->Param("agn_triggering_cold_mass"); + + auto fluid = hydro_pkg->Param("fluid"); + if (fluid == Fluid::euler) { + agn_triggering.ReduceColdMass(cold_mass, md, dt, + hydro_pkg->Param("eos")); + } else if (fluid == Fluid::glmmhd) { + agn_triggering.ReduceColdMass(cold_mass, md, dt, + hydro_pkg->Param("eos")); + } else { + PARTHENON_FAIL("AGNTriggeringReduceTriggeringQuantities: Unknown EOS"); + } + + hydro_pkg->UpdateParam("agn_triggering_cold_mass", cold_mass); + break; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + Real total_mass = hydro_pkg->Param("agn_triggering_total_mass"); + Real mass_weighted_density = + hydro_pkg->Param("agn_triggering_mass_weighted_density"); + Real mass_weighted_velocity = + hydro_pkg->Param("agn_triggering_mass_weighted_velocity"); + Real mass_weighted_cs = + hydro_pkg->Param("agn_triggering_mass_weighted_cs"); + + agn_triggering.ReduceBondiTriggeringQuantities( + total_mass, mass_weighted_density, mass_weighted_velocity, mass_weighted_cs, md); + + hydro_pkg->UpdateParam("agn_triggering_total_mass", total_mass); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_density", mass_weighted_density); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_velocity", + mass_weighted_velocity); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_cs", mass_weighted_cs); + break; + } + case AGNTriggeringMode::NONE: { + break; + } + } + return TaskStatus::complete; +} + +parthenon::TaskStatus +AGNTriggeringMPIReduceTriggering(parthenon::StateDescriptor *hydro_pkg) { +#ifdef MPI_PARALLEL + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + switch (agn_triggering.triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + + Real accretion_rate = hydro_pkg->Param("agn_triggering_cold_mass"); + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &accretion_rate, 1, + MPI_PARTHENON_REAL, MPI_SUM, MPI_COMM_WORLD)); + hydro_pkg->UpdateParam("agn_triggering_cold_mass", accretion_rate); + break; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + Real triggering_quantities[] = { + hydro_pkg->Param("agn_triggering_total_mass"), + hydro_pkg->Param("agn_triggering_mass_weighted_density"), + hydro_pkg->Param("agn_triggering_mass_weighted_velocity"), + hydro_pkg->Param("agn_triggering_mass_weighted_cs"), + }; + + PARTHENON_MPI_CHECK(MPI_Allreduce(MPI_IN_PLACE, &triggering_quantities, 4, + MPI_PARTHENON_REAL, MPI_SUM, MPI_COMM_WORLD)); + + hydro_pkg->UpdateParam("agn_triggering_total_mass", triggering_quantities[0]); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_density", + triggering_quantities[1]); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_velocity", + triggering_quantities[2]); + hydro_pkg->UpdateParam("agn_triggering_mass_weighted_cs", triggering_quantities[3]); + break; + } + case AGNTriggeringMode::NONE: { + break; + } + } +#endif + return TaskStatus::complete; +} + +parthenon::TaskStatus +AGNTriggeringFinalizeTriggering(parthenon::MeshData *md, + const parthenon::SimTime &tm) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + const auto &agn_triggering = hydro_pkg->Param("agn_triggering"); + + // Append quantities to file if applicable + if (agn_triggering.write_to_file_ && parthenon::Globals::my_rank == 0) { + std::ofstream triggering_file; + triggering_file.open(agn_triggering.triggering_filename_, std::ofstream::app); + + triggering_file << tm.time << " " << tm.dt << " " + << agn_triggering.GetAccretionRate(hydro_pkg.get()) << " "; + + switch (agn_triggering.triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + + triggering_file << hydro_pkg->Param("agn_triggering_cold_mass"); + break; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + const auto &total_mass = hydro_pkg->Param("agn_triggering_total_mass"); + const auto &avg_density = + hydro_pkg->Param("agn_triggering_mass_weighted_density") / total_mass; + const auto &avg_velocity = + hydro_pkg->Param("agn_triggering_mass_weighted_velocity") / total_mass; + const auto &avg_cs = + hydro_pkg->Param("agn_triggering_mass_weighted_cs") / total_mass; + triggering_file << total_mass << " " << avg_density << " " << avg_velocity << " " + << avg_cs; + break; + } + case AGNTriggeringMode::NONE: { + break; + } + } + + triggering_file << std::endl; + triggering_file.close(); + } + + // Remove accreted gas if using a Bondi-like mode + if (agn_triggering.remove_accreted_mass_) { + switch (agn_triggering.triggering_mode_) { + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + auto fluid = hydro_pkg->Param("fluid"); + if (fluid == Fluid::euler) { + agn_triggering.RemoveBondiAccretedGas(md, tm.dt, + hydro_pkg->Param("eos")); + } else if (fluid == Fluid::glmmhd) { + agn_triggering.RemoveBondiAccretedGas( + md, tm.dt, hydro_pkg->Param("eos")); + } else { + PARTHENON_FAIL("AGNTriggeringFinalizeTriggering: Unknown EOS"); + } + break; + } + case AGNTriggeringMode::COLD_GAS: // Already removed during reduction + case AGNTriggeringMode::NONE: { + break; + } + } + } + + return TaskStatus::complete; +} + +// Limit timestep to a factor of the cold gas accretion time for Cold Gas +// triggered cooling, or a factor of the time to accrete the total mass for +// Bondi-like accretion +parthenon::Real +AGNTriggering::EstimateTimeStep(parthenon::MeshData *md) const { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + switch (triggering_mode_) { + case AGNTriggeringMode::COLD_GAS: { + return accretion_cfl_ * cold_t_acc_; + } + case AGNTriggeringMode::BOOSTED_BONDI: + case AGNTriggeringMode::BOOTH_SCHAYE: { + // Test the Bondi-like triggering methods + const Real total_mass = hydro_pkg->Param("agn_triggering_total_mass"); + if (total_mass == 0) { + // TODO(forrestglines)During the first timestep, the total mass and + // accretion rate has not yet been reduced. However, since accreted mass is + // removed during that reduction, the timestep is needed to execute that + // reduction. As a compromise, we ignore the timestep constraint during the + // first timestep, assuming that accretion is slow initially + return std::numeric_limits::max(); + } + const Real mdot = GetAccretionRate(hydro_pkg.get()); + return accretion_cfl_ * total_mass / mdot; + } + case AGNTriggeringMode::NONE: { + return std::numeric_limits::max(); + } + } + return std::numeric_limits::max(); +} + +} // namespace cluster diff --git a/src/pgen/cluster/agn_triggering.hpp b/src/pgen/cluster/agn_triggering.hpp new file mode 100644 index 00000000..8236a291 --- /dev/null +++ b/src/pgen/cluster/agn_triggering.hpp @@ -0,0 +1,131 @@ +#ifndef CLUSTER_AGN_TRIGGERING_HPP_ +#define CLUSTER_AGN_TRIGGERING_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file agn_triggering.hpp +// \brief Class for computing AGN triggering from Bondi-like and cold gas accretion + +// parthenon headers +#include +#include +#include +#include +#include +#include + +// AthenaPK headers +#include "../../units.hpp" +#include "jet_coords.hpp" +#include "utils/error_checking.hpp" + +namespace cluster { + +enum class AGNTriggeringMode { NONE, COLD_GAS, BOOSTED_BONDI, BOOTH_SCHAYE }; + +AGNTriggeringMode ParseAGNTriggeringMode(const std::string &mode_str); + +/************************************************************ + * AGN Triggering class : For computing the mass triggering the AGN + ************************************************************/ +class AGNTriggering { + private: + const parthenon::Real gamma_; + parthenon::Real mean_molecular_mass_; + + public: + const AGNTriggeringMode triggering_mode_; + + const parthenon::Real accretion_radius_; + + // Parameters for cold-gas triggering + const parthenon::Real cold_temp_thresh_; + const parthenon::Real cold_t_acc_; + + // Parameters necessary for Boosted Bondi accretion + const parthenon::Real bondi_alpha_; //(Only for boosted Bondi) + const parthenon::Real bondi_M_smbh_; + + // Additional parameters for Booth Schaye + const parthenon::Real bondi_n0_; + const parthenon::Real bondi_beta_; + + // Used in timestep estimation + const parthenon::Real accretion_cfl_; + + // Useful for debugging + const bool remove_accreted_mass_; + + // Write triggering quantities (accretion rate or Bondi quantities) to file at + // every timestep. Intended for testing quantities at every timestep, since + // this file does not work across restarts, and since these quantities are + // included in Parthenon phdf outputs. + const bool write_to_file_; + const std::string triggering_filename_; + + AGNTriggering(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg, + const std::string &block = "problem/cluster/agn_triggering"); + + // Compute Cold gas accretion rate within the accretion radius for cold gas triggering + // and simultaneously remove cold gas (updating conserveds and primitives) + template + void ReduceColdMass(parthenon::Real &cold_mass, + parthenon::MeshData *md, const parthenon::Real dt, + const EOS eos) const; + + // Compute Mass-weighted total density, velocity, and sound speed and total mass + // for Bondi accretion + void ReduceBondiTriggeringQuantities(parthenon::Real &total_mass, + parthenon::Real &mass_weighted_density, + parthenon::Real &mass_weighted_velocity, + parthenon::Real &mass_weighted_cs, + parthenon::MeshData *md) const; + + // Remove gas consistent with Bondi accretion + /// i.e. proportional to the accretion rate, weighted by the local gas mass + template + void RemoveBondiAccretedGas(parthenon::MeshData *md, + const parthenon::Real dt, const EOS eos) const; + + // Compute and return the current AGN accretion rate from already globally + // reduced quantities + parthenon::Real GetAccretionRate(parthenon::StateDescriptor *hydro_pkg) const; + + friend parthenon::TaskStatus + AGNTriggeringResetTriggering(parthenon::StateDescriptor *hydro_pkg); + + friend parthenon::TaskStatus + AGNTriggeringReduceTriggering(parthenon::MeshData *md, + const parthenon::Real dt); + + friend parthenon::TaskStatus + AGNTriggeringMPIReduceTriggering(parthenon::StateDescriptor *hydro_pkg); + + friend parthenon::TaskStatus + AGNTriggeringFinalizeTriggering(parthenon::MeshData *md, + const parthenon::SimTime &tm); + + // Limit timestep to a factor of the cold gas accretion time for Cold Gas + // triggered cooling, or a factor of the time to accrete the total mass for + // Bondi-like accretion + parthenon::Real EstimateTimeStep(parthenon::MeshData *md) const; +}; + +parthenon::TaskStatus AGNTriggeringResetTriggering(parthenon::StateDescriptor *hydro_pkg); + +parthenon::TaskStatus +AGNTriggeringReduceTriggering(parthenon::MeshData *md, + const parthenon::Real dt); + +parthenon::TaskStatus +AGNTriggeringMPIReduceTriggering(parthenon::StateDescriptor *hydro_pkg); + +parthenon::TaskStatus +AGNTriggeringFinalizeTriggering(parthenon::MeshData *md, + const parthenon::SimTime &tm); + +} // namespace cluster + +#endif // CLUSTER_AGN_TRIGGERING_HPP_ diff --git a/src/pgen/cluster/cluster_clips.cpp b/src/pgen/cluster/cluster_clips.cpp new file mode 100644 index 00000000..3b11227f --- /dev/null +++ b/src/pgen/cluster/cluster_clips.cpp @@ -0,0 +1,175 @@ + +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file agn_triggering.cpp +// \brief Class for computing AGN triggering from Bondi-like and cold gas accretion + +// Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" +#include + +// AthenaPK headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" + +namespace cluster { +using namespace parthenon; + +template +void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt, const EOS eos); + +void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + auto fluid = hydro_pkg->Param("fluid"); + if (fluid == Fluid::euler) { + ApplyClusterClips(md, tm, beta_dt, hydro_pkg->Param("eos")); + } else if (fluid == Fluid::glmmhd) { + ApplyClusterClips(md, tm, beta_dt, hydro_pkg->Param("eos")); + } else { + PARTHENON_FAIL("Cluster::ApplyClusterClips: Unknown EOS"); + } +} + +template +void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt, const EOS eos) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // Apply clips -- ceilings on temperature, velocity, alfven velocity, and + // density floor -- within a radius of the AGN + const auto &dfloor = hydro_pkg->Param("cluster_dfloor"); + const auto &eceil = hydro_pkg->Param("cluster_eceil"); + const auto &vceil = hydro_pkg->Param("cluster_vceil"); + const auto &vAceil = hydro_pkg->Param("cluster_vAceil"); + const auto &clip_r = hydro_pkg->Param("cluster_clip_r"); + + if (clip_r > 0 && (dfloor > 0 || eceil < std::numeric_limits::infinity() || + vceil < std::numeric_limits::infinity() || + vAceil < std::numeric_limits::infinity())) { + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + const auto nscalars = hydro_pkg->Param("nscalars"); + + const Real clip_r2 = SQR(clip_r); + const Real vceil2 = SQR(vceil); + const Real vAceil2 = SQR(vAceil); + const Real gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + + const bool magnetic_fields = (hydro_pkg->Param("fluid") == Fluid::glmmhd); + Real added_dfloor_mass = 0.0, removed_vceil_energy = 0.0, added_vAceil_mass = 0.0, + removed_eceil_energy = 0.0; + + Kokkos::parallel_reduce( + "Cluster::ApplyClusterClips", + Kokkos::MDRangePolicy>( + DevExecSpace(), {0, kb.s, jb.s, ib.s}, + {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real &added_dfloor_mass_team, Real &removed_vceil_energy_team, + Real &added_vAceil_mass_team, Real &removed_eceil_energy_team) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const Real r2 = + SQR(coords.Xc<1>(i)) + SQR(coords.Xc<2>(j)) + SQR(coords.Xc<3>(k)); + + if (r2 < clip_r2) { + // Cell falls within clipping radius + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + + if (dfloor > 0) { + const Real rho = prim(IDN, k, j, i); + if (rho < dfloor) { + added_dfloor_mass_team += (dfloor - rho) * coords.CellVolume(k, j, i); + cons(IDN, k, j, i) = dfloor; + prim(IDN, k, j, i) = dfloor; + } + } + + if (vceil < std::numeric_limits::infinity()) { + // Apply velocity ceiling + const Real v2 = SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + + SQR(prim(IV3, k, j, i)); + if (v2 > vceil2) { + // Fix the velocity to the velocity ceiling + const Real v = sqrt(v2); + cons(IM1, k, j, i) *= vceil / v; + cons(IM2, k, j, i) *= vceil / v; + cons(IM3, k, j, i) *= vceil / v; + prim(IV1, k, j, i) *= vceil / v; + prim(IV2, k, j, i) *= vceil / v; + prim(IV3, k, j, i) *= vceil / v; + + // Remove kinetic energy + const Real removed_energy = 0.5 * prim(IDN, k, j, i) * (v2 - vceil2); + removed_vceil_energy_team += removed_energy * coords.CellVolume(k, j, i); + cons(IEN, k, j, i) -= removed_energy; + } + } + + if (magnetic_fields && vAceil2 < std::numeric_limits::infinity()) { + // Apply Alfven velocity ceiling by raising density + const Real rho = prim(IDN, k, j, i); + const Real B2 = (SQR(prim(IB1, k, j, i)) + SQR(prim(IB2, k, j, i)) + + SQR(prim(IB3, k, j, i))); + + // compute Alfven mach number + const Real va2 = (B2 / rho); + + if (va2 > vAceil2) { + // Increase the density to match the alfven velocity ceiling + const Real rho_new = std::sqrt(B2 / vAceil2); + added_vAceil_mass_team += (rho_new - rho) * coords.CellVolume(k, j, i); + cons(IDN, k, j, i) = rho_new; + prim(IDN, k, j, i) = rho_new; + } + } + + if (eceil < std::numeric_limits::infinity()) { + // Apply internal energy ceiling as a pressure ceiling + const Real internal_e = prim(IPR, k, j, i) / (gm1 * prim(IDN, k, j, i)); + if (internal_e > eceil) { + const Real removed_energy = prim(IDN, k, j, i) * (internal_e - eceil); + removed_eceil_energy_team += removed_energy * coords.CellVolume(k, j, i); + cons(IEN, k, j, i) -= removed_energy; + prim(IPR, k, j, i) = gm1 * prim(IDN, k, j, i) * eceil; + } + } + } + }, + added_dfloor_mass, removed_vceil_energy, added_vAceil_mass, removed_eceil_energy); + + // Add the freshly added mass/removed energy to running totals + hydro_pkg->UpdateParam("added_dfloor_mass", + added_dfloor_mass + + hydro_pkg->Param("added_dfloor_mass")); + hydro_pkg->UpdateParam("removed_vceil_energy", + removed_vceil_energy + + hydro_pkg->Param("removed_vceil_energy")); + hydro_pkg->UpdateParam("added_vAceil_mass", + added_vAceil_mass + + hydro_pkg->Param("added_vAceil_mass")); + hydro_pkg->UpdateParam("removed_eceil_energy", + removed_eceil_energy + + hydro_pkg->Param("removed_eceil_energy")); + } +} + +} // namespace cluster \ No newline at end of file diff --git a/src/pgen/cluster/cluster_clips.hpp b/src/pgen/cluster/cluster_clips.hpp new file mode 100644 index 00000000..74351f0a --- /dev/null +++ b/src/pgen/cluster/cluster_clips.hpp @@ -0,0 +1,27 @@ + + +#ifndef CLUSTER_CLUSTER_CLIPS_HPP_ +#define CLUSTER_CLUSTER_CLIPS_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster_clips.hpp +// \brief Class for applying floors and ceils and reducing removed/added mass/energy + +// C++ headers +#include // sqrt() + +// Parthenon headers +#include "basic_types.hpp" +#include "mesh/mesh.hpp" + +namespace cluster { + +void ApplyClusterClips(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt); + +} + +#endif // CLUSTER_AGN_TRIGGERING_HPP_ \ No newline at end of file diff --git a/src/pgen/cluster/cluster_gravity.hpp b/src/pgen/cluster/cluster_gravity.hpp index 2c7c2a18..e35e60e4 100644 --- a/src/pgen/cluster/cluster_gravity.hpp +++ b/src/pgen/cluster/cluster_gravity.hpp @@ -1,12 +1,12 @@ +#ifndef CLUSTER_CLUSTER_GRAVITY_HPP_ +#define CLUSTER_CLUSTER_GRAVITY_HPP_ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file cluster_gravity.hpp // \brief Class for defining gravitational acceleration for a cluster+bcg+smbh -#ifndef CLUSTER_CLUSTER_GRAVITY_HPP_ -#define CLUSTER_CLUSTER_GRAVITY_HPP_ // Parthenon headers #include @@ -17,8 +17,7 @@ namespace cluster { // Types of BCG's -enum class BCG { NONE, MATHEWS, HERNQUIST }; -// Mathews BCG: Mathews 2006 DOI: 10.1086/499119 +enum class BCG { NONE, HERNQUIST }; // Hernquiest BCG: Hernquist 1990 DOI:10.1086/168845 /************************************************************ @@ -33,73 +32,96 @@ class ClusterGravity { bool include_smbh_g_; // NFW Parameters - parthenon::Real R_nfw_s_; - parthenon::Real - GMC_nfw_; // G , Mass, and Constants rolled into one, to minimize footprint + parthenon::Real r_nfw_s_; + // G , Mass, and Constants rolled into one + parthenon::Real g_const_nfw_; + parthenon::Real rho_const_nfw_; // BCG Parameters parthenon::Real alpha_bcg_s_; parthenon::Real beta_bcg_s_; - parthenon::Real R_bcg_s_; - parthenon::Real - GMC_bcg_; // G , Mass, and Constants rolled into one, to minimize footprint + parthenon::Real r_bcg_s_; + // G , Mass, and Constants rolled into one + parthenon::Real g_const_bcg_; + parthenon::Real rho_const_bcg_; // SMBH Parameters - parthenon::Real - GMC_smbh_; // G , Mass, and Constants rolled into one, to minimize footprint + // G , Mass, and Constants rolled into one + parthenon::Real g_const_smbh_; // Radius underwhich to truncate parthenon::Real smoothing_r_; // Static Helper functions to calculate constants to minimize in-kernel work static parthenon::Real calc_R_nfw_s(const parthenon::Real rho_crit, - const parthenon::Real M_nfw_200, + const parthenon::Real m_nfw_200, const parthenon::Real c_nfw) { const parthenon::Real rho_nfw_0 = 200 / 3. * rho_crit * pow(c_nfw, 3.) / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); const parthenon::Real R_nfw_s = - pow(M_nfw_200 / (4 * M_PI * rho_nfw_0 * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))), + pow(m_nfw_200 / (4 * M_PI * rho_nfw_0 * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))), 1. / 3.); return R_nfw_s; } - static parthenon::Real calc_GMC_nfw(const parthenon::Real gravitational_constant, - const parthenon::Real M_nfw_200, - const parthenon::Real c_nfw) { - return gravitational_constant * M_nfw_200 / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); + static parthenon::Real calc_g_const_nfw(const parthenon::Real gravitational_constant, + const parthenon::Real m_nfw_200, + const parthenon::Real c_nfw) { + return gravitational_constant * m_nfw_200 / (log(1 + c_nfw) - c_nfw / (1 + c_nfw)); } - static parthenon::Real calc_GMC_bcg(const parthenon::Real gravitational_constant, - BCG which_bcg_g, const parthenon::Real M_bcg_s, - const parthenon::Real R_bcg_s, - const parthenon::Real alpha_bcg_s, - const parthenon::Real beta_bcg_s) { + static parthenon::Real calc_rho_const_nfw(const parthenon::Real gravitational_constant, + const parthenon::Real m_nfw_200, + const parthenon::Real c_nfw) { + return m_nfw_200 / (4 * M_PI * (log(1 + c_nfw) - c_nfw / (1 + c_nfw))); + } + static parthenon::Real calc_g_const_bcg(const parthenon::Real gravitational_constant, + BCG which_bcg_g, const parthenon::Real m_bcg_s, + const parthenon::Real r_bcg_s, + const parthenon::Real alpha_bcg_s, + const parthenon::Real beta_bcg_s) { switch (which_bcg_g) { case BCG::NONE: return 0; - case BCG::MATHEWS: - return 1 / (R_bcg_s * R_bcg_s); case BCG::HERNQUIST: - return gravitational_constant * M_bcg_s / (R_bcg_s * R_bcg_s); + return gravitational_constant * m_bcg_s / (r_bcg_s * r_bcg_s); + } + return NAN; + } + static parthenon::Real calc_rho_const_bcg(const parthenon::Real gravitational_constant, + BCG which_bcg_g, + const parthenon::Real m_bcg_s, + const parthenon::Real r_bcg_s, + const parthenon::Real alpha_bcg_s, + const parthenon::Real beta_bcg_s) { + switch (which_bcg_g) { + case BCG::NONE: + return 0; + case BCG::HERNQUIST: + return m_bcg_s * r_bcg_s / (2 * M_PI); } return NAN; } static KOKKOS_INLINE_FUNCTION parthenon::Real - calc_GMC_smbh(const parthenon::Real gravitational_constant, - const parthenon::Real M_smbh) { - return gravitational_constant * M_smbh; + calc_g_const_smbh(const parthenon::Real gravitational_constant, + const parthenon::Real m_smbh) { + return gravitational_constant * m_smbh; } public: + // ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) + // is called from cluster.cpp to add the ClusterGravity object to hydro_pkg + // + // ClusterGravity(parthenon::ParameterInput *pin) is used in SNIAFeedback to + // calculate the BCG density profile ClusterGravity(parthenon::ParameterInput *pin) { Units units(pin); // Determine which element to include - include_nfw_g_ = pin->GetOrAddBoolean("problem/cluster", "include_nfw_g", false); + include_nfw_g_ = + pin->GetOrAddBoolean("problem/cluster/gravity", "include_nfw_g", false); const std::string which_bcg_g_str = - pin->GetOrAddString("problem/cluster", "which_bcg_g", "NONE"); + pin->GetOrAddString("problem/cluster/gravity", "which_bcg_g", "NONE"); if (which_bcg_g_str == "NONE") { which_bcg_g_ = BCG::NONE; - } else if (which_bcg_g_str == "MATHEWS") { - which_bcg_g_ = BCG::MATHEWS; } else if (which_bcg_g_str == "HERNQUIST") { which_bcg_g_ = BCG::HERNQUIST; } else { @@ -109,7 +131,8 @@ class ClusterGravity { PARTHENON_FAIL(msg); } - include_smbh_g_ = pin->GetOrAddBoolean("problem/cluster", "include_smbh_g", false); + include_smbh_g_ = + pin->GetOrAddBoolean("problem/cluster/gravity", "include_smbh_g", false); // Initialize the NFW Profile const parthenon::Real hubble_parameter = pin->GetOrAddReal( @@ -118,25 +141,32 @@ class ClusterGravity { (8 * M_PI * units.gravitational_constant()); const parthenon::Real M_nfw_200 = - pin->GetOrAddReal("problem/cluster", "M_nfw_200", 8.5e14 * units.msun()); - const parthenon::Real c_nfw = pin->GetOrAddReal("problem/cluster", "c_nfw", 6.81); - R_nfw_s_ = calc_R_nfw_s(rho_crit, M_nfw_200, c_nfw); - GMC_nfw_ = calc_GMC_nfw(units.gravitational_constant(), M_nfw_200, c_nfw); - - // Initialize the NFW Profile - alpha_bcg_s_ = pin->GetOrAddReal("problem/cluster", "alpha_bcg_s", 0.1); - beta_bcg_s_ = pin->GetOrAddReal("problem/cluster", "beta_bcg_s", 1.43); + pin->GetOrAddReal("problem/cluster/gravity", "m_nfw_200", 8.5e14 * units.msun()); + const parthenon::Real c_nfw = + pin->GetOrAddReal("problem/cluster/gravity", "c_nfw", 6.81); + r_nfw_s_ = calc_R_nfw_s(rho_crit, M_nfw_200, c_nfw); + g_const_nfw_ = calc_g_const_nfw(units.gravitational_constant(), M_nfw_200, c_nfw); + + // Initialize the BCG Profile + alpha_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "alpha_bcg_s", 0.1); + beta_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "beta_bcg_s", 1.43); const parthenon::Real M_bcg_s = - pin->GetOrAddReal("problem/cluster", "M_bcg_s", 7.5e10 * units.msun()); - R_bcg_s_ = pin->GetOrAddReal("problem/cluster", "R_bcg_s", 4 * units.kpc()); - GMC_bcg_ = calc_GMC_bcg(units.gravitational_constant(), which_bcg_g_, M_bcg_s, - R_bcg_s_, alpha_bcg_s_, beta_bcg_s_); + pin->GetOrAddReal("problem/cluster/gravity", "m_bcg_s", 7.5e10 * units.msun()); + r_bcg_s_ = pin->GetOrAddReal("problem/cluster/gravity", "r_bcg_s", 4 * units.kpc()); + g_const_bcg_ = calc_g_const_bcg(units.gravitational_constant(), which_bcg_g_, M_bcg_s, + r_bcg_s_, alpha_bcg_s_, beta_bcg_s_); const parthenon::Real m_smbh = - pin->GetOrAddReal("problem/cluster", "m_smbh", 3.4e8 * units.msun()); - GMC_smbh_ = calc_GMC_smbh(units.gravitational_constant(), m_smbh), + pin->GetOrAddReal("problem/cluster/gravity", "m_smbh", 3.4e8 * units.msun()); + g_const_smbh_ = calc_g_const_smbh(units.gravitational_constant(), m_smbh), - smoothing_r_ = pin->GetOrAddReal("problem/cluster", "g_smoothing_radius", 0.0); + smoothing_r_ = + pin->GetOrAddReal("problem/cluster/gravity", "g_smoothing_radius", 0.0); + } + + ClusterGravity(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg) + : ClusterGravity(pin) { + hydro_pkg->AddParam<>("cluster_gravity", *this); } // Inline functions to compute gravitational acceleration @@ -150,32 +180,57 @@ class ClusterGravity { // Add NFW gravity if (include_nfw_g_) { - g_r += GMC_nfw_ * (log(1 + r / R_nfw_s_) - r / (r + R_nfw_s_)) / r2; + g_r += g_const_nfw_ * (log(1 + r / r_nfw_s_) - r / (r + r_nfw_s_)) / r2; } // Add BCG gravity switch (which_bcg_g_) { case BCG::NONE: break; - case BCG::MATHEWS: { - const parthenon::Real s_bcg = 0.9; - g_r += GMC_bcg_ * // Note: *cm**3*s**-2 //To make units work - pow(pow(r / R_bcg_s_, 0.5975 / 3.206e-7 * s_bcg) + - pow(pow(r / R_bcg_s_, 1.849 / 1.861e-6), s_bcg), - -1 / s_bcg); - } break; case BCG::HERNQUIST: - g_r += GMC_bcg_ / ((1 + r / R_bcg_s_) * (1 + r / R_bcg_s_)); + g_r += g_const_bcg_ / ((1 + r / r_bcg_s_) * (1 + r / r_bcg_s_)); break; } // Add SMBH, point mass gravity if (include_smbh_g_) { - g_r += GMC_smbh_ / r2; + g_r += g_const_smbh_ / r2; } return g_r; } + // Inline functions to compute density + KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r_in) const + __attribute__((always_inline)) { + + const parthenon::Real r = std::max(r_in, smoothing_r_); + + parthenon::Real rho = 0; + + // Add NFW gravity + if (include_nfw_g_) { + rho += rho_const_nfw_ / (r * pow(r + r_nfw_s_, 2)); + } + + // Add BCG gravity + switch (which_bcg_g_) { + case BCG::NONE: + break; + case BCG::HERNQUIST: + rho += rho_const_bcg_ / (r * pow(r + r_bcg_s_, 3)); + break; + } + + // SMBH, point mass gravity -- density is not defined. Throw an error + if (include_smbh_g_ && r <= smoothing_r_) { + Kokkos::abort("ClusterGravity::SMBH density is not defined"); + } + + return rho; + } + + // SNIAFeedback needs to be a friend to disable the SMBH and NFW + friend class SNIAFeedback; }; } // namespace cluster diff --git a/src/pgen/cluster/cluster_reductions.cpp b/src/pgen/cluster/cluster_reductions.cpp new file mode 100644 index 00000000..9e9edb4b --- /dev/null +++ b/src/pgen/cluster/cluster_reductions.cpp @@ -0,0 +1,101 @@ + +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster_reductions.cpp +// \brief Cluster-specific reductions to compute the total cold gas and maximum radius +// of AGN feedback + +// Parthenon headers +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/mesh.hpp" +#include "parthenon_array_generic.hpp" +#include "utils/error_checking.hpp" +#include + +// AthenaPK headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" + +namespace cluster { +using namespace parthenon; + +parthenon::Real LocalReduceColdGas(parthenon::MeshData *md) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &cold_thresh = hydro_pkg->Param("reduction_cold_threshold"); + auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + const auto e_thresh = + cold_thresh / mbar_over_kb / (hydro_pkg->Param("AdiabaticIndex") - 1.0); + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + + const Real gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + + Real cold_gas = 0.0; + + Kokkos::parallel_reduce( + "LocalReduceColdGas", + Kokkos::MDRangePolicy>( + DevExecSpace(), {0, kb.s, jb.s, ib.s}, + {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real &cold_gas_team) { + auto &prim = prim_pack(b); + const auto &coords = prim_pack.GetCoords(b); + + const Real internal_e = prim(IPR, k, j, i) / (gm1 * prim(IDN, k, j, i)); + if (internal_e < e_thresh) { + cold_gas_team += prim(IDN, k, j, i) * coords.CellVolume(k, j, i); + } + }, + cold_gas); + return cold_gas; +} + +parthenon::Real LocalReduceAGNExtent(parthenon::MeshData *md) { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const auto &tracer_thresh = hydro_pkg->Param("reduction_agn_tracer_threshold"); + + // Grab some necessary variables + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + + Real max_r2 = 0.0; + + Kokkos::parallel_reduce( + "LocalReduceAGNExtent", + Kokkos::MDRangePolicy>( + DevExecSpace(), {0, kb.s, jb.s, ib.s}, + {cons_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real &max_r2_team) { + auto &cons = cons_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const auto r2 = SQR(coords.Xc<1>(k, j, i)) + SQR(coords.Xc<2>(k, j, i)) + + SQR(coords.Xc<3>(k, j, i)); + if (cons(nhydro, k, j, i) / cons(IDN, k, j, i) > tracer_thresh && r2 > max_r2) { + max_r2_team = r2; + } + }, + Kokkos::Max(max_r2)); + + return std::sqrt(max_r2); +} + +} // namespace cluster \ No newline at end of file diff --git a/src/pgen/cluster/cluster_reductions.hpp b/src/pgen/cluster/cluster_reductions.hpp new file mode 100644 index 00000000..4fd98bc2 --- /dev/null +++ b/src/pgen/cluster/cluster_reductions.hpp @@ -0,0 +1,24 @@ +#ifndef CLUSTER_CLUSTER_REDUCTIONS_HPP_ +#define CLUSTER_CLUSTER_REDUCTIONS_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file cluster_reductions.hpp +// \brief Cluster-specific reductions to compute the total cold gas and maximum radius +// of AGN feedback + +// parthenon headers +#include +#include + +namespace cluster { + +parthenon::Real LocalReduceColdGas(parthenon::MeshData *md); + +parthenon::Real LocalReduceAGNExtent(parthenon::MeshData *md); + +} // namespace cluster + +#endif // CLUSTER_CLUSTER_REDUCTIONS_HPP_ diff --git a/src/pgen/cluster/cluster_utils.hpp b/src/pgen/cluster/cluster_utils.hpp new file mode 100644 index 00000000..e76e2e84 --- /dev/null +++ b/src/pgen/cluster/cluster_utils.hpp @@ -0,0 +1,56 @@ +//======================================================================================== +//// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +///// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +///// Licensed under the 3-clause BSD License, see LICENSE file for details +/////======================================================================================== +//! \file cluster_utils.hpp +// \brief Utilities for galaxy cluster functions +#ifndef CLUSTER_CLUSTER_UTILS_HPP_ +#define CLUSTER_CLUSTER_UTILS_HPP_ + +// parthenon headers +#include + +// AthenaPK headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" +#include "utils/error_checking.hpp" + +namespace cluster { + +// Add a density to the conserved variables while keeping velocity fixed +template +KOKKOS_INLINE_FUNCTION void +AddDensityToConsAtFixedVel(const parthenon::Real density, View4D &cons, + const View4D &prim, const int &k, const int &j, const int &i) { + // Add density such that velocity is fixed + cons(IDN, k, j, i) += density; + cons(IM1, k, j, i) += density * prim(IV1, k, j, i); + cons(IM2, k, j, i) += density * prim(IV2, k, j, i); + cons(IM3, k, j, i) += density * prim(IV3, k, j, i); + cons(IEN, k, j, i) += + density * (0.5 * (SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + + SQR(prim(IV3, k, j, i)))); +} + +// Add a density to the conserved variables while keeping velocity and +// temperature ( propto pressure/density) fixed +template +KOKKOS_INLINE_FUNCTION void +AddDensityToConsAtFixedVelTemp(const parthenon::Real density, View4D &cons, + const View4D &prim, const Real adiabaticIndex, + const int &k, const int &j, const int &i) { + // Add density such that velocity and temperature (propto pressure/density) is fixed + cons(IDN, k, j, i) += density; + cons(IM1, k, j, i) += density * prim(IV1, k, j, i); + cons(IM2, k, j, i) += density * prim(IV2, k, j, i); + cons(IM3, k, j, i) += density * prim(IV3, k, j, i); + cons(IEN, k, j, i) += + density * (0.5 * (SQR(prim(IV1, k, j, i)) + SQR(prim(IV2, k, j, i)) + + SQR(prim(IV3, k, j, i))) + + 1 / (adiabaticIndex - 1.0) * prim(IPR, k, j, i) / prim(IDN, k, j, i)); +} + +} // namespace cluster + +#endif // CLUSTER_CLUSTER_UTILS_HPP_ diff --git a/src/pgen/cluster/entropy_profiles.hpp b/src/pgen/cluster/entropy_profiles.hpp index b45ef632..1fd75580 100644 --- a/src/pgen/cluster/entropy_profiles.hpp +++ b/src/pgen/cluster/entropy_profiles.hpp @@ -1,12 +1,12 @@ +#ifndef CLUSTER_ENTROPY_PROFILES_HPP_ +#define CLUSTER_ENTROPY_PROFILES_HPP_ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file entropy profiles.hpp // \brief Classes defining initial entropy profile -#ifndef CLUSTER_ENTROPY_PROFILES_HPP_ -#define CLUSTER_ENTROPY_PROFILES_HPP_ // Parthenon headers #include @@ -17,26 +17,26 @@ namespace cluster { class ACCEPTEntropyProfile { - private: - // Entropy Profile - parthenon::Real K_0_, K_100_, R_K_, alpha_K_; public: + // Entropy Profile + parthenon::Real k_0_, k_100_, r_k_, alpha_k_; + ACCEPTEntropyProfile(parthenon::ParameterInput *pin) { Units units(pin); - K_0_ = pin->GetOrAddReal("problem/cluster", "K_0", + k_0_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "k_0", 20 * units.kev() * units.cm() * units.cm()); - K_100_ = pin->GetOrAddReal("problem/cluster", "K_100", + k_100_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "k_100", 120 * units.kev() * units.cm() * units.cm()); - R_K_ = pin->GetOrAddReal("problem/cluster", "R_K", 100 * units.kpc()); - alpha_K_ = pin->GetOrAddReal("problem/cluster", "alpha_K", 1.75); + r_k_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "r_k", 100 * units.kpc()); + alpha_k_ = pin->GetOrAddReal("problem/cluster/entropy_profile", "alpha_k", 1.75); } // Get entropy from radius, using broken power law profile for entropy - parthenon::Real K_from_r(const parthenon::Real r) const { - const parthenon::Real K = K_0_ + K_100_ * pow(r / R_K_, alpha_K_); - return K; + KOKKOS_INLINE_FUNCTION parthenon::Real K_from_r(const parthenon::Real r) const { + const parthenon::Real k = k_0_ + k_100_ * pow(r / r_k_, alpha_k_); + return k; } }; diff --git a/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp b/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp index 8df0250d..9c88269d 100644 --- a/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp +++ b/src/pgen/cluster/hydrostatic_equilibrium_sphere.cpp @@ -1,6 +1,6 @@ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file hydrostatic_equilbirum_sphere.cpp @@ -12,6 +12,7 @@ // C++ headers #include +#include // Parthenon headers #include @@ -36,6 +37,7 @@ using namespace parthenon; template HydrostaticEquilibriumSphere:: HydrostaticEquilibriumSphere(ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg, GravitationalField gravitational_field, EntropyProfile entropy_profile) : gravitational_field_(gravitational_field), entropy_profile_(entropy_profile) { @@ -44,37 +46,35 @@ HydrostaticEquilibriumSphere:: mh_ = units.mh(); k_boltzmann_ = units.k_boltzmann(); - const Real He_mass_fraction = pin->GetReal("hydro", "He_mass_fraction"); - const Real H_mass_fraction = 1.0 - He_mass_fraction; + mu_ = hydro_pkg->Param("mu"); + mu_e_ = hydro_pkg->Param("mu_e"); - mu_ = 1 / (He_mass_fraction * 3. / 4. + (1 - He_mass_fraction) * 2); - mu_e_ = 1 / (He_mass_fraction * 2. / 4. + (1 - He_mass_fraction)); - - R_fix_ = - pin->GetOrAddReal("problem/cluster", "R_fix", 1953.9724519818478 * units.kpc()); - rho_fix_ = pin->GetOrAddReal("problem/cluster", "rho_fix", + r_fix_ = pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", "r_fix", + 1953.9724519818478 * units.kpc()); + rho_fix_ = pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", "rho_fix", 8.607065015897638e-30 * units.g() / pow(units.kpc(), 3)); const Real gam = pin->GetReal("hydro", "gamma"); const Real gm1 = (gam - 1.0); - R_sampling_ = pin->GetOrAddReal("problem/cluster", "R_sampling", 4.0); - max_dR_ = pin->GetOrAddReal("problem/cluster", "max_dR", 1e-3); + r_sampling_ = + pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", "r_sampling", 4.0); // Test out the HSE sphere if requested - const bool test_he_sphere = - pin->GetOrAddBoolean("problem/cluster", "test_he_sphere", false); + const bool test_he_sphere = pin->GetOrAddBoolean( + "problem/cluster/hydrostatic_equilibrium", "test_he_sphere", false); if (test_he_sphere) { - const Real test_he_sphere_R_start = pin->GetOrAddReal( - "problem/cluster", "test_he_sphere_R_start_kpc", 1e-3 * units.kpc()); - const Real test_he_sphere_R_end = pin->GetOrAddReal( - "problem/cluster", "test_he_sphere_R_end_kpc", 4000 * units.kpc()); - const int test_he_sphere_n_r = - pin->GetOrAddInteger("problem/cluster", "test_he_sphere_n_r", 4000); + const Real test_he_sphere_r_start = + pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", + "test_he_sphere_r_start", 1e-3 * units.kpc()); + const Real test_he_sphere_r_end = + pin->GetOrAddReal("problem/cluster/hydrostatic_equilibrium", + "test_he_sphere_r_end", 4000 * units.kpc()); + const int test_he_sphere_n_r = pin->GetOrAddInteger( + "problem/cluster/hydrostatic_equilibrium", "test_he_sphere_n_r", 4000); if (Globals::my_rank == 0) { - typedef Kokkos::View View1D; - auto P_rho_profile = generate_P_rho_profile( - test_he_sphere_R_start, test_he_sphere_R_end, test_he_sphere_n_r); + auto P_rho_profile = generate_P_rho_profile( + test_he_sphere_r_start, test_he_sphere_r_end, test_he_sphere_n_r); std::ofstream test_he_file; test_he_file.open("test_he_sphere.dat"); @@ -82,88 +82,48 @@ HydrostaticEquilibriumSphere:: test_he_file.close(); } } -} - -/************************************************************ - * PRhoProfile::P_from_r - ************************************************************/ -template -template -Real HydrostaticEquilibriumSphere::PRhoProfile< - View1D>::P_from_r(const Real r) const { - - // Determine indices in R bounding r - const int i_r = - static_cast(floor((n_R_ - 1) / (R_end_ - R_start_) * (r - R_start_))); - - if (r < R_(i_r) - kRTol || r > R_(i_r + 1) + kRTol) { - std::stringstream msg; - msg << "### FATAL ERROR in function [HydrostaticEquilibriumSphere::PRhoProfile]" - << std::endl - << "R(i_r) to R_(i_r+1) does not contain r" << std::endl - << "R(i_r) R_r R(i_r+1):" << R_(i_r) << " " << r << " " << R_(i_r + 1) - << std::endl; - PARTHENON_FAIL(msg); - } - - // Linearly interpolate Pressure from P - const Real P_r = (P_(i_r) * (R_(i_r + 1) - r) + P_(i_r + 1) * (r - R_(i_r))) / - (R_(i_r + 1) - R_(i_r)); - return P_r; -} - -/************************************************************ - * PRhoProfile::rho_from_r - ************************************************************/ -template -template -Real HydrostaticEquilibriumSphere::PRhoProfile< - View1D>::rho_from_r(const Real r) const { - - // Get pressure first - const Real P_r = P_from_r(r); - // Compute entropy and pressure here - const Real K_r = sphere_.entropy_profile_.K_from_r(r); - const Real rho_r = sphere_.rho_from_P_K(P_r, K_r); - return rho_r; + hydro_pkg->AddParam<>("hydrostatic_equilibirum_sphere", *this); } /************************************************************ * PRhoProfile::write_to_ostream ************************************************************/ -template -template -std::ostream & -HydrostaticEquilibriumSphere::PRhoProfile< - View1D>::write_to_ostream(std::ostream &os) const { - - const dP_dr_from_r_P_functor dP_dr_func(sphere_); - for (int i = 0; i < R_.extent(0); i++) { - const Real r = R_(i); - const Real P = P_(i); - const Real K = sphere_.entropy_profile_.K_from_r(r); - const Real rho = sphere_.rho_from_P_K(P, K); +template +std::ostream &PRhoProfile::write_to_ostream( + std::ostream &os) const { + + const typename HydrostaticEquilibriumSphere< + GravitationalField, EntropyProfile>::dP_dr_from_r_P_functor dP_dr_func(sphere_); + + auto host_r = Kokkos::create_mirror_view(r_); + Kokkos::deep_copy(host_r, r_); + auto host_p = Kokkos::create_mirror_view(p_); + Kokkos::deep_copy(host_p, p_); + + for (int i = 0; i < host_r.extent(0); i++) { + const Real r = host_r(i); + const Real p = host_p(i); + const Real k = sphere_.entropy_profile_.K_from_r(r); + const Real rho = sphere_.rho_from_P_K(p, k); const Real n = sphere_.n_from_rho(rho); const Real ne = sphere_.ne_from_rho(rho); - const Real T = sphere_.T_from_rho_P(rho, P); + const Real temp = sphere_.T_from_rho_P(rho, p); const Real g = sphere_.gravitational_field_.g_from_r(r); - const Real dP_dr = dP_dr_func(r, P); + const Real dP_dr = dP_dr_func(r, p); - os << r << " " << P << " " << K << " " << rho << " " << n << " " << ne << " " << T + os << r << " " << p << " " << k << " " << rho << " " << n << " " << ne << " " << temp << " " << g << " " << dP_dr << std::endl; } return os; } /************************************************************ - * HydrostaticEquilibriumSphere::generate_P_rho_profile(x,y,z) + *HydrostaticEquilibriumSphere::generate_P_rho_profile(x,y,z) ************************************************************/ -template -template -typename HydrostaticEquilibriumSphere::template PRhoProfile -HydrostaticEquilibriumSphere::generate_P_rho_profile( +template +PRhoProfile +HydrostaticEquilibriumSphere::generate_P_rho_profile( IndexRange ib, IndexRange jb, IndexRange kb, parthenon::UniformCartesian coords) const { @@ -174,19 +134,26 @@ HydrostaticEquilibriumSphere::generate_P_rho ************************************************************/ // Determine spacing of grid (WARNING assumes equispaced grid in x,y,z) - PARTHENON_REQUIRE(coords.Dxc<1>(0) == coords.Dxc<1>(1), "No equidistant grid in x1dir"); - PARTHENON_REQUIRE(coords.Dxc<2>(0) == coords.Dxc<2>(1), "No equidistant grid in x2dir"); - PARTHENON_REQUIRE(coords.Dxc<3>(0) == coords.Dxc<3>(1), "No equidistant grid in x3dir"); - PARTHENON_REQUIRE(coords.Dxc<1>(0) == coords.Dxc<2>(1), - "No equidistant grid between x1 and x2 dir"); - PARTHENON_REQUIRE(coords.Dxc<2>(0) == coords.Dxc<3>(1), - "No equidistant grid between x2 and x3 dir"); - const Real dR = std::min(coords.Dxc<1>(0) / R_sampling_, max_dR_); + PARTHENON_REQUIRE(std::abs(coords.Dxc<1>(0) - coords.Dxc<1>(1)) < + 10 * std::numeric_limits::epsilon(), + "No equidistant grid in x1dir"); + PARTHENON_REQUIRE(std::abs(coords.Dxc<2>(0) - coords.Dxc<2>(1)) < + 10 * std::numeric_limits::epsilon(), + "No equidistant grid in x2dir"); + PARTHENON_REQUIRE(std::abs(coords.Dxc<3>(0) - coords.Dxc<3>(1)) < + 10 * std::numeric_limits::epsilon(), + "No equidistant grid in x3dir"); + // Resolution of profile on this meshbock -- use 1/r_sampling_ of resolution + // or 1/r_sampling_ of r_k, whichever is smaller + const Real dr = + std::min(std::min(coords.Dxc<1>(0), std::min(coords.Dxc<2>(0), coords.Dxc<3>(0))) / + r_sampling_, + entropy_profile_.r_k_ / r_sampling_); // Loop through mesh for minimum and maximum radius // Make sure to include R_fix_ - Real R_start = R_fix_; - Real R_end = R_fix_; + Real r_start = r_fix_; + Real r_end = r_fix_; for (int k = kb.s; k <= kb.e; k++) { for (int j = jb.s; j <= jb.e; j++) { for (int i = ib.s; i <= ib.e; i++) { @@ -194,66 +161,66 @@ HydrostaticEquilibriumSphere::generate_P_rho const Real r = sqrt(coords.Xc<1>(i) * coords.Xc<1>(i) + coords.Xc<2>(j) * coords.Xc<2>(j) + coords.Xc<3>(k) * coords.Xc<3>(k)); - R_start = std::min(r, R_start); - R_end = std::max(r, R_end); + r_start = std::min(r, r_start); + r_end = std::max(r, r_end); } } } // Add some room for R_start and R_end - R_start = std::max(0.0, R_start - R_sampling_ * dR); - R_end += R_sampling_ * dR; + r_start = std::max(0.0, r_start - r_sampling_ * dr); + r_end += r_sampling_ * dr; // Compute number of cells needed - const unsigned int n_R = static_cast(ceil((R_end - R_start) / dR)); + const auto n_r = static_cast(ceil((r_end - r_start) / dr)); // Make R_end consistent - R_end = R_start + dR * (n_R - 1); + r_end = r_start + dr * (n_r - 1); - return generate_P_rho_profile(R_start, R_end, n_R); + return generate_P_rho_profile(r_start, r_end, n_r); } /************************************************************ * HydrostaticEquilibriumSphere::generate_P_rho_profile(Ri,Re,nR) ************************************************************/ -template -template -typename HydrostaticEquilibriumSphere::template PRhoProfile -HydrostaticEquilibriumSphere::generate_P_rho_profile( - const Real R_start, const Real R_end, const unsigned int n_R) const { +template +PRhoProfile +HydrostaticEquilibriumSphere::generate_P_rho_profile( + const Real r_start, const Real r_end, const unsigned int n_r) const { // Array of radii along which to compute the profile - View1D R("R", n_R); - const Real dR = (R_end - R_start) / (n_R - 1.0); + ParArray1D device_r("PRhoProfile r", n_r); + auto r = Kokkos::create_mirror_view(device_r); + const Real dr = (r_end - r_start) / (n_r - 1.0); // Use a linear R - possibly adapt if using a mesh with logrithmic r - for (int i = 0; i < n_R; i++) { - R(i) = R_start + i * dR; + for (int i = 0; i < n_r; i++) { + r(i) = r_start + i * dr; } /************************************************************ * Integrate Pressure inward and outward from virial radius ************************************************************/ // Create array for pressure - View1D P("P", n_R); + ParArray1D device_p("PRhoProfile p", n_r); + auto p = Kokkos::create_mirror_view(device_p); - const Real K_fix = entropy_profile_.K_from_r(R_fix_); - const Real P_fix = P_from_rho_K(rho_fix_, K_fix); + const Real k_fix = entropy_profile_.K_from_r(r_fix_); + const Real p_fix = P_from_rho_K(rho_fix_, k_fix); // Integrate P inward from R_fix_ - Real Ri = R_fix_; // Start Ri at R_fix_ first - Real Pi = P_fix; // Start with pressure at R_fix_ + Real r_i = r_fix_; // Start Ri at R_fix_ first + Real p_i = p_fix; // Start with pressure at R_fix_ // Find the index in R right before R_fix_ - int i_fix = static_cast(floor((n_R - 1) / (R_end - R_start) * (R_fix_ - R_start))); - if (R_fix_ < R(i_fix) - kRTol || R_fix_ > R(i_fix + 1) + kRTol) { + int i_fix = static_cast(floor((n_r - 1) / (r_end - r_start) * (r_fix_ - r_start))); + if (r_fix_ < r(i_fix) - kRTol || r_fix_ > r(i_fix + 1) + kRTol) { std::stringstream msg; msg << "### FATAL ERROR in function " "[HydrostaticEquilibriumSphere::generate_P_rho_profile]" << std::endl - << "R(i_fix) to R_(i_fix+1) does not contain R_fix_" << std::endl - << "R(i_fix) R_fix_ R(i_fix+1):" << R(i_fix) << " " << R_fix_ << " " - << R(i_fix + 1) << std::endl; + << "r(i_fix) to r_(i_fix+1) does not contain r_fix_" << std::endl + << "r(i_fix) r_fix_ r(i_fix+1):" << r(i_fix) << " " << r_fix_ << " " + << r(i_fix + 1) << std::endl; PARTHENON_FAIL(msg); } @@ -261,65 +228,34 @@ HydrostaticEquilibriumSphere::generate_P_rho // Make is the i right before R_fix_ for (int i = i_fix + 1; i > 0; i--) { // Move is up one, to account for initial R_fix_ - P(i - 1) = step_rk4(Ri, R(i - 1), Pi, dP_dr_from_r_P); - Ri = R(i - 1); - Pi = P(i - 1); + p(i - 1) = step_rk4(r_i, r(i - 1), p_i, dP_dr_from_r_P); + r_i = r(i - 1); + p_i = p(i - 1); } // Integrate P outward from R_fix_ - Ri = R_fix_; // Start Ri at R_fix_ first - Pi = P_fix; // Start with pressure at R_fix_ + r_i = r_fix_; // Start Ri at R_fix_ first + p_i = p_fix; // Start with pressure at R_fix_ // Make is the i right after R_fix_ - for (int i = i_fix; i < n_R - 1; + for (int i = i_fix; i < n_r - 1; i++) { // Move is back one, to account for initial R_fix_ - P(i + 1) = step_rk4(Ri, R(i + 1), Pi, dP_dr_from_r_P); - Ri = R(i + 1); - Pi = P(i + 1); + p(i + 1) = step_rk4(r_i, r(i + 1), p_i, dP_dr_from_r_P); + r_i = r(i + 1); + p_i = p(i + 1); } - return PRhoProfile(R, P, *this); + Kokkos::deep_copy(device_r, r); + Kokkos::deep_copy(device_p, p); + + return PRhoProfile(device_r, device_p, r(0), + r(n_r - 1), *this); } // Instantiate HydrostaticEquilibriumSphere template class HydrostaticEquilibriumSphere; // Instantiate PRhoProfile -template class HydrostaticEquilibriumSphere:: - PRhoProfile>; -#if (defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP)) -template class HydrostaticEquilibriumSphere:: - PRhoProfile>; -#endif - -// Instantiate generate_P_rho_profile -template HydrostaticEquilibriumSphere::PRhoProfile< - Kokkos::View> - HydrostaticEquilibriumSphere:: - generate_P_rho_profile>( - parthenon::IndexRange, parthenon::IndexRange, parthenon::IndexRange, - parthenon::UniformCartesian) const; -template HydrostaticEquilibriumSphere::PRhoProfile< - Kokkos::View> -HydrostaticEquilibriumSphere:: - generate_P_rho_profile>( - const parthenon::Real, const parthenon::Real, const unsigned int) const; -#if (defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP)) -template HydrostaticEquilibriumSphere::PRhoProfile< - Kokkos::View> - HydrostaticEquilibriumSphere:: - generate_P_rho_profile< - Kokkos::View>( - parthenon::IndexRange, parthenon::IndexRange, parthenon::IndexRange, - parthenon::UniformCartesian) const; -template HydrostaticEquilibriumSphere::PRhoProfile< - Kokkos::View> -HydrostaticEquilibriumSphere:: - generate_P_rho_profile>( - const parthenon::Real, const parthenon::Real, const unsigned int) const; -#endif +template class PRhoProfile; } // namespace cluster diff --git a/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp b/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp index b3527cfa..fe1ceb34 100644 --- a/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp +++ b/src/pgen/cluster/hydrostatic_equilibrium_sphere.hpp @@ -1,14 +1,13 @@ +#ifndef CLUSTER_HYDROSTATIC_EQUILIBRIUM_SPHERE_HPP_ +#define CLUSTER_HYDROSTATIC_EQUILIBRIUM_SPHERE_HPP_ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 2021, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file hydrostatic_equilbirum_sphere // \brief Class for initializing a sphere in hydrostatic equiblibrium -#ifndef CLUSTER_HYDROSTATIC_EQUILIBRIUM_SPHERE_HPP_ -#define CLUSTER_HYDROSTATIC_EQUILIBRIUM_SPHERE_HPP_ - // Parthenon headers #include #include @@ -18,6 +17,9 @@ namespace cluster { +template +class PRhoProfile; + /************************************************************ * Hydrostatic Equilbrium Spnere Class, * for initializing a sphere in hydrostatic equiblibrium @@ -39,13 +41,13 @@ class HydrostaticEquilibriumSphere { parthenon::Real mh_, k_boltzmann_; // Density to fix baryons at a radius (change to temperature?) - parthenon::Real R_fix_, rho_fix_; + parthenon::Real r_fix_, rho_fix_; // Molecular weights parthenon::Real mu_, mu_e_; - // R mesh sampling parameters - parthenon::Real R_sampling_, max_dR_; + // R mesh sampling parameter + parthenon::Real r_sampling_; /************************************************************ * Functions to build the cluster model @@ -56,35 +58,36 @@ class HydrostaticEquilibriumSphere { // Get pressure from density and entropy, using ideal gas law and definition // of entropy - parthenon::Real P_from_rho_K(const parthenon::Real rho, const parthenon::Real K) const { - const parthenon::Real P = - K * pow(mu_ / mu_e_, 2. / 3.) * pow(rho / (mu_ * mh_), 5. / 3.); - return P; + KOKKOS_INLINE_FUNCTION parthenon::Real P_from_rho_K(const parthenon::Real rho, + const parthenon::Real k) const { + const parthenon::Real p = k * pow(rho / mh_, 5. / 3.) / (mu_ * pow(mu_e_, 2. / 3.)); + return p; } // Get density from pressure and entropy, using ideal gas law and definition // of entropy - parthenon::Real rho_from_P_K(const parthenon::Real P, const parthenon::Real K) const { - const parthenon::Real rho = - pow(P / K, 3. / 5.) * mu_ * mh_ / pow(mu_ / mu_e_, 2. / 5); + KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_P_K(const parthenon::Real p, + const parthenon::Real k) const { + const parthenon::Real rho = pow(mu_ * p / k, 3. / 5.) * mh_ * pow(mu_e_, 2. / 5); return rho; } // Get total number density from density - parthenon::Real n_from_rho(const parthenon::Real rho) const { + KOKKOS_INLINE_FUNCTION parthenon::Real n_from_rho(const parthenon::Real rho) const { const parthenon::Real n = rho / (mu_ * mh_); return n; } // Get electron number density from density - parthenon::Real ne_from_rho(const parthenon::Real rho) const { + KOKKOS_INLINE_FUNCTION parthenon::Real ne_from_rho(const parthenon::Real rho) const { const parthenon::Real ne = mu_ / mu_e_ * n_from_rho(rho); return ne; } // Get the temperature from density and pressure - parthenon::Real T_from_rho_P(const parthenon::Real rho, const parthenon::Real P) const { - const parthenon::Real T = P / (n_from_rho(rho) * k_boltzmann_); + KOKKOS_INLINE_FUNCTION parthenon::Real T_from_rho_P(const parthenon::Real rho, + const parthenon::Real p) const { + const parthenon::Real T = p / (n_from_rho(rho) * k_boltzmann_); return T; } @@ -102,11 +105,11 @@ class HydrostaticEquilibriumSphere { dP_dr_from_r_P_functor( const HydrostaticEquilibriumSphere &sphere) : sphere_(sphere) {} - parthenon::Real operator()(const parthenon::Real r, const parthenon::Real P) const { + parthenon::Real operator()(const parthenon::Real r, const parthenon::Real p) const { const parthenon::Real g = sphere_.gravitational_field_.g_from_r(r); - const parthenon::Real K = sphere_.entropy_profile_.K_from_r(r); - const parthenon::Real rho = sphere_.rho_from_P_K(P, K); + const parthenon::Real k = sphere_.entropy_profile_.K_from_r(r); + const parthenon::Real rho = sphere_.rho_from_P_K(p, k); const parthenon::Real dP_dr = -rho * g; return dP_dr; } @@ -129,40 +132,69 @@ class HydrostaticEquilibriumSphere { public: HydrostaticEquilibriumSphere(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg, GravitationalField gravitational_field, EntropyProfile entropy_profile); - template - class PRhoProfile { - private: - const View1D R_; - const View1D P_; - const HydrostaticEquilibriumSphere &sphere_; + PRhoProfile + generate_P_rho_profile(parthenon::IndexRange ib, parthenon::IndexRange jb, + parthenon::IndexRange kb, + parthenon::UniformCartesian coords) const; - const int n_R_; - const parthenon::Real R_start_, R_end_; + PRhoProfile + generate_P_rho_profile(const parthenon::Real r_start, const parthenon::Real r_end, + const unsigned int n_R) const; - public: - PRhoProfile(const View1D R, const View1D P, - const HydrostaticEquilibriumSphere &sphere) - : R_(R), P_(P), sphere_(sphere), n_R_(R_.extent(0)), R_start_(R_(0)), - R_end_(R_(n_R_ - 1)) {} - - parthenon::Real P_from_r(const parthenon::Real r) const; - parthenon::Real rho_from_r(const parthenon::Real r) const; - std::ostream &write_to_ostream(std::ostream &os) const; - }; + template + friend class PRhoProfile; +}; - template - PRhoProfile generate_P_rho_profile(parthenon::IndexRange ib, - parthenon::IndexRange jb, - parthenon::IndexRange kb, - parthenon::UniformCartesian coords) const; +template +class PRhoProfile { + private: + const parthenon::ParArray1D r_; + const parthenon::ParArray1D p_; + const HydrostaticEquilibriumSphere sphere_; + + const int n_r_; + const parthenon::Real r_start_, r_end_; + + public: + PRhoProfile( + const parthenon::ParArray1D &r, + const parthenon::ParArray1D &p, const parthenon::Real r_start, + const parthenon::Real r_end, + const HydrostaticEquilibriumSphere &sphere) + : r_(r), p_(p), sphere_(sphere), n_r_(r_.extent(0)), r_start_(r_start), + r_end_(r_end) {} + + KOKKOS_INLINE_FUNCTION parthenon::Real P_from_r(const parthenon::Real r) const { + // Determine indices in R bounding r + const int i_r = + static_cast(floor((n_r_ - 1) / (r_end_ - r_start_) * (r - r_start_))); + + if (r < r_(i_r) - sphere_.kRTol || r > r_(i_r + 1) + sphere_.kRTol) { + Kokkos::abort("PRhoProfile::P_from_r R(i_r) to R_(i_r+1) does not contain r"); + } + + // Linearly interpolate Pressure from P + const parthenon::Real P_r = + (p_(i_r) * (r_(i_r + 1) - r) + p_(i_r + 1) * (r - r_(i_r))) / + (r_(i_r + 1) - r_(i_r)); - template - PRhoProfile generate_P_rho_profile(const parthenon::Real R_start, - const parthenon::Real R_end, - const unsigned int n_R) const; + return P_r; + } + + KOKKOS_INLINE_FUNCTION parthenon::Real rho_from_r(const parthenon::Real r) const { + using parthenon::Real; + // Get pressure first + const Real p_r = P_from_r(r); + // Compute entropy and pressure here + const Real k_r = sphere_.entropy_profile_.K_from_r(r); + const Real rho_r = sphere_.rho_from_P_K(p_r, k_r); + return rho_r; + } + std::ostream &write_to_ostream(std::ostream &os) const; }; } // namespace cluster diff --git a/src/pgen/cluster/jet_coords.hpp b/src/pgen/cluster/jet_coords.hpp new file mode 100644 index 00000000..d7eb1c26 --- /dev/null +++ b/src/pgen/cluster/jet_coords.hpp @@ -0,0 +1,116 @@ +#ifndef CLUSTER_JET_COORDS_HPP_ +#define CLUSTER_JET_COORDS_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file jet_coords.hpp +// \brief Class for working with precesing jet + +// Parthenon headers +#include "Kokkos_Macros.hpp" +#include +#include +#include +#include + +namespace cluster { + +/************************************************************ + * Jet Coordinates Class, for computing cylindrical coordinates in reference to + * a jet along a fixed tilted axis + * Lightweight object intended for inlined computation, within kernels. + ************************************************************/ +class JetCoords { + private: + // cos and sin of angle of the jet axis off the z-axis + const parthenon::Real cos_theta_jet_axis_, sin_theta_jet_axis_; + // cos and sin of angle of the jet axis around the z-axis + const parthenon::Real cos_phi_jet_axis_, sin_phi_jet_axis_; + + public: + explicit JetCoords(const parthenon::Real theta_jet_axis, + const parthenon::Real phi_jet_axis) + : cos_theta_jet_axis_(cos(theta_jet_axis)), + sin_theta_jet_axis_(sin(theta_jet_axis)), cos_phi_jet_axis_(cos(phi_jet_axis)), + sin_phi_jet_axis_(sin(phi_jet_axis)) {} + + // Convert simulation cartesian coordinates to jet cylindrical coordinates + KOKKOS_INLINE_FUNCTION void + SimCartToJetCylCoords(const parthenon::Real x_sim, const parthenon::Real y_sim, + const parthenon::Real z_sim, parthenon::Real &r_jet, + parthenon::Real &cos_theta_jet, parthenon::Real &sin_theta_jet, + parthenon::Real &h_jet) const __attribute__((always_inline)) { + + // Position in jet-cartesian coordinates + const parthenon::Real x_jet = x_sim * cos_phi_jet_axis_ * cos_theta_jet_axis_ + + y_sim * sin_phi_jet_axis_ * cos_theta_jet_axis_ - + z_sim * sin_theta_jet_axis_; + const parthenon::Real y_jet = -x_sim * sin_phi_jet_axis_ + y_sim * cos_phi_jet_axis_; + const parthenon::Real z_jet = x_sim * sin_theta_jet_axis_ * cos_phi_jet_axis_ + + y_sim * sin_phi_jet_axis_ * sin_theta_jet_axis_ + + z_sim * cos_theta_jet_axis_; + + // Position in jet-cylindrical coordinates + r_jet = sqrt(pow(fabs(x_jet), 2) + pow(fabs(y_jet), 2)); + // Setting cos_theta and sin_theta to 0 for r = 0 as all places where + // those variables are used (SimCardToJetCylCoords) an r = 0 leads to the x and y + // component being 0, too. + cos_theta_jet = (r_jet != 0) ? x_jet / r_jet : 0; + sin_theta_jet = (r_jet != 0) ? y_jet / r_jet : 0; + h_jet = z_jet; + } + + // Convert jet cylindrical vector to simulation cartesian vector + KOKKOS_INLINE_FUNCTION void JetCylToSimCartVector( + const parthenon::Real cos_theta_jet, const parthenon::Real sin_theta_jet, + const parthenon::Real v_r_jet, const parthenon::Real v_theta_jet, + const parthenon::Real v_h_jet, parthenon::Real &v_x_sim, parthenon::Real &v_y_sim, + parthenon::Real &v_z_sim) const __attribute__((always_inline)) { + // The vector in jet-cartesian coordinates + const parthenon::Real v_x_jet = v_r_jet * cos_theta_jet - v_theta_jet * sin_theta_jet; + const parthenon::Real v_y_jet = v_r_jet * sin_theta_jet + v_theta_jet * cos_theta_jet; + const parthenon::Real v_z_jet = v_h_jet; + + // Multiply v_jet by the DCM matrix to take Jet cartesian to Simulation Cartesian + v_x_sim = v_x_jet * cos_phi_jet_axis_ * cos_theta_jet_axis_ - + v_y_jet * sin_phi_jet_axis_ + + v_z_jet * sin_theta_jet_axis_ * cos_phi_jet_axis_; + v_y_sim = v_x_jet * sin_phi_jet_axis_ * cos_theta_jet_axis_ + + v_y_jet * cos_phi_jet_axis_ + + v_z_jet * sin_phi_jet_axis_ * sin_theta_jet_axis_; + v_z_sim = -v_x_jet * sin_theta_jet_axis_ + v_z_jet * cos_theta_jet_axis_; + } +}; +/************************************************************ + * Jet Coordinates Factory Class + * A factory for creating JetCoords objects given a time + ************************************************************/ +class JetCoordsFactory { + private: + // Jet-axis Radians off the z-axis + const parthenon::Real theta_jet_axis_; + // Precesion rate of Jet-axis, radians/time + const parthenon::Real phi_dot_jet_axis_; + // Initial precession offset in radians of Jet-axis (Useful for testing) + const parthenon::Real phi0_jet_axis_; + + public: + explicit JetCoordsFactory(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg, + const std::string &block = "problem/cluster/precessing_jet") + : theta_jet_axis_(pin->GetOrAddReal(block, "jet_theta", 0)), + phi_dot_jet_axis_(pin->GetOrAddReal(block, "jet_phi_dot", 0)), + phi0_jet_axis_(pin->GetOrAddReal(block, "jet_phi0", 0)) { + hydro_pkg->AddParam<>("jet_coords_factory", *this); + } + + JetCoords CreateJetCoords(const parthenon::Real time) const { + return JetCoords(theta_jet_axis_, phi0_jet_axis_ + time * phi_dot_jet_axis_); + } +}; + +} // namespace cluster + +#endif // CLUSTER_JET_COORDS_HPP_ diff --git a/src/pgen/cluster/magnetic_tower.cpp b/src/pgen/cluster/magnetic_tower.cpp new file mode 100644 index 00000000..1679ffd7 --- /dev/null +++ b/src/pgen/cluster/magnetic_tower.cpp @@ -0,0 +1,319 @@ +//======================================================================================== +//// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +///// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +///// Licensed under the 3-clause BSD License, see LICENSE file for details +/////======================================================================================== +//! \file magnetic_tower.cpp +// \brief Class for defining magnetic towers + +// Parthenon headers +#include +#include +#include +#include +#include +#include +#include + +// Athena headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../main.hpp" +#include "../../units.hpp" +#include "cluster_utils.hpp" +#include "magnetic_tower.hpp" +#include "utils/error_checking.hpp" + +namespace cluster { +using namespace parthenon; + +void MagneticTower::AddSrcTerm(parthenon::Real field_to_add, parthenon::Real mass_to_add, + parthenon::MeshData *md, + const parthenon::SimTime &tm) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + if (field_to_add == 0 && mass_to_add == 0) { + return; // Nothing to do + } + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + if (hydro_pkg->Param("fluid") != Fluid::glmmhd) { + PARTHENON_FAIL("MagneticTower::AddSrcTerm: Only Fluid::glmmhd is supported"); + } + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + const auto &A_pack = md->PackVariables(std::vector{"magnetic_tower_A"}); + + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + + // Scale density_to_add to match mass_to_add when integrated over all space + PARTHENON_REQUIRE_THROWS( + mass_to_add == 0.0 || l_mass_scale_ > 0.0, + "Trying to inject mass with the tower model but 0 mass lengthscale. Either disable " + "tower mass injection or set positive lengthscale."); + const auto density_to_add = + mass_to_add > 0.0 ? mass_to_add / (pow(l_mass_scale_, 3) * pow(M_PI, 3. / 2.)) + : 0.0; + + const JetCoords jet_coords = + hydro_pkg->Param("jet_coords_factory").CreateJetCoords(tm.time); + const MagneticTowerObj mt = + MagneticTowerObj(field_to_add, alpha_, l_scale_, offset_, thickness_, + density_to_add, l_mass_scale_, jet_coords, potential_); + + const auto &eos = hydro_pkg->Param("eos"); + + // Construct magnetic vector potential then compute magnetic fields + IndexRange a_ib = ib; + a_ib.s -= 1; + a_ib.e += 1; + IndexRange a_jb = jb; + a_jb.s -= 1; + a_jb.e += 1; + IndexRange a_kb = kb; + a_kb.s -= 1; + a_kb.e += 1; + + // Construct the magnetic tower potential + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddFieldSrcTerm::ConstructPotential", + parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, a_kb.s, a_kb.e, a_jb.s, + a_jb.e, a_ib.s, a_ib.e, + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + // Compute and apply potential + auto &A = A_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + Real a_x_, a_y_, a_z_; + mt.PotentialInSimCart(coords.Xc<1>(i), coords.Xc<2>(j), coords.Xc<3>(k), a_x_, + a_y_, a_z_); + + A(0, k, j, i) = a_x_; + A(1, k, j, i) = a_y_; + A(2, k, j, i) = a_z_; + }); + + // Take the curl of the potential and apply the new magnetic field + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::MagneticFieldSrcTerm::ApplyPotential", + parthenon::DevExecSpace(), 0, cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, + ib.e, KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + auto &A = A_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + // Take the curl of a to compute the magnetic field + const Real b_x = + (A(2, k, j + 1, i) - A(2, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0 - + (A(1, k + 1, j, i) - A(1, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0; + const Real b_y = + (A(0, k + 1, j, i) - A(0, k - 1, j, i)) / coords.Dxc<3>(k) / 2.0 - + (A(2, k, j, i + 1) - A(2, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0; + const Real b_z = + (A(1, k, j, i + 1) - A(1, k, j, i - 1)) / coords.Dxc<1>(i) / 2.0 - + (A(0, k, j + 1, i) - A(0, k, j - 1, i)) / coords.Dxc<2>(j) / 2.0; + + // Add the magnetic field to the conserved variables + cons(IB1, k, j, i) += b_x; + cons(IB2, k, j, i) += b_y; + cons(IB3, k, j, i) += b_z; + + // Add the magnetic field energy given the existing field in prim + // dE_B = 1/2*( 2*dt*B_old*B_new + dt**2*B_new**2) + cons(IEN, k, j, i) += prim(IB1, k, j, i) * b_x + prim(IB2, k, j, i) * b_y + + prim(IB3, k, j, i) * b_z + + 0.5 * (b_x * b_x + b_y * b_y + b_z * b_z); + + // Add density + const auto cell_delta_rho = + density_to_add > 0.0 + ? mt.DensityFromSimCart(coords.Xc<1>(i), coords.Xc<2>(j), coords.Xc<3>(k)) + : 0.0; + cons(IDN, k, j, i) += cell_delta_rho; + }); +} + +// Compute the increase to magnetic energy (1/2*B**2) over local meshes. Adds +// to linear_contrib and quadratic_contrib +// increases relative to B0 and B0**2. Necessary for scaling magnetic fields +// to inject a specified magnetic energy +void MagneticTower::ReducePowerContribs(parthenon::Real &linear_contrib, + parthenon::Real &quadratic_contrib, + parthenon::MeshData *md, + const parthenon::SimTime &tm) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + + const JetCoords jet_coords = + hydro_pkg->Param("jet_coords_factory").CreateJetCoords(tm.time); + + // Make a construct a copy of this with field strength 1 to send to the device + const MagneticTowerObj mt = MagneticTowerObj(1, alpha_, l_scale_, offset_, thickness_, + 0, l_mass_scale_, jet_coords, potential_); + + // Get the reduction of the linear and quadratic contributions ready + Real linear_contrib_red, quadratic_contrib_red; + + Kokkos::parallel_reduce( + "MagneticTowerScaleFactor", + Kokkos::MDRangePolicy>( + DevExecSpace(), {0, kb.s, jb.s, ib.s}, + {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real &llinear_contrib_red, Real &lquadratic_contrib_red) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const Real cell_volume = coords.CellVolume(k, j, i); + + // Compute the magnetic field at cell centers directly + Real b_x, b_y, b_z; + mt.FieldInSimCart(coords.Xc<1>(i), coords.Xc<2>(j), coords.Xc<3>(k), b_x, b_y, + b_z); + + // increases B**2 by 2*B0*Bnew + dt**2*Bnew**2) + llinear_contrib_red += (prim(IB1, k, j, i) * b_x + prim(IB2, k, j, i) * b_y + + prim(IB3, k, j, i) * b_z) * + cell_volume; + lquadratic_contrib_red += 0.5 * (b_x * b_x + b_y * b_y + b_z * b_z) * cell_volume; + }, + linear_contrib_red, quadratic_contrib_red); + + linear_contrib += linear_contrib_red; + quadratic_contrib += quadratic_contrib_red; +} + +// Add magnetic potential to provided potential +template +void MagneticTower::AddInitialFieldToPotential(parthenon::MeshBlock *pmb, + parthenon::IndexRange kb, + parthenon::IndexRange jb, + parthenon::IndexRange ib, + const View4D &A) const { + if (initial_field_ == 0) { + return; // Nothing to do + } + + auto hydro_pkg = pmb->packages.Get("Hydro"); + const auto &coords = pmb->coords; + + const JetCoords jet_coords = + hydro_pkg->Param("jet_coords_factory").CreateJetCoords(0.0); + const MagneticTowerObj mt(initial_field_, alpha_, l_scale_, offset_, thickness_, 0, + l_mass_scale_, jet_coords, potential_); + + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "MagneticTower::AddInitialFieldToPotential", + parthenon::DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &k, const int &j, const int &i) { + // Compute and apply potential + Real a_x, a_y, a_z; + mt.PotentialInSimCart(coords.Xc<1>(i), coords.Xc<2>(j), coords.Xc<3>(k), a_x, a_y, + a_z); + A(0, k, j, i) += a_x; + A(1, k, j, i) += a_y; + A(2, k, j, i) += a_z; + }); +} + +// Instantiate the template definition in this source file +template void +MagneticTower::AddInitialFieldToPotential<>(MeshBlock *pmb, IndexRange kb, IndexRange jb, + IndexRange ib, + const ParArray4D &A) const; + +// Add the fixed_field_rate (and associated magnetic energy) to the +// conserved variables for all meshblocks with a MeshData +void MagneticTower::FixedFieldSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm) const { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + if (fixed_field_rate_ != 0) { + AddSrcTerm(fixed_field_rate_ * beta_dt, fixed_mass_rate_ * beta_dt, md, tm); + } +} + +// Add the specified magnetic power (and associated magnetic field) to the +// conserved variables for all meshblocks with a MeshData +void MagneticTower::PowerSrcTerm(const parthenon::Real power, + const parthenon::Real mass_rate, + parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm) const { + if (power == 0) { + // Nothing to inject, return + return; + } + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + const Real linear_contrib = hydro_pkg->Param("magnetic_tower_linear_contrib"); + const Real quadratic_contrib = + hydro_pkg->Param("magnetic_tower_quadratic_contrib"); + if (linear_contrib == 0 && quadratic_contrib == 0) { + PARTHENON_FAIL("MagneticTowerModel::PowerSrcTerm mt_linear_contrib " + "and mt_quadratic_contrib are both zero. " + "(Has MagneticTowerReducePowerContribs been called?)"); + } + + const Real disc = + linear_contrib * linear_contrib + 4 * quadratic_contrib * beta_dt * power; + if (disc < 0 || quadratic_contrib == 0) { + std::stringstream msg; + msg << "MagneticTowerModel::PowerSrcTerm No field rate is viable" + << " linear_contrib: " << std::to_string(linear_contrib) + << " quadratic_contrib: " << std::to_string(quadratic_contrib); + PARTHENON_FAIL(msg); + } + const Real field_to_add = (-linear_contrib + sqrt(disc)) / (2 * quadratic_contrib); + const Real mass_to_add = mass_rate * beta_dt; + + AddSrcTerm(field_to_add, mass_to_add, md, tm); +} + +parthenon::TaskStatus +MagneticTowerResetPowerContribs(parthenon::StateDescriptor *hydro_pkg) { + hydro_pkg->UpdateParam("magnetic_tower_linear_contrib", 0.0); + hydro_pkg->UpdateParam("magnetic_tower_quadratic_contrib", 0.0); + return TaskStatus::complete; +} + +parthenon::TaskStatus +MagneticTowerReducePowerContribs(parthenon::MeshData *md, + const parthenon::SimTime &tm) { + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + const auto &magnetic_tower = hydro_pkg->Param("magnetic_tower"); + + parthenon::Real linear_contrib = + hydro_pkg->Param("magnetic_tower_linear_contrib"); + parthenon::Real quadratic_contrib = + hydro_pkg->Param("magnetic_tower_quadratic_contrib"); + magnetic_tower.ReducePowerContribs(linear_contrib, quadratic_contrib, md, tm); + + hydro_pkg->UpdateParam("magnetic_tower_linear_contrib", linear_contrib); + hydro_pkg->UpdateParam("magnetic_tower_quadratic_contrib", quadratic_contrib); + return TaskStatus::complete; +} + +} // namespace cluster diff --git a/src/pgen/cluster/magnetic_tower.hpp b/src/pgen/cluster/magnetic_tower.hpp new file mode 100644 index 00000000..ebcc1d44 --- /dev/null +++ b/src/pgen/cluster/magnetic_tower.hpp @@ -0,0 +1,270 @@ +#ifndef CLUSTER_MAGNETIC_TOWER_HPP_ +#define CLUSTER_MAGNETIC_TOWER_HPP_ +//======================================================================================== +//// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +///// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +///// Licensed under the 3-clause BSD License, see LICENSE file for details +/////======================================================================================== +//! \file magnetic_tower.hpp +// \brief Class for defining magnetic towers + +// parthenon headers +#include +#include +#include +#include +#include +#include + +#include "jet_coords.hpp" +#include "utils/error_checking.hpp" + +namespace cluster { + +enum class MagneticTowerPotential { undefined, li, donut }; +/************************************************************ + * Magnetic Tower Object, for computing magnetic field, vector potential at a + * fixed time with a fixed field + * Lightweight object intended for inlined computation within kernels + ************************************************************/ +class MagneticTowerObj { + private: + const parthenon::Real field_; + const parthenon::Real alpha_, l_scale_; + const parthenon::Real offset_, thickness_; + + const parthenon::Real density_, l_mass_scale2_; + + JetCoords jet_coords_; + // Note that this eventually might better be a template parameter, but while the number + // of potentials implemented is limited (and similarly complex) this should currently + // not be a performance concern. + const MagneticTowerPotential potential_; + + public: + MagneticTowerObj(const parthenon::Real field, const parthenon::Real alpha, + const parthenon::Real l_scale, const parthenon::Real offset, + const parthenon::Real thickness, const parthenon::Real density, + const parthenon::Real l_mass_scale, const JetCoords jet_coords, + const MagneticTowerPotential potential) + : field_(field), alpha_(alpha), l_scale_(l_scale), offset_(offset), + thickness_(thickness), density_(density), l_mass_scale2_(SQR(l_mass_scale)), + jet_coords_(jet_coords), potential_(potential) { + PARTHENON_REQUIRE(l_scale > 0, + "Magnetic Tower Length scale must be strictly postitive"); + PARTHENON_REQUIRE( + l_mass_scale >= 0, + "Magnetic Tower Mass Length scale must be zero (disabled) or postitive"); + } + + // Compute Jet Potential in jet cylindrical coordinates + KOKKOS_INLINE_FUNCTION void + PotentialInJetCyl(const parthenon::Real r, const parthenon::Real h, + parthenon::Real &a_r, parthenon::Real &a_theta, + parthenon::Real &a_h) const __attribute__((always_inline)) { + if (potential_ == MagneticTowerPotential::donut) { + const parthenon::Real exp_r2_h2 = exp(-pow(r / l_scale_, 2)); + // Compute the potential in jet cylindrical coordinates + a_r = 0.0; + a_theta = 0.0; + if (fabs(h) >= 0.001 && fabs(h) <= offset_ + thickness_) { + a_h = field_ * l_scale_ * exp_r2_h2; + } else { + a_h = 0.0; + } + } else if (potential_ == MagneticTowerPotential::li) { + const parthenon::Real exp_r2_h2 = exp(-pow(r / l_scale_, 2) - pow(h / l_scale_, 2)); + // Compute the potential in jet cylindrical coordinates + a_r = 0.0; + a_theta = field_ * l_scale_ * (r / l_scale_) * exp_r2_h2; + a_h = field_ * l_scale_ * alpha_ / 2.0 * exp_r2_h2; + } else { + PARTHENON_FAIL("Unknown magnetic tower potential."); + } + } + + // Compute Magnetic Potential in simulation Cartesian coordinates + KOKKOS_INLINE_FUNCTION void + PotentialInSimCart(const parthenon::Real x, const parthenon::Real y, + const parthenon::Real z, parthenon::Real &a_x, parthenon::Real &a_y, + parthenon::Real &a_z) const __attribute__((always_inline)) { + // Compute the jet cylindrical coordinates + parthenon::Real r, cos_theta, sin_theta, h; + jet_coords_.SimCartToJetCylCoords(x, y, z, r, cos_theta, sin_theta, h); + + // Compute the potential in jet cylindrical coordinates + parthenon::Real a_r, a_theta, a_h; + PotentialInJetCyl(r, h, a_r, a_theta, a_h); + + // Convert vector potential from jet cylindrical to simulation cartesian + jet_coords_.JetCylToSimCartVector(cos_theta, sin_theta, a_r, a_theta, a_h, a_x, a_y, + a_z); + } + + // Compute Magnetic Fields in Jet cylindrical Coordinates + KOKKOS_INLINE_FUNCTION void + FieldInJetCyl(const parthenon::Real r, const parthenon::Real h, parthenon::Real &b_r, + parthenon::Real &b_theta, parthenon::Real &b_h) const + __attribute__((always_inline)) { + if (potential_ == MagneticTowerPotential::donut) { + const parthenon::Real exp_r2_h2 = exp(-pow(r / l_scale_, 2)); + // Compute the field in jet cylindrical coordinates + b_r = 0.0; + if (fabs(h) >= 0.001 && fabs(h) <= offset_ + thickness_) { + b_theta = 2.0 * field_ * r / l_scale_ * exp_r2_h2; + } else { + b_theta = 0.0; + } + b_h = 0.0; + } else if (potential_ == MagneticTowerPotential::li) { + const parthenon::Real exp_r2_h2 = exp(-pow(r / l_scale_, 2) - pow(h / l_scale_, 2)); + // Compute the field in jet cylindrical coordinates + b_r = field_ * 2 * (h / l_scale_) * (r / l_scale_) * exp_r2_h2; + b_theta = field_ * alpha_ * (r / l_scale_) * exp_r2_h2; + b_h = field_ * 2 * (1 - pow(r / l_scale_, 2)) * exp_r2_h2; + } else { + PARTHENON_FAIL("Unknown magnetic tower potential."); + } + } + + // Compute Magnetic field in Simulation Cartesian coordinates + KOKKOS_INLINE_FUNCTION void + FieldInSimCart(const parthenon::Real x, const parthenon::Real y, + const parthenon::Real z, parthenon::Real &b_x, parthenon::Real &b_y, + parthenon::Real &b_z) const __attribute__((always_inline)) { + // Compute the jet cylindrical coordinates + parthenon::Real r, cos_theta, sin_theta, h; + jet_coords_.SimCartToJetCylCoords(x, y, z, r, cos_theta, sin_theta, h); + + // Compute the magnetic field in jet_coords + parthenon::Real b_r, b_theta, b_h; + FieldInJetCyl(r, h, b_r, b_theta, b_h); + + // Convert potential to cartesian + jet_coords_.JetCylToSimCartVector(cos_theta, sin_theta, b_r, b_theta, b_h, b_x, b_y, + b_z); + } + + // Compute Density injection from Simulation Cartesian coordinates + KOKKOS_INLINE_FUNCTION parthenon::Real DensityFromSimCart(const parthenon::Real x, + const parthenon::Real y, + const parthenon::Real z) const + __attribute__((always_inline)) { + // Compute the jet cylindrical coordinates + parthenon::Real r, cos_theta, sin_theta, h; + jet_coords_.SimCartToJetCylCoords(x, y, z, r, cos_theta, sin_theta, h); + + return density_ * exp(-(SQR(r) + SQR(h)) / l_mass_scale2_); + } +}; + +/************************************************************ + * Magnetic Tower Model, for initializing a magnetic tower and tasks related to + * injecting a magnetic tower as a source term + ************************************************************/ +class MagneticTower { + public: + const parthenon::Real alpha_, l_scale_; + const parthenon::Real offset_, thickness_; + + const parthenon::Real initial_field_; + const parthenon::Real fixed_field_rate_; + + const parthenon::Real fixed_mass_rate_; + const parthenon::Real l_mass_scale_; + + MagneticTowerPotential potential_; + + MagneticTower(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg, + const std::string &block = "problem/cluster/magnetic_tower") + : alpha_(pin->GetOrAddReal(block, "li_alpha", 0)), + l_scale_(pin->GetOrAddReal(block, "l_scale", 0)), + offset_(pin->GetOrAddReal(block, "donut_offset", 0)), + thickness_(pin->GetOrAddReal(block, "donut_thickness", 0)), + initial_field_(pin->GetOrAddReal(block, "initial_field", 0)), + fixed_field_rate_(pin->GetOrAddReal(block, "fixed_field_rate", 0)), + fixed_mass_rate_(pin->GetOrAddReal(block, "fixed_mass_rate", 0)), + l_mass_scale_(pin->GetOrAddReal(block, "l_mass_scale", 0)), + potential_(MagneticTowerPotential::undefined) { + hydro_pkg->AddParam("magnetic_tower_linear_contrib", 0.0, true); + hydro_pkg->AddParam("magnetic_tower_quadratic_contrib", 0.0, true); + + const auto potential_str = pin->GetOrAddString(block, "potential_type", "undefined"); + + if (potential_str == "donut") { + potential_ = MagneticTowerPotential::donut; + PARTHENON_REQUIRE_THROWS(offset_ >= 0.0 && thickness_ > 0.0, + "Incompatible combination of donut_offset and " + "donut_thickness for magnetic donut feedback.") + PARTHENON_REQUIRE_THROWS(alpha_ == 0.0, + "Please disable (set to zero) tower li_alpha " + "for the donut model"); + } else if (potential_str == "li") { + potential_ = MagneticTowerPotential::li; + PARTHENON_REQUIRE_THROWS(offset_ <= 0.0 && thickness_ <= 0.0, + "Please disable (set to zero) tower offset and thickness " + "for the Li tower model"); + } + + // Vector potential is only locally used, so no need to + // communicate/restrict/prolongate/fluxes/etc + parthenon::Metadata m({parthenon::Metadata::Cell, parthenon::Metadata::Derived, + parthenon::Metadata::OneCopy}, + std::vector({3})); + hydro_pkg->AddField("magnetic_tower_A", m); + + // Finally, add object to params (should be done last as otherwise modification within + // this function would not survive). + hydro_pkg->AddParam<>("magnetic_tower", *this); + } + + // Add initial magnetic field to provided potential with a single meshblock + template + void AddInitialFieldToPotential(parthenon::MeshBlock *pmb, parthenon::IndexRange kb, + parthenon::IndexRange jb, parthenon::IndexRange ib, + const View4D &A) const; + + // Add the fixed_field_rate (and associated magnetic energy) to the + // conserved variables for all meshblocks within a MeshData + void FixedFieldSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm) const; + + // Add the specified magnetic power (and associated magnetic field) to the + // conserved variables for all meshblocks within a MeshData + void PowerSrcTerm(const parthenon::Real power, const parthenon::Real mass_rate, + parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm) const; + + // Add the specified magnetic field (and associated magnetic energy) to the + // conserved variables for all meshblocks with a MeshData + void AddSrcTerm(parthenon::Real field_to_add, parthenon::Real mass_to_add, + parthenon::MeshData *md, + const parthenon::SimTime &tm) const; + + // Compute the increase to magnetic energy (1/2*B**2) over local meshes. Adds + // to linear_contrib and quadratic_contrib + // increases relative to B0 and B0**2. Necessary for scaling magnetic fields + // to inject a specified magnetic energy + void ReducePowerContribs(parthenon::Real &linear_contrib, + parthenon::Real &quadratic_contrib, + parthenon::MeshData *md, + const parthenon::SimTime &tm) const; + + friend parthenon::TaskStatus + MagneticTowerResetPowerContribs(parthenon::StateDescriptor *hydro_pkg); + + friend parthenon::TaskStatus + MagneticTowerReducePowerContribs(parthenon::MeshData *md, + const parthenon::SimTime &tm); +}; + +parthenon::TaskStatus +MagneticTowerResetPowerContribs(parthenon::StateDescriptor *hydro_pkg); +parthenon::TaskStatus +MagneticTowerReducePowerContribs(parthenon::MeshData *md, + const parthenon::SimTime &tm); + +} // namespace cluster + +#endif // CLUSTER_MAGNETIC_TOWER_HPP_ diff --git a/src/pgen/cluster/snia_feedback.cpp b/src/pgen/cluster/snia_feedback.cpp new file mode 100644 index 00000000..ed41657f --- /dev/null +++ b/src/pgen/cluster/snia_feedback.cpp @@ -0,0 +1,121 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file snia_feedback.cpp +// \brief Class for injecting SNIA feedback following BCG density + +#include + +// Parthenon headers +#include +#include +#include +#include +#include +#include + +// Athena headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" +#include "../../main.hpp" +#include "../../units.hpp" +#include "cluster_gravity.hpp" +#include "cluster_utils.hpp" +#include "snia_feedback.hpp" + +namespace cluster { +using namespace parthenon; + +SNIAFeedback::SNIAFeedback(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg) + : power_per_bcg_mass_( + pin->GetOrAddReal("problem/cluster/snia_feedback", "power_per_bcg_mass", 0.0)), + mass_rate_per_bcg_mass_(pin->GetOrAddReal("problem/cluster/snia_feedback", + "mass_rate_per_bcg_mass", 0.0)), + bcg_gravity_(pin), disabled_(pin->GetOrAddBoolean("problem/cluster/snia_feedback", + "disabled", false)) { + + // Initialize the gravity from the cluster + // Turn off the NFW and SMBH to get just the BCG gravity + bcg_gravity_.include_nfw_g_ = false; + bcg_gravity_.include_smbh_g_ = false; + + PARTHENON_REQUIRE(disabled_ || bcg_gravity_.which_bcg_g_ != BCG::NONE, + "BCG must be defined for SNIA Feedback to be enabled"); + hydro_pkg->AddParam("snia_feedback", *this); +} + +void SNIAFeedback::FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm) const { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + auto fluid = hydro_pkg->Param("fluid"); + if (fluid == Fluid::euler) { + FeedbackSrcTerm(md, beta_dt, tm, hydro_pkg->Param("eos")); + } else if (fluid == Fluid::glmmhd) { + FeedbackSrcTerm(md, beta_dt, tm, hydro_pkg->Param("eos")); + } else { + PARTHENON_FAIL("SNIAFeedback::FeedbackSrcTerm: Unknown EOS"); + } +} +template +void SNIAFeedback::FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm, const EOS &eos) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + if ((power_per_bcg_mass_ == 0 && mass_rate_per_bcg_mass_ == 0) || disabled_) { + // No AGN feedback, return + return; + } + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + const auto nscalars = hydro_pkg->Param("nscalars"); + + const Real energy_per_bcg_mass = power_per_bcg_mass_ * beta_dt; + const Real mass_per_bcg_mass = mass_rate_per_bcg_mass_ * beta_dt; + + const ClusterGravity bcg_gravity = bcg_gravity_; + + //////////////////////////////////////////////////////////////////////////////// + + // Constant volumetric heating + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "SNIAFeedback::FeedbackSrcTerm", parthenon::DevExecSpace(), 0, + cons_pack.GetDim(5) - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const Real x = coords.Xc<1>(i); + const Real y = coords.Xc<2>(j); + const Real z = coords.Xc<3>(k); + + const Real r = sqrt(x * x + y * y + z * z); + + const Real bcg_density = bcg_gravity.rho_from_r(r); + + const Real snia_energy_density = energy_per_bcg_mass * bcg_density; + const Real snia_mass_density = mass_per_bcg_mass * bcg_density; + + cons(IEN, k, j, i) += snia_energy_density; + AddDensityToConsAtFixedVel(snia_mass_density, cons, prim, k, j, i); + + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + }); +} + +} // namespace cluster diff --git a/src/pgen/cluster/snia_feedback.hpp b/src/pgen/cluster/snia_feedback.hpp new file mode 100644 index 00000000..41fda0ce --- /dev/null +++ b/src/pgen/cluster/snia_feedback.hpp @@ -0,0 +1,50 @@ +#ifndef CLUSTER_SNIA_FEEDBACK_HPP_ +#define CLUSTER_SNIA_FEEDBACK_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file snia_feedback.hpp +// \brief Class for injecting SNIA feedback following BCG density + +// parthenon headers +#include +#include +#include +#include +#include + +#include "jet_coords.hpp" + +namespace cluster { + +/************************************************************ + * AGNFeedback + ************************************************************/ +class SNIAFeedback { + public: + // Power and Mass to inject per mass in the BCG + parthenon::Real power_per_bcg_mass_; // energy/(mass*time) + parthenon::Real mass_rate_per_bcg_mass_; // 1/(time) + + // ClusterGravity object to calculate BCG density + ClusterGravity bcg_gravity_; + + const bool disabled_; + + SNIAFeedback(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg); + + void FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm) const; + + // Apply the feedback from SNIAe tied to the BCG density + template + void FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm, + const EOS &eos) const; +}; + +} // namespace cluster + +#endif // CLUSTER_SNIA_FEEDBACK_HPP_ diff --git a/src/pgen/cluster/stellar_feedback.cpp b/src/pgen/cluster/stellar_feedback.cpp new file mode 100644 index 00000000..aa130d65 --- /dev/null +++ b/src/pgen/cluster/stellar_feedback.cpp @@ -0,0 +1,176 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file stellar_feedback.cpp +// \brief Class for magic heating modeling star formation + +#include + +// Parthenon headers +#include +#include +#include +#include +#include +#include + +// Athena headers +#include "../../eos/adiabatic_glmmhd.hpp" +#include "../../eos/adiabatic_hydro.hpp" +#include "../../main.hpp" +#include "../../units.hpp" +#include "cluster_gravity.hpp" +#include "cluster_utils.hpp" +#include "stellar_feedback.hpp" +#include "utils/error_checking.hpp" + +namespace cluster { +using namespace parthenon; + +StellarFeedback::StellarFeedback(parthenon::ParameterInput *pin, + parthenon::StateDescriptor *hydro_pkg) + : stellar_radius_( + pin->GetOrAddReal("problem/cluster/stellar_feedback", "stellar_radius", 0.0)), + exclusion_radius_( + pin->GetOrAddReal("problem/cluster/stellar_feedback", "exclusion_radius", 0.0)), + efficiency_( + pin->GetOrAddReal("problem/cluster/stellar_feedback", "efficiency", 0.0)), + number_density_threshold_(pin->GetOrAddReal("problem/cluster/stellar_feedback", + "number_density_threshold", 0.0)), + temperatue_threshold_(pin->GetOrAddReal("problem/cluster/stellar_feedback", + "temperature_threshold", 0.0)) { + if (stellar_radius_ == 0.0 && exclusion_radius_ == 0.0 && efficiency_ == 0.0 && + number_density_threshold_ == 0.0 && temperatue_threshold_ == 0.0) { + disabled_ = true; + } else { + disabled_ = false; + } + + if (!disabled_ && exclusion_radius_ == 0.0) { + // If exclusion_radius_ is not specified, use AGN triggering accretion radius. + // If both are zero, the PARTHENON_REQUIRE will fail + exclusion_radius_ = + pin->GetOrAddReal("problem/cluster/agn_triggering", "accretion_radius", 0); + } + + PARTHENON_REQUIRE(disabled_ || + (stellar_radius_ != 0.0 && exclusion_radius_ != 0.0 && + efficiency_ != 0.00 && number_density_threshold_ != 0.0 && + temperatue_threshold_ != 0.0), + "Enabling stellar feedback requires setting all parameters."); + + hydro_pkg->AddParam("stellar_feedback", *this); +} + +void StellarFeedback::FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm) const { + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + auto fluid = hydro_pkg->Param("fluid"); + if (fluid == Fluid::euler) { + FeedbackSrcTerm(md, beta_dt, tm, hydro_pkg->Param("eos")); + } else if (fluid == Fluid::glmmhd) { + FeedbackSrcTerm(md, beta_dt, tm, hydro_pkg->Param("eos")); + } else { + PARTHENON_FAIL("StellarFeedback::FeedbackSrcTerm: Unknown EOS"); + } +} +template +void StellarFeedback::FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, + const parthenon::SimTime &tm, + const EOS &eos_) const { + using parthenon::IndexDomain; + using parthenon::IndexRange; + using parthenon::Real; + + auto hydro_pkg = md->GetBlockData(0)->GetBlockPointer()->packages.Get("Hydro"); + + if (disabled_) { + // No stellar feedback, return + return; + } + + // Grab some necessary variables + const auto &prim_pack = md->PackVariables(std::vector{"prim"}); + const auto &cons_pack = md->PackVariables(std::vector{"cons"}); + IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); + const auto nhydro = hydro_pkg->Param("nhydro"); + const auto nscalars = hydro_pkg->Param("nscalars"); + + // const auto gm1 = (hydro_pkg->Param("AdiabaticIndex") - 1.0); + const auto units = hydro_pkg->Param("units"); + const auto mbar = hydro_pkg->Param("mu") * units.mh(); + const auto mbar_over_kb = hydro_pkg->Param("mbar_over_kb"); + + const auto mass_to_energy = efficiency_ * SQR(units.speed_of_light()); + const auto stellar_radius = stellar_radius_; + const auto exclusion_radius = exclusion_radius_; + const auto temperature_threshold = temperatue_threshold_; + const auto number_density_threshold = number_density_threshold_; + + const auto eos = eos_; + + //////////////////////////////////////////////////////////////////////////////// + + Real stellar_mass = 0.0; + + // Constant volumetric heating, reduce mass removed + Kokkos::parallel_reduce( + "StellarFeedback::FeedbackSrcTerm", + Kokkos::MDRangePolicy>( + DevExecSpace(), {0, kb.s, jb.s, ib.s}, + {prim_pack.GetDim(5), kb.e + 1, jb.e + 1, ib.e + 1}, + {1, 1, 1, ib.e + 1 - ib.s}), + KOKKOS_LAMBDA(const int &b, const int &k, const int &j, const int &i, + Real &stellar_mass_team) { + auto &cons = cons_pack(b); + auto &prim = prim_pack(b); + const auto &coords = cons_pack.GetCoords(b); + + const auto x = coords.Xc<1>(i); + const auto y = coords.Xc<2>(j); + const auto z = coords.Xc<3>(k); + + const auto r = sqrt(x * x + y * y + z * z); + if (r > stellar_radius || r <= exclusion_radius) { + return; + } + + auto number_density = prim(IDN, k, j, i) / mbar; + if (number_density < number_density_threshold) { + return; + } + + auto temp = mbar_over_kb * prim(IPR, k, j, i) / prim(IDN, k, j, i); + if (temp > temperature_threshold) { + return; + } + + // All conditions to convert mass to energy are met + const auto cell_delta_rho = number_density_threshold * mbar - prim(IDN, k, j, i); + stellar_mass_team -= cell_delta_rho * coords.CellVolume(k, j, i); + + // First remove density at fixed temperature + AddDensityToConsAtFixedVelTemp(cell_delta_rho, cons, prim, eos.GetGamma(), k, j, + i); + // Then add thermal energy + const auto cell_delta_energy_density = -mass_to_energy * cell_delta_rho; + PARTHENON_REQUIRE( + cell_delta_energy_density > 0.0, + "Sanity check failed. Added thermal energy should be positive."); + cons(IEN, k, j, i) += cell_delta_energy_density; + + // Update prims + eos.ConsToPrim(cons, prim, nhydro, nscalars, k, j, i); + }, + stellar_mass); + hydro_pkg->UpdateParam( + "stellar_mass", stellar_mass + hydro_pkg->Param("stellar_mass")); +} + +} // namespace cluster diff --git a/src/pgen/cluster/stellar_feedback.hpp b/src/pgen/cluster/stellar_feedback.hpp new file mode 100644 index 00000000..e3d4b20c --- /dev/null +++ b/src/pgen/cluster/stellar_feedback.hpp @@ -0,0 +1,51 @@ +#ifndef CLUSTER_STELLAR_FEEDBACK_HPP_ +#define CLUSTER_STELLAR_FEEDBACK_HPP_ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//! \file stellar_feedback.hpp +// \brief Class for injecting Stellar feedback following BCG density + +// parthenon headers +#include +#include +#include +#include +#include + +#include "jet_coords.hpp" + +namespace cluster { + +/************************************************************ + * StellarFeedback + ************************************************************/ +class StellarFeedback { + private: + // feedback parameters in code units + const parthenon::Real stellar_radius_; // length + parthenon::Real exclusion_radius_; // length + const parthenon::Real efficiency_; // dimless + const parthenon::Real number_density_threshold_; // 1/(length^3) + const parthenon::Real temperatue_threshold_; // K + + bool disabled_; + + public: + StellarFeedback(parthenon::ParameterInput *pin, parthenon::StateDescriptor *hydro_pkg); + + void FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm) const; + + // Apply stellar feedback following cold gas density above a density threshold + template + void FeedbackSrcTerm(parthenon::MeshData *md, + const parthenon::Real beta_dt, const parthenon::SimTime &tm, + const EOS &eos) const; +}; + +} // namespace cluster + +#endif // CLUSTER_STELLAR_FEEDBACK_HPP_ diff --git a/src/pgen/orszag_tang.cpp b/src/pgen/orszag_tang.cpp index baa014e3..a034c73f 100644 --- a/src/pgen/orszag_tang.cpp +++ b/src/pgen/orszag_tang.cpp @@ -1,6 +1,7 @@ +//======================================================================================== // AthenaPK - a performance portable block structured AMR MHD code -// Copyright (c) 2021, Athena Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena Parthenon Collaboration. All rights reserved. // Licensed under the 3-Clause License (the "LICENSE") //======================================================================================== //! \file orszag_tang.cpp @@ -41,11 +42,15 @@ void ProblemGenerator(MeshBlock *pmb, ParameterInput *pin) { "ProblemGenerator: Orszag-Tang", kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { u(IDN, k, j, i) = d0; + // Note the different signs in this pgen compared to the the eqn mentioned in the + // original paper (and other codes). + // They are related to our domain going from -0.5 to 0.5 (for symmetry reason) + // rather than 0 to 2pi (i.e., the sign for single wave sinus is flipped). u(IM1, k, j, i) = d0 * v0 * std::sin(2.0 * M_PI * coords.Xc<2>(j)); u(IM2, k, j, i) = -d0 * v0 * std::sin(2.0 * M_PI * coords.Xc<1>(i)); u(IM3, k, j, i) = 0.0; - u(IB1, k, j, i) = -B0 * std::sin(2.0 * M_PI * coords.Xc<2>(j)); + u(IB1, k, j, i) = B0 * std::sin(2.0 * M_PI * coords.Xc<2>(j)); u(IB2, k, j, i) = B0 * std::sin(4.0 * M_PI * coords.Xc<1>(i)); u(IB3, k, j, i) = 0.0; diff --git a/src/pgen/pgen.hpp b/src/pgen/pgen.hpp index da1b555e..182d8497 100644 --- a/src/pgen/pgen.hpp +++ b/src/pgen/pgen.hpp @@ -96,9 +96,15 @@ void RandomBlasts(MeshData *md, const parthenon::SimTime &tm, const Real); namespace cluster { using namespace parthenon::driver::prelude; -void InitUserMeshData(Mesh *mesh, ParameterInput *pin); -void ProblemGenerator(MeshBlock *pmb, parthenon::ParameterInput *pin); -void ClusterSrcTerm(MeshData *md, const parthenon::SimTime, const Real beta_dt); +void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg); +void InitUserMeshData(ParameterInput *pin); +void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md); +void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin); +void ClusterUnsplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt); +void ClusterSplitSrcTerm(MeshData *md, const parthenon::SimTime &tm, + const Real beta_dt); +parthenon::Real ClusterEstimateTimestep(MeshData *md); } // namespace cluster namespace sod { diff --git a/src/pgen/turbulence.cpp b/src/pgen/turbulence.cpp index d74fac64..58882f52 100644 --- a/src/pgen/turbulence.cpp +++ b/src/pgen/turbulence.cpp @@ -1,6 +1,6 @@ //======================================================================================== // AthenaPK - a performance portable block structured AMR astrophysical MHD code. -// Copyright (c) 20212-2022, Athena-Parthenon Collaboration. All rights reserved. +// Copyright (c) 2021-2023, Athena-Parthenon Collaboration. All rights reserved. // Licensed under the 3-clause BSD License, see LICENSE file for details //======================================================================================== //! \file turbulence.cpp @@ -27,21 +27,14 @@ // AthenaPK headers #include "../main.hpp" #include "../units.hpp" +#include "../utils/few_modes_ft.hpp" namespace turbulence { using namespace parthenon::package::prelude; - -typedef Kokkos::complex Complex; using parthenon::DevMemSpace; using parthenon::ParArray2D; - -// Defining these "globally" as they are fixed across all blocks -ParArray2D accel_hat_, accel_hat_new_; -ParArray2D k_vec_; -Kokkos::View random_num_; -Kokkos::View random_num_host; -std::mt19937 rng; -std::uniform_real_distribution<> dist(-1.0, 1.0); +using utils::few_modes_ft::Complex; +using utils::few_modes_ft::FewModesFT; // TODO(?) until we are able to process multiple variables in a single hst function call // we'll use this enum to identify the various vars. @@ -120,85 +113,61 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg pkg->UpdateParam(parthenon::hist_param_key, hst_vars); // Step 2. Add appropriate fields required by this pgen - std::vector acc_labels(3); - acc_labels[0] = "Acceleration1"; - acc_labels[1] = "Acceleration2"; - acc_labels[2] = "Acceleration3"; - // Using OneCopy here to save memory. We typically don't need to update/evolve the // acceleration field for various stages in a cycle as the "model" error of the // turbulence driver is larger than the numerical one any way. This may need to be // changed if an "as close as possible" comparison between methods/codes is the goal and // not turbulence from a physical point of view. Metadata m({Metadata::Cell, Metadata::Derived, Metadata::OneCopy}, - std::vector({3}), acc_labels); + std::vector({3})); pkg->AddField("acc", m); auto num_modes = pin->GetInteger("problem/turbulence", "num_modes"); // number of wavemodes - if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { - std::cout << "### WARNING using more than 100 explicit modes will significantly " - << "increase the runtime." << std::endl - << "If many modes are required in the acceleration field consider using " - << "the driving mechanism based on full FFTs." << std::endl; - } - pkg->AddParam<>("turbulence/num_modes", num_modes); - - const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); - const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); - const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx1}), "phases_i"); - pkg->AddField("phases_i", m); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx2}), "phases_j"); - pkg->AddField("phases_j", m); - m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, - std::vector({2, num_modes, nx3}), "phases_k"); - pkg->AddField("phases_k", m); uint32_t rseed = pin->GetOrAddInteger("problem/turbulence", "rseed", -1); // seed for random number. pkg->AddParam<>("turbulence/rseed", rseed); - auto kpeak = + auto k_peak = pin->GetOrAddReal("problem/turbulence", "kpeak", 0.0); // peak of the forcing spec - pkg->AddParam<>("turbulence/kpeak", kpeak); + pkg->AddParam<>("turbulence/kpeak", k_peak); auto accel_rms = pin->GetReal("problem/turbulence", "accel_rms"); // turbulence amplitude pkg->AddParam<>("turbulence/accel_rms", accel_rms); - auto tcorr = + auto t_corr = pin->GetReal("problem/turbulence", "corr_time"); // forcing autocorrelation time - pkg->AddParam<>("turbulence/tcorr", tcorr); + pkg->AddParam<>("turbulence/t_corr", t_corr); Real sol_weight = pin->GetReal("problem/turbulence", "sol_weight"); // solenoidal weight pkg->AddParam<>("turbulence/sol_weight", sol_weight); - // Acceleration field in Fourier space using complex to real transform. - accel_hat_ = ParArray2D("accel_hat", 3, num_modes); - accel_hat_new_ = ParArray2D("accel_hat_new", 3, num_modes); - // list of wavenumber vectors - k_vec_ = ParArray2D("k_vec", 3, num_modes); - auto k_vec_host = Kokkos::create_mirror_view(k_vec_); + auto k_vec = ParArray2D("k_vec", 3, num_modes); + auto k_vec_host = Kokkos::create_mirror_view(k_vec); for (int j = 0; j < 3; j++) { for (int i = 1; i <= num_modes; i++) { k_vec_host(j, i - 1) = pin->GetInteger("modes", "k_" + std::to_string(i) + "_" + std::to_string(j)); } } - Kokkos::deep_copy(k_vec_, k_vec_host); + Kokkos::deep_copy(k_vec, k_vec_host); - random_num_ = Kokkos::View("random_num", 3, - num_modes, 2); - random_num_host = Kokkos::create_mirror_view(random_num_); + auto few_modes_ft = FewModesFT(pin, pkg, "turbulence", num_modes, k_vec, k_peak, + sol_weight, t_corr, rseed); + // object must be mutable to update the internal state of the RNG + pkg->AddParam<>("turbulence/few_modes_ft", few_modes_ft, true); // Check if this is is a restart and restore previous state if (pin->DoesParameterExist("problem/turbulence", "accel_hat_0_0_r")) { + // Need to extract mutable object from Params here as the original few_modes_ft above + // and the one in Params are different instances + auto *pfew_modes_ft = pkg->MutableParam("turbulence/few_modes_ft"); // Restore (common) acceleration field in spectral space - auto accel_hat_host = Kokkos::create_mirror_view(accel_hat_); + auto accel_hat = pfew_modes_ft->GetVarHat(); + auto accel_hat_host = Kokkos::create_mirror_view(accel_hat); for (int i = 0; i < 3; i++) { for (int m = 0; m < num_modes; m++) { auto real = @@ -210,133 +179,26 @@ void ProblemInitPackageData(ParameterInput *pin, parthenon::StateDescriptor *pkg accel_hat_host(i, m) = Complex(real, imag); } } - Kokkos::deep_copy(accel_hat_, accel_hat_host); + Kokkos::deep_copy(accel_hat, accel_hat_host); // Restore state of random number gen { std::istringstream iss(pin->GetString("problem/turbulence", "state_rng")); - iss >> rng; + pfew_modes_ft->RestoreRNG(iss); } // Restore state of dist { std::istringstream iss(pin->GetString("problem/turbulence", "state_dist")); - iss >> dist; + pfew_modes_ft->RestoreDist(iss); } - - } else { - // init RNG - rng.seed(rseed); } } +// SetPhases is used as InitMeshBlockUserData because phases need to be reset on remeshing void SetPhases(MeshBlock *pmb, ParameterInput *pin) { - auto pm = pmb->pmy_mesh; auto hydro_pkg = pmb->packages.Get("Hydro"); - - // The following restriction could technically be lifted if the turbulence driver is - // directly embedded in the hydro driver rather than a user defined source as well as - // fixing the pack_size=-1 when using the Mesh- (not MeshBlock-)based problem generator. - // The restriction stems from requiring a collective MPI comm to normalize the - // acceleration and magnetic field, respectively. Note, that the restriction does not - // apply here, but for the ProblemGenerator() and Driving() function below. The check is - // just added here for convenience as this function is called during problem - // initializtion. From my (pgrete) point of view, it's currently cleaner to keep things - // separate and not touch the main driver at the expense of using one pack per rank -- - // which is typically fastest on devices anyway. - const auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size"); - PARTHENON_REQUIRE_THROWS(pack_size == -1, - "Turbulence pgen currently needs parthenon/mesh/pack_size=-1 " - "to work because of global reductions.") - - auto Lx = pm->mesh_size.x1max - pm->mesh_size.x1min; - auto Ly = pm->mesh_size.x2max - pm->mesh_size.x2min; - auto Lz = pm->mesh_size.x3max - pm->mesh_size.x3min; - // should also be easily fixed, just need to double check transforms and volume - // weighting everywhere - if ((Lx != 1.0) || (Ly != 1.0) || (Lz != 1.0)) { - std::stringstream msg; - msg << "### FATAL ERROR in turbulence driver" << std::endl - << "Only domain sizes with edge lengths of 1 are supported." << std::endl; - throw std::runtime_error(msg.str().c_str()); - } - - auto gnx1 = pm->mesh_size.nx1; - auto gnx2 = pm->mesh_size.nx2; - auto gnx3 = pm->mesh_size.nx3; - // as above, this restriction should/could be easily lifted - if ((gnx1 != gnx2) || (gnx2 != gnx3)) { - std::stringstream msg; - msg << "### FATAL ERROR in turbulence driver" << std::endl - << "Only cubic mesh sizes are supported." << std::endl; - throw std::runtime_error(msg.str().c_str()); - } - - const auto nx1 = pmb->block_size.nx1; - const auto nx2 = pmb->block_size.nx2; - const auto nx3 = pmb->block_size.nx3; - - const auto gis = pmb->loc.lx1 * pmb->block_size.nx1; - const auto gjs = pmb->loc.lx2 * pmb->block_size.nx2; - const auto gks = pmb->loc.lx3 * pmb->block_size.nx3; - - const auto num_modes = hydro_pkg->Param("turbulence/num_modes"); - - // make local ref to capure in lambda - auto &k_vec = k_vec_; - - Complex I(0.0, 1.0); - - auto &base = pmb->meshblock_data.Get(); - auto &phases_i = base->Get("phases_i").data; - auto &phases_j = base->Get("phases_j").data; - auto &phases_k = base->Get("phases_k").data; - - pmb->par_for( - "forcing: calc phases_i", 0, nx1 - 1, KOKKOS_LAMBDA(int i) { - Real gi = static_cast(i + gis); - Real w_kx; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_kx = k_vec(0, m) * 2. * M_PI / static_cast(gnx1); - // adjust phase factor to Complex->Real IFT: u_hat*(k) = u_hat(-k) - if (k_vec(0, m) == 0.0) { - phase = 0.5 * Kokkos::exp(I * w_kx * gi); - } else { - phase = Kokkos::exp(I * w_kx * gi); - } - phases_i(i, m, 0) = phase.real(); - phases_i(i, m, 1) = phase.imag(); - } - }); - - pmb->par_for( - "forcing: calc phases_j", 0, nx2 - 1, KOKKOS_LAMBDA(int j) { - Real gj = static_cast(j + gjs); - Real w_ky; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_ky = k_vec(1, m) * 2. * M_PI / static_cast(gnx2); - phase = Kokkos::exp(I * w_ky * gj); - phases_j(j, m, 0) = phase.real(); - phases_j(j, m, 1) = phase.imag(); - } - }); - - pmb->par_for( - "forcing: calc phases_k", 0, nx3 - 1, KOKKOS_LAMBDA(int k) { - Real gk = static_cast(k + gks); - Real w_kz; - Complex phase; - - for (int m = 0; m < num_modes; m++) { - w_kz = k_vec(2, m) * 2. * M_PI / static_cast(gnx3); - phase = Kokkos::exp(I * w_kz * gk); - phases_k(k, m, 0) = phase.real(); - phases_k(k, m, 1) = phase.imag(); - } - }); + auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); + few_modes_ft.SetPhases(pmb, pin); } //======================================================================================== @@ -480,150 +342,9 @@ void ProblemGenerator(Mesh *pmesh, ParameterInput *pin, MeshData *md) { void Generate(MeshData *md, Real dt) { auto pmb = md->GetBlockData(0)->GetBlockPointer(); auto hydro_pkg = pmb->packages.Get("Hydro"); - - const auto num_modes = hydro_pkg->Param("turbulence/num_modes"); - - Complex I(0.0, 1.0); - auto &random_num = random_num_; - - // get a set of random numbers from the CPU so that they are deterministic - // when run on GPUs - Real v1, v2, v_sqr; - for (int n = 0; n < 3; n++) - for (int m = 0; m < num_modes; m++) { - do { - v1 = dist(rng); - v2 = dist(rng); - v_sqr = v1 * v1 + v2 * v2; - } while (v_sqr >= 1.0 || v_sqr == 0.0); - - random_num_host(n, m, 0) = v1; - random_num_host(n, m, 1) = v2; - } - Kokkos::deep_copy(random_num, random_num_host); - - // make local ref to capure in lambda - auto &k_vec = k_vec_; - auto &accel_hat = accel_hat_; - auto &accel_hat_new = accel_hat_new_; - - const auto kpeak = hydro_pkg->Param("turbulence/kpeak"); - // generate new power spectrum (injection) - pmb->par_for( - "forcing: new power spec", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - Real kmag, tmp, norm, v_sqr; - - Real kx = k_vec(0, m); - Real ky = k_vec(1, m); - Real kz = k_vec(2, m); - - kmag = std::sqrt(kx * kx + ky * ky + kz * kz); - - accel_hat_new(n, m) = Complex(0., 0.); - - tmp = std::pow(kmag / kpeak, 2.) * (2. - std::pow(kmag / kpeak, 2.)); - if (tmp < 0.) tmp = 0.; - v_sqr = SQR(random_num(n, m, 0)) + SQR(random_num(n, m, 1)); - norm = std::sqrt(-2.0 * std::log(v_sqr) / v_sqr); - - accel_hat_new(n, m) = - Complex(tmp * norm * random_num(n, m, 0), tmp * norm * random_num(n, m, 1)); - }); - - // enforce symmetry of complex to real transform - pmb->par_for( - "forcing: enforce symmetry", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - if (k_vec(0, m) == 0.) { - for (int m2 = 0; m2 < m; m2++) { - if (k_vec(1, m) == -k_vec(1, m2) && k_vec(2, m) == -k_vec(2, m2)) - accel_hat_new(n, m) = - Complex(accel_hat_new(n, m2).real(), -accel_hat_new(n, m2).imag()); - } - } - }); - - const auto sol_weight = hydro_pkg->Param("turbulence/sol_weight"); - // project - pmb->par_for( - "forcing: projection", 0, num_modes - 1, KOKKOS_LAMBDA(const int m) { - Real kmag; - - Real kx = k_vec(0, m); - Real ky = k_vec(1, m); - Real kz = k_vec(2, m); - - kmag = std::sqrt(kx * kx + ky * ky + kz * kz); - - // setting kmag to 1 as a "continue" doesn't work within the parallel_for - // construct and it doesn't affect anything (there should never be power in the - // k=0 mode) - if (kmag == 0.) kmag = 1.; - - // make it a unit vector - kx /= kmag; - ky /= kmag; - kz /= kmag; - - Complex dot(accel_hat_new(0, m).real() * kx + accel_hat_new(1, m).real() * ky + - accel_hat_new(2, m).real() * kz, - accel_hat_new(0, m).imag() * kx + accel_hat_new(1, m).imag() * ky + - accel_hat_new(2, m).imag() * kz); - - accel_hat_new(0, m) = Complex(accel_hat_new(0, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * kx, - accel_hat_new(0, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * kx); - accel_hat_new(1, m) = Complex(accel_hat_new(1, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * ky, - accel_hat_new(1, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * ky); - accel_hat_new(2, m) = Complex(accel_hat_new(2, m).real() * sol_weight + - (1. - 2. * sol_weight) * dot.real() * kz, - accel_hat_new(2, m).imag() * sol_weight + - (1. - 2. * sol_weight) * dot.imag() * kz); - }); - - // evolve - const auto tcorr = hydro_pkg->Param("turbulence/tcorr"); - Real c_drift = std::exp(-dt / tcorr); - Real c_diff = std::sqrt(1.0 - c_drift * c_drift); - - pmb->par_for( - "forcing: evolve spec", 0, 2, 0, num_modes - 1, - KOKKOS_LAMBDA(const int n, const int m) { - accel_hat(n, m) = Complex( - accel_hat(n, m).real() * c_drift + accel_hat_new(n, m).real() * c_diff, - accel_hat(n, m).imag() * c_drift + accel_hat_new(n, m).imag() * c_diff); - }); - - IndexRange ib = md->GetBlockData(0)->GetBoundsI(IndexDomain::interior); - IndexRange jb = md->GetBlockData(0)->GetBoundsJ(IndexDomain::interior); - IndexRange kb = md->GetBlockData(0)->GetBoundsK(IndexDomain::interior); - auto acc_pack = md->PackVariables(std::vector{"acc"}); - auto phases_i = md->PackVariables(std::vector{"phases_i"}); - auto phases_j = md->PackVariables(std::vector{"phases_j"}); - auto phases_k = md->PackVariables(std::vector{"phases_k"}); - // implictly assuming cubic box of size L=1 - pmb->par_for( - "Inverse FT", 0, acc_pack.GetDim(5) - 1, 0, 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { - Complex phase, phase_i, phase_j, phase_k; - acc_pack(b, n, k, j, i) = 0.0; - - for (int m = 0; m < num_modes; m++) { - phase_i = - Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); - phase_j = - Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); - phase_k = - Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); - phase = phase_i * phase_j * phase_k; - acc_pack(b, n, k, j, i) += 2. * (accel_hat(n, m).real() * phase.real() - - accel_hat(n, m).imag() * phase.imag()); - } - }); + // Must be mutable so the internal RNG state is updated + auto *few_modes_ft = hydro_pkg->MutableParam("turbulence/few_modes_ft"); + few_modes_ft->Generate(md, dt, "acc"); } //---------------------------------------------------------------------------------------- @@ -726,24 +447,16 @@ void Driving(MeshData *md, const parthenon::SimTime &tm, const Real dt) { Perturb(md, dt); } -void Cleanup() { - // Ensure the Kokkos views are gargabe collected before finalized is called - k_vec_ = {}; - accel_hat_ = {}; - accel_hat_new_ = {}; - random_num_ = {}; - random_num_host = {}; -} - void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { auto hydro_pkg = pmb->packages.Get("Hydro"); - const auto num_modes = hydro_pkg->Param("turbulence/num_modes"); - // Store (common) acceleration field in spectral space + auto few_modes_ft = hydro_pkg->Param("turbulence/few_modes_ft"); + auto var_hat = few_modes_ft.GetVarHat(); auto accel_hat_host = - Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), accel_hat_); + Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), var_hat); + const auto num_modes = few_modes_ft.GetNumModes(); for (int i = 0; i < 3; i++) { for (int m = 0; m < num_modes; m++) { pin->SetReal("problem/turbulence", @@ -755,17 +468,11 @@ void UserWorkBeforeOutput(MeshBlock *pmb, ParameterInput *pin) { } } // store state of random number gen - { - std::ostringstream oss; - oss << rng; - pin->SetString("problem/turbulence", "state_rng", oss.str()); - } + auto state_rng = few_modes_ft.GetRNGState(); + pin->SetString("problem/turbulence", "state_rng", state_rng); // store state of distribution - { - std::ostringstream oss; - oss << dist; - pin->SetString("problem/turbulence", "state_dist", oss.str()); - } + auto state_dist = few_modes_ft.GetDistState(); + pin->SetString("problem/turbulence", "state_dist", state_dist); } } // namespace turbulence diff --git a/src/units.hpp b/src/units.hpp index 77b0043a..a089e632 100644 --- a/src/units.hpp +++ b/src/units.hpp @@ -130,4 +130,4 @@ class Units { parthenon::Real microgauss() const { return microgauss_cgs / code_magnetic_cgs(); } }; -#endif // PHYSICAL_CONSTANTS_HPP_ +#endif // UNITS_HPP_ diff --git a/src/utils/few_modes_ft.cpp b/src/utils/few_modes_ft.cpp new file mode 100644 index 00000000..f477b211 --- /dev/null +++ b/src/utils/few_modes_ft.cpp @@ -0,0 +1,406 @@ +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//======================================================================================== +//! \file few_modes_ft.cpp +// \brief Helper functions for an inverse (explicit complex to real) FT + +// C++ headers +#include +#include + +// Parthenon headers +#include "basic_types.hpp" +#include "config.hpp" +#include "globals.hpp" +#include "kokkos_abstraction.hpp" +#include "mesh/domain.hpp" +#include "mesh/meshblock_pack.hpp" + +// AthenaPK headers +#include "../main.hpp" +#include "few_modes_ft.hpp" +#include "utils/error_checking.hpp" + +namespace utils::few_modes_ft { +using Complex = Kokkos::complex; +using parthenon::IndexRange; +using parthenon::Metadata; + +FewModesFT::FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, + std::string prefix, int num_modes, ParArray2D k_vec, + Real k_peak, Real sol_weight, Real t_corr, uint32_t rseed, + bool fill_ghosts) + : prefix_(prefix), num_modes_(num_modes), k_vec_(k_vec), k_peak_(k_peak), + t_corr_(t_corr), fill_ghosts_(fill_ghosts) { + + if ((num_modes > 100) && (parthenon::Globals::my_rank == 0)) { + std::cout << "### WARNING using more than 100 explicit modes will significantly " + << "increase the runtime." << std::endl + << "If many modes are required in the transform field consider using " + << "the driving mechanism based on full FFTs." << std::endl; + } + // Ensure that all all wavevectors can be represented on the root grid + const auto gnx1 = pin->GetInteger("parthenon/mesh", "nx1"); + const auto gnx2 = pin->GetInteger("parthenon/mesh", "nx2"); + const auto gnx3 = pin->GetInteger("parthenon/mesh", "nx3"); + // Need to make this comparison on the host as (for some reason) an extended cuda device + // lambda cannot live in the constructor of an object. + auto k_vec_host = k_vec.GetHostMirrorAndCopy(); + for (int i = 0; i < num_modes; i++) { + PARTHENON_REQUIRE(std::abs(k_vec_host(0, i)) <= gnx1 / 2, "k_vec x1 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_host(1, i)) <= gnx2 / 2, "k_vec x2 mode too large"); + PARTHENON_REQUIRE(std::abs(k_vec_host(2, i)) <= gnx3 / 2, "k_vec x3 mode too large"); + } + + const auto nx1 = pin->GetInteger("parthenon/meshblock", "nx1"); + const auto nx2 = pin->GetInteger("parthenon/meshblock", "nx2"); + const auto nx3 = pin->GetInteger("parthenon/meshblock", "nx3"); + const auto ng_tot = fill_ghosts_ ? 2 * parthenon::Globals::nghost : 0; + auto m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, + std::vector({2, num_modes, nx1 + ng_tot}), prefix + "_phases_i"); + pkg->AddField(prefix + "_phases_i", m); + m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, + std::vector({2, num_modes, nx2 + ng_tot}), prefix + "_phases_j"); + pkg->AddField(prefix + "_phases_j", m); + m = Metadata({Metadata::None, Metadata::Derived, Metadata::OneCopy}, + std::vector({2, num_modes, nx3 + ng_tot}), prefix + "_phases_k"); + pkg->AddField(prefix + "_phases_k", m); + + // Variable (e.g., acceleration field for turbulence driver) in Fourier space using + // complex to real transform. + var_hat_ = ParArray2D(prefix + "_var_hat", 3, num_modes); + var_hat_new_ = ParArray2D(prefix + "_var_hat_new", 3, num_modes); + + PARTHENON_REQUIRE((sol_weight == -1.0) || (sol_weight >= 0.0 && sol_weight <= 1.0), + "sol_weight for projection in few modes fft module needs to be " + "between 0.0 and 1.0 or set to -1.0 (to disable projection).") + sol_weight_ = sol_weight; + + random_num_ = Kokkos::View( + "random_num", 3, num_modes, 2); + random_num_host_ = Kokkos::create_mirror_view(random_num_); + + rng_.seed(rseed); + dist_ = std::uniform_real_distribution<>(-1.0, 1.0); +} + +void FewModesFT::SetPhases(MeshBlock *pmb, ParameterInput *pin) { + auto pm = pmb->pmy_mesh; + auto hydro_pkg = pmb->packages.Get("Hydro"); + + // The following restriction could technically be lifted if the turbulence driver is + // directly embedded in the hydro driver rather than a user defined source as well as + // fixing the pack_size=-1 when using the Mesh- (not MeshBlock-)based problem generator. + // The restriction stems from requiring a collective MPI comm to normalize the + // acceleration and magnetic field, respectively. Note, that the restriction does not + // apply here, but for the ProblemGenerator() and Driving() function below. The check is + // just added here for convenience as this function is called during problem + // initializtion. From my (pgrete) point of view, it's currently cleaner to keep things + // separate and not touch the main driver at the expense of using one pack per rank -- + // which is typically fastest on devices anyway. + const auto pack_size = pin->GetInteger("parthenon/mesh", "pack_size"); + PARTHENON_REQUIRE_THROWS(pack_size == -1, + "Few modes FT currently needs parthenon/mesh/pack_size=-1 " + "to work because of global reductions.") + + auto Lx1 = pm->mesh_size.x1max - pm->mesh_size.x1min; + auto Lx2 = pm->mesh_size.x2max - pm->mesh_size.x2min; + auto Lx3 = pm->mesh_size.x3max - pm->mesh_size.x3min; + + // Adjust (logical) grid size at levels other than the root level. + // This is required for simulation with mesh refinement so that the phases calculated + // below take the logical grid size into account. For example, the local phases at level + // 1 should be calculated assuming a grid that is twice as large as the root grid. + const auto root_level = pm->GetRootLevel(); + auto gnx1 = pm->mesh_size.nx1 * std::pow(2, pmb->loc.level() - root_level); + auto gnx2 = pm->mesh_size.nx2 * std::pow(2, pmb->loc.level() - root_level); + auto gnx3 = pm->mesh_size.nx3 * std::pow(2, pmb->loc.level() - root_level); + + // Restriction should also be easily fixed, just need to double check transforms and + // volume weighting everywhere + PARTHENON_REQUIRE_THROWS(((gnx1 == gnx2) && (gnx2 == gnx3)) && + ((Lx1 == Lx2) && (Lx2 == Lx3)), + "FMFT has only been tested with cubic meshes and constant " + "dx/dy/dz. Remove this warning at your own risk.") + + const auto nx1 = pmb->block_size.nx1; + const auto nx2 = pmb->block_size.nx2; + const auto nx3 = pmb->block_size.nx3; + + const auto gis = pmb->loc.lx1() * pmb->block_size.nx1; + const auto gjs = pmb->loc.lx2() * pmb->block_size.nx2; + const auto gks = pmb->loc.lx3() * pmb->block_size.nx3; + + // make local ref to capure in lambda + const auto num_modes = num_modes_; + auto &k_vec = k_vec_; + + Complex I(0.0, 1.0); + + auto &base = pmb->meshblock_data.Get(); + auto &phases_i = base->Get(prefix_ + "_phases_i").data; + auto &phases_j = base->Get(prefix_ + "_phases_j").data; + auto &phases_k = base->Get(prefix_ + "_phases_k").data; + + const auto ng = fill_ghosts_ ? parthenon::Globals::nghost : 0; + pmb->par_for( + "FMFT: calc phases_i", 0, nx1 - 1 + 2 * ng, KOKKOS_LAMBDA(int i) { + Real gi = static_cast((i + gis - ng) % static_cast(gnx1)); + Real w_kx; + Complex phase; + + for (int m = 0; m < num_modes; m++) { + w_kx = k_vec(0, m) * 2. * M_PI / static_cast(gnx1); + // adjust phase factor to Complex->Real IFT: u_hat*(k) = u_hat(-k) + if (k_vec(0, m) == 0.0) { + phase = 0.5 * Kokkos::exp(I * w_kx * gi); + } else { + phase = Kokkos::exp(I * w_kx * gi); + } + phases_i(i, m, 0) = phase.real(); + phases_i(i, m, 1) = phase.imag(); + } + }); + + pmb->par_for( + "FMFT: calc phases_j", 0, nx2 - 1 + 2 * ng, KOKKOS_LAMBDA(int j) { + Real gj = static_cast((j + gjs - ng) % static_cast(gnx2)); + Real w_ky; + Complex phase; + + for (int m = 0; m < num_modes; m++) { + w_ky = k_vec(1, m) * 2. * M_PI / static_cast(gnx2); + phase = Kokkos::exp(I * w_ky * gj); + phases_j(j, m, 0) = phase.real(); + phases_j(j, m, 1) = phase.imag(); + } + }); + + pmb->par_for( + "FMFT: calc phases_k", 0, nx3 - 1 + 2 * ng, KOKKOS_LAMBDA(int k) { + Real gk = static_cast((k + gks - ng) % static_cast(gnx3)); + Real w_kz; + Complex phase; + + for (int m = 0; m < num_modes; m++) { + w_kz = k_vec(2, m) * 2. * M_PI / static_cast(gnx3); + phase = Kokkos::exp(I * w_kz * gk); + phases_k(k, m, 0) = phase.real(); + phases_k(k, m, 1) = phase.imag(); + } + }); +} + +void FewModesFT::Generate(MeshData *md, const Real dt, + const std::string &var_name) { + auto pmb = md->GetBlockData(0)->GetBlockPointer(); + + const auto num_modes = num_modes_; + + Complex I(0.0, 1.0); + auto &random_num = random_num_; + + // get a set of random numbers from the CPU so that they are deterministic + // when run on GPUs + Real v1, v2, v_sqr; + for (int n = 0; n < 3; n++) + for (int m = 0; m < num_modes; m++) { + do { + v1 = dist_(rng_); + v2 = dist_(rng_); + v_sqr = v1 * v1 + v2 * v2; + } while (v_sqr >= 1.0 || v_sqr == 0.0); + + random_num_host_(n, m, 0) = v1; + random_num_host_(n, m, 1) = v2; + } + Kokkos::deep_copy(random_num, random_num_host_); + + // make local ref to capure in lambda + auto &k_vec = k_vec_; + auto &var_hat = var_hat_; + auto &var_hat_new = var_hat_new_; + + const auto kpeak = k_peak_; + + // generate new power spectrum (injection) + pmb->par_for( + "FMFT: new power spec", 0, 2, 0, num_modes - 1, + KOKKOS_LAMBDA(const int n, const int m) { + Real kmag, tmp, norm, v_sqr; + + Real kx = k_vec(0, m); + Real ky = k_vec(1, m); + Real kz = k_vec(2, m); + + kmag = std::sqrt(kx * kx + ky * ky + kz * kz); + + var_hat_new(n, m) = Complex(0., 0.); + + tmp = std::pow(kmag / kpeak, 2.) * (2. - std::pow(kmag / kpeak, 2.)); + if (tmp < 0.) tmp = 0.; + v_sqr = SQR(random_num(n, m, 0)) + SQR(random_num(n, m, 1)); + norm = std::sqrt(-2.0 * std::log(v_sqr) / v_sqr); + + var_hat_new(n, m) = + Complex(tmp * norm * random_num(n, m, 0), tmp * norm * random_num(n, m, 1)); + }); + + // enforce symmetry of complex to real transform + pmb->par_for( + "forcing: enforce symmetry", 0, 2, 0, num_modes - 1, + KOKKOS_LAMBDA(const int n, const int m) { + if (k_vec(0, m) == 0.) { + for (int m2 = 0; m2 < m; m2++) { + if (k_vec(1, m) == -k_vec(1, m2) && k_vec(2, m) == -k_vec(2, m2)) + var_hat_new(n, m) = + Complex(var_hat_new(n, m2).real(), -var_hat_new(n, m2).imag()); + } + } + }); + + const auto sol_weight = sol_weight_; + if (sol_weight_ >= 0.0) { + // project + pmb->par_for( + "forcing: projection", 0, num_modes - 1, KOKKOS_LAMBDA(const int m) { + Real kmag; + + Real kx = k_vec(0, m); + Real ky = k_vec(1, m); + Real kz = k_vec(2, m); + + kmag = std::sqrt(kx * kx + ky * ky + kz * kz); + + // setting kmag to 1 as a "continue" doesn't work within the parallel_for + // construct and it doesn't affect anything (there should never be power in the + // k=0 mode) + if (kmag == 0.) kmag = 1.; + + // make it a unit vector + kx /= kmag; + ky /= kmag; + kz /= kmag; + + Complex dot(var_hat_new(0, m).real() * kx + var_hat_new(1, m).real() * ky + + var_hat_new(2, m).real() * kz, + var_hat_new(0, m).imag() * kx + var_hat_new(1, m).imag() * ky + + var_hat_new(2, m).imag() * kz); + + var_hat_new(0, m) = Complex(var_hat_new(0, m).real() * sol_weight + + (1. - 2. * sol_weight) * dot.real() * kx, + var_hat_new(0, m).imag() * sol_weight + + (1. - 2. * sol_weight) * dot.imag() * kx); + var_hat_new(1, m) = Complex(var_hat_new(1, m).real() * sol_weight + + (1. - 2. * sol_weight) * dot.real() * ky, + var_hat_new(1, m).imag() * sol_weight + + (1. - 2. * sol_weight) * dot.imag() * ky); + var_hat_new(2, m) = Complex(var_hat_new(2, m).real() * sol_weight + + (1. - 2. * sol_weight) * dot.real() * kz, + var_hat_new(2, m).imag() * sol_weight + + (1. - 2. * sol_weight) * dot.imag() * kz); + }); + } + + // evolve + const auto c_drift = std::exp(-dt / t_corr_); + const auto c_diff = std::sqrt(1.0 - c_drift * c_drift); + + pmb->par_for( + "FMFT: evolve spec", 0, 2, 0, num_modes - 1, + KOKKOS_LAMBDA(const int n, const int m) { + var_hat(n, m) = + Complex(var_hat(n, m).real() * c_drift + var_hat_new(n, m).real() * c_diff, + var_hat(n, m).imag() * c_drift + var_hat_new(n, m).imag() * c_diff); + }); + + auto domain = fill_ghosts_ ? IndexDomain::entire : IndexDomain::interior; + IndexRange ib = md->GetBlockData(0)->GetBoundsI(domain); + IndexRange jb = md->GetBlockData(0)->GetBoundsJ(domain); + IndexRange kb = md->GetBlockData(0)->GetBoundsK(domain); + auto var_pack = md->PackVariables(std::vector{var_name}); + auto phases_i = md->PackVariables(std::vector{prefix_ + "_phases_i"}); + auto phases_j = md->PackVariables(std::vector{prefix_ + "_phases_j"}); + auto phases_k = md->PackVariables(std::vector{prefix_ + "_phases_k"}); + + // implictly assuming cubic box of size L=1 + parthenon::par_for( + DEFAULT_LOOP_PATTERN, "FMFT: Inverse FT", parthenon::DevExecSpace(), 0, + md->NumBlocks() - 1, 0, 2, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int n, const int k, const int j, const int i) { + Complex phase, phase_i, phase_j, phase_k; + var_pack(b, n, k, j, i) = 0.0; + + for (int m = 0; m < num_modes; m++) { + phase_i = + Complex(phases_i(b, 0, i - ib.s, m, 0), phases_i(b, 0, i - ib.s, m, 1)); + phase_j = + Complex(phases_j(b, 0, j - jb.s, m, 0), phases_j(b, 0, j - jb.s, m, 1)); + phase_k = + Complex(phases_k(b, 0, k - kb.s, m, 0), phases_k(b, 0, k - kb.s, m, 1)); + phase = phase_i * phase_j * phase_k; + var_pack(b, n, k, j, i) += 2. * (var_hat(n, m).real() * phase.real() - + var_hat(n, m).imag() * phase.imag()); + } + }); +} + +// Creates a random set of wave vectors with k_mag within k_peak/2 and 2*k_peak +ParArray2D MakeRandomModes(const int num_modes, const Real k_peak, + uint32_t rseed = 31224) { + auto k_vec = parthenon::ParArray2D("k_vec", 3, num_modes); + auto k_vec_h = Kokkos::create_mirror_view_and_copy(parthenon::HostMemSpace(), k_vec); + + const int k_low = std::floor(k_peak / 2); + const int k_high = std::ceil(2 * k_peak); + + std::mt19937 rng; + rng.seed(rseed); + std::uniform_int_distribution<> dist(-k_high, k_high); + + int n_mode = 0; + int n_attempt = 0; + constexpr int max_attempts = 1000000; + Real kx1, kx2, kx3, k_mag, ampl; + bool mode_exists = false; + while (n_mode < num_modes && n_attempt < max_attempts) { + n_attempt += 1; + + kx1 = dist(rng); + kx2 = dist(rng); + kx3 = dist(rng); + k_mag = std::sqrt(SQR(kx1) + SQR(kx2) + SQR(kx3)); + + // Expected amplitude of the spectral function. If this is changed, it also needs to + // be changed in the FMFT class (or abstracted). + ampl = SQR(k_mag / k_peak) * (2.0 - SQR(k_mag / k_peak)); + + // Check is mode was already picked by chance + mode_exists = false; + for (int n_mode_exsist = 0; n_mode_exsist < n_mode; n_mode_exsist++) { + if (k_vec_h(0, n_mode_exsist) == kx1 && k_vec_h(1, n_mode_exsist) == kx2 && + k_vec_h(2, n_mode_exsist) == kx3) { + mode_exists = true; + } + } + + // kx1 < 0.0 because we use a explicit symmetric Complex to Real transform + if (ampl < 0 || k_mag < k_low || k_mag > k_high || mode_exists || kx1 < 0.0) { + continue; + } + k_vec_h(0, n_mode) = kx1; + k_vec_h(1, n_mode) = kx2; + k_vec_h(2, n_mode) = kx3; + n_mode++; + } + PARTHENON_REQUIRE_THROWS( + n_attempt < max_attempts, + "Cluster init did not succeed in calculating perturbation modes.") + Kokkos::deep_copy(k_vec, k_vec_h); + + return k_vec; +} +} // namespace utils::few_modes_ft \ No newline at end of file diff --git a/src/utils/few_modes_ft.hpp b/src/utils/few_modes_ft.hpp new file mode 100644 index 00000000..ce17401c --- /dev/null +++ b/src/utils/few_modes_ft.hpp @@ -0,0 +1,71 @@ + +//======================================================================================== +// AthenaPK - a performance portable block structured AMR astrophysical MHD code. +// Copyright (c) 2023, Athena-Parthenon Collaboration. All rights reserved. +// Licensed under the 3-clause BSD License, see LICENSE file for details +//======================================================================================== +//======================================================================================== +//! \file few_modes_ft.hpp +// \brief Helper functions for an inverse (explicit complex to real) FT + +// Parthenon headers +#include "basic_types.hpp" +#include "config.hpp" +#include +#include +#include + +// AthenaPK headers +#include "../main.hpp" +#include "mesh/domain.hpp" + +namespace utils::few_modes_ft { +using namespace parthenon::package::prelude; +using parthenon::Real; +using Complex = Kokkos::complex; +using parthenon::IndexRange; +using parthenon::ParArray2D; + +class FewModesFT { + private: + int num_modes_; + std::string prefix_; + ParArray2D var_hat_, var_hat_new_; + ParArray2D k_vec_; + Real k_peak_; // peak of the power spectrum + Kokkos::View random_num_; + Kokkos::View random_num_host_; + std::mt19937 rng_; + std::uniform_real_distribution<> dist_; + Real sol_weight_; // power in solenoidal modes for projection. Set to negative to + // disable projection + Real t_corr_; // correlation time for evolution of Ornstein-Uhlenbeck process + bool fill_ghosts_; // if the inverse transform should also fill ghost zones + + public: + FewModesFT(parthenon::ParameterInput *pin, parthenon::StateDescriptor *pkg, + std::string prefix, int num_modes, ParArray2D k_vec, Real k_peak, + Real sol_weight, Real t_corr, uint32_t rseed, bool fill_ghosts = false); + + ParArray2D GetVarHat() { return var_hat_; } + int GetNumModes() { return num_modes_; } + void SetPhases(MeshBlock *pmb, ParameterInput *pin); + void Generate(MeshData *md, const Real dt, const std::string &var_name); + void RestoreRNG(std::istringstream &iss) { iss >> rng_; } + void RestoreDist(std::istringstream &iss) { iss >> dist_; } + std::string GetRNGState() { + std::ostringstream oss; + oss << rng_; + return oss.str(); + } + std::string GetDistState() { + std::ostringstream oss; + oss << dist_; + return oss.str(); + } +}; + +// Creates a random set of wave vectors with k_mag within k_peak/2 and 2*k_peak +ParArray2D MakeRandomModes(const int num_modes, const Real k_peak, uint32_t rseed); + +} // namespace utils::few_modes_ft \ No newline at end of file diff --git a/tst/regression/CMakeLists.txt b/tst/regression/CMakeLists.txt index 6af8846b..3461a817 100644 --- a/tst/regression/CMakeLists.txt +++ b/tst/regression/CMakeLists.txt @@ -35,7 +35,7 @@ setup_test_serial("performance" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ --driver_input ${PROJECT_SOURCE_DIR}/inputs/linear_wave3d.in --num_steps 21" "performance") setup_test_both("cluster_hse" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ - --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/hse.in --num_steps 1" "convergence") + --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/hse.in --num_steps 2" "convergence") setup_test_serial("cluster_tabular_cooling" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/cooling.in --num_steps 11" "convergence") @@ -48,3 +48,12 @@ setup_test_both("aniso_therm_cond_ring_multid" "--driver ${PROJECT_BINARY_DIR}/b setup_test_both("field_loop" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ --driver_input ${PROJECT_SOURCE_DIR}/inputs/field_loop.in --num_steps 12" "convergence") + +setup_test_both("cluster_magnetic_tower" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ + --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/magnetic_tower.in --num_steps 4" "convergence") + +setup_test_both("cluster_hydro_agn_feedback" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ + --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/hydro_agn_feedback.in --num_steps 5" "convergence") + +setup_test_both("cluster_agn_triggering" "--driver ${PROJECT_BINARY_DIR}/bin/athenaPK \ + --driver_input ${PROJECT_SOURCE_DIR}/inputs/cluster/agn_triggering.in --num_steps 3" "convergence") diff --git a/tst/regression/test_suites/cluster_agn_triggering/__init__.py b/tst/regression/test_suites/cluster_agn_triggering/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/regression/test_suites/cluster_agn_triggering/cluster_agn_triggering.py b/tst/regression/test_suites/cluster_agn_triggering/cluster_agn_triggering.py new file mode 100644 index 00000000..49fdaafd --- /dev/null +++ b/tst/regression/test_suites/cluster_agn_triggering/cluster_agn_triggering.py @@ -0,0 +1,438 @@ +# ======================================================================================== +# AthenaPK - a performance portable block structured AMR MHD code +# Copyright (c) 2020-2021, Athena Parthenon Collaboration. All rights reserved. +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== +# (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +# ======================================================================================== + +# Modules +import math +import numpy as np +import matplotlib + +matplotlib.use("agg") +import matplotlib.pylab as plt +import sys +import os +import utils.test_case +import unyt +import itertools + +""" To prevent littering up imported folders with .pyc files or __pycache_ folder""" +sys.dont_write_bytecode = True + + +class TestCase(utils.test_case.TestCaseAbs): + def __init__(self): + + # Define cluster parameters + # Setup units + unyt.define_unit("code_length", (1, "Mpc")) + unyt.define_unit("code_mass", (1e14, "Msun")) + unyt.define_unit("code_time", (1, "Gyr")) + self.code_length = unyt.unyt_quantity(1, "code_length") + self.code_mass = unyt.unyt_quantity(1, "code_mass") + self.code_time = unyt.unyt_quantity(1, "code_time") + + self.tlim = unyt.unyt_quantity(0.1, "code_time") + + # Setup constants + self.k_b = unyt.kb_cgs + self.G = unyt.G_cgs + self.m_u = unyt.amu + + self.adiabatic_index = 5.0 / 3.0 + self.He_mass_fraction = 0.25 + self.mu = 1 / ( + self.He_mass_fraction * 3.0 / 4.0 + (1 - self.He_mass_fraction) * 2 + ) + self.mean_molecular_mass = self.mu * self.m_u + + # Define the initial uniform gas + self.uniform_gas_rho = unyt.unyt_quantity(1e-22, "g/cm**3") + self.uniform_gas_ux = unyt.unyt_quantity(60000, "cm/s") + self.uniform_gas_uy = unyt.unyt_quantity(40000, "cm/s") + self.uniform_gas_uz = unyt.unyt_quantity(-50000, "cm/s") + self.uniform_gas_pres = unyt.unyt_quantity(1e-10, "dyne/cm**2") + + self.uniform_gas_Mx = self.uniform_gas_rho * self.uniform_gas_ux + self.uniform_gas_My = self.uniform_gas_rho * self.uniform_gas_uy + self.uniform_gas_Mz = self.uniform_gas_rho * self.uniform_gas_uz + self.uniform_gas_energy_density = 1.0 / 2.0 * self.uniform_gas_rho * ( + self.uniform_gas_ux**2 + + self.uniform_gas_uy**2 + + self.uniform_gas_uz**2 + ) + self.uniform_gas_pres / (self.adiabatic_index - 1.0) + + self.uniform_gas_vel = np.sqrt( + self.uniform_gas_ux**2 + + self.uniform_gas_uy**2 + + self.uniform_gas_uz**2 + ) + + self.uniform_gas_temp = ( + self.mu * self.m_u / self.k_b * self.uniform_gas_pres / self.uniform_gas_rho + ) + + # SMBH parameters (for Bondi-like accretion) + self.M_smbh = unyt.unyt_quantity(1e8, "Msun") + + # Triggering parameters + self.accretion_radius = unyt.unyt_quantity(20, "kpc") + self.cold_temp_thresh = self.uniform_gas_temp * 1.01 + self.cold_t_acc = unyt.unyt_quantity(100, "Myr") + self.bondi_alpha = 100 + self.bondi_beta = 2 + self.bondi_n0 = 0.05 * (self.uniform_gas_rho / self.mean_molecular_mass) + + self.norm_tol = 1e-3 + self.linf_accretion_rate_tol = 1e-3 + + self.step_params_list = ["COLD_GAS", "BOOSTED_BONDI", "BOOTH_SCHAYE"] + self.steps = len(self.step_params_list) + + def Prepare(self, parameters, step): + """ + Any preprocessing that is needed before the drive is run can be done in + this method + + This includes preparing files or any other pre processing steps that + need to be implemented. The method also provides access to the + parameters object which controls which parameters are being used to run + the driver. + + It is possible to append arguments to the driver_cmd_line_args if it is + desired to override the parthenon input file. Each element in the list + is simply a string of the form '/=', where the + contents of the string are exactly what one would type on the command + line run running a parthenon driver. + + As an example if the following block was uncommented it would overwrite + any of the parameters that were specified in the parthenon input file + parameters.driver_cmd_line_args = ['output1/file_type=vtk', + 'output1/variable=cons', + 'output1/dt=0.4', + 'time/tlim=0.4', + 'mesh/nx1=400'] + """ + triggering_mode = self.step_params_list[step - 1] + output_id = triggering_mode + + parameters.driver_cmd_line_args = [ + f"parthenon/output2/id={output_id}", + f"parthenon/output2/dt={self.tlim.in_units('code_time').v}", + f"parthenon/time/tlim={self.tlim.in_units('code_time').v}", + f"hydro/gamma={self.adiabatic_index}", + f"hydro/He_mass_fraction={self.He_mass_fraction}", + f"units/code_length_cgs={self.code_length.in_units('cm').v}", + f"units/code_mass_cgs={self.code_mass.in_units('g').v}", + f"units/code_time_cgs={self.code_time.in_units('s').v}", + f"problem/cluster/uniform_gas/init_uniform_gas=true", + f"problem/cluster/uniform_gas/rho={self.uniform_gas_rho.in_units('code_mass*code_length**-3').v}", + f"problem/cluster/uniform_gas/ux={self.uniform_gas_ux.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uy={self.uniform_gas_uy.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uz={self.uniform_gas_uz.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/pres={self.uniform_gas_pres.in_units('code_mass*code_length**-1*code_time**-2').v}", + f"problem/cluster/gravity/m_smbh={self.M_smbh.in_units('code_mass').v}", + f"problem/cluster/agn_triggering/triggering_mode={triggering_mode}", + f"problem/cluster/agn_triggering/accretion_radius={self.accretion_radius.in_units('code_length').v}", + f"problem/cluster/agn_triggering/cold_temp_thresh={self.cold_temp_thresh.in_units('K').v}", + f"problem/cluster/agn_triggering/cold_t_acc={self.cold_t_acc.in_units('code_time').v}", + f"problem/cluster/agn_triggering/bondi_alpha={self.bondi_alpha}", + f"problem/cluster/agn_triggering/bondi_beta={self.bondi_beta}", + f"problem/cluster/agn_triggering/bondi_n0={self.bondi_n0.in_units('code_length**-3').v}", + f"problem/cluster/agn_triggering/write_to_file=true", + f"problem/cluster/agn_triggering/triggering_filename={triggering_mode}_triggering.dat", + ] + + return parameters + + def Analyse(self, parameters): + """ + Analyze the output and determine if the test passes. + + This function is called after the driver has been executed. It is + responsible for reading whatever data it needs and making a judgment + about whether or not the test passes. It takes no inputs. Output should + be True (test passes) or False (test fails). + + The parameters that are passed in provide the paths to relevant + locations and commands. Of particular importance is the path to the + output folder. All files from a drivers run should appear in and output + folder located in + parthenon/tst/regression/test_suites/test_name/output. + + It is possible in this function to read any of the output files such as + hdf5 output and compare them to expected quantities. + + """ + analyze_status = True + + for step in range(1, self.steps + 1): + triggering_mode = self.step_params_list[step - 1] + output_id = triggering_mode + step_status = True + + print(f"Testing {output_id}") + + # Read the triggering data produced by the sim, replicate the + # integration of triggering to determine the final state of the gas + sim_data = np.loadtxt(f"{triggering_mode}_triggering.dat") + + sim_times = unyt.unyt_array(sim_data[:, 0], "code_time") + sim_dts = unyt.unyt_array(sim_data[:, 1], "code_time") + sim_accretion_rate = unyt.unyt_array(sim_data[:, 2], "code_mass/code_time") + + if triggering_mode == "COLD_GAS": + sim_cold_mass = unyt.unyt_array(sim_data[:, 3], "code_mass") + elif ( + triggering_mode == "BOOSTED_BONDI" or triggering_mode == "BOOTH_SCHAYE" + ): + sim_total_mass = unyt.unyt_array(sim_data[:, 3], "code_mass") + sim_avg_density = unyt.unyt_array( + sim_data[:, 4], "code_mass/code_length**3" + ) + sim_avg_velocity = unyt.unyt_array( + sim_data[:, 5], "code_length/code_time" + ) + sim_avg_cs = unyt.unyt_array(sim_data[:, 6], "code_length/code_time") + else: + raise Exception( + f"Triggering mode {triggering_mode} not supported in analysis" + ) + + n_times = sim_data.shape[0] + + analytic_density = unyt.unyt_array( + np.empty(n_times + 1), "code_mass*code_length**-3" + ) + analytic_pressure = unyt.unyt_array( + np.empty(n_times + 1), "code_mass/(code_length*code_time**2)" + ) + analytic_accretion_rate = unyt.unyt_array( + np.empty(n_times), "code_mass*code_time**-1" + ) + + analytic_density[0] = self.uniform_gas_rho.in_units( + "code_mass*code_length**-3" + ) + analytic_pressure[0] = self.uniform_gas_pres.in_units( + "code_mass/(code_length*code_time**2)" + ) + + accretion_volume = 4.0 / 3.0 * np.pi * self.accretion_radius**3 + + for i in range(n_times): + dt = sim_dts[i] + + if triggering_mode == "COLD_GAS": + # Temperature should stay fixed below cold gas threshold + accretion_rate = ( + analytic_density[i] * accretion_volume / self.cold_t_acc + ) + elif ( + triggering_mode == "BOOSTED_BONDI" + or triggering_mode == "BOOTH_SCHAYE" + ): + + if triggering_mode == "BOOSTED_BONDI": + alpha = self.bondi_alpha + elif triggering_mode == "BOOTH_SCHAYE": + n = analytic_density[i] / (self.mu * self.m_u) + if n <= self.bondi_n0: + alpha = 1.0 + else: + alpha = (n / self.bondi_n0) ** self.bondi_beta + else: + raise Exception( + f"Triggering mode {triggering_mode} not supported in analysis" + ) + + cs2 = ( + self.adiabatic_index + * self.uniform_gas_pres + / self.uniform_gas_rho + ) + accretion_rate = ( + alpha + * ( + 2 + * np.pi + * unyt.G_cgs**2 + * self.M_smbh**2 + * analytic_density[i] + ) + / (self.uniform_gas_vel**2 + cs2) ** (3.0 / 2.0) + ) + else: + raise Exception( + f"Triggering mode {triggering_mode} not supported in analysis" + ) + + accretion_rate_density = accretion_rate / accretion_volume + + analytic_density[i + 1] = ( + analytic_density[i] - accretion_rate_density * dt + ).in_units(analytic_density.units) + analytic_pressure[i + 1] = ( + analytic_pressure[i] + - accretion_rate_density + * dt + * analytic_pressure[i] + / analytic_density[i] + ).in_units(analytic_pressure.units) + analytic_accretion_rate[i] = accretion_rate.in_units( + analytic_accretion_rate.units + ) + + # Compare the analytic accretion_rate + accretion_rate_err = np.abs( + (analytic_accretion_rate - sim_accretion_rate) / analytic_accretion_rate + ) + + if np.max(accretion_rate_err) > self.linf_accretion_rate_tol: + analyze_status = False + print( + f"{triggering_mode} linf_accretion_rate_err {np.max(accretion_rate_err)}" + f" exceeds tolerance {self.linf_accretion_rate_tol}" + f" at i={np.argmax(accretion_rate_err)}" + f" time={sim_times[np.argmax(accretion_rate_err)]}" + ) + + final_rho = analytic_density[-1] + final_pres = analytic_pressure[-1] + final_Mx = self.uniform_gas_ux * final_rho + final_My = self.uniform_gas_uy * final_rho + final_Mz = self.uniform_gas_uz * final_rho + final_energy_density = 1.0 / 2.0 * final_rho * ( + self.uniform_gas_ux**2 + + self.uniform_gas_uy**2 + + self.uniform_gas_uz**2 + ) + final_pres / (self.adiabatic_index - 1.0) + + def accretion_mask(Z, Y, X, inner_state, outer_state): + pos_cart = unyt.unyt_array((X, Y, Z), "code_length") + + r = np.sqrt(np.sum(pos_cart**2, axis=0)) + + state = inner_state * (r < self.accretion_radius) + outer_state * ( + r >= self.accretion_radius + ) + + return state + + # Check that the initial and final outputs match the expected tower + sys.path.insert( + 1, + parameters.parthenon_path + + "/scripts/python/packages/parthenon_tools/parthenon_tools", + ) + + try: + import compare_analytic + except ModuleNotFoundError: + print("Couldn't find module to analyze Parthenon hdf5 files.") + return False + + initial_analytic_components = { + "cons_density": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_rho.in_units("code_mass/code_length**3").v, + "cons_momentum_density_1": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_Mx.in_units( + "code_mass*code_length**-2*code_time**-1" + ).v, + "cons_momentum_density_2": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_My.in_units( + "code_mass*code_length**-2*code_time**-1" + ).v, + "cons_momentum_density_3": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_Mz.in_units( + "code_mass*code_length**-2*code_time**-1" + ).v, + "cons_total_energy_density": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_energy_density.in_units( + "code_mass*code_length**-1*code_time**-2" + ).v, + "prim_velocity_1": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_ux.in_units("code_length*code_time**-1").v, + "prim_velocity_2": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_uy.in_units("code_length*code_time**-1").v, + "prim_velocity_3": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_uz.in_units("code_length*code_time**-1").v, + "prim_pressure": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_pres.in_units( + "code_mass/(code_length*code_time**2)" + ).v, + } + + final_analytic_components = { + "cons_density": lambda Z, Y, X, time: accretion_mask( + Z, Y, X, final_rho, self.uniform_gas_rho + ) + .in_units("code_mass/code_length**3") + .v, + "cons_momentum_density_1": lambda Z, Y, X, time: accretion_mask( + Z, Y, X, final_Mx, self.uniform_gas_Mx + ) + .in_units("code_mass*code_length**-2*code_time**-1") + .v, + "cons_momentum_density_2": lambda Z, Y, X, time: accretion_mask( + Z, Y, X, final_My, self.uniform_gas_My + ) + .in_units("code_mass*code_length**-2*code_time**-1") + .v, + "cons_momentum_density_3": lambda Z, Y, X, time: accretion_mask( + Z, Y, X, final_Mz, self.uniform_gas_Mz + ) + .in_units("code_mass*code_length**-2*code_time**-1") + .v, + "cons_total_energy_density": lambda Z, Y, X, time: accretion_mask( + Z, Y, X, final_energy_density, self.uniform_gas_energy_density + ) + .in_units("code_mass*code_length**-1*code_time**-2") + .v, + "prim_velocity_1": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_ux.in_units("code_length*code_time**-1").v, + "prim_velocity_2": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_uy.in_units("code_length*code_time**-1").v, + "prim_velocity_3": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_uz.in_units("code_length*code_time**-1").v, + "prim_pressure": lambda Z, Y, X, time: accretion_mask( + Z, Y, X, final_pres, self.uniform_gas_pres + ) + .in_units("code_mass/(code_length*code_time**2)") + .v, + } + + phdf_files = [ + f"{parameters.output_path}/parthenon.{output_id}.00000.phdf", + f"{parameters.output_path}/parthenon.{output_id}.final.phdf", + ] + + # Use a very loose tolerance, linf relative error + analytic_status = True + for analytic_components, phdf_file in zip( + (initial_analytic_components, final_analytic_components), phdf_files + ): + analytic_status &= compare_analytic.compare_analytic( + phdf_file, + analytic_components, + err_func=lambda gold, test: compare_analytic.norm_err_func( + gold, test, norm_ord=np.inf, relative=True + ), + tol=self.norm_tol, + ) + + analyze_status &= analytic_status + + return analyze_status diff --git a/tst/regression/test_suites/cluster_hse/cluster_hse.py b/tst/regression/test_suites/cluster_hse/cluster_hse.py index bc90b5f8..0f567e4f 100644 --- a/tst/regression/test_suites/cluster_hse/cluster_hse.py +++ b/tst/regression/test_suites/cluster_hse/cluster_hse.py @@ -33,12 +33,16 @@ class TestCase(utils.test_case.TestCaseAbs): def __init__(self): - # Define cluster parameters # Setup units unyt.define_unit("code_length", (1, "Mpc")) unyt.define_unit("code_mass", (1e14, "Msun")) unyt.define_unit("code_time", (1, "Gyr")) + unyt.define_unit("code_velocity", (1, "code_length/code_time")) + unyt.define_unit( + "code_magnetic", + (np.sqrt(4 * np.pi), "(code_mass/code_length)**0.5/code_time"), + ) self.code_length = unyt.unyt_quantity(1, "code_length") self.code_mass = unyt.unyt_quantity(1, "code_mass") self.code_time = unyt.unyt_quantity(1, "code_time") @@ -59,14 +63,14 @@ def __init__(self): # NFW parameters self.c_nfw = 6.0 - self.M_nfw_200 = unyt.unyt_quantity(1e15, "Msun") + self.m_nfw_200 = unyt.unyt_quantity(1e15, "Msun") # BCG parameters - self.M_bcg_s = unyt.unyt_quantity(1e11, "Msun") - self.R_bcg_s = unyt.unyt_quantity(4, "kpc") + self.m_bcg_s = unyt.unyt_quantity(1e11, "Msun") + self.r_bcg_s = unyt.unyt_quantity(4, "kpc") # SMBH parameters - self.M_smbh = unyt.unyt_quantity(1e8, "Msun") + self.m_smbh = unyt.unyt_quantity(1e8, "Msun") # Smooth gravity at origin, for numerical reasons self.g_smoothing_radius = unyt.unyt_quantity(0.0, "code_length") @@ -92,6 +96,18 @@ def __init__(self): self.norm_tol = 1e-3 + self.sigma_v = unyt.unyt_quantity(75.0, "km/s") + char_v_lengthscale = unyt.unyt_quantity(100.0, "kpc") + # Note that the box scale is set in the input file directly (-0.1 to 0.1), + # so if the input file changes, the following line should change, too. + l_box = 0.2 * self.code_length + self.k_peak_v = l_box / char_v_lengthscale + + self.sigma_b = unyt.unyt_quantity(1e-8, "G") + # using a different one than for the velocity + char_b_lengthscale = unyt.unyt_quantity(50.0, "kpc") + self.k_peak_b = l_box / char_b_lengthscale + def Prepare(self, parameters, step): """ Any preprocessing that is needed before the drive is run can be done in @@ -124,46 +140,132 @@ def Prepare(self, parameters, step): f"units/code_mass_cgs={self.code_mass.in_units('g').v}", f"units/code_time_cgs={self.code_time.in_units('s').v}", f"problem/cluster/hubble_parameter={self.hubble_parameter.in_units('1/code_time').v}", - f"problem/cluster/include_nfw_g={self.include_nfw_g}", - f"problem/cluster/which_bcg_g={self.which_bcg_g}", - f"problem/cluster/include_smbh_g={self.include_smbh_g}", - f"problem/cluster/c_nfw={self.c_nfw}", - f"problem/cluster/M_nfw_200={self.M_nfw_200.in_units('code_mass').v}", - f"problem/cluster/M_bcg_s={self.M_bcg_s.in_units('code_mass').v}", - f"problem/cluster/R_bcg_s={self.R_bcg_s.in_units('code_length').v}", - f"problem/cluster/M_smbh={self.M_smbh.in_units('code_mass').v}", - f"problem/cluster/g_smoothing_radius={self.g_smoothing_radius.in_units('code_length').v}", - f"problem/cluster/K_0={self.K_0.in_units('code_length**4*code_mass/code_time**2').v}", - f"problem/cluster/K_100={self.K_100.in_units('code_length**4*code_mass/code_time**2').v}", - f"problem/cluster/R_K={self.R_K.in_units('code_length').v}", - f"problem/cluster/alpha_K={self.alpha_K}", - f"problem/cluster/R_fix={self.R_fix.in_units('code_length').v}", - f"problem/cluster/rho_fix={self.rho_fix.in_units('code_mass/code_length**3').v}", - f"problem/cluster/R_sampling={self.R_sampling}", - f"problem/cluster/max_dR={self.max_dR}", + f"problem/cluster/gravity/include_nfw_g={self.include_nfw_g}", + f"problem/cluster/gravity/which_bcg_g={self.which_bcg_g}", + f"problem/cluster/gravity/include_smbh_g={self.include_smbh_g}", + f"problem/cluster/gravity/c_nfw={self.c_nfw}", + f"problem/cluster/gravity/m_nfw_200={self.m_nfw_200.in_units('code_mass').v}", + f"problem/cluster/gravity/m_bcg_s={self.m_bcg_s.in_units('code_mass').v}", + f"problem/cluster/gravity/r_bcg_s={self.r_bcg_s.in_units('code_length').v}", + f"problem/cluster/gravity/m_smbh={self.m_smbh.in_units('code_mass').v}", + f"problem/cluster/gravity/g_smoothing_radius={self.g_smoothing_radius.in_units('code_length').v}", + f"problem/cluster/entropy_profile/k_0={self.K_0.in_units('code_length**4*code_mass/code_time**2').v}", + f"problem/cluster/entropy_profile/k_100={self.K_100.in_units('code_length**4*code_mass/code_time**2').v}", + f"problem/cluster/entropy_profile/r_k={self.R_K.in_units('code_length').v}", + f"problem/cluster/entropy_profile/alpha_k={self.alpha_K}", + f"problem/cluster/hydrostatic_equilibrium/r_fix={self.R_fix.in_units('code_length').v}", + f"problem/cluster/hydrostatic_equilibrium/rho_fix={self.rho_fix.in_units('code_mass/code_length**3').v}", + f"problem/cluster/hydrostatic_equilibrium/r_sampling={self.R_sampling}", + f"hydro/fluid={'euler' if step == 2 else 'glmmhd'}", + f"problem/cluster/init_perturb/sigma_v={0.0 if step == 2 else self.sigma_v.in_units('code_velocity').v}", + f"problem/cluster/init_perturb/k_peak_v={0.0 if step == 2 else self.k_peak_v.v}", + f"problem/cluster/init_perturb/sigma_b={0.0 if step == 2 else self.sigma_b.in_units('code_magnetic').v}", + f"problem/cluster/init_perturb/k_peak_b={0.0 if step == 2 else self.k_peak_b.v}", + f"parthenon/output2/id={'prim' if step == 2 else 'prim_perturb'}", + f"parthenon/time/nlim={-1 if step == 2 else 1}", ] return parameters def Analyse(self, parameters): - """ - Analyze the output and determine if the test passes. + analyze_status = self.AnalyseHSE(parameters) + analyze_status &= self.AnalyseInitPert(parameters) + return analyze_status - This function is called after the driver has been executed. It is - responsible for reading whatever data it needs and making a judgment - about whether or not the test passes. It takes no inputs. Output should - be True (test passes) or False (test fails). + def AnalyseInitPert(self, parameters): + analyze_status = True + sys.path.insert( + 1, + parameters.parthenon_path + + "/scripts/python/packages/parthenon_tools/parthenon_tools", + ) - The parameters that are passed in provide the paths to relevant - locations and commands. Of particular importance is the path to the - output folder. All files from a drivers run should appear in and output - folder located in - parthenon/tst/regression/test_suites/test_name/output. + try: + import phdf + except ModuleNotFoundError: + print("Couldn't find module to load Parthenon hdf5 files.") + return False - It is possible in this function to read any of the output files such as - hdf5 output and compare them to expected quantities. + data_file = phdf.phdf( + f"{parameters.output_path}/parthenon.prim_perturb.00000.phdf" + ) + dx = data_file.xf[:, 1:] - data_file.xf[:, :-1] + dy = data_file.yf[:, 1:] - data_file.yf[:, :-1] + dz = data_file.zf[:, 1:] - data_file.zf[:, :-1] + + # create array of volume with (block, k, j, i) indices + cell_vol = np.empty( + ( + data_file.x.shape[0], + data_file.z.shape[1], + data_file.y.shape[1], + data_file.x.shape[1], + ) + ) + for block in range(dx.shape[0]): + dz3d, dy3d, dx3d = np.meshgrid( + dz[block], dy[block], dx[block], indexing="ij" + ) + cell_vol[block, :, :, :] = dx3d * dy3d * dz3d - """ + # flatten array as prim var are also flattended + cell_vol = cell_vol.ravel() + + prim = data_file.Get("prim") + + # FIXME: For now this is hard coded - a component mapping should be done by phdf + prim_col_dict = { + "velocity_1": 1, + "velocity_2": 2, + "velocity_3": 3, + "magnetic_field_1": 5, + "magnetic_field_2": 6, + "magnetic_field_3": 7, + } + + vx = prim[prim_col_dict["velocity_1"]] + vy = prim[prim_col_dict["velocity_2"]] + vz = prim[prim_col_dict["velocity_3"]] + + # volume weighted rms velocity + rms_v = np.sqrt( + np.sum((vx**2 + vy**2 + vz**2) * cell_vol) / np.sum(cell_vol) + ) + + sigma_v_match = np.isclose( + rms_v, self.sigma_v.in_units("code_velocity").v, rtol=1e-14, atol=1e-14 + ) + + if not sigma_v_match: + analyze_status = False + print( + f"ERROR: velocity perturbations don't match\n" + f"Expected {self.sigma_v.in_units('code_velocity')} but got {rms_v}\n" + ) + + bx = prim[prim_col_dict["magnetic_field_1"]] + by = prim[prim_col_dict["magnetic_field_2"]] + bz = prim[prim_col_dict["magnetic_field_3"]] + + # volume weighted rms magnetic field + rms_b = np.sqrt( + np.sum((bx**2 + by**2 + bz**2) * cell_vol) / np.sum(cell_vol) + ) + + sigma_b_match = np.isclose( + rms_b, self.sigma_b.in_units("code_magnetic").v, rtol=1e-14, atol=1e-14 + ) + + if not sigma_b_match: + analyze_status = False + print( + f"ERROR: magnetic field perturbations don't match\n" + f"Expected {self.sigma_b.in_units('code_magnetic')} but got {rms_b}\n" + ) + + return analyze_status + + def AnalyseHSE(self, parameters): analyze_status = True self.Yp = self.He_mass_fraction @@ -187,7 +289,7 @@ def Analyse(self, parameters): / (np.log(1 + self.c_nfw) - self.c_nfw / (1 + self.c_nfw)) ) self.R_nfw_s = ( - self.M_nfw_200 + self.m_nfw_200 / ( 4 * np.pi @@ -228,25 +330,25 @@ def T_from_rho_P(rho, P): def g_nfw_from_r(r): return ( self.G - * self.M_nfw_200 + * self.m_nfw_200 / (np.log(1 + self.c_nfw) - self.c_nfw / (1 + self.c_nfw)) * (np.log(1 + r / self.R_nfw_s) - r / (r + self.R_nfw_s)) / r**2 ) def g_bcg_hernquist_from_r(r): - # M_bcg = 8*self.M_bcg_s*(r/self.R_bcg_s)**2/( 2*( 1 + r/self.R_bcg_s)**2) - # return G*M_bcg/r**2 + # m_bcg = 8*self.m_bcg_s*(r/self.r_bcg_s)**2/( 2*( 1 + r/self.r_bcg_s)**2) + # return G*m_bcg/r**2 g = ( self.G - * self.M_bcg_s - / (self.R_bcg_s**2) - / (2 * (1 + r / self.R_bcg_s) ** 2) + * self.m_bcg_s + / (self.r_bcg_s**2) + / (2 * (1 + r / self.r_bcg_s) ** 2) ) return g def g_smbh_from_r(r): - return self.G * self.M_smbh / r**2 + return self.G * self.m_smbh / r**2 def g_from_r(r, include_gs): g = unyt.unyt_array(np.zeros_like(r), "code_length*code_time**-2") diff --git a/tst/regression/test_suites/cluster_hydro_agn_feedback/__init__.py b/tst/regression/test_suites/cluster_hydro_agn_feedback/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/regression/test_suites/cluster_hydro_agn_feedback/cluster_hydro_agn_feedback.py b/tst/regression/test_suites/cluster_hydro_agn_feedback/cluster_hydro_agn_feedback.py new file mode 100644 index 00000000..29dc4d69 --- /dev/null +++ b/tst/regression/test_suites/cluster_hydro_agn_feedback/cluster_hydro_agn_feedback.py @@ -0,0 +1,517 @@ +# ======================================================================================== +# AthenaPK - a performance portable block structured AMR MHD code +# Copyright (c) 2020-2021, Athena Parthenon Collaboration. All rights reserved. +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== +# (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +# ======================================================================================== + +# Modules +import math +import numpy as np +import matplotlib + +matplotlib.use("agg") +import matplotlib.pylab as plt +import sys +import os +import utils.test_case +import unyt +import itertools + + +class PrecessedJetCoords: + def __init__(self, theta, phi): + self.theta = theta + self.phi = phi + + # Axis of the jet + self.jet_n = np.array( + ( + np.sin(self.theta) * np.cos(self.phi), + np.sin(self.theta) * np.sin(self.phi), + np.cos(self.theta), + ) + ) + + def cart_to_rho_h(self, pos_cart): + """ + Convert from cartesian coordinates to jet coordinates + """ + + pos_h = np.sum(pos_cart * self.jet_n[:, None], axis=0) + pos_rho = np.linalg.norm(pos_cart - pos_h * self.jet_n[:, None], axis=0) + return (pos_rho, pos_h) + + +class ZJetCoords: + def __init__(self): + self.jet_n = np.array((0, 0, 1.0)) + + def cart_to_rho_h(self, pos_cart): + """ + Convert from cartesian coordinates to jet coordinates + """ + + pos_rho = np.linalg.norm(pos_cart[:2], axis=0) + pos_h = pos_cart[2] + + return pos_rho, pos_h + + +""" To prevent littering up imported folders with .pyc files or __pycache_ folder""" +sys.dont_write_bytecode = True + + +class TestCase(utils.test_case.TestCaseAbs): + def __init__(self): + + # Define cluster parameters + # Setup units + unyt.define_unit("code_length", (1, "Mpc")) + unyt.define_unit("code_mass", (1e14, "Msun")) + unyt.define_unit("code_time", (1, "Gyr")) + self.code_length = unyt.unyt_quantity(1, "code_length") + self.code_mass = unyt.unyt_quantity(1, "code_mass") + self.code_time = unyt.unyt_quantity(1, "code_time") + + self.tlim = unyt.unyt_quantity(5e-3, "code_time") + + # Setup constants + self.k_b = unyt.kb_cgs + self.G = unyt.G_cgs + self.m_u = unyt.amu + + self.adiabatic_index = 5.0 / 3.0 + self.He_mass_fraction = 0.25 + + # Define the initial uniform gas + self.uniform_gas_rho = unyt.unyt_quantity(1e-24, "g/cm**3") + self.uniform_gas_ux = unyt.unyt_quantity(0, "cm/s") + self.uniform_gas_uy = unyt.unyt_quantity(0, "cm/s") + self.uniform_gas_uz = unyt.unyt_quantity(0, "cm/s") + self.uniform_gas_pres = unyt.unyt_quantity(1e-10, "dyne/cm**2") + + self.uniform_gas_Mx = self.uniform_gas_rho * self.uniform_gas_ux + self.uniform_gas_My = self.uniform_gas_rho * self.uniform_gas_uy + self.uniform_gas_Mz = self.uniform_gas_rho * self.uniform_gas_uz + self.uniform_gas_energy_density = 1.0 / 2.0 * self.uniform_gas_rho * ( + self.uniform_gas_ux**2 + + self.uniform_gas_uy**2 + + self.uniform_gas_uz**2 + ) + self.uniform_gas_pres / (self.adiabatic_index - 1.0) + + # The precessing jet + self.jet_phi0 = 1.2 + self.jet_phi_dot = 0 + self.jet_theta = 0.4 + self.precessed_jet_coords = PrecessedJetCoords(self.jet_theta, self.jet_phi0) + self.zjet_coords = ZJetCoords() + + # Feedback parameters + self.fixed_power = unyt.unyt_quantity(2e46, "erg/s") + self.agn_thermal_radius = unyt.unyt_quantity(100, "kpc") + self.efficiency = 1.0e-3 + self.jet_temperature = unyt.unyt_quantity(1e7, "K") + self.jet_radius = unyt.unyt_quantity(50, "kpc") + self.jet_thickness = unyt.unyt_quantity(50, "kpc") + self.jet_offset = unyt.unyt_quantity(10, "kpc") + + mu = 1.0 / (3.0 / 4.0 * self.He_mass_fraction + (1 - self.He_mass_fraction) * 2) + self.jet_internal_e = ( + self.jet_temperature + * unyt.boltzmann_constant + / (mu * unyt.amu * (self.adiabatic_index - 1.0)) + ) + + self.norm_tol = 1e-3 + + self.steps = 5 + self.step_params_list = list( + itertools.product( + ("thermal_only", "kinetic_only", "combined"), (True, False) + ) + ) + # Remove ("thermal_only",True) since it is redudant, jet precession is + # irrelevant with only thermal feedback + self.step_params_list.remove(("thermal_only", True)) + + def Prepare(self, parameters, step): + """ + Any preprocessing that is needed before the drive is run can be done in + this method + + This includes preparing files or any other pre processing steps that + need to be implemented. The method also provides access to the + parameters object which controls which parameters are being used to run + the driver. + + It is possible to append arguments to the driver_cmd_line_args if it is + desired to override the parthenon input file. Each element in the list + is simply a string of the form '/=', where the + contents of the string are exactly what one would type on the command + line run running a parthenon driver. + + As an example if the following block was uncommented it would overwrite + any of the parameters that were specified in the parthenon input file + parameters.driver_cmd_line_args = ['output1/file_type=vtk', + 'output1/variable=cons', + 'output1/dt=0.4', + 'time/tlim=0.4', + 'mesh/nx1=400'] + """ + feedback_mode, precessed_jet = self.step_params_list[step - 1] + output_id = f"{feedback_mode}_precessed_{precessed_jet}" + + if feedback_mode == "thermal_only": + agn_kinetic_fraction = 0.0 + agn_thermal_fraction = 1.0 + elif feedback_mode == "kinetic_only": + agn_kinetic_fraction = 1.0 + agn_thermal_fraction = 0.0 + elif feedback_mode == "combined": + agn_kinetic_fraction = 0.5 + agn_thermal_fraction = 0.5 + else: + raise Exception(f"Feedback mode {feedback_mode} not supported in analysis") + + parameters.driver_cmd_line_args = [ + f"parthenon/output2/id={output_id}", + f"parthenon/output2/dt={self.tlim.in_units('code_time').v}", + f"parthenon/time/tlim={self.tlim.in_units('code_time').v}", + f"hydro/gamma={self.adiabatic_index}", + f"hydro/He_mass_fraction={self.He_mass_fraction}", + f"units/code_length_cgs={self.code_length.in_units('cm').v}", + f"units/code_mass_cgs={self.code_mass.in_units('g').v}", + f"units/code_time_cgs={self.code_time.in_units('s').v}", + f"problem/cluster/uniform_gas/init_uniform_gas=true", + f"problem/cluster/uniform_gas/rho={self.uniform_gas_rho.in_units('code_mass*code_length**-3').v}", + f"problem/cluster/uniform_gas/ux={self.uniform_gas_ux.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uy={self.uniform_gas_uy.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uz={self.uniform_gas_uz.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/pres={self.uniform_gas_pres.in_units('code_mass*code_length**-1*code_time**-2').v}", + f"problem/cluster/precessing_jet/jet_phi0={self.jet_phi0 if precessed_jet else 0}", + f"problem/cluster/precessing_jet/jet_phi_dot={self.jet_phi_dot if precessed_jet else 0}", + f"problem/cluster/precessing_jet/jet_theta={self.jet_theta if precessed_jet else 0}", + f"problem/cluster/agn_feedback/fixed_power={self.fixed_power.in_units('code_mass*code_length**2/code_time**3').v}", + f"problem/cluster/agn_feedback/efficiency={self.efficiency}", + f"problem/cluster/agn_feedback/thermal_fraction={agn_thermal_fraction}", + f"problem/cluster/agn_feedback/kinetic_fraction={agn_kinetic_fraction}", + f"problem/cluster/agn_feedback/magnetic_fraction=0", + f"problem/cluster/agn_feedback/thermal_radius={self.agn_thermal_radius.in_units('code_length').v}", + f"problem/cluster/agn_feedback/kinetic_jet_temperature={self.jet_temperature.in_units('K').v}", + f"problem/cluster/agn_feedback/kinetic_jet_radius={self.jet_radius.in_units('code_length').v}", + f"problem/cluster/agn_feedback/kinetic_jet_thickness={self.jet_thickness.in_units('code_length').v}", + f"problem/cluster/agn_feedback/kinetic_jet_offset={self.jet_offset.in_units('code_length').v}", + ] + + return parameters + + def Analyse(self, parameters): + """ + Analyze the output and determine if the test passes. + + This function is called after the driver has been executed. It is + responsible for reading whatever data it needs and making a judgment + about whether or not the test passes. It takes no inputs. Output should + be True (test passes) or False (test fails). + + The parameters that are passed in provide the paths to relevant + locations and commands. Of particular importance is the path to the + output folder. All files from a drivers run should appear in and output + folder located in + parthenon/tst/regression/test_suites/test_name/output. + + It is possible in this function to read any of the output files such as + hdf5 output and compare them to expected quantities. + + """ + analyze_status = True + + self.Yp = self.He_mass_fraction + self.mu = 1 / (self.Yp * 3.0 / 4.0 + (1 - self.Yp) * 2) + self.mu_e = 1 / (self.Yp * 2.0 / 4.0 + (1 - self.Yp)) + + for step in range(1, self.steps + 1): + feedback_mode, precessed_jet = self.step_params_list[step - 1] + output_id = f"{feedback_mode}_precessed_{precessed_jet}" + step_status = True + + print(f"Testing {output_id}") + + if precessed_jet is True: + jet_coords = self.precessed_jet_coords + else: + jet_coords = self.zjet_coords + + if feedback_mode == "thermal_only": + agn_kinetic_fraction = 0.0 + agn_thermal_fraction = 1.0 + elif feedback_mode == "kinetic_only": + agn_kinetic_fraction = 1.0 + agn_thermal_fraction = 0.0 + elif feedback_mode == "combined": + agn_kinetic_fraction = 0.5 + agn_thermal_fraction = 0.5 + else: + raise Exception( + f"Feedback mode {feedback_mode} not supported in analysis" + ) + + jet_density = ( + (agn_kinetic_fraction * self.fixed_power) + / (self.efficiency * unyt.c_cgs**2) + / (2 * np.pi * self.jet_radius**2 * self.jet_thickness) + ) + + jet_velocity = np.sqrt( + 2 + * ( + self.efficiency * unyt.c_cgs**2 + - (1 - self.efficiency) * self.jet_internal_e + ) + ) + jet_feedback = ( + self.fixed_power + * agn_kinetic_fraction + / (2 * np.pi * self.jet_radius**2 * self.jet_thickness) + ) + + def kinetic_feedback(Z, Y, X, time): + if not hasattr(time, "units"): + time = unyt.unyt_quantity(time, "code_time") + R, H = jet_coords.cart_to_rho_h(np.array((X, Y, Z))) + R = unyt.unyt_array(R, "code_length") + H = unyt.unyt_array(H, "code_length") + + # sign_jet = np.piecewise(H, [H <= 0, H > 0], [1, -1]).v #Backwards jet REMOVEME + sign_jet = np.piecewise(H, [H <= 0, H > 0], [-1, 1]).v + inside_jet = ( + np.piecewise( + R, + [ + R <= self.jet_radius, + ], + [1, 0], + ) + * np.piecewise( + H, + [ + np.abs(H) >= self.jet_offset, + ], + [1, 0], + ) + * np.piecewise( + H, + [ + np.abs(H) <= self.jet_offset + self.jet_thickness, + ], + [1, 0], + ) + ).v + + drho = inside_jet * time * jet_density + dMx = ( + inside_jet + * time + * sign_jet + * jet_density + * jet_velocity + * jet_coords.jet_n[0] + ) + dMy = ( + inside_jet + * time + * sign_jet + * jet_density + * jet_velocity + * jet_coords.jet_n[1] + ) + dMz = ( + inside_jet + * time + * sign_jet + * jet_density + * jet_velocity + * jet_coords.jet_n[2] + ) + + # Note: Final density should be correct by thermal mass injected + # final_density = (time*jet_density + self.uniform_gas_rho) + # final_velocity = (time*jet_density*jet_velocity)/( final_density) + # dKE = inside_jet * 0.5 * final_density * final_velocity**2 + # dTE = inside_jet * time * jet_density * self.uniform_gas_pres / (self.uniform_gas_rho*(self.adiabatic_index - 1.0)) + dE = jet_feedback * time * inside_jet + + # DELETEME + # print(dKE.max().in_units("code_mass*code_length**-1*code_time**-2"), + # dTE.max().in_units("code_mass*code_length**-1*code_time**-2")) + # print(dE.max()/( + # time* agn_kinetic_fraction*self.fixed_power/(2*np.pi*self.jet_radius**2*self.jet_thickness))) + + return drho, dMx, dMy, dMz, dE + + def thermal_feedback(Z, Y, X, time): + if not hasattr(time, "units"): + time = unyt.unyt_quantity(time, "code_time") + R = np.sqrt(X**2 + Y**2 + Z**2) + inside_sphere = np.piecewise( + R, + [ + R <= self.agn_thermal_radius.in_units("code_length"), + ], + [1, 0], + ) + dE = ( + inside_sphere + * time + * ( + self.fixed_power + * agn_thermal_fraction + / (4.0 / 3.0 * np.pi * self.agn_thermal_radius**3) + ) + ) + + drho = ( + inside_sphere + * time + * ( + self.fixed_power + / (self.efficiency * unyt.c_cgs**2) + * agn_thermal_fraction + / (4.0 / 3.0 * np.pi * self.agn_thermal_radius**3) + ) + ) + # Assume no velocity, no change in momentum with mass injection + return drho, dE + + def agn_feedback(Z, Y, X, dt): + drho_k, dMx_k, dMy_k, dMz_k, dE_k = kinetic_feedback(Z, Y, X, dt) + drho_t, dE_t = thermal_feedback(Z, Y, X, dt) + + drho = drho_k + drho_t + dMx = dMx_k + dMy = dMy_k + dMz = dMz_k + dE = dE_k + dE_t + + return drho, dMx, dMy, dMz, dE + + # Check that the initial and final outputs match the expected tower + sys.path.insert( + 1, + parameters.parthenon_path + + "/scripts/python/packages/parthenon_tools/parthenon_tools", + ) + + try: + import compare_analytic + except ModuleNotFoundError: + print("Couldn't find module to analyze Parthenon hdf5 files.") + return False + + initial_analytic_components = { + "cons_density": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_rho.in_units("code_mass/code_length**3").v, + "cons_momentum_density_1": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_Mx.in_units( + "code_mass*code_length**-2*code_time**-1" + ).v, + "cons_momentum_density_2": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_My.in_units( + "code_mass*code_length**-2*code_time**-1" + ).v, + "cons_momentum_density_3": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_Mz.in_units( + "code_mass*code_length**-2*code_time**-1" + ).v, + "cons_total_energy_density": lambda Z, Y, X, time: np.ones_like(Z) + * self.uniform_gas_energy_density.in_units( + "code_mass*code_length**-1*code_time**-2" + ).v, + } + + final_analytic_components = { + "cons_density": lambda Z, Y, X, time: ( + self.uniform_gas_rho + agn_feedback(Z, Y, X, time)[0] + ) + .in_units("code_mass/code_length**3") + .v, + "cons_momentum_density_1": lambda Z, Y, X, time: ( + self.uniform_gas_Mx + agn_feedback(Z, Y, X, time)[1] + ) + .in_units("code_mass*code_length**-2*code_time**-1") + .v, + "cons_momentum_density_2": lambda Z, Y, X, time: ( + self.uniform_gas_My + agn_feedback(Z, Y, X, time)[2] + ) + .in_units("code_mass*code_length**-2*code_time**-1") + .v, + "cons_momentum_density_3": lambda Z, Y, X, time: ( + self.uniform_gas_Mz + agn_feedback(Z, Y, X, time)[3] + ) + .in_units("code_mass*code_length**-2*code_time**-1") + .v, + "cons_total_energy_density": lambda Z, Y, X, time: ( + self.uniform_gas_energy_density + agn_feedback(Z, Y, X, time)[4] + ) + .in_units("code_mass*code_length**-1*code_time**-2") + .v, + } + + phdf_files = [ + f"{parameters.output_path}/parthenon.{output_id}.00000.phdf", + f"{parameters.output_path}/parthenon.{output_id}.final.phdf", + ] + + def zero_corrected_linf_err(gold, test): + non_zero_linf = np.max( + np.abs((gold[gold != 0] - test[gold != 0]) / gold[gold != 0]), + initial=0, + ) + zero_linf = np.max( + np.abs((gold[gold == 0] - test[gold == 0])), initial=0 + ) + + return np.max((non_zero_linf, zero_linf)) + + # Use a very loose tolerance, linf relative error + # initial_analytic_status, final_analytic_status = [ + # compare_analytic.compare_analytic( + # phdf_file, + # analytic_components, + # err_func=zero_corrected_linf_err, + # tol=1e-3, + # ) + # for analytic_components, phdf_file in zip( + # (initial_analytic_components, final_analytic_components), phdf_files + # ) + # ] + initial_analytic_status = compare_analytic.compare_analytic( + phdf_files[0], + initial_analytic_components, + err_func=zero_corrected_linf_err, + tol=1e-6, + ) + final_analytic_status = compare_analytic.compare_analytic( + phdf_files[1], + final_analytic_components, + err_func=zero_corrected_linf_err, + tol=1e-3, + ) + + print(" Initial analytic status", initial_analytic_status) + print(" Final analytic status", final_analytic_status) + + analyze_status &= initial_analytic_status & final_analytic_status + + return analyze_status diff --git a/tst/regression/test_suites/cluster_magnetic_tower/__init__.py b/tst/regression/test_suites/cluster_magnetic_tower/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tst/regression/test_suites/cluster_magnetic_tower/cluster_magnetic_tower.py b/tst/regression/test_suites/cluster_magnetic_tower/cluster_magnetic_tower.py new file mode 100644 index 00000000..c80ef354 --- /dev/null +++ b/tst/regression/test_suites/cluster_magnetic_tower/cluster_magnetic_tower.py @@ -0,0 +1,638 @@ +# ======================================================================================== +# AthenaPK - a performance portable block structured AMR MHD code +# Copyright (c) 2020-2021, Athena Parthenon Collaboration. All rights reserved. +# Licensed under the 3-clause BSD License, see LICENSE file for details +# ======================================================================================== +# (C) (or copyright) 2020. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 for Los +# Alamos National Laboratory (LANL), which is operated by Triad National Security, LLC +# for the U.S. Department of Energy/National Nuclear Security Administration. All rights +# in the program are reserved by Triad National Security, LLC, and the U.S. Department +# of Energy/National Nuclear Security Administration. The Government is granted for +# itself and others acting on its behalf a nonexclusive, paid-up, irrevocable worldwide +# license in this material to reproduce, prepare derivative works, distribute copies to +# the public, perform publicly and display publicly, and to permit others to do so. +# ======================================================================================== + +# Modules +import math +import numpy as np +import matplotlib + +matplotlib.use("agg") +import matplotlib.pylab as plt +import sys +import os +import utils.test_case +import unyt +import itertools + + +class PrecessedJetCoords: + # Note: Does note rotate the vector around the jet axis, only rotates the vector to `z_hat` + def __init__(self, phi_jet, theta_jet): + self.phi_jet = phi_jet + self.theta_jet = theta_jet + + def cart_to_jet_coords(self, pos_sim): + """ + Convert from simulation cartesian coordinates to jet cylindrical coordinates + """ + + x_sim = pos_sim[0] + y_sim = pos_sim[1] + z_sim = pos_sim[2] + + x_jet = ( + x_sim * np.cos(self.phi_jet) * np.cos(self.theta_jet) + + y_sim * np.sin(self.phi_jet) * np.cos(self.theta_jet) + - z_sim * np.sin(self.theta_jet) + ) + y_jet = -x_sim * np.sin(self.phi_jet) + y_sim * np.cos(self.phi_jet) + z_jet = ( + x_sim * np.sin(self.theta_jet) * np.cos(self.phi_jet) + + y_sim * np.sin(self.phi_jet) * np.sin(self.theta_jet) + + z_sim * np.cos(self.theta_jet) + ) + + r_jet = np.sqrt(x_jet**2 + y_jet**2) + theta_jet = np.arctan2(y_jet, x_jet) + h_jet = z_jet + + return (r_jet, theta_jet, h_jet) + + def jet_to_cart_vec(self, pos_sim, vec_jet): + """ + Convert vector in jet cylindrical coordinates to simulation cartesian coordinates + """ + + r_pos, theta_pos, h_pos = self.cart_to_jet_coords(pos_sim) + + v_x_jet = vec_jet[0] * np.cos(theta_pos) - vec_jet[1] * np.sin(theta_pos) + v_y_jet = vec_jet[0] * np.sin(theta_pos) + vec_jet[1] * np.cos(theta_pos) + v_z_jet = vec_jet[2] + + v_x_sim = ( + v_x_jet * np.cos(self.phi_jet) * np.cos(self.theta_jet) + - v_y_jet * np.sin(self.phi_jet) + + v_z_jet * np.sin(self.theta_jet) * np.cos(self.phi_jet) + ) + v_y_sim = ( + v_x_jet * np.sin(self.phi_jet) * np.cos(self.theta_jet) + + v_y_jet * np.cos(self.phi_jet) + + v_z_jet * np.sin(self.phi_jet) * np.sin(self.theta_jet) + ) + v_z_sim = -v_x_jet * np.sin(self.theta_jet) + v_z_jet * np.cos(self.theta_jet) + + return (v_x_sim, v_y_sim, v_z_sim) + + +class ZJetCoords: + def __init__(self): + pass + + def cart_to_jet_coords(self, pos_cart): + """ + Convert from cartesian coordinates to jet coordinates + """ + + pos_rho = np.sqrt(pos_cart[0] ** 2 + pos_cart[1] ** 2) + pos_theta = np.arctan2(pos_cart[1], pos_cart[0]) + pos_theta[pos_rho == 0] = 0 + pos_h = pos_cart[2] + + return (pos_rho, pos_theta, pos_h) + + def jet_to_cart_vec(self, pos_cart, vec_jet): + + vec_rho = vec_jet[0] + vec_theta = vec_jet[1] + vec_h = vec_jet[2] + + r_pos, theta_pos, h_pos = self.cart_to_jet_coords(pos_cart) + + # Compute vector in cartesian coords + vec_x = vec_rho * np.cos(theta_pos) - vec_theta * np.sin(theta_pos) + vec_y = vec_rho * np.sin(theta_pos) + vec_theta * np.cos(theta_pos) + vec_z = vec_h + + return (vec_x, vec_y, vec_z) + + +""" To prevent littering up imported folders with .pyc files or __pycache_ folder""" +sys.dont_write_bytecode = True + + +class TestCase(utils.test_case.TestCaseAbs): + def __init__(self): + + # Define cluster parameters + # Setup units + unyt.define_unit("code_length", (1, "Mpc")) + unyt.define_unit("code_mass", (1e14, "Msun")) + unyt.define_unit("code_time", (1, "Gyr")) + self.code_length = unyt.unyt_quantity(1, "code_length") + self.code_mass = unyt.unyt_quantity(1, "code_mass") + self.code_time = unyt.unyt_quantity(1, "code_time") + + self.tlim = unyt.unyt_quantity(1e-2, "code_time") + + # Setup constants + self.k_b = unyt.kb_cgs + self.G = unyt.G_cgs + self.m_u = unyt.amu + + self.adiabatic_index = 5.0 / 3.0 + self.He_mass_fraction = 0.25 + + # Define the initial uniform gas + self.uniform_gas_rho = unyt.unyt_quantity(1e-24, "g/cm**3") + self.uniform_gas_ux = unyt.unyt_quantity(0, "cm/s") + self.uniform_gas_uy = unyt.unyt_quantity(0, "cm/s") + self.uniform_gas_uz = unyt.unyt_quantity(0, "cm/s") + self.uniform_gas_pres = unyt.unyt_quantity(1e-10, "dyne/cm**2") + + self.uniform_gas_Mx = self.uniform_gas_rho * self.uniform_gas_ux + self.uniform_gas_My = self.uniform_gas_rho * self.uniform_gas_uy + self.uniform_gas_Mz = self.uniform_gas_rho * self.uniform_gas_uz + self.uniform_gas_energy_density = 1.0 / 2.0 * self.uniform_gas_rho * ( + self.uniform_gas_ux**2 + + self.uniform_gas_uy**2 + + self.uniform_gas_uz**2 + ) + self.uniform_gas_pres / (self.adiabatic_index - 1.0) + + # Efficiency of power to accretion rate (controls rate of mass injection for this test) + # self.efficiency = 0 + self.efficiency = 1e-3 + + # The precessing jet + self.theta_jet = 0.2 + self.phi_dot_jet = 0 # Use phi_dot = 0 for stationary jet + self.phi_jet0 = 1 # Offset initial jet + self.precessed_jet_coords = PrecessedJetCoords(self.phi_jet0, self.theta_jet) + self.zjet_coords = ZJetCoords() + + # Initial and Feedback shared parameters + self.magnetic_tower_alpha = 20 + # self.magnetic_tower_l_scale = unyt.unyt_quantity(1,"code_length") + self.magnetic_tower_l_scale = unyt.unyt_quantity(10, "kpc") + self.magnetic_tower_l_mass_scale = unyt.unyt_quantity(5, "kpc") + + # The Initial Tower + self.initial_magnetic_tower_field = unyt.unyt_quantity(1e-6, "G") + + # The Feedback Tower + # For const field tests + self.feedback_magnetic_tower_field = unyt.unyt_quantity(1e-4, "G/Gyr") + # For const energy tests + self.feedback_magnetic_tower_power = unyt.unyt_quantity(1e44, "erg/s") + # For const field tests + # self.feedback_magnetic_tower_mass = unyt.unyt_quantity(0,"g/s") + self.feedback_magnetic_tower_mass = self.feedback_magnetic_tower_power / ( + self.efficiency * unyt.c_cgs**2 + ) + + self.energy_density_tol = 1e-2 + + # Tolerance of linf error of magnetic fields, total energy density, and density + self.linf_analytic_tol = 5e-2 + + # Tolerance on total initial and final magnetic energy + self.b_eng_initial_tol = 1e-2 + self.b_eng_final_tol = 1e-2 + + # Tolerance in max divergence over magnetic tower field scale + self.divB_tol = 1e-11 + + self.steps = 4 + self.step_params_list = list( + itertools.product(("const_field", "const_power"), (True, False)) + ) + + def Prepare(self, parameters, step): + """ + Any preprocessing that is needed before the drive is run can be done in + this method + + This includes preparing files or any other pre processing steps that + need to be implemented. The method also provides access to the + parameters object which controls which parameters are being used to run + the driver. + + It is possible to append arguments to the driver_cmd_line_args if it is + desired to override the parthenon input file. Each element in the list + is simply a string of the form '/=', where the + contents of the string are exactly what one would type on the command + line run running a parthenon driver. + + As an example if the following block was uncommented it would overwrite + any of the parameters that were specified in the parthenon input file + parameters.driver_cmd_line_args = ['output1/file_type=vtk', + 'output1/variable=cons', + 'output1/dt=0.4', + 'time/tlim=0.4', + 'mesh/nx1=400'] + """ + feedback_mode, precessed_jet = self.step_params_list[step - 1] + output_id = f"{feedback_mode}_precessed_{precessed_jet}" + + if feedback_mode == "const_power": + fixed_power = self.feedback_magnetic_tower_power.in_units( + "code_mass*code_length**2/code_time**3" + ).v + fixed_field_rate = 0 + fixed_mass_rate = 0 + else: + fixed_power = 0 + fixed_field_rate = self.feedback_magnetic_tower_field.in_units( + "sqrt(code_mass)/sqrt(code_length)/code_time**2" + ).v + fixed_mass_rate = self.feedback_magnetic_tower_mass.in_units( + "code_mass/code_time" + ).v + + parameters.driver_cmd_line_args = [ + f"parthenon/output2/id={output_id}", + f"parthenon/output2/dt={self.tlim.in_units('code_time').v}", + f"parthenon/time/tlim={self.tlim.in_units('code_time').v}", + f"hydro/gamma={self.adiabatic_index}", + f"hydro/He_mass_fraction={self.He_mass_fraction}", + f"units/code_length_cgs={self.code_length.in_units('cm').v}", + f"units/code_mass_cgs={self.code_mass.in_units('g').v}", + f"units/code_time_cgs={self.code_time.in_units('s').v}", + f"problem/cluster/uniform_gas/init_uniform_gas=true", + f"problem/cluster/uniform_gas/rho={self.uniform_gas_rho.in_units('code_mass*code_length**-3').v}", + f"problem/cluster/uniform_gas/ux={self.uniform_gas_ux.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uy={self.uniform_gas_uy.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uz={self.uniform_gas_uz.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/pres={self.uniform_gas_pres.in_units('code_mass*code_length**-1*code_time**-2').v}", + f"problem/cluster/precessing_jet/jet_theta={self.theta_jet if precessed_jet else 0}", + f"problem/cluster/precessing_jet/jet_phi_dot={self.phi_dot_jet if precessed_jet else 0}", + f"problem/cluster/precessing_jet/jet_phi0={self.phi_jet0 if precessed_jet else 0}", + f"problem/cluster/agn_feedback/fixed_power={fixed_power}", + f"problem/cluster/agn_feedback/efficiency={self.efficiency}", + f"problem/cluster/agn_feedback/magnetic_fraction=1", + f"problem/cluster/agn_feedback/kinetic_fraction=0", + f"problem/cluster/agn_feedback/thermal_fraction=0", + f"problem/cluster/magnetic_tower/li_alpha={self.magnetic_tower_alpha}", + f"problem/cluster/magnetic_tower/l_scale={self.magnetic_tower_l_scale.in_units('code_length').v}", + f"problem/cluster/magnetic_tower/initial_field={self.initial_magnetic_tower_field.in_units('sqrt(code_mass)/sqrt(code_length)/code_time').v}", + f"problem/cluster/magnetic_tower/fixed_field_rate={fixed_field_rate}", + f"problem/cluster/magnetic_tower/fixed_mass_rate={fixed_mass_rate}", + f"problem/cluster/magnetic_tower/l_mass_scale={self.magnetic_tower_l_mass_scale.in_units('code_length').v}", + ] + + return parameters + + def Analyse(self, parameters): + """ + Analyze the output and determine if the test passes. + + This function is called after the driver has been executed. It is + responsible for reading whatever data it needs and making a judgment + about whether or not the test passes. It takes no inputs. Output should + be True (test passes) or False (test fails). + + The parameters that are passed in provide the paths to relevant + locations and commands. Of particular importance is the path to the + output folder. All files from a drivers run should appear in and output + folder located in + parthenon/tst/regression/test_suites/test_name/output. + + It is possible in this function to read any of the output files such as + hdf5 output and compare them to expected quantities. + + """ + analyze_status = True + + self.Yp = self.He_mass_fraction + self.mu = 1 / (self.Yp * 3.0 / 4.0 + (1 - self.Yp) * 2) + self.mu_e = 1 / (self.Yp * 2.0 / 4.0 + (1 - self.Yp)) + + magnetic_units = "sqrt(code_mass)/sqrt(code_length)/code_time" + + for step in range(1, self.steps + 1): + feedback_mode, precessed_jet = self.step_params_list[step - 1] + output_id = f"{feedback_mode}_precessed_{precessed_jet}" + step_status = True + + print(">" * 20) + print(f"Testing {output_id}") + print(">" * 20) + + B0_initial = self.initial_magnetic_tower_field + # Compute the initial magnetic energy + b_eng_initial_anyl = ( + np.pi ** (3.0 / 2.0) + / (8 * np.sqrt(2)) + * (5 + self.magnetic_tower_alpha**2) + * self.magnetic_tower_l_scale**3 + * B0_initial**2 + ) + + if feedback_mode == "const_field": + B0_final = ( + self.feedback_magnetic_tower_field * self.tlim + + self.initial_magnetic_tower_field + ) + # Estimate the final magnetic field using the total energy of the tower out to inifinity + b_eng_final_anyl = ( + np.pi ** (3.0 / 2.0) + / (8 * np.sqrt(2)) + * (5 + self.magnetic_tower_alpha**2) + * self.magnetic_tower_l_scale**3 + * B0_final**2 + ) + injected_mass = self.feedback_magnetic_tower_mass * self.tlim + elif feedback_mode == "const_power": + # Estimate the final magnetic field using the total energy of the tower out to inifinity + # Slightly inaccurate due to finite domain + B0_final = np.sqrt( + ( + b_eng_initial_anyl + + self.feedback_magnetic_tower_power * self.tlim + ) + / ( + np.pi ** (3.0 / 2.0) + / (8 * np.sqrt(2)) + * (5 + self.magnetic_tower_alpha**2) + * self.magnetic_tower_l_scale**3 + ) + ) + b_eng_final_anyl = ( + self.feedback_magnetic_tower_power * self.tlim + + ( + np.pi ** (3.0 / 2.0) + / (8 * np.sqrt(2)) + * (5 + self.magnetic_tower_alpha**2) + * self.magnetic_tower_l_scale**3 + ) + * B0_initial**2 + ) + injected_mass = ( + self.feedback_magnetic_tower_power + / (self.efficiency * unyt.c_cgs**2) + * self.tlim + ) + else: + raise Exception( + f"Feedback mode {feedback_mode} not supported in analysis" + ) + + rho0_final = injected_mass / ( + self.magnetic_tower_l_mass_scale**3 * np.pi ** (3.0 / 2.0) + ) + + if precessed_jet is True: + jet_coords = self.precessed_jet_coords + else: + jet_coords = self.zjet_coords + + def field_func(Z, Y, X, B0): + l = self.magnetic_tower_l_scale + alpha = self.magnetic_tower_alpha + + pos_cart = unyt.unyt_array((X, Y, Z), "code_length") + R, Theta, H = jet_coords.cart_to_jet_coords(pos_cart) + + B_r = ( + B0 * 2 * (H / l) * (R / l) * np.exp(-((R / l) ** 2) - (H / l) ** 2) + ) + B_theta = B0 * alpha * (R / l) * np.exp(-((R / l) ** 2) - (H / l) ** 2) + B_h = ( + B0 * 2 * (1 - (R / l) ** 2) * np.exp(-((R / l) ** 2) - (H / l) ** 2) + ) + B_jet = unyt.unyt_array((B_r, B_theta, B_h), magnetic_units) + + B_x, B_y, B_z = jet_coords.jet_to_cart_vec(pos_cart, B_jet) + + return unyt.unyt_array((B_x, B_y, B_z), magnetic_units) + + b_energy_func = lambda Z, Y, X, B0: 0.5 * np.sum( + field_func(Z, Y, X, B0) ** 2, axis=0 + ) + + def density_func(Z, Y, X, rho0): + pos_cart = unyt.unyt_array((X, Y, Z), "code_length") + R, Theta, H = jet_coords.cart_to_jet_coords(pos_cart) + + Density = self.uniform_gas_rho + rho0 * np.exp( + -(R**2 + H**2) / self.magnetic_tower_l_mass_scale**2 + ) + return Density + + def internal_energy_density_func(Z, Y, X, rho0): + pos_cart = unyt.unyt_array((X, Y, Z), "code_length") + R, Theta, H = jet_coords.cart_to_jet_coords(pos_cart) + + rho_e = self.uniform_gas_energy_density + 1.0 / ( + self.adiabatic_index - 1 + ) * ( + rho0 + * np.exp(-(R**2 + H**2) / self.magnetic_tower_l_mass_scale**2) + * (self.uniform_gas_pres / self.uniform_gas_rho) + ) + return rho_e + + # Check that the initial and final outputs match the expected tower + sys.path.insert( + 1, + parameters.parthenon_path + + "/scripts/python/packages/parthenon_tools/parthenon_tools", + ) + + try: + import compare_analytic + import phdf + except ModuleNotFoundError: + print("Couldn't find module to compare Parthenon hdf5 files.") + return False + + ######################################## + # Compare to the analytically expected densities, total energy + # densities, and magnetic fields + ######################################## + + phdf_filenames = [ + f"{parameters.output_path}/parthenon.{output_id}.00000.phdf", + f"{parameters.output_path}/parthenon.{output_id}.final.phdf", + ] + + # Create a relative L-Inf errpr function, ignore where zero in gold data + rel_linf_err_func = lambda gold, test: compare_analytic.norm_err_func( + gold, test, norm_ord=np.inf, relative=True, ignore_gold_zero=True + ) + + # Create a linf error function scaled by a magnetic field + # Avoids relative comparisons in areas where magnetic field is close to zero + def B_scaled_linf_err(gold, test, B0): + err_val = np.abs((gold - test) / B0) + return err_val.max() + + # Use a very loose tolerance, linf relative error + analytic_statuses = [] + for B_field, rho0, phdf_filename, label in zip( + (B0_initial, B0_final), + (unyt.unyt_quantity(0, "code_mass*code_length**-3"), rho0_final), + phdf_filenames, + ("Initial", "Final"), + ): + + # Construct lambda functions for initial and final analytically + # expected density and total energy density + densities_analytic_components = { + "cons_density": lambda Z, Y, X, time: density_func(Z, Y, X, rho0) + .in_units("code_mass*code_length**-3") + .v, + "cons_total_energy_density": lambda Z, Y, X, time: ( + internal_energy_density_func(Z, Y, X, rho0) + + b_energy_func(Z, Y, X, B_field) + ) + .in_units("code_mass*code_length**-1*code_time**-2") + .v, + } + + # Compare the simulation and analytic density and total energy density + densities_analytic_status = compare_analytic.compare_analytic( + phdf_filename, + densities_analytic_components, + err_func=rel_linf_err_func, + tol=self.linf_analytic_tol, + ) + + # Construct lambda functions for initial and final analytically expected magnetic fields + field_analytic_components = { + "cons_magnetic_field_1": lambda Z, Y, X, time: field_func( + Z, Y, X, B_field + ) + .in_units(magnetic_units)[0] + .v, + "cons_magnetic_field_2": lambda Z, Y, X, time: field_func( + Z, Y, X, B_field + ) + .in_units(magnetic_units)[1] + .v, + "cons_magnetic_field_3": lambda Z, Y, X, time: field_func( + Z, Y, X, B_field + ) + .in_units(magnetic_units)[2] + .v, + } + + # Compare the simulation and analytic magnetic fields, + # scaled by magnetic tower field scale + field_analytic_status = compare_analytic.compare_analytic( + phdf_filename, + field_analytic_components, + err_func=lambda gold, test: B_scaled_linf_err( + gold, test, B_field.in_units(magnetic_units).v[()] + ), + tol=self.linf_analytic_tol, + ) + + analytic_status = densities_analytic_status and field_analytic_status + if not analytic_status: + print(f"{label} Analytic comparison failed\n") + + analytic_statuses.append(analytic_status) + + analyze_status &= np.all(analytic_statuses) + + for phdf_filename, b_eng_anyl, B0, b_eng_tol, label in zip( + phdf_filenames, + (b_eng_initial_anyl, b_eng_final_anyl), + (B0_initial, B0_final), + (self.b_eng_initial_tol, self.b_eng_final_tol), + ("Initial", "Final"), + ): + + ######################################## + # Compare with the analytically expected total magnetic energy + ######################################## + phdf_file = phdf.phdf(phdf_filename) + + # Get the cell volumes from phdf_file + xf = phdf_file.xf + yf = phdf_file.yf + zf = phdf_file.zf + cell_vols = unyt.unyt_array( + np.einsum("ai,aj,ak->aijk", np.diff(zf), np.diff(yf), np.diff(xf)), + "code_length**3", + ) + + # Get the magnetic energy from phdf_file + B = unyt.unyt_array( + list( + phdf_file.GetComponents( + [ + "cons_magnetic_field_1", + "cons_magnetic_field_2", + "cons_magnetic_field_3", + ], + flatten=False, + ).values() + ), + magnetic_units, + ) + b_eng = np.sum(0.5 * np.sum(B**2, axis=0) * cell_vols) + + # Get the estimated magnetic energy from the expected mt_tower field + + Z, Y, X = phdf_file.GetVolumeLocations(flatten=False) + Z = unyt.unyt_array(Z, "code_length") + Y = unyt.unyt_array(Y, "code_length") + X = unyt.unyt_array(X, "code_length") + b_eng_numer = np.sum(b_energy_func(Z, Y, X, B0) * cell_vols) + + b_eng_anyl_rel_err = np.abs((b_eng - b_eng_anyl) / b_eng_anyl) + b_eng_numer_rel_err = np.abs((b_eng - b_eng_numer) / b_eng_numer) + + if b_eng_anyl_rel_err > b_eng_tol: + print( + f"{label} Analytically Integrated Relative Energy Error: {b_eng_anyl_rel_err} exceeds tolerance {b_eng_tol}", + f"Analytic {'>' if b_eng_anyl > b_eng else '<'} Simulation", + ) + analyze_status = False + if b_eng_numer_rel_err > b_eng_tol: + print( + f"{label} Numerically Integrated Relative Energy Error: {b_eng_numer_rel_err} exceeds tolerance {b_eng_tol}", + f"Numerical {'>' if b_eng_numer > b_eng else '<'} Simulation", + ) + analyze_status = False + + ######################################## + # Check divB + ######################################## + + # FIXME: This computation of the fluxes would work better with 1 ghostzone from the simulation + # Compute cell lengths (note: these are NGridxNBlockSide) + dxf = np.diff(xf, axis=1) + dyf = np.diff(yf, axis=1) + dzf = np.diff(zf, axis=1) + + dBxdx = ( + 0.5 + * (B[0, :, :, :, 2:] - B[0, :, :, :, :-2])[:, 1:-1, 1:-1, :] + / dxf[:, np.newaxis, np.newaxis, 1:-1] + ) + dBydy = ( + 0.5 + * (B[1, :, :, 2:, :] - B[1, :, :, :-2, :])[:, 1:-1, :, 1:-1] + / dyf[:, np.newaxis, 1:-1, np.newaxis] + ) + dBzdz = ( + 0.5 + * (B[2, :, 2:, :, :] - B[2, :, :-2, :, :])[:, :, 1:-1, 1:-1] + / dzf[:, 1:-1, np.newaxis, np.newaxis] + ) + + divB = dBxdx + dBydy + dBzdz + + if np.max(divB) / B0 > self.divB_tol: + print( + f"{label} Max div B Error: Max divB/B0 {np.max(divB)/B0} exceeds tolerance {self.divB_tol}" + ) + analyze_status = False + + return analyze_status diff --git a/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py b/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py index c9eaa4f8..e04cdc60 100644 --- a/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py +++ b/tst/regression/test_suites/cluster_tabular_cooling/cluster_tabular_cooling.py @@ -36,7 +36,6 @@ class TestCase(utils.test_case.TestCaseAbs): def __init__(self): - # Define cluster parameters # Setup units unyt.define_unit("code_length", (1, "Mpc")) @@ -164,12 +163,12 @@ def Prepare(self, parameters, step): f"units/code_length_cgs={self.code_length.in_units('cm').v}", f"units/code_mass_cgs={self.code_mass.in_units('g').v}", f"units/code_time_cgs={self.code_time.in_units('s').v}", - f"problem/cluster/init_uniform_gas=true", - f"problem/cluster/uniform_gas_rho={self.uniform_gas_rho.in_units('code_mass*code_length**-3').v}", - f"problem/cluster/uniform_gas_ux={self.uniform_gas_ux.in_units('code_length*code_time**-1').v}", - f"problem/cluster/uniform_gas_uy={self.uniform_gas_uy.in_units('code_length*code_time**-1').v}", - f"problem/cluster/uniform_gas_uz={self.uniform_gas_uz.in_units('code_length*code_time**-1').v}", - f"problem/cluster/uniform_gas_pres={self.uniform_gas_pres.in_units('code_mass*code_length**-1*code_time**-2').v}", + f"problem/cluster/uniform_gas/init_uniform_gas=true", + f"problem/cluster/uniform_gas/rho={self.uniform_gas_rho.in_units('code_mass*code_length**-3').v}", + f"problem/cluster/uniform_gas/ux={self.uniform_gas_ux.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uy={self.uniform_gas_uy.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/uz={self.uniform_gas_uz.in_units('code_length*code_time**-1').v}", + f"problem/cluster/uniform_gas/pres={self.uniform_gas_pres.in_units('code_mass*code_length**-1*code_time**-2').v}", f"cooling/table_filename={table_filename}", f"cooling/log_temp_col=0", f"cooling/log_lambda_col=1", @@ -322,10 +321,10 @@ def zero_corrected_linf_err(gold, test): } rho = unyt.unyt_array( - prim[:, prim_col_dict["density"]], "code_mass*code_length**-3" + prim[prim_col_dict["density"], :], "code_mass*code_length**-3" ) pres = unyt.unyt_array( - prim[:, prim_col_dict["pressure"]], + prim[prim_col_dict["pressure"], :], "code_mass*code_length**-1*code_time**-2", )