Skip to content

Commit

Permalink
Merge branch 'development' into exact_riemann_merge
Browse files Browse the repository at this point in the history
  • Loading branch information
zingale authored Jul 27, 2024
2 parents d060b06 + d8dd29e commit 6f6083b
Show file tree
Hide file tree
Showing 45 changed files with 3,352 additions and 119 deletions.
1 change: 1 addition & 0 deletions .github/workflows/good_defines.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
ALLOW_GPU_PRINTF
AMREX_DEBUG
AMREX_PARTICLES
AMREX_SPACEDIM
Expand Down
31 changes: 31 additions & 0 deletions Docs/source/mpi_plus_x.rst
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,37 @@ To enable this, compile with::
USE_HIP = TRUE


Printing Warnings from GPU Kernels
==================================

.. index:: USE_GPU_PRINTF

Castro will output warnings if several assumptions are violated (often
triggering a retry in the process). On GPUs, printing from a kernel
(using ``printf()``) can increase the number of registers a kernel needs,
causing performance problems. As a result, warnings are disabled by
wrapping them in ``#ifndef AMREX_USE_GPU``.

However, for debugging GPU runs, sometimes we want to see these
warnings. The build option ``USE_GPU_PRINTF=TRUE`` will enable these
(by setting the preprocessor flag ``ALLOW_GPU_PRINTF``).

.. note::

Not every warning has been enabled for GPUs.

.. tip::

On AMD architectures, it seems necessary to use unbuffered I/O. This
can be accomplished in the job submission script (for SLURM) by doing

::

srun -u ./Castro...




Working at Supercomputing Centers
=================================

Expand Down
4 changes: 4 additions & 0 deletions Exec/Make.Castro
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,10 @@ ifeq ($(USE_GPU),TRUE)
endif
endif

ifeq ($(USE_GPU_PRINTF),TRUE)
DEFINES += -DALLOW_GPU_PRINTF
endif

CASTRO_AUTO_SOURCE_DIR := $(TmpBuildDir)/castro_sources/$(optionsSuffix).EXE


Expand Down
132 changes: 66 additions & 66 deletions Exec/science/Detonation/ci-benchmarks/sdc_det_plt00040_extrema.out
Original file line number Diff line number Diff line change
@@ -1,79 +1,79 @@
plotfile = det_x_plt00040
time = 5.1558159140336702e-06
variables minimum value maximum value
density 185260165.55 216617920.63
xmom -63255523503 2.9519346608e+16
density 185259874.21 216582994.41
xmom -87234240329 2.9512596512e+16
ymom 0 0
zmom 0 0
rho_E 1.3062473821e+26 2.7891343298e+26
rho_e 1.3062473821e+26 2.7748842906e+26
Temp 50000000.026 7845854931.5
rho_H1 2.1204369735e-22 0.020000115169
rho_He3 0.0017224919811 0.02102205401
rho_He4 94366339.651 200001593.93
rho_C12 0.020000000216 21203829.102
rho_N14 1.999999523e-22 9.5660401467
rho_O16 0.02 19209.488217
rho_Ne20 0.02 3464.1596069
rho_Mg24 0.02 23322.458406
rho_Si28 0.02 2015700.9685
rho_S32 0.02 1656602.4644
rho_Ar36 0.02 822007.67426
rho_Ca40 0.02 724910.24417
rho_Ti44 0.02 34179.295535
rho_Cr48 0.02 78164.189237
rho_Fe52 0.02 278034.23746
rho_Fe54 0.02 95026946.079
rho_Ni56 0.02 2239485.5719
rho_n 2.1204369735e-22 234580.3659
rho_p 0.019999995438 3610755.3081
rho_enuc -4.6808330292e+29 3.576498348e+32
pressure 5.5236728651e+25 1.1610548795e+26
kineng 0 2.0467474279e+24
soundspeed 612864631.21 895226097.3
Gamma_1 1.3599756287 1.3819899024
MachNumber 0 0.16110770916
uplusc 612864631.21 999815567.76
uminusc -895226406.71 -612860598.79
entropy 98214767.758 336273469.46
rho_E 1.3062473821e+26 2.7893784326e+26
rho_e 1.3062473821e+26 2.7751188263e+26
Temp 50000000.026 7845854662.6
rho_H1 2.1207307865e-22 0.020000135717
rho_He3 0.0017225013714 0.02102428438
rho_He4 94358881.486 200001799.59
rho_C12 0.020000000216 21276964.112
rho_N14 1.9999983474e-22 197.76685905
rho_O16 0.02 19209.489937
rho_Ne20 0.02 4937.0489429
rho_Mg24 0.02 23322.104606
rho_Si28 0.02 2015343.4134
rho_S32 0.02 1656264.8265
rho_Ar36 0.02 821829.69632
rho_Ca40 0.02 725030.13946
rho_Ti44 0.02 34172.699418
rho_Cr48 0.02 78131.560865
rho_Fe52 0.02 277298.50875
rho_Fe54 0.02 94994585.838
rho_Ni56 0.02 2238978.4894
rho_n 2.1207307865e-22 234580.14761
rho_p 0.019999995438 3609721.5282
rho_enuc -4.6900452682e+29 3.5791669148e+32
pressure 5.5236728651e+25 1.1610544319e+26
kineng 0 2.0455630311e+24
soundspeed 612864631.21 895226067.36
Gamma_1 1.3599756137 1.3820271358
MachNumber 0 0.16114390973
uplusc 612864631.21 999569362.18
uminusc -895226468.14 -612860078.6
entropy 98214767.758 336273451.62
magvort 0 0
divu -97879.470548 34017.033805
eint_E 6.5312369103e+17 1.380434464e+18
eint_e 6.5312369103e+17 1.380434464e+18
logden 8.2677820478 8.3356943826
StateErr_0 185260165.55 216617920.63
StateErr_1 50000000.026 7845854931.5
divu -97746.019905 33801.828895
eint_E 6.5312369103e+17 1.3804343879e+18
eint_e 6.5312369103e+17 1.3804343879e+18
logden 8.2677813648 8.3356243538
StateErr_0 185259874.21 216582994.41
StateErr_1 50000000.026 7845854662.6
StateErr_2 1e-30 9.9999779324e-11
X(H1) 1e-30 9.9999779324e-11
X(He3) 8.9861142144e-12 9.9999601254e-11
X(He4) 0.48402941802 0.9999999982
X(C12) 1.0000000108e-10 0.09999745037
X(N14) 1e-30 4.5113532098e-08
X(O16) 1e-10 9.6047388579e-05
X(Ne20) 1e-10 1.6337008127e-05
X(Mg24) 1e-10 0.00011119757992
X(Si28) 1e-10 0.010055548658
X(S32) 1e-10 0.0083462743796
X(Ar36) 1e-10 0.0041650262219
X(Ca40) 1e-10 0.0037107481152
X(Ti44) 1e-10 0.00017303069502
X(Cr48) 1e-10 0.00040092438857
X(Fe52) 1e-10 0.0013387480381
X(Fe54) 1e-10 0.46489869655
X(Ni56) 1e-10 0.010338413209
X(n) 1e-30 0.0011729011118
X(p) 9.999997703e-11 0.017599035758
abar 4.000000001 6.7313315116
Ye 0.49998670822 0.50001557428
x_velocity -316.27744461 138671594.27
X(He3) 8.9857380636e-12 9.9999601254e-11
X(He4) 0.48403393423 0.9999999982
X(C12) 1.0000000108e-10 0.10032845398
X(N14) 1e-30 9.3254108586e-07
X(O16) 1e-10 9.604742181e-05
X(Ne20) 1e-10 2.3279941869e-05
X(Mg24) 1e-10 0.00011116921103
X(Si28) 1e-10 0.01005607083
X(S32) 1e-10 0.0083471961707
X(Ar36) 1e-10 0.0041657025725
X(Ca40) 1e-10 0.0037098568497
X(Ti44) 1e-10 0.00017298041086
X(Cr48) 1e-10 0.00040079244474
X(Fe52) 1e-10 0.0013382570124
X(Fe54) 1e-10 0.46490351089
X(Ni56) 1e-10 0.010337739099
X(n) 1e-30 0.0011729003976
X(p) 9.9999977071e-11 0.017596365447
abar 4.000000001 6.7314014168
Ye 0.49998670317 0.50001557187
x_velocity -436.17123215 138623047.3
y_velocity 0 0
z_velocity 0 0
t_sound_t_enuc 3.4411968038e-13 0.97442435993
enuc -2.4887686639e+21 1.6510630042e+24
magvel 0 138671594.27
radvel -316.27744461 138671594.27
circvel 0 2.4494897428
magmom 0 2.9519346608e+16
t_sound_t_enuc 3.4412406685e-13 0.97531384976
enuc -2.4936491578e+21 1.6525613771e+24
magvel 0 138623047.3
radvel -436.17123215 138623047.3
circvel 0 2
magmom 0 2.9512596512e+16
angular_momentum_x 0 0
angular_momentum_y 0 0
angular_momentum_z 0 0
Expand Down
47 changes: 47 additions & 0 deletions Exec/science/Detonation/shock_paper/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# Shock Burning Experiments

This directory is meant to explore shock burning with detonation. Compile as:

```
make USE_SIMPLIFIED_SDC=TRUE USE_SHOCK_VAR=TRUE NETWORK_DIR=aprox13 -j 4
```

Then the script `setup_runs.py` will setup a suite of simulations with
the following resolutions into separate directories (using the
`inputs-shock-burn.template`):


| resolution | base grid | levels (4x jumps) |
| ------------ | ----------- | ------------------- |
| 24 km | 48 | 1 |
| 12 km | 96 | 1 |
| 6 km | 192 | 1 |
| 3 km | 384 | 1 |
| 1.5 km | 768 | 1 |
| 0.1875 km | 6144 | 1 |
| 2343.74 cm | 12288 | 2 |

you can set the value of the shock detection threshold there
and the directory names will reflect that setting.

## plotting

The following scripts can make useful plots (some use the
`detonation.py` module as support):

* `det_speed_comp.py` : plot detonation speed vs. resolution, using
simple differencing to estimate the detonation speed from the last 3
plotfiles.

* `profile_compare.py` : given a list of pruns (from different
resolutions), make a plot showing all of their profiles together.

* `profiles.py` : for a single run, plot profiles of T and enuc for
several times.

* `show_shock_flag.py` : simply plot the shock variable on top of T
and enuc profiles.

* `zoom_summary.py` : given a list of runs (from different
resolutions), plot the last plotfile from each run, zoomed in on
where the peak energy generation is.
74 changes: 74 additions & 0 deletions Exec/science/Detonation/shock_paper/det_speed_comp.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
#!/usr/bin/env python

import matplotlib.pyplot as plt

import detonation

runs = [("res24.0km", 24),
("res12.0km", 12),
("res6.0km", 6),
("res3.0km", 3),
("res1.5km", 1.5),
("res0.1875km", 0.1875)] #,
#("res0.024km", 0.024)] #,
#("res0.003km", 0.003)]

nsb1_runs = [("res24.0km_noshockburn_1", 24),
("res12.0km_noshockburn_1", 12),
("res6.0km_noshockburn_1", 6),
("res3.0km_noshockburn_1", 3),
("res1.5km_noshockburn_1", 1.5),
("res0.1875km_noshockburn_1", 0.1875)] #,

nsb23_runs = [("res24.0km_noshockburn_0.666", 24),
("res12.0km_noshockburn_0.666", 12),
("res6.0km_noshockburn_0.666", 6),
("res3.0km_noshockburn_0.666", 3),
("res1.5km_noshockburn_0.666", 1.5),
("res0.1875km_noshockburn_0.666", 0.1875)] #,

res = []
v = []
dv = []

for ddir, dx in runs:
res.append(dx)
d = detonation.Detonation(ddir)
v.append(d.v)
dv.append(d.v_sigma)

nsb23_res = []
nsb23_v = []
nsb23_dv = []

for ddir, dx in nsb23_runs:
nsb23_res.append(dx)
d = detonation.Detonation(ddir)
nsb23_v.append(d.v)
nsb23_dv.append(d.v_sigma)

nsb1_res = []
nsb1_v = []
nsb1_dv = []

for ddir, dx in nsb1_runs:
nsb1_res.append(dx)
d = detonation.Detonation(ddir)
nsb1_v.append(d.v)
nsb1_dv.append(d.v_sigma)


fig, ax = plt.subplots()

ax.errorbar(res, v, yerr=dv, fmt="o", label="burning in shocks allowed")
ax.errorbar(nsb23_res, nsb23_v, yerr=nsb23_dv, fmt="d", label="no shock burning (f=2/3)")
ax.errorbar(nsb1_res, nsb1_v, yerr=nsb1_dv, fmt="d", label="no shock burning (f=1)")

ax.set_xlabel(r"$\Delta x$ (km)")
ax.set_ylabel(r"$v_\mathrm{det}$ (cm / s)")

ax.legend()

ax.set_xscale("log")

fig.savefig("det_speeds.png")
Loading

0 comments on commit 6f6083b

Please sign in to comment.