Skip to content

Commit

Permalink
PopIII Simulation (#368)
Browse files Browse the repository at this point in the history
This PR will add a PopIII simulation to quokka. It will only test that
the simulation starts successfully and runs for a few timesteps - it's
not meant to be a _production_ simulation.

---------

Co-authored-by: Piyush Sharda <[email protected]>
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Piyush Sharda <[email protected]>
Co-authored-by: Piyush Sharda <[email protected]>
Co-authored-by: Ben Wibking <[email protected]>
Co-authored-by: Piyush Sharda <[email protected]>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
  • Loading branch information
8 people authored Feb 5, 2024
1 parent ae42169 commit 93686aa
Show file tree
Hide file tree
Showing 14 changed files with 767 additions and 20 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/clang-tidy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
config_file: src/.clang-tidy
build_dir: build
apt_packages: libopenmpi-dev,libhdf5-mpi-dev,python3-dev,python3-numpy,python3-matplotlib
cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF
cmake_command: cmake . -B build -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DQUOKKA_PYTHON=ON -DQUOKKA_OPENPMD=ON -DopenPMD_USE_ADIOS2=OFF -DAMReX_SPACEDIM=3
split_workflow: true

# Uploads an artefact containing clang_fixes.json
Expand Down
2 changes: 1 addition & 1 deletion .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,4 +24,4 @@ ci:
autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
autoupdate_schedule: weekly
skip: []
submodules: false
submodules: false
13 changes: 7 additions & 6 deletions src/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,14 @@ add_subdirectory(RadhydroUniformAdvecting)
add_subdirectory(RadhydroPulse)
add_subdirectory(RadhydroPulseMG)

add_subdirectory(ODEIntegration)
add_subdirectory(BinaryOrbitCIC)
add_subdirectory(Cooling)
add_subdirectory(ShockCloud)
add_subdirectory(PassiveScalar)
add_subdirectory(FCQuantities)
add_subdirectory(SphericalCollapse)
add_subdirectory(NSCBC)
add_subdirectory(StarCluster)
add_subdirectory(ODEIntegration)
add_subdirectory(PassiveScalar)
add_subdirectory(PrimordialChem)
add_subdirectory(BinaryOrbitCIC)
add_subdirectory(PopIII)
add_subdirectory(ShockCloud)
add_subdirectory(StarCluster)
add_subdirectory(SphericalCollapse)
16 changes: 15 additions & 1 deletion src/Chemistry.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,13 +31,18 @@ namespace quokka::chemistry

AMREX_GPU_DEVICE void chemburner(burn_t &chemstate, const Real dt);

template <typename problem_t> void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed)
template <typename problem_t> void computeChemistry(amrex::MultiFab &mf, const Real dt, const Real max_density_allowed, const Real min_density_allowed)
{

const BL_PROFILE("Chemistry::computeChemistry()");
for (amrex::MFIter iter(mf); iter.isValid(); ++iter) {
const amrex::Box &indexRange = iter.validbox();
auto const &state = mf.array(iter);

if (dt < 0) {
amrex::Abort("Cannot do chemistry with dt < 0!");
}

amrex::ParallelFor(indexRange, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept {
const Real rho = state(i, j, k, HydroSystem<problem_t>::density_index);
const Real xmom = state(i, j, k, HydroSystem<problem_t>::x1Momentum_index);
Expand All @@ -64,6 +69,11 @@ template <typename problem_t> void computeChemistry(amrex::MultiFab &mf, const R
chemstate.xn[nn] = inmfracs[nn];
}

// dont do chemistry in cells with densities below the minimum density specified
if (rho < min_density_allowed) {
return;
}

// stop the test if we have reached very high densities
if (rho > max_density_allowed) {
amrex::Abort("Density exceeded max_density_allowed!");
Expand All @@ -82,6 +92,10 @@ template <typename problem_t> void computeChemistry(amrex::MultiFab &mf, const R
// which would otherwise slow down compilation due to the large RHS file
chemburner(chemstate, dt);

if (std::isnan(chemstate.xn[0]) || std::isnan(chemstate.rho)) {
amrex::Abort("Burner returned NAN");
}

if (!chemstate.success) {
amrex::Abort("VODE integration was unsuccessful!");
}
Expand Down
78 changes: 78 additions & 0 deletions src/PopIII/CMakeLists.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
if (AMReX_SPACEDIM EQUAL 3)
# Define a custom target that runs the Python script to produce the input perturbations file


set(microphysics_network_name "primordial_chem") #this will override network_name to primordial_chem for this directory only
setup_target_for_microphysics_compilation(${microphysics_network_name} "${CMAKE_CURRENT_BINARY_DIR}/")

#use the BEFORE keyword so that these files get priority in compilation for targets in this directory
#this is critical to ensure the correct Microphysics files are linked to primordial chem target
include_directories(BEFORE ${primordial_chem_dirs} "${CMAKE_CURRENT_BINARY_DIR}/" "includes/extern_parameters.H" "includes/network_properties.H")

add_executable(popiii popiii.cpp ../TurbDataReader.cpp ../main.cpp "${openPMDSources}" ../GrackleDataReader.cpp ../CloudyCooling.cpp ../Chemistry.cpp ${primordial_chem_sources})
target_compile_definitions(popiii PUBLIC PRIMORDIAL_CHEM) #this will add #define PRIMORDIAL_CHEM

if(AMReX_GPU_BACKEND MATCHES "CUDA")
setup_target_for_cuda_compilation(popiii)
endif()

#need h5py to run perturbations.py file below
#purpose of this code is to check whether the h5py python package is installed
execute_process(
COMMAND Python3::Interpreter -c "h5py"
RESULT_VARIABLE EXIT_CODE
OUTPUT_QUIET
)

#re-run cmake if this file is changed
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/tests/PopIII.in)

# Read the 'PopIII.in' file line by line
file(READ ${CMAKE_SOURCE_DIR}/tests/PopIII.in pop_in_contents)

# Split the file contents into lines
string(REPLACE "\n" ";" pop_in_lines "${pop_in_contents}")

# Iterate through the lines and look for lines starting with 'amr.n_cell'
foreach(line IN LISTS pop_in_lines)
if (line MATCHES "^amr\\.n_cell.*")
set(match_line "${line}")
break()
endif()
endforeach()

if (match_line)
message(STATUS "picked line is ${match_line}")
# Use a regular expression to find consecutive digit
string(REGEX MATCHALL "[0-9]+" digits_list "${match_line}")
# Iterate through the matched digits and extract the first one
list(GET digits_list 0 first_digits)
# Check if matched digits were found
if (first_digits)
#first_digits give the box size, but you also want box size / 2 as kmax in the script below
math(EXPR int_first_digits "${first_digits}")
# Check if the conversion was successful
if (int_first_digits)
# Divide the integer by two
math(EXPR result "${int_first_digits} / 2")
else()
message(FATAL_ERROR "Conversion to integer failed, so cannot find kmax!")
endif()
message(STATUS "Will create initial perturbations of box size ${first_digits} and kmax ${result}")

else()
message(FATAL_ERROR "No box size found to create initial perturbations for!")
endif()
else()
message(FATAL_ERROR "No matching line found in ${CMAKE_SOURCE_DIR}/tests/PopIII.in!")
endif()


add_test(NAME ComputePerturbations COMMAND python3 ${CMAKE_SOURCE_DIR}/src/perturbation.py --kmin=2 --kmax=${result} --size=${first_digits} --alpha=1.8 --f_solenoidal=0.66667 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
add_test(NAME PopIII COMMAND popiii PopIII.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
set_tests_properties(ComputePerturbations PROPERTIES FIXTURES_SETUP PopIII_fixture)
set_tests_properties(PopIII PROPERTIES FIXTURES_REQUIRED PopIII_fixture)

# AMR test only works on Setonix because Gadi and avatar do not have enough memory per GPU
# add_test(NAME PopIIIAMR COMMAND popiii popiii_AMR.in WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/tests)
endif()
30 changes: 30 additions & 0 deletions src/PopIII/ascent_actions.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
-
action: "add_pipelines"
pipelines:
pl1:
f2:
type: "slice"
params:
point:
x: 0.
y: 0.
z: 0.
normal:
x: 0.0
y: 0.0
z: 1.0
-
action: "add_scenes"
scenes:
s1:
plots:
p1:
type: "pseudocolor"
field: "log_density"
pipeline: "pl1"
renders:
r1:
image_prefix: "dens%05d"
annotations: "true"
camera:
zoom: 1.5
Loading

0 comments on commit 93686aa

Please sign in to comment.