Catalyst support (AMReX-Codes#4011)

## Summary This PR completes the work of @andrewcombs and @c-wetterer-nelson from AMReX-Codes#3444 by adding a docker image that includes ParaView and can be used for testing. It also adds a minor fix a02c1a5 discovered when testing these changes with the current master of WarpX. ## Additional background Supersedes and closes AMReX-Codes#3444 ## Checklist The proposed changes: - [x] fix a bug or incorrect behavior in AMReX - [x] add new capabilities to AMReX - [ ] changes answers in the test suite to more than roundoff level - [ ] are likely to significantly affect the results of downstream AMReX users - [ ] include documentation in the code and/or rst files, if appropriate --------- Co-authored-by: Andrew Combs <[email protected]> Co-authored-by: Corey Wetterer-Nelson <[email protected]>
WeiqunZhang · Jul 30, 2024 · 83ecf62 · 83ecf62
1 parent a8c5255
commit 83ecf62
Show file tree

Hide file tree

Showing 13 changed files with 108 additions and 17 deletions.
diff --git a/.github/workflows/catalyst.yml b/.github/workflows/catalyst.yml
@@ -0,0 +1,33 @@
+name: 🐧 Catalyst
+
+on: [push, pull_request]
+
+concurrency:
+ group: ${{ github.ref }}-${{ github.head_ref }}-insituvis-catalyst
+ cancel-in-progress: true
+
+jobs:
+ catalyst:
+ name: Catalyst
+ runs-on: ubuntu-22.04
+ if: github.event.pull_request.draft == false
+ env:
+ CXX: g++
+ CC: gcc
+ CMAKE_PREFIX: "/opt/conduit;/opt/catalyst"
+ container:
+ image: kitware/paraview:ci-catalyst-amrex-warpx-20240701
+ steps:
+ - uses: actions/checkout@v4
+ - name: Configure
+ run: |
+ cmake -S . -B build \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -DAMReX_ENABLE_TESTS=ON \
+ -DAMReX_FORTRAN=OFF \
+ -DAMReX_CATALYST=ON \
+ -DAMReX_CONDUIT=ON
+ - name: Build
+ run: |
+ cmake --build build -j 4
+
diff --git a/Docs/sphinx_documentation/source/BuildingAMReX.rst b/Docs/sphinx_documentation/source/BuildingAMReX.rst
@@ -508,6 +508,8 @@ The list of available options is reported in the :ref:`table <tab:cmakevar>` bel
  +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
  | AMReX_CONDUIT | Enable Conduit support | NO | YES, NO |
  +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+ | AMReX_CATALYST | Enable Catalyst support | NO | YES, NO |
+ +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
  | AMReX_ASCENT | Enable Ascent support | NO | YES, NO |
  +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
  | AMReX_HYPRE | Enable HYPRE interfaces | NO | YES, NO |

diff --git a/Src/CMakeLists.txt b/Src/CMakeLists.txt
@@ -43,7 +43,6 @@ include(AMReXParallelBackends)
 # Add definitions
 #
 include(AMReXSetDefines)
-
 #
 # Find and link third party libraries if needed
 #
@@ -188,7 +187,7 @@ if (AMReX_SENSEI)
  add_subdirectory(Extern/SENSEI)
 endif ()
 
-if (AMReX_CONDUIT OR AMReX_ASCENT)
+if (AMReX_CONDUIT OR AMReX_ASCENT OR AMReX_CATALYST)
  add_subdirectory(Extern/Conduit)
 endif ()
 

diff --git a/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H b/Src/Extern/Conduit/AMReX_Conduit_Blueprint_ParticlesI.H
@@ -172,25 +172,30 @@ ParticleTileToBlueprint(const ParticleTile<ParticleType,
  } else {
  const auto &soa = ptile.GetStructOfArrays();
 
- // for soa entries, we can use standard strides,
- // since these are contiguous arrays
 
- // id is the first int entry
- conduit::Node &n_f_id = n_fields[topology_name + "_id"];
+ if(!soa.GetIntData().empty())
+ {
 
- n_f_id["topology"] = topology_name;
- n_f_id["association"] = "element";
- n_f_id["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
- num_particles);
+ // for soa entries, we can use standard strides,
+ // since these are contiguous arrays
 
- // cpu is the second int entry
- conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+  // id is the first int entry
+  conduit::Node &n_f_id = n_fields[topology_name + "_id"];
 
- n_f_cpu["topology"] = topology_name;
- n_f_cpu["association"] = "element";
- n_f_cpu["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
- num_particles);
+  n_f_id["topology"] = topology_name;
+  n_f_id["association"] = "element";
+  n_f_id["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
+  num_particles);
 
+ // cpu is the second int entry
+ conduit::Node &n_f_cpu = n_fields[topology_name + "_cpu"];
+
+ n_f_cpu["topology"] = topology_name;
+ n_f_cpu["association"] = "element";
+ n_f_cpu["values"].set_external(const_cast<int*>(&soa.GetIntData(0)[0]),
+ num_particles);
+
+ }
  }
 
  // --------------------------------

diff --git a/Src/Extern/Conduit/CMakeLists.txt b/Src/Extern/Conduit/CMakeLists.txt
@@ -4,6 +4,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
  # TODO: Particles PR merges another file
  target_sources(amrex_${D}d
  PRIVATE
+ AMReX_Conduit_Blueprint_ParticlesI.H
  AMReX_Conduit_Blueprint.H
  AMReX_Conduit_Blueprint.cpp
  )

diff --git a/Src/Extern/Conduit/Make.package b/Src/Extern/Conduit/Make.package
@@ -3,7 +3,7 @@
 #
 
 CEXE_sources += AMReX_Conduit_Blueprint.cpp
-
+CEXE_headers += AMReX_Conduit_Blueprint_ParticlesI.H
 CEXE_headers += AMReX_Conduit_Blueprint.H
 
 VPATH_LOCATIONS += $(AMREX_HOME)/Src/Extern/Conduit

diff --git a/Tools/CMake/AMReXConfig.cmake.in b/Tools/CMake/AMReXConfig.cmake.in
@@ -79,6 +79,7 @@ set(AMReX_PARTICLES_FOUND @AMReX_PARTICLES@)
 set(AMReX_P@AMReX_PARTICLES_PRECISION@_FOUND ON)
 set(AMReX_SENSEI_FOUND @AMReX_SENSEI@)
 set(AMReX_CONDUIT_FOUND @AMReX_CONDUIT@)
+set(AMReX_CATALYST_FOUND @AMReX_CATALYST@)
 set(AMReX_ASCENT_FOUND @AMReX_ASCENT@)
 set(AMReX_HYPRE_FOUND @AMReX_HYPRE@)
 set(AMReX_PETSC_FOUND @AMReX_PETSC@)
@@ -134,6 +135,7 @@ set(AMReX_PARTICLES_PRECISION @AMReX_PARTICLES_PRECISION@)
 set(AMReX_SENSEI @AMReX_SENSEI@)
 set(AMReX_NO_SENSEI_AMR_INST @AMReX_NO_SENSEI_AMR_INST@)
 set(AMReX_CONDUIT @AMReX_CONDUIT@)
+set(AMReX_CATALYST @AMReX_CATALYST@)
 set(AMReX_ASCENT @AMReX_ASCENT@)
 set(AMReX_HYPRE @AMReX_HYPRE@)
 set(AMReX_PETSC @AMReX_PETSC@)
@@ -202,6 +204,10 @@ if (@AMReX_ASCENT@)
  find_dependency(Ascent REQUIRED)
 endif ()
 
+if (@AMReX_CATALYST@)
+ find_dependency(Catalyst REQUIRED)
+endif ()
+
 if (@AMReX_CONDUIT@)
  find_dependency(Conduit REQUIRED)
 endif ()

diff --git a/Tools/CMake/AMReXOptions.cmake b/Tools/CMake/AMReXOptions.cmake
@@ -320,6 +320,11 @@ print_option( AMReX_NO_SENSEI_AMR_INST )
 option( AMReX_CONDUIT "Enable Conduit support" OFF )
 print_option( AMReX_CONDUIT )
 
+# Catalyst
+cmake_dependent_option( AMReX_CATALYST "Enable Catalyst support" OFF 
+ "AMReX_CONDUIT" OFF )
+print_option( AMReX_CATALYST )
+
 # Ascent
 cmake_dependent_option( AMReX_ASCENT "Enable Ascent support" OFF
  "AMReX_CONDUIT" OFF )

diff --git a/Tools/CMake/AMReXSetDefines.cmake b/Tools/CMake/AMReXSetDefines.cmake
@@ -148,6 +148,9 @@ add_amrex_define( AMREX_NO_SENSEI_AMR_INST NO_LEGACY IF AMReX_NO_SENSEI_AMR_INST
 # Conduit Support
 add_amrex_define( AMREX_USE_CONDUIT NO_LEGACY IF AMReX_CONDUIT )
 
+# Catalyst Support
+add_amrex_define( AMREX_USE_CATALYST NO_LEGACY IF AMReX_CATALYST )
+
 # Ascent Support
 add_amrex_define( AMREX_USE_ASCENT NO_LEGACY IF AMReX_ASCENT )
 

diff --git a/Tools/CMake/AMReXThirdPartyLibraries.cmake b/Tools/CMake/AMReXThirdPartyLibraries.cmake
@@ -90,6 +90,15 @@ if (AMReX_ASCENT) # Ascent will find conduit, so check for Ascent first
  endforeach()
 endif ()
 
+#
+# Catalyst
+#
+if (AMReX_CATALYST) 
+ find_package(Catalyst REQUIRED PATHS "$ENV{CATALYST_IMPLEMENTATION_PATHS}")
+ foreach(D IN LISTS AMReX_SPACEDIM)
+ target_link_libraries(amrex_${D}d PUBLIC catalyst::catalyst)
+ endforeach()
+endif ()
 
 #
 # Conduit

diff --git a/Tools/CMake/AMReX_Config_ND.H.in b/Tools/CMake/AMReX_Config_ND.H.in
@@ -42,6 +42,7 @@
 #cmakedefine AMREX_USE_SENSEI_INSITU
 #cmakedefine AMREX_NO_SENSEI_AMR_INST
 #cmakedefine AMREX_USE_CONDUIT
+#cmakedefine AMREX_USE_CATALYST
 #cmakedefine AMREX_USE_ASCENT
 #cmakedefine AMREX_USE_EB
 #cmakedefine AMREX_USE_CUDA

diff --git a/Tools/GNUMake/Make.defs b/Tools/GNUMake/Make.defs
@@ -216,6 +216,12 @@ else
  USE_CONDUIT := FALSE
 endif
 
+ifdef USE_CATALYST
+ USE_CATALYST := $(strip $(USE_CATALYST))
+else
+ USE_CATALYST := FALSE
+endif
+
 ifdef USE_ASCENT
  USE_ASCENT := $(strip $(USE_ASCENT))
 else
@@ -1057,6 +1063,11 @@ ifeq ($(USE_CONDUIT),TRUE)
  include $(AMREX_HOME)/Tools/GNUMake/packages/Make.conduit
 endif
 
+ifeq ($(USE_CATALYST),TRUE)
+ $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst...)
+ include $(AMREX_HOME)/Tools/GNUMake/packages/Make.catalyst
+endif
+
 ifeq ($(USE_ASCENT),TRUE)
  $(info Loading $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent...)
  include $(AMREX_HOME)/Tools/GNUMake/packages/Make.ascent

diff --git a/Tools/GNUMake/packages/Make.catalyst b/Tools/GNUMake/packages/Make.catalyst
@@ -0,0 +1,16 @@
+#########################################################
+# Catalyst (https://gitlab.kitware.com/paraview/catalyst) Support 
+#########################################################
+
+CPPFLAGS += -DAMREX_USE_CATALYST
+
+ifdef CATALYST_DIR
+ INCLUDE_LOCATIONS += $(CATALYST_DIR)/include
+ VPATH_LOCATIONS += $(CATALYST_DIR)/include
+ LIBRARY_LOCATIONS += $(CATALYST_DIR)/lib
+ LIBRARIES += -Wl,-rpath,$(CATALYST_DIR)/lib
+
+ LIBRARIES += -lcatalyst
+
+endif
+