Merge branch 'development' into add_1d_cylinder

.github/workflows/codeql.yml

@@ -31,16 +31,22 @@ jobs:
  - name: Checkout
  uses: actions/checkout@v4
 
+ - uses: actions/setup-python@v5
+ name: Install Python
+ with:
+ python-version: '3.x'
+
  - name: Install Packages (C++)
  if: ${{ matrix.language == 'cpp' }}
  run: |
  sudo apt-get update
  sudo apt-get install --yes cmake openmpi-bin libopenmpi-dev libhdf5-openmpi-dev libadios-openmpi-dev ccache
 
  python -m pip install --upgrade pip
+ python -m pip install --upgrade pipx
  python -m pip install --upgrade wheel
  python -m pip install --upgrade cmake
- export CMAKE="$HOME/.local/bin/cmake" && echo "CMAKE=$CMAKE" >> $GITHUB_ENV
+ python -m pipx install cmake
 
  - name: Set Up Cache
  if: ${{ matrix.language == 'cpp' }}
@@ -54,7 +60,7 @@ jobs:
  - name: Configure (C++)
  if: ${{ matrix.language == 'cpp' }}
  run: |
- $CMAKE -S . -B build -DWarpX_OPENPMD=ON
+ cmake -S . -B build -DWarpX_OPENPMD=ON
 
  - name: Initialize CodeQL
  uses: github/codeql-action/init@v3
@@ -75,15 +81,15 @@ jobs:
  export CCACHE_MAXSIZE=100M
  ccache -z
 
- $CMAKE --build build -j 4
+ cmake --build build -j 4
 
  ccache -s
  du -hs ~/.cache/ccache
 
  # Make sure CodeQL has something to do
  touch Source/Utils/WarpXVersion.cpp
  export CCACHE_DISABLE=1
- $CMAKE --build build -j 4
+ cmake --build build -j 4
 
  - name: Perform CodeQL Analysis
  uses: github/codeql-action/analyze@v3

.github/workflows/cuda.yml

@@ -135,7 +135,7 @@ jobs:
  which nvcc || echo "nvcc not in PATH!"
 
  git clone https://github.com/AMReX-Codes/amrex.git ../amrex
- cd ../amrex && git checkout --detach 24.10 && cd -
+ cd ../amrex && git checkout --detach e1222803739ed2342b9ff6fc2d57316ff0d6cb0c && cd -
  make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4
 
  ccache -s

.github/workflows/intel.yml

@@ -184,6 +184,7 @@ jobs:
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DWarpX_COMPUTE=SYCL \
  -DWarpX_EB=ON \
+ -DWarpX_FFT=ON \
  -DWarpX_PYTHON=ON \
  -DWarpX_MPI=OFF \
  -DWarpX_OPENPMD=ON \

.github/workflows/macos.yml

@@ -22,13 +22,16 @@ jobs:
  #CMAKE_GENERATOR: Ninja
  steps:
  - uses: actions/checkout@v4
- - name: install dependencies
+ - uses: actions/setup-python@v5
+ name: Install Python
+ with:
+ python-version: '3.x'
+ - name: install brew dependencies
  run: |
  set +e
  brew unlink gcc
  brew update
  brew upgrade || true
- brew install --overwrite python
  brew install ccache
  brew install fftw
  brew install libomp
@@ -39,12 +42,12 @@ jobs:
  set -e
  brew tap openpmd/openpmd
  brew install openpmd-api
-
- python3 -m venv py-venv
- source py-venv/bin/activate
+ - name: install pip dependencies
+ run: |
  python3 -m pip install --upgrade pip
  python3 -m pip install --upgrade build packaging setuptools wheel
  python3 -m pip install --upgrade mpi4py
+ python3 -m pip install --upgrade -r Regression/requirements.txt
  - name: CCache Cache
  uses: actions/cache@v4
  with:
@@ -60,8 +63,6 @@ jobs:
  export CCACHE_SLOPPINESS=time_macros
  ccache -z
 
- source py-venv/bin/activate
-
  cmake -S . -B build_dp \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
  -DWarpX_EB=OFF \
@@ -71,7 +72,6 @@ jobs:
 
  cmake -S . -B build_sp \
  -DCMAKE_VERBOSE_MAKEFILE=ON \
- -DPython_EXECUTABLE=$(which python3) \
  -DWarpX_EB=OFF \
  -DWarpX_PYTHON=ON \
  -DWarpX_OPENPMD=ON \
@@ -85,7 +85,6 @@ jobs:
 
  - name: run pywarpx
  run: |
- source py-venv/bin/activate
  export OMP_NUM_THREADS=1
 
  mpirun -n 2 Examples/Physics_applications/laser_acceleration/inputs_test_3d_laser_acceleration_picmi.py

.pre-commit-config.yaml

@@ -18,7 +18,7 @@ exclude: '^share/openPMD/thirdParty'
 # See https://pre-commit.com/hooks.html for more hooks
 repos:
 - repo: https://github.com/pre-commit/pre-commit-hooks
- rev: v4.6.0
+ rev: v5.0.0
  hooks:
  - id: trailing-whitespace
  args: [--markdown-linebreak-ext=md]
@@ -69,7 +69,7 @@ repos:
 # Python: Ruff linter & formatter
 # https://docs.astral.sh/ruff/
 - repo: https://github.com/astral-sh/ruff-pre-commit
- rev: v0.6.8
+ rev: v0.6.9
  hooks:
  # Run the linter
  - id: ruff

CMakeLists.txt

@@ -724,9 +724,9 @@ endforeach()
 #
 if(WarpX_PYTHON)
  set(PY_PIP_OPTIONS "-v" CACHE STRING
- "Additional parameters to pass to `pip`")
+ "Additional parameters to pass to `pip` as ; separated list")
  set(PY_PIP_INSTALL_OPTIONS "" CACHE STRING
- "Additional parameters to pass to `pip install`")
+ "Additional parameters to pass to `pip install` as ; separated list")
 
  # ensure all targets are built before we package them in a wheel
  set(pyWarpX_INSTALL_TARGET_NAMES)
@@ -749,7 +749,8 @@ if(WarpX_PYTHON)
  ${CMAKE_COMMAND} -E rm -f -r warpx-whl
  COMMAND
  ${CMAKE_COMMAND} -E env PYWARPX_LIB_DIR=$<TARGET_FILE_DIR:pyWarpX_${WarpX_DIMS_LAST}>
- ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} wheel --no-build-isolation --no-deps --wheel-dir=warpx-whl ${WarpX_SOURCE_DIR}
+ ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} wheel --no-build-isolation --no-deps --wheel-dir=warpx-whl "${WarpX_SOURCE_DIR}"
+ COMMAND_EXPAND_LISTS VERBATIM
  WORKING_DIRECTORY
  ${WarpX_BINARY_DIR}
  DEPENDS
@@ -764,6 +765,7 @@ if(WarpX_PYTHON)
  endif()
  add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_requirements
  ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install ${PY_PIP_INSTALL_OPTIONS} -r "${WarpX_SOURCE_DIR}/${pyWarpX_REQUIREMENT_FILE}"
+ COMMAND_EXPAND_LISTS VERBATIM
  WORKING_DIRECTORY
  ${WarpX_BINARY_DIR}
  )
@@ -781,6 +783,7 @@ if(WarpX_PYTHON)
  add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install
  ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
  ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx
+ COMMAND_EXPAND_LISTS VERBATIM
  WORKING_DIRECTORY
  ${WarpX_BINARY_DIR}
  DEPENDS
@@ -794,6 +797,7 @@ if(WarpX_PYTHON)
  add_custom_target(${WarpX_CUSTOM_TARGET_PREFIX}pip_install_nodeps
  ${CMAKE_COMMAND} -E env WARPX_MPI=${WarpX_MPI}
  ${Python_EXECUTABLE} -m pip ${PY_PIP_OPTIONS} install --force-reinstall --no-index --no-deps ${PY_PIP_INSTALL_OPTIONS} --find-links=warpx-whl pywarpx
+ COMMAND_EXPAND_LISTS VERBATIM
  WORKING_DIRECTORY
  ${WarpX_BINARY_DIR}
  DEPENDS

Docs/source/developers/checksum.rst

@@ -1,32 +1,36 @@
 .. _developers-checksum:
 
-Checksum regression tests
-=========================
+Checksums on Tests
+==================
 
-WarpX has checksum regression tests: as part of CI testing, when running a given test, the checksum module computes one aggregated number per field (``Ex_checksum = np.sum(np.abs(Ex))``) and compares it to a reference (benchmark). This should be sensitive enough to make the test fail if your PR causes a significant difference, print meaningful error messages, and give you a chance to fix a bug or reset the benchmark if needed.
+When running an automated test, we often compare the data of final time step of the test with expected values to catch accidental changes.
+Instead of relying on reference files that we would have to store in their full size, we calculate an aggregate checksum.
 
-The checksum module is located in ``Regression/Checksum/``, and the benchmarks are stored as human-readable `JSON <https://www.json.org/json-en.html>`__ files in ``Regression/Checksum/benchmarks_json/``, with one file per benchmark (for instance, test ``Langmuir_2d`` has a corresponding benchmark ``Regression/Checksum/benchmarks_json/Langmuir_2d.json``).
+For this purpose, the checksum Python module computes one aggregated number per field (e.g., the sum of the absolute values of the array elements) and compares it to a reference value (benchmark).
+This should be sensitive enough to make the test fail if your PR causes a significant difference, print meaningful error messages, and give you a chance to fix a bug or reset the benchmark if needed.
 
-For more details on the implementation, the Python files in ``Regression/Checksum/`` should be well documented.
+The checksum module is located in ``Regression/Checksum/``, and the benchmarks are stored as human-readable `JSON <https://www.json.org/json-en.html>`__ files in ``Regression/Checksum/benchmarks_json/``, with one file per benchmark (for example, the test ``test_2d_langmuir_multi`` has a corresponding benchmark ``Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json``).
 
-From a user point of view, you should only need to use ``checksumAPI.py``. It contains Python functions that can be imported and used from an analysis Python script. It can also be executed directly as a Python script. Here are recipes for the main tasks related to checksum regression tests in WarpX CI.
+For more details on the implementation, please refer to the Python implementation in ``Regression/Checksum/``.
 
-Include a checksum regression test in an analysis Python script
----------------------------------------------------------------
+From a user point of view, you should only need to use ``checksumAPI.py``, which contains Python functions that can be imported and used from an analysis Python script or can also be executed directly as a Python script.
+
+How to compare checksums in your analysis script
+------------------------------------------------
 
 This relies on the function ``evaluate_checksum``:
 
 .. autofunction:: checksumAPI.evaluate_checksum
 
-For an example, see
+Here's an example:
 
-.. literalinclude:: ../../../Examples/analysis_default_regression.py
+.. literalinclude:: ../../../Examples/Tests/embedded_circle/analysis.py
  :language: python
 
-This can also be included in an existing analysis script. Note that the plotfile must be ``<test name>_plt?????``, as is generated by the CI framework.
+This can also be included as part of an existing analysis script.
 
-Evaluate a checksum regression test from a bash terminal
---------------------------------------------------------
+How to evaluate checksums from the command line
+-----------------------------------------------
 
 You can execute ``checksumAPI.py`` as a Python script for that, and pass the plotfile that you want to evaluate, as well as the test name (so the script knows which benchmark to compare it to).
 
@@ -41,11 +45,8 @@ See additional options
 * ``--rtol`` relative tolerance for the comparison
 * ``--atol`` absolute tolerance for the comparison (a sum of both is used by ``numpy.isclose()``)
 
-Create/Reset a benchmark with new values that you know are correct
-------------------------------------------------------------------
-
-Create/Reset a benchmark from a plotfile generated locally
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+How to create or reset checksums with local benchmark values
+------------------------------------------------------------
 
 This is using ``checksumAPI.py`` as a Python script.
 
@@ -65,8 +66,8 @@ Since this will automatically change the JSON file stored on the repo, make a se
  git add <test name>.json
  git commit -m "reset benchmark for <test name> because ..." --author="Tools <[email protected]>"
 
-Automated reset of a list of test benchmarks
---------------------------------------------
+How to reset checksums for a list of tests with local benchmark values
+----------------------------------------------------------------------
 
 If you set the environment variable ``export CHECKSUM_RESET=ON`` before running tests that are compared against existing benchmarks, the test analysis will reset the benchmarks to the new values, skipping the comparison.
 
@@ -80,8 +81,8 @@ With `CTest <https://cmake.org/cmake/help/latest/manual/ctest.1.html>`__ (coming
  # ... check and commit changes ...
 
 
-Reset a benchmark from the Azure pipeline output on Github
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+How to reset checksums for a list of tests with benchmark values from the Azure pipeline output
+-----------------------------------------------------------------------------------------------
 
 Alternatively, the benchmarks can be reset using the output of the Azure continuous intergration (CI) tests on Github. The output can be accessed by following the steps below:
 

Docs/source/developers/testing.rst

@@ -1,35 +1,45 @@
 .. _developers-testing:
 
-Testing the code
+Testing the Code
 ================
 
-When adding a new feature, you want to make sure that (i) you did not break the existing code and (ii) your contribution gives correct results. While the code is tested regularly remotely (on the cloud when commits are pushed to an open PR, and every night on local clusters), it can also be useful to run tests on your custom input file. This section details how to use both automated and custom tests.
+When proposing a code change, you want to make sure that
 
-Continuous Integration in WarpX
--------------------------------
+* the code change does not break the existing code;
+* the code change gives correct results (numerics, physics, etc.).
 
-Configuration
-^^^^^^^^^^^^^
+WarpX follows the continuous integration (CI) software development practice, where automated builds and tests are run after merging code changes into the main branch.
 
-Our regression tests are run with `CTest <https://cmake.org/cmake/help/book/mastering-cmake/chapter/Testing%20With%20CMake%20and%20CTest.html#>`__, an executable that comes with CMake.
-
-The test suite is ready to run once you have configured and built WarpX with CMake, following the instructions that you find in our :ref:`Users <install-cmake>` or :ref:`Developers <building-cmake>` sections.
-
-A test that requires a build option that was not configured and built will be skipped automatically. For example, if you configure and build WarpX in 1D only, any test of dimensionality other than 1D, which would require WarpX to be configured and built in the corresponding dimensionality, will be skipped automatically.
+While the code is tested regularly remotely (on the cloud when commits are pushed to an open PR, and every night on local clusters), it can also be useful to run tests on your custom input file.
 
 How to run pre-commit tests locally
 -----------------------------------
 
-When proposing code changes to Warpx, we perform a couple of automated stylistic and correctness checks on the code change.
-You can run those locally before you push to save some time, install them once like this:
+First, when proposing a code change, we perform a couple of automated style and correctness checks.
+
+If you install the ``pre-commit`` tool on your local machine via
 
 .. code-block:: sh
 
  python -m pip install -U pre-commit
  pre-commit install
 
+the style and correctness checks will run automatically on your local machine, after you commit the change and before you push.
+
+If you do not install the ``pre-commit`` tool on your local machine, these checks will run automatically as part of our CI workflows and a commit containing style and correctness changes might be added automatically to your branch.
+In that case, you will need to pull that automated commit before pushing further changes.
+
 See `pre-commit.com <https://pre-commit.com>`__ and our ``.pre-commit-config.yaml`` file in the repository for more details.
 
+How to configure the automated tests
+------------------------------------
+
+Our regression tests are run with `CTest <https://cmake.org/cmake/help/book/mastering-cmake/chapter/Testing%20With%20CMake%20and%20CTest.html#>`__, an executable that comes with CMake.
+
+The test suite is ready to run once you have configured and built WarpX with CMake, following the instructions that you find in our :ref:`Users <install-cmake>` or :ref:`Developers <building-cmake>` sections.
+
+A test that requires a build option that was not configured and built will be skipped automatically. For example, if you configure and build WarpX in 1D only, any test of dimensionality other than 1D, which would require WarpX to be configured and built in the corresponding dimensionality, will be skipped automatically.
+
 How to run automated tests locally
 ----------------------------------
 
@@ -107,7 +117,15 @@ If you modify the code base locally and want to assess the effects of your code
 How to add automated tests
 --------------------------
 
-As mentioned above, the input files and scripts used by the automated tests can be found in the `Examples <https://github.com/ECP-WarpX/WarpX/tree/development/Examples>`__ directory, either under `Physics_applications <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Physics_applications>`__ or `Tests <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Tests>`__.
+An automated test typically consists of the following components:
+
+* input file or PICMI input script;
+* analysis script;
+* checksum file.
+
+To learn more about how to use checksums in automated tests, please see the corresponding section :ref:`Checksums on Tests <developers-checksum>`.
+
+As mentioned above, the input files and scripts used by the automated tests can be found in the `Examples <https://github.com/ECP-WarpX/WarpX/tree/development/Examples>`__ directory, under either `Physics_applications <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Physics_applications>`__ or `Tests <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Tests>`__.
 
 Each test directory must contain a file named ``CMakeLists.txt`` where all tests associated with the input files and scripts in that directory must be listed.
 
@@ -173,7 +191,8 @@ A new test can be added by adding a corresponding entry in ``CMakeLists.txt`` as
 
 If you need a new Python package dependency for testing, please add it in `Regression/requirements.txt <https://github.com/ECP-WarpX/WarpX/blob/development/Regression/requirements.txt>`__.
 
-Sometimes two or more tests share a large number of input parameters. The shared input parameters can be collected in a "base" input file that can be passed as a runtime parameter in the actual test input files through the parameter ``FILE``.
+Sometimes two or more tests share a large number of input parameters.
+The shared input parameters can be collected in a "base" input file that can be passed as a runtime parameter in the actual test input files through the parameter ``FILE``.
 
 If the new test is added in a new directory that did not exist before, please add the name of that directory with the command ``add_subdirectory`` in `Physics_applications/CMakeLists.txt <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Physics_applications/CMakeLists.txt>`__ or `Tests/CMakeLists.txt <https://github.com/ECP-WarpX/WarpX/tree/development/Examples/Tests/CMakeLists.txt>`__, depending on where the new test directory is located.
 

Docs/source/developers/workflows.rst

@@ -8,7 +8,7 @@ Workflows
 
  profiling
  testing
- documentation
  checksum
- local_compile
  run_clang_tidy_locally
+ local_compile
+ documentation