From 57fbee98b8d5efa29ce69b33e8d63c0b099789ff Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 19 Jul 2023 16:04:01 +0200 Subject: [PATCH 01/49] Spack-solvable versions, add tests to Spack workflow --- .github/workflows/spack.yml | 21 ++++++++++----------- doc/contrib/release.rst | 6 +++--- scripts/build_spack_package.sh | 4 ++-- spack/package.py | 11 +++++------ 4 files changed, 20 insertions(+), 22 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index bd5ed162d0..1f306b0cd7 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -28,15 +28,14 @@ jobs: key: spack-cache-${{ matrix.os }}-${{ github.run_id }} restore-keys: spack-cache-${{ matrix.os }}- - - name: Build Arbor's Spack package against the develop branch - run: arbor/scripts/build_spack_package.sh arbor develop + - name: Build Arbor's Spack package against the latest release + run: arbor/scripts/build_spack_package.sh arbor latest_release - # build_spack_package.sh only builds, does not install, therefore can't run Python code. - # - name: Run Python examples - # run: | - # cd arbor - # scripts/run_python_examples.sh - # - name: Test executables - # run: | - # cd arbor - # scripts/test_executables.sh + - name: Run Python examples + run: | + cd arbor + scripts/run_python_examples.sh + - name: Test executables + run: | + cd arbor + scripts/test_executables.sh diff --git a/doc/contrib/release.rst b/doc/contrib/release.rst index c0e2ae65f6..309ad418a1 100644 --- a/doc/contrib/release.rst +++ b/doc/contrib/release.rst @@ -45,14 +45,14 @@ Update tags/versions and test - Double check that all examples/tutorials/etc are covered by CI - Check Python/pip/PyPi metadata and scripts, e.g. ``setup.py``, ``pyproject.toml`` -#. Create new temp-branch ending in ``-rc``. E.g. ``v0.6-rc`` +#. Create new temp-branch ending in ``-rc``. E.g. ``v0.9.0-rc`` #. Bump the ``VERSION`` file: - For as long as `scikit-build-core` does not support loading fields from external files, also bump in `pyproject.toml` - See also :ref:`dev-version` - Append ``-rc``. (Make sure there's no ``-dev``) -#. Create a draft PR. Tag and push with ``-rc``. E.g. ``v0.6-rc`` +#. Create a draft PR. Tag and push with ``-rc``. E.g. ``v0.9.0-rc`` - on cmdline: ``git tag -a TAGNAME`` - ``git push upstream refs/tags/TAGNAME`` @@ -126,7 +126,7 @@ Post Release #. Update Zenodo with authors and changelog created in previous step and submit. -#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just release ``3.14``, change ``VERSION`` to ``3.15-dev`` +#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just released ``3.14.15``, change ``VERSION`` to ``3.15.16-dev``. Make the number portion always consists of a 3-tuple. Shorter versions are uninstallable by Spack (``spack install arbor@0.8`` will install v0.8.1, due to anything short than a 3-tuple being interpreted as a version range). - Update ``spack/package.py``. The checksum of the targz is the sha256sum. - Include changes such as to ``CITATIONS``, ``doc/index.rst`` in postrel PR. Copy Zenodo BibTex export to ``CITATIONS``. diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index f5e52650bc..1833178949 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -61,5 +61,5 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR -ARBOR_VERSION=$(cat "$ARBOR_DIR/VERSION") -spack dev-build arbor@${ARBOR_VERSION} +python +spack dev-build arbor +python +spack load arbor diff --git a/spack/package.py b/spack/package.py index dad2ef24c7..8cba901ade 100644 --- a/spack/package.py +++ b/spack/package.py @@ -21,26 +21,25 @@ class Arbor(CMakePackage, CudaPackage): url = "https://github.com/arbor-sim/arbor/releases/download/v0.8.1/arbor-v0.8.1-full.tar.gz" maintainers = ["thorstenhater", "brenthuisman", "haampie"] - version("master", branch="master", submodules=True) - if DEVVERSION: - version(DEVVERSION) + version("master", branch="master") + version("develop") version( "0.8.1", sha256="caebf96676ace6a9c50436541c420ca4bb53f0639dcab825de6fa370aacf6baa", url="https://github.com/arbor-sim/arbor/releases/download/v0.8.1/arbor-v0.8.1-full.tar.gz", ) version( - "0.8", + "0.8.0", sha256="18df5600308841616996a9de93b55a105be0f59692daa5febd3a65aae5bc2c5d", url="https://github.com/arbor-sim/arbor/releases/download/v0.8/arbor-v0.8-full.tar.gz", ) version( - "0.7", + "0.7.0", sha256="c3a6b7193946aee882bb85f9c38beac74209842ee94e80840968997ba3b84543", url="https://github.com/arbor-sim/arbor/releases/download/v0.7/arbor-v0.7-full.tar.gz", ) version( - "0.6", + "0.6.0", sha256="4cd333b18effc8833428ddc0b99e7dc976804771bc85da90034c272c7019e1e8", url="https://github.com/arbor-sim/arbor/releases/download/v0.6/arbor-v0.6-full.tar.gz", ) From c6b34ae6994acdfad1d79c60830ae71202de0150 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 19 Jul 2023 16:08:15 +0200 Subject: [PATCH 02/49] install fresh deps, if available --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 1833178949..3b84881a38 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -61,5 +61,5 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR -spack dev-build arbor +python +spack dev-build arbor +python --fresh spack load arbor From 3cf50c26f3900fd81cf53c165ba278cb2feeaa99 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 19 Jul 2023 16:10:35 +0200 Subject: [PATCH 03/49] Tell Spack to install develop version of Arbor, which is the locally available one --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 3b84881a38..12462653af 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -61,5 +61,5 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR -spack dev-build arbor +python --fresh +spack dev-build arbor@develop +python --fresh spack load arbor From ee001f79a498cf084d5fab523266d3055c7eeab2 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 19 Jul 2023 16:18:59 +0200 Subject: [PATCH 04/49] TODO: can --fresh refer only to deps? --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 12462653af..c3f077fba7 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -61,5 +61,5 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR -spack dev-build arbor@develop +python --fresh +spack dev-build arbor@develop +python spack load arbor From f1c20d128ce7f77011aefa03e003494b650b3c44 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 20 Jul 2023 10:07:20 +0200 Subject: [PATCH 05/49] Maybe: spack load doesnt persist over GA steps --- .github/workflows/spack.yml | 16 ++++++++-------- scripts/build_spack_package.sh | 1 + 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 1f306b0cd7..c38496ac3a 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -31,11 +31,11 @@ jobs: - name: Build Arbor's Spack package against the latest release run: arbor/scripts/build_spack_package.sh arbor latest_release - - name: Run Python examples - run: | - cd arbor - scripts/run_python_examples.sh - - name: Test executables - run: | - cd arbor - scripts/test_executables.sh + # - name: Run Python examples + # run: | + # cd arbor + # scripts/run_python_examples.sh + # - name: Test executables + # run: | + # cd arbor + # scripts/test_executables.sh diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index c3f077fba7..515dbceafb 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -63,3 +63,4 @@ cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR spack dev-build arbor@develop +python spack load arbor +scripts/run_python_examples.sh From 82342fec6f88574068c964a6b82c7725133a8e84 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 20 Jul 2023 17:11:32 +0200 Subject: [PATCH 06/49] Persist Spack across GH Action steps --- .github/workflows/spack.yml | 21 +++++++++++++-------- scripts/build_spack_package.sh | 6 ++++-- spack/package.py | 5 +++++ 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index c38496ac3a..c927a9abb2 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -29,13 +29,18 @@ jobs: restore-keys: spack-cache-${{ matrix.os }}- - name: Build Arbor's Spack package against the latest release + id: setup_spack run: arbor/scripts/build_spack_package.sh arbor latest_release - # - name: Run Python examples - # run: | - # cd arbor - # scripts/run_python_examples.sh - # - name: Test executables - # run: | - # cd arbor - # scripts/test_executables.sh + - name: Run Python examples + run: | + source ${{ steps.setup_spack.outputs.SOURCE_SPACK }}" + spack load arbor + cd arbor + scripts/run_python_examples.sh + - name: Test executables + run: | + source ${{ steps.setup_spack.outputs.SOURCE_SPACK }}" + spack load arbor + cd arbor + scripts/test_executables.sh diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 515dbceafb..0600815f94 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -50,6 +50,10 @@ esac mkdir ~/.spack cp $ARBOR_DIR/spack/config.yaml ~/.spack +if [[ ! -z "$GITHUB_ACTIONS" ]]; then + echo "SOURCE_SPACK=`realpath share/spack/setup-env.sh`" >> "$GITHUB_OUTPUT" +fi + source $SPACK_DIR/share/spack/setup-env.sh spack repo create $SPACK_CUSTOM_REPO @@ -62,5 +66,3 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR spack dev-build arbor@develop +python -spack load arbor -scripts/run_python_examples.sh diff --git a/spack/package.py b/spack/package.py index 8cba901ade..3715f3de70 100644 --- a/spack/package.py +++ b/spack/package.py @@ -131,3 +131,8 @@ def cmake_args(self): args.append("-DARB_CXX_FLAGS_TARGET=" + opt_flags) return args + + @run_after('install', when=+python) + @on_package_attributes(run_tests=True) + def install_test(self): + python('-c', 'import arbor') From f66e989b59ae49c8111c84593edcb227ef633d1b Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 20 Jul 2023 17:16:06 +0200 Subject: [PATCH 07/49] Persist Spack across GH Action steps 2 --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 0600815f94..0f22f51d58 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -51,7 +51,7 @@ mkdir ~/.spack cp $ARBOR_DIR/spack/config.yaml ~/.spack if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "SOURCE_SPACK=`realpath share/spack/setup-env.sh`" >> "$GITHUB_OUTPUT" + echo "SOURCE_SPACK=`realpath $SPACK_DIR/share/spack/setup-env.sh`" >> "$GITHUB_OUTPUT" fi source $SPACK_DIR/share/spack/setup-env.sh From 8ef9e8ea01c98cdf4e07fda4ffbfb9b2476cea81 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 20 Jul 2023 18:53:25 +0200 Subject: [PATCH 08/49] black --- spack/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spack/package.py b/spack/package.py index 3715f3de70..4d9e4634fd 100644 --- a/spack/package.py +++ b/spack/package.py @@ -132,7 +132,7 @@ def cmake_args(self): return args - @run_after('install', when=+python) + @run_after("install", when="+python") @on_package_attributes(run_tests=True) def install_test(self): - python('-c', 'import arbor') + python("-c", "import arbor") From 9e353c44e73d98804c9a04f4142b38b520fc0789 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 21 Jul 2023 09:06:14 +0200 Subject: [PATCH 09/49] Without quotes? --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 0f22f51d58..d373ebf24c 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -51,7 +51,7 @@ mkdir ~/.spack cp $ARBOR_DIR/spack/config.yaml ~/.spack if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "SOURCE_SPACK=`realpath $SPACK_DIR/share/spack/setup-env.sh`" >> "$GITHUB_OUTPUT" + echo SOURCE_SPACK=`realpath $SPACK_DIR/share/spack/setup-env.sh` >> "$GITHUB_OUTPUT" fi source $SPACK_DIR/share/spack/setup-env.sh From 50d8f100e489bbc9e0c4035df2590ae080e6646d Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 21 Jul 2023 10:16:57 +0200 Subject: [PATCH 10/49] quotes are hard to find --- .github/workflows/spack.yml | 4 ++-- scripts/build_spack_package.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index c927a9abb2..2e08eb85c2 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -34,13 +34,13 @@ jobs: - name: Run Python examples run: | - source ${{ steps.setup_spack.outputs.SOURCE_SPACK }}" + source ${{ steps.setup_spack.outputs.SOURCE_SPACK }} spack load arbor cd arbor scripts/run_python_examples.sh - name: Test executables run: | - source ${{ steps.setup_spack.outputs.SOURCE_SPACK }}" + source ${{ steps.setup_spack.outputs.SOURCE_SPACK }} spack load arbor cd arbor scripts/test_executables.sh diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index d373ebf24c..0f22f51d58 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -51,7 +51,7 @@ mkdir ~/.spack cp $ARBOR_DIR/spack/config.yaml ~/.spack if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo SOURCE_SPACK=`realpath $SPACK_DIR/share/spack/setup-env.sh` >> "$GITHUB_OUTPUT" + echo "SOURCE_SPACK=`realpath $SPACK_DIR/share/spack/setup-env.sh`" >> "$GITHUB_OUTPUT" fi source $SPACK_DIR/share/spack/setup-env.sh From 4cdf312e87b561deae81e27f118bad7cd64e207c Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 21 Jul 2023 12:01:28 +0200 Subject: [PATCH 11/49] It aint pretty, but... --- .github/workflows/spack.yml | 15 +-------------- scripts/build_spack_package.sh | 10 ++++++---- scripts/run_python_examples.sh | 3 +++ scripts/test_executables.sh | 6 ++++++ 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 2e08eb85c2..bfec7a5ecf 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -28,19 +28,6 @@ jobs: key: spack-cache-${{ matrix.os }}-${{ github.run_id }} restore-keys: spack-cache-${{ matrix.os }}- - - name: Build Arbor's Spack package against the latest release + - name: Build and test Arbor on latest Spack release id: setup_spack run: arbor/scripts/build_spack_package.sh arbor latest_release - - - name: Run Python examples - run: | - source ${{ steps.setup_spack.outputs.SOURCE_SPACK }} - spack load arbor - cd arbor - scripts/run_python_examples.sh - - name: Test executables - run: | - source ${{ steps.setup_spack.outputs.SOURCE_SPACK }} - spack load arbor - cd arbor - scripts/test_executables.sh diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 0f22f51d58..2d530ec332 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -50,10 +50,6 @@ esac mkdir ~/.spack cp $ARBOR_DIR/spack/config.yaml ~/.spack -if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "SOURCE_SPACK=`realpath $SPACK_DIR/share/spack/setup-env.sh`" >> "$GITHUB_OUTPUT" -fi - source $SPACK_DIR/share/spack/setup-env.sh spack repo create $SPACK_CUSTOM_REPO @@ -66,3 +62,9 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR spack dev-build arbor@develop +python + +if [[ ! -z "$GITHUB_ACTIONS" ]]; then + spack load arbor + scripts/run_python_examples.sh >> $GITHUB_STEP_SUMMARY + scripts/test_executables.sh >> $GITHUB_STEP_SUMMARY +fi diff --git a/scripts/run_python_examples.sh b/scripts/run_python_examples.sh index 4ae6d262b2..3b216863ba 100755 --- a/scripts/run_python_examples.sh +++ b/scripts/run_python_examples.sh @@ -15,6 +15,9 @@ $PREFIX python3 -m pip install -r python/example/example_requirements.txt -U runpyex () { echo "=== Executing $1 ======================================" $PREFIX python3 python/example/$* + if [[ ! -z "$GITHUB_ACTIONS" ]]; then + echo "### Example $1: OK." >> $GITHUB_STEP_SUMMARY + fi } runpyex brunel.py -n 400 -m 100 -e 20 -p 0.1 -w 1.2 -d 1 -g 0.5 -l 5 -t 100 -s 1 -G 50 -S 123 diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index 98cadf4b18..6be4177ca8 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -6,7 +6,13 @@ set -Eeuo pipefail echo "=== Executing modcc test ======================================" modcc python/example/cat/dummy.mod test -f "dummy.hpp" +if [[ ! -z "$GITHUB_ACTIONS" ]]; then + echo "### modcc: OK." >> $GITHUB_STEP_SUMMARY +fi echo "=== Executing a-b-c test ======================================" arbor-build-catalogue -v cat python/example/cat ./scripts/test-catalogue.py ./cat-catalogue.so +if [[ ! -z "$GITHUB_ACTIONS" ]]; then + echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY +fi From e211a094ab90bf1285c231320b823d4c3210832d Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 21 Jul 2023 12:23:54 +0200 Subject: [PATCH 12/49] Prevent Arbor from entering Spack cache --- scripts/build_spack_package.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 2d530ec332..bd68353390 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -67,4 +67,6 @@ if [[ ! -z "$GITHUB_ACTIONS" ]]; then spack load arbor scripts/run_python_examples.sh >> $GITHUB_STEP_SUMMARY scripts/test_executables.sh >> $GITHUB_STEP_SUMMARY + ## make sure there's no Arbor in cache. + spack uninstall -faR arbor fi From 075a56d836a4014961b0928362ed32561e3803d5 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 21 Jul 2023 13:54:54 +0200 Subject: [PATCH 13/49] generate Spack cache --- scripts/test_executables.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index 6be4177ca8..5cb1e86e70 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -10,9 +10,9 @@ if [[ ! -z "$GITHUB_ACTIONS" ]]; then echo "### modcc: OK." >> $GITHUB_STEP_SUMMARY fi -echo "=== Executing a-b-c test ======================================" -arbor-build-catalogue -v cat python/example/cat -./scripts/test-catalogue.py ./cat-catalogue.so -if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY -fi +# echo "=== Executing a-b-c test ======================================" +# arbor-build-catalogue -v cat python/example/cat +# ./scripts/test-catalogue.py ./cat-catalogue.so +# if [[ ! -z "$GITHUB_ACTIONS" ]]; then +# echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY +# fi From d98727ef2bf7bb4622d7af1826606b210c8b55ed Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 21 Jul 2023 15:32:05 +0200 Subject: [PATCH 14/49] by all means --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index bd68353390..7aff4d762b 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -68,5 +68,5 @@ if [[ ! -z "$GITHUB_ACTIONS" ]]; then scripts/run_python_examples.sh >> $GITHUB_STEP_SUMMARY scripts/test_executables.sh >> $GITHUB_STEP_SUMMARY ## make sure there's no Arbor in cache. - spack uninstall -faR arbor + spack uninstall -yafR arbor fi From 52a47b78afd255afa9ee77245027e67728f75f30 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Tue, 25 Jul 2023 10:45:31 +0200 Subject: [PATCH 15/49] a-b-c verbose=off prints MORE info --- scripts/test_executables.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index 5cb1e86e70..d8fc41daa9 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -10,9 +10,9 @@ if [[ ! -z "$GITHUB_ACTIONS" ]]; then echo "### modcc: OK." >> $GITHUB_STEP_SUMMARY fi -# echo "=== Executing a-b-c test ======================================" -# arbor-build-catalogue -v cat python/example/cat -# ./scripts/test-catalogue.py ./cat-catalogue.so -# if [[ ! -z "$GITHUB_ACTIONS" ]]; then -# echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY -# fi +echo "=== Executing a-b-c test ======================================" +arbor-build-catalogue cat python/example/cat +./scripts/test-catalogue.py ./cat-catalogue.so +if [[ ! -z "$GITHUB_ACTIONS" ]]; then + echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY +fi From 1ea55f92ce0f9eb4e67b429a8ae9005cc759f486 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Tue, 25 Jul 2023 12:42:26 +0200 Subject: [PATCH 16/49] better a-b-c error message? --- scripts/arbor-build-catalogue | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/scripts/arbor-build-catalogue b/scripts/arbor-build-catalogue index 89e9ac40f8..2461005345 100755 --- a/scripts/arbor-build-catalogue +++ b/scripts/arbor-build-catalogue @@ -311,14 +311,11 @@ with TemporaryDirectory() as tmp: cmake_cmd = "cmake .." make_cmd = "make" - if verbose: - out, err = (None, None) - make_cmd += " VERBOSE=1" - else: - out, err = (sp.PIPE, sp.PIPE) + # if verbose: + make_cmd += " VERBOSE=1" try: - sp.run(cmake_cmd, shell=True, check=True, stdout=out, stderr=err) - sp.run(make_cmd, shell=True, check=True, stdout=out, stderr=err) + sp.run(cmake_cmd, shell=True, check=True, capture_output = True, text = True) + sp.run(make_cmd, shell=True, check=True, capture_output = True, text = True) shutil.copy2(f"{name}-catalogue.so", pwd) except sp.CalledProcessError as e: import traceback as tb @@ -327,9 +324,9 @@ with TemporaryDirectory() as tmp: # Not in verbose mode, so we have captured the # `stdout` and `stderr` and can print it to the user. sys.stdout.write("Build log:\n") - sys.stdout.write(e.stdout.decode()) + sys.stdout.write(e.stdout) sys.stderr.write(tb.format_exc() + " Error:\n\n") - sys.stderr.write(e.stderr.decode()) + sys.stderr.write(e.stderr) else: # In verbose mode the outputs weren't captured and # have been streamed to `stdout` and `stderr` already. From 4e384321501bfba2081c8e35ff9a25fa69672514 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Tue, 25 Jul 2023 13:44:19 +0200 Subject: [PATCH 17/49] twas Github stealing stdout all along --- scripts/arbor-build-catalogue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/arbor-build-catalogue b/scripts/arbor-build-catalogue index 2461005345..ebd5d382e6 100755 --- a/scripts/arbor-build-catalogue +++ b/scripts/arbor-build-catalogue @@ -311,8 +311,8 @@ with TemporaryDirectory() as tmp: cmake_cmd = "cmake .." make_cmd = "make" - # if verbose: - make_cmd += " VERBOSE=1" + if verbose: + make_cmd += " VERBOSE=1" try: sp.run(cmake_cmd, shell=True, check=True, capture_output = True, text = True) sp.run(make_cmd, shell=True, check=True, capture_output = True, text = True) From 23edbb64a84929e3280ca918e206fe4711e9d76b Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Tue, 25 Jul 2023 14:26:07 +0200 Subject: [PATCH 18/49] a-b-c: remove useless try block. force nonspack c++ compiler --- scripts/arbor-build-catalogue | 18 +++++++++--------- scripts/test_executables.sh | 2 +- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/arbor-build-catalogue b/scripts/arbor-build-catalogue index ebd5d382e6..974f554dae 100755 --- a/scripts/arbor-build-catalogue +++ b/scripts/arbor-build-catalogue @@ -15,20 +15,20 @@ config = A.config() prefix = Path(config["prefix"]) CXX = Path(config["CXX"]) if not CXX.exists(): - try: - # Example <>/lib/python3.10/site-packages/arbor - altern = "c++" - print( - f"Warning: prefix '{CXX}' does not exist, falling back to '{altern}'.", - file=sys.stderr, - ) - CXX = altern - except: + # Example <>/lib/python3.10/site-packages/arbor + altern = "c++" + from shutil import which + if which(altern) == None: print( f"Error: Neither prefix '{CXX}' nor fallback '{altern}' exist. Please provide a path to a C++ compiler with --cxx", file=sys.stderr, ) exit(-1) + print( + f"Warning: prefix '{CXX}' does not exist, falling back to '{altern}'.", + file=sys.stderr, + ) + CXX = altern def parse_arguments(): diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index d8fc41daa9..07cb9358ec 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -12,7 +12,7 @@ fi echo "=== Executing a-b-c test ======================================" arbor-build-catalogue cat python/example/cat -./scripts/test-catalogue.py ./cat-catalogue.so +./scripts/test-catalogue.py ./cat-catalogue.so --cxx=c++ if [[ ! -z "$GITHUB_ACTIONS" ]]; then echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY fi From 47844b486b3921b804ae080d7b83fb6f92cb0668 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Tue, 25 Jul 2023 15:35:04 +0200 Subject: [PATCH 19/49] add flags to correct place. --- scripts/build_spack_package.sh | 8 +++++--- scripts/test_executables.sh | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 7aff4d762b..1b36a418eb 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -61,12 +61,14 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR -spack dev-build arbor@develop +python +spack dev-build arbor@develop +python --test root if [[ ! -z "$GITHUB_ACTIONS" ]]; then + ## Spack runs Spack tests in the Spack build env. + ## Below tests are meant to verify install actually happened correctly, including pip install of packages spack load arbor - scripts/run_python_examples.sh >> $GITHUB_STEP_SUMMARY - scripts/test_executables.sh >> $GITHUB_STEP_SUMMARY + scripts/run_python_examples.sh + scripts/test_executables.sh ## make sure there's no Arbor in cache. spack uninstall -yafR arbor fi diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index 07cb9358ec..04347d971d 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -11,8 +11,8 @@ if [[ ! -z "$GITHUB_ACTIONS" ]]; then fi echo "=== Executing a-b-c test ======================================" -arbor-build-catalogue cat python/example/cat -./scripts/test-catalogue.py ./cat-catalogue.so --cxx=c++ +arbor-build-catalogue --cxx=c++ cat python/example/cat +./scripts/test-catalogue.py ./cat-catalogue.so if [[ ! -z "$GITHUB_ACTIONS" ]]; then echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY fi From b9edb63691093b61f7155f9485b64cae21eeb59f Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Tue, 25 Jul 2023 15:52:51 +0200 Subject: [PATCH 20/49] does order matter for Spack? --- scripts/build_spack_package.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 1b36a418eb..3c2098d40e 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -61,7 +61,7 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR -spack dev-build arbor@develop +python --test root +spack dev-build --test root arbor@develop +python if [[ ! -z "$GITHUB_ACTIONS" ]]; then ## Spack runs Spack tests in the Spack build env. From 6879c77983f21657d0dd6320e08ae679dbffff62 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 10:36:32 +0200 Subject: [PATCH 21/49] Use BUILD_TESTING --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d2a7d20acc..b60385070c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -114,7 +114,7 @@ include(ErrorTarget) # reguired for add_error_target # Set release as the default build type (CMake default is debug.) -if (NOT CMAKE_BUILD_TYPE) +if (NOT CMAKE_BUILD_TYPE AND NOT BUILD_TESTING) set(CMAKE_BUILD_TYPE release CACHE STRING "Choose the type of build." FORCE) # Set the possible values of build type for cmake-gui set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "debug" "release") From ad0375c9f0bf7464abba58b17f86bfcb2629ec48 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 14:16:41 +0200 Subject: [PATCH 22/49] Better GH Actions workflow for Spack --- .github/workflows/spack.yml | 49 +++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index bfec7a5ecf..41ca11bd31 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -12,7 +12,8 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - python-version: [3.8] + python-version: [3.10] + spack-version: [develop, latest_master] steps: - name: Checkout uses: actions/checkout@v3 @@ -25,9 +26,43 @@ jobs: uses: actions/cache@v3 with: path: ~/.spack-cache - key: spack-cache-${{ matrix.os }}-${{ github.run_id }} - restore-keys: spack-cache-${{ matrix.os }}- - - - name: Build and test Arbor on latest Spack release - id: setup_spack - run: arbor/scripts/build_spack_package.sh arbor latest_release + key: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}-${{ github.run_id }} + restore-keys: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}- + - name: Get Spack (develop) + if: ${{ matrix.config.spack-version == "develop" }} + run: | + git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git + - name: Get Spack (latest_release) + if: ${{ matrix.config.spack-version == "latest_release" }} + run: | + wget "$(curl -sH "Accept: application/vnd.github.v3+json" https://api.github.com/repos/spack/spack/releases/latest | grep browser_download_url | cut -d '"' -f 4)" + tar xfz spack*.tar.gz + mv spack*/ spack + - name: Prep + run: | + mkdir ~/.spack + cp arbor/spack/config.yaml ~/.spack + source spack/share/spack/setup-env.sh + spack repo create custom_repo + mkdir -p custom_repo/packages/arbor + spack repo add custom_repo + spack reindex + cp arbor/spack/package.py custom_repo/packages/arbor + - name: Build Arbor and run unit tests through Spack + run: | + source spack/share/spack/setup-env.sh + spack dev-build --test root arbor@develop +python + - name: Build Arbor and run uni tests + run: | + source spack/share/spack/setup-env.sh + spack load arbor + cd arbor + scripts/run_python_examples.sh + scripts/test_executables.sh + spack uninstall -yafR arbor + - name: Spack Failure: upload logs + uses: actions/upload-artifact@v3 + if: ${{ failure() }} + with: + name: Spack logs + path: arbor/*.txt From 57bcc90c22dd4895c6f17c4b8fb25c6667d7de83 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 14:19:49 +0200 Subject: [PATCH 23/49] whelp --- .github/workflows/spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 41ca11bd31..9a491c2039 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -60,7 +60,7 @@ jobs: scripts/run_python_examples.sh scripts/test_executables.sh spack uninstall -yafR arbor - - name: Spack Failure: upload logs + - name: Failure, upload logs uses: actions/upload-artifact@v3 if: ${{ failure() }} with: From 673ef8602ec18d4dee6811e33384e64e9600ee6a Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 14:20:39 +0200 Subject: [PATCH 24/49] whelp2 --- .github/workflows/spack.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 9a491c2039..3b2eb00d28 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -29,11 +29,11 @@ jobs: key: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}-${{ github.run_id }} restore-keys: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}- - name: Get Spack (develop) - if: ${{ matrix.config.spack-version == "develop" }} + if: ${{ matrix.config.spack-version == develop }} run: | git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git - name: Get Spack (latest_release) - if: ${{ matrix.config.spack-version == "latest_release" }} + if: ${{ matrix.config.spack-version == latest_release }} run: | wget "$(curl -sH "Accept: application/vnd.github.v3+json" https://api.github.com/repos/spack/spack/releases/latest | grep browser_download_url | cut -d '"' -f 4)" tar xfz spack*.tar.gz From 23f3e6959eeb4de1086d378778dcf671b7fc7ceb Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 14:21:51 +0200 Subject: [PATCH 25/49] single quotes? --- .github/workflows/spack.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 3b2eb00d28..1c64110468 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -29,11 +29,11 @@ jobs: key: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}-${{ github.run_id }} restore-keys: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}- - name: Get Spack (develop) - if: ${{ matrix.config.spack-version == develop }} + if: ${{ matrix.config.spack-version == 'develop' }} run: | git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git - name: Get Spack (latest_release) - if: ${{ matrix.config.spack-version == latest_release }} + if: ${{ matrix.config.spack-version == 'latest_release' }} run: | wget "$(curl -sH "Accept: application/vnd.github.v3+json" https://api.github.com/repos/spack/spack/releases/latest | grep browser_download_url | cut -d '"' -f 4)" tar xfz spack*.tar.gz From b13c9cf24eb0268ac10ac066429fb12388c33e92 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 14:36:50 +0200 Subject: [PATCH 26/49] single quotes everywhere --- .github/workflows/spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 1c64110468..219b13d5f5 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: os: [ubuntu-22.04] - python-version: [3.10] + python-version: ['3.10'] spack-version: [develop, latest_master] steps: - name: Checkout From 6b7e2cd79f9ad1f0bbda1f4ed9f72678b2dc0c4c Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 14:56:02 +0200 Subject: [PATCH 27/49] single quotes: its never enough --- .github/workflows/spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 219b13d5f5..f2827fb1a0 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -13,7 +13,7 @@ jobs: matrix: os: [ubuntu-22.04] python-version: ['3.10'] - spack-version: [develop, latest_master] + spack-version: ['develop', 'latest_release'] steps: - name: Checkout uses: actions/checkout@v3 From b53d84bcac99b865df4986ea11b61509ba2b4820 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 15:17:40 +0200 Subject: [PATCH 28/49] sloppy copy past --- .github/workflows/spack.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index f2827fb1a0..cacd977f72 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -29,11 +29,11 @@ jobs: key: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}-${{ github.run_id }} restore-keys: spack-cache-${{ matrix.os }}-${{ matrix.spack-version }}- - name: Get Spack (develop) - if: ${{ matrix.config.spack-version == 'develop' }} + if: ${{ matrix.spack-version == 'develop' }} run: | git clone -c feature.manyFiles=true --depth 1 https://github.com/spack/spack.git - name: Get Spack (latest_release) - if: ${{ matrix.config.spack-version == 'latest_release' }} + if: ${{ matrix.spack-version == 'latest_release' }} run: | wget "$(curl -sH "Accept: application/vnd.github.v3+json" https://api.github.com/repos/spack/spack/releases/latest | grep browser_download_url | cut -d '"' -f 4)" tar xfz spack*.tar.gz From 83f99a01694c51b77de960cbe395ea1165f8998f Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Wed, 26 Jul 2023 17:15:18 +0200 Subject: [PATCH 29/49] maybe --- .github/workflows/spack.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index cacd977f72..ad03b69c99 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -56,7 +56,6 @@ jobs: run: | source spack/share/spack/setup-env.sh spack load arbor - cd arbor scripts/run_python_examples.sh scripts/test_executables.sh spack uninstall -yafR arbor From 561abbca12fdc738ba78387d7c1202c939bfd228 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 27 Jul 2023 14:18:26 +0200 Subject: [PATCH 30/49] geen idee --- .github/workflows/spack.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index ad03b69c99..a32afbfe21 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -54,6 +54,12 @@ jobs: spack dev-build --test root arbor@develop +python - name: Build Arbor and run uni tests run: | + pwd + ls + ls ./arbor + cd ~/arbor + pwd + ls source spack/share/spack/setup-env.sh spack load arbor scripts/run_python_examples.sh From 6d35dc881ca65f49289981ff267c92567f28d2df Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 27 Jul 2023 17:12:07 +0200 Subject: [PATCH 31/49] naming things --- .github/workflows/spack.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index a32afbfe21..36ff8defc2 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -14,6 +14,7 @@ jobs: os: [ubuntu-22.04] python-version: ['3.10'] spack-version: ['develop', 'latest_release'] + fail-fast: false steps: - name: Checkout uses: actions/checkout@v3 @@ -49,10 +50,6 @@ jobs: spack reindex cp arbor/spack/package.py custom_repo/packages/arbor - name: Build Arbor and run unit tests through Spack - run: | - source spack/share/spack/setup-env.sh - spack dev-build --test root arbor@develop +python - - name: Build Arbor and run uni tests run: | pwd ls @@ -60,6 +57,10 @@ jobs: cd ~/arbor pwd ls + source spack/share/spack/setup-env.sh + spack dev-build --test root arbor@develop +python + - name: Load Arbor and verify installation, Python Examples. + run: | source spack/share/spack/setup-env.sh spack load arbor scripts/run_python_examples.sh From d62d1c6ba855ff0ae698cd190142acf4003773ef Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 27 Jul 2023 17:19:05 +0200 Subject: [PATCH 32/49] more debug --- .github/workflows/spack.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 36ff8defc2..1bb9aaa20c 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -51,11 +51,17 @@ jobs: cp arbor/spack/package.py custom_repo/packages/arbor - name: Build Arbor and run unit tests through Spack run: | + echo "pwd" pwd + echo "ls" ls + echo "ls ./arbor" ls ./arbor + echo "cd ~/arbor" cd ~/arbor + echo "pwd" pwd + echo "ls" ls source spack/share/spack/setup-env.sh spack dev-build --test root arbor@develop +python From f5603d59de624cef375f1e51b57a19730c2f99cc Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 27 Jul 2023 17:24:07 +0200 Subject: [PATCH 33/49] more debug2 --- .github/workflows/spack.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 1bb9aaa20c..9a056f0185 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -51,22 +51,22 @@ jobs: cp arbor/spack/package.py custom_repo/packages/arbor - name: Build Arbor and run unit tests through Spack run: | - echo "pwd" + echo "CMD: pwd" pwd - echo "ls" + echo "CMD: ls" ls - echo "ls ./arbor" + echo "CMD: ls ./arbor" ls ./arbor - echo "cd ~/arbor" - cd ~/arbor - echo "pwd" - pwd - echo "ls" - ls source spack/share/spack/setup-env.sh spack dev-build --test root arbor@develop +python - name: Load Arbor and verify installation, Python Examples. run: | + echo "CMD: pwd" + pwd + echo "CMD: ls" + ls + echo "CMD: ls ./arbor" + ls ./arbor source spack/share/spack/setup-env.sh spack load arbor scripts/run_python_examples.sh From 57d6fc4e429726e0cf665904a82e30be4e79a796 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Thu, 27 Jul 2023 18:38:49 +0200 Subject: [PATCH 34/49] I still think this should work --- .github/workflows/spack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 9a056f0185..d882235c6d 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -69,6 +69,7 @@ jobs: ls ./arbor source spack/share/spack/setup-env.sh spack load arbor + cd ./arbor scripts/run_python_examples.sh scripts/test_executables.sh spack uninstall -yafR arbor From 32657cae4b844c09f99268d1a43084b60dba26d3 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 09:48:09 +0200 Subject: [PATCH 35/49] this may be it --- .github/workflows/spack.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index d882235c6d..d7d7c934e0 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -58,6 +58,7 @@ jobs: echo "CMD: ls ./arbor" ls ./arbor source spack/share/spack/setup-env.sh + cd arbor spack dev-build --test root arbor@develop +python - name: Load Arbor and verify installation, Python Examples. run: | @@ -69,7 +70,7 @@ jobs: ls ./arbor source spack/share/spack/setup-env.sh spack load arbor - cd ./arbor + cd arbor scripts/run_python_examples.sh scripts/test_executables.sh spack uninstall -yafR arbor From 0921820715e21711b922d8f12ef7d88ea98a2764 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 11:32:53 +0200 Subject: [PATCH 36/49] disable spack-arbor unit tests for now and see if the rest works --- .github/workflows/spack.yml | 21 ++++++--------------- scripts/build_spack_package.sh | 11 +---------- 2 files changed, 7 insertions(+), 25 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index d7d7c934e0..92023b7b8f 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -51,23 +51,12 @@ jobs: cp arbor/spack/package.py custom_repo/packages/arbor - name: Build Arbor and run unit tests through Spack run: | - echo "CMD: pwd" - pwd - echo "CMD: ls" - ls - echo "CMD: ls ./arbor" - ls ./arbor source spack/share/spack/setup-env.sh cd arbor - spack dev-build --test root arbor@develop +python + spack dev-build arbor@develop +python + ## spack dev-build --test root arbor@develop +python - name: Load Arbor and verify installation, Python Examples. run: | - echo "CMD: pwd" - pwd - echo "CMD: ls" - ls - echo "CMD: ls ./arbor" - ls ./arbor source spack/share/spack/setup-env.sh spack load arbor cd arbor @@ -78,5 +67,7 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ failure() }} with: - name: Spack logs - path: arbor/*.txt + name: spack-log-${{ spack-version }} + path: | + arbor/*.txt + !arbor/CMakeLists.txt diff --git a/scripts/build_spack_package.sh b/scripts/build_spack_package.sh index 3c2098d40e..9263d38ec1 100755 --- a/scripts/build_spack_package.sh +++ b/scripts/build_spack_package.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash # checks out Spack and Arbor and builds it with the package.py from Arbor's repo # Spack can be the latest release or the develop branch +# Also runs unit tests. set -Eeuo pipefail @@ -62,13 +63,3 @@ spack reindex cp $ARBOR_DIR/spack/package.py $SPACK_CUSTOM_REPO/packages/arbor cd $ARBOR_DIR spack dev-build --test root arbor@develop +python - -if [[ ! -z "$GITHUB_ACTIONS" ]]; then - ## Spack runs Spack tests in the Spack build env. - ## Below tests are meant to verify install actually happened correctly, including pip install of packages - spack load arbor - scripts/run_python_examples.sh - scripts/test_executables.sh - ## make sure there's no Arbor in cache. - spack uninstall -yafR arbor -fi From 80d9cbed518aa3e2582ca1b3682626af48f17feb Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 11:39:58 +0200 Subject: [PATCH 37/49] fix syntax --- .github/workflows/spack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 92023b7b8f..ca93307abb 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -67,7 +67,7 @@ jobs: uses: actions/upload-artifact@v3 if: ${{ failure() }} with: - name: spack-log-${{ spack-version }} + name: spack-log-${{ matrix.spack-version }} path: | arbor/*.txt !arbor/CMakeLists.txt From 8a98d507041f485b7ae0bd31f62f240728cbe43d Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 13:14:35 +0200 Subject: [PATCH 38/49] a-b-c test: was setting `c++` required? --- scripts/test_executables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index 04347d971d..d8fc41daa9 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -11,7 +11,7 @@ if [[ ! -z "$GITHUB_ACTIONS" ]]; then fi echo "=== Executing a-b-c test ======================================" -arbor-build-catalogue --cxx=c++ cat python/example/cat +arbor-build-catalogue cat python/example/cat ./scripts/test-catalogue.py ./cat-catalogue.so if [[ ! -z "$GITHUB_ACTIONS" ]]; then echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY From f0475cf854a9b8bf3ad3915969fd660ad13cb179 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 13:21:29 +0200 Subject: [PATCH 39/49] remove github debugging --- scripts/run_python_examples.sh | 4 +--- scripts/test_executables.sh | 8 ++------ 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/run_python_examples.sh b/scripts/run_python_examples.sh index 3b216863ba..8661a8bdf0 100755 --- a/scripts/run_python_examples.sh +++ b/scripts/run_python_examples.sh @@ -15,9 +15,7 @@ $PREFIX python3 -m pip install -r python/example/example_requirements.txt -U runpyex () { echo "=== Executing $1 ======================================" $PREFIX python3 python/example/$* - if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "### Example $1: OK." >> $GITHUB_STEP_SUMMARY - fi + echo "### Example $1: OK." } runpyex brunel.py -n 400 -m 100 -e 20 -p 0.1 -w 1.2 -d 1 -g 0.5 -l 5 -t 100 -s 1 -G 50 -S 123 diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index d8fc41daa9..421d004a7a 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -6,13 +6,9 @@ set -Eeuo pipefail echo "=== Executing modcc test ======================================" modcc python/example/cat/dummy.mod test -f "dummy.hpp" -if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "### modcc: OK." >> $GITHUB_STEP_SUMMARY -fi +ech "executable modcc: OK." echo "=== Executing a-b-c test ======================================" arbor-build-catalogue cat python/example/cat ./scripts/test-catalogue.py ./cat-catalogue.so -if [[ ! -z "$GITHUB_ACTIONS" ]]; then - echo "### a-b-c: OK." >> $GITHUB_STEP_SUMMARY -fi +echo "executable a-b-c: OK." From f0d2aa303e419c01600cf63eea24fc4c0c5f830d Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 13:23:08 +0200 Subject: [PATCH 40/49] re-add setting cxx for modcc test --- scripts/test_executables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index 421d004a7a..ed7b60a47f 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -9,6 +9,6 @@ test -f "dummy.hpp" ech "executable modcc: OK." echo "=== Executing a-b-c test ======================================" -arbor-build-catalogue cat python/example/cat +arbor-build-catalogue --cxx=c++ cat python/example/cat ./scripts/test-catalogue.py ./cat-catalogue.so echo "executable a-b-c: OK." From 2c5e9fdb278f86a2e43d4756e1153de7afdc4394 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 13:36:47 +0200 Subject: [PATCH 41/49] typo --- scripts/test_executables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/test_executables.sh b/scripts/test_executables.sh index ed7b60a47f..58a411536c 100755 --- a/scripts/test_executables.sh +++ b/scripts/test_executables.sh @@ -6,7 +6,7 @@ set -Eeuo pipefail echo "=== Executing modcc test ======================================" modcc python/example/cat/dummy.mod test -f "dummy.hpp" -ech "executable modcc: OK." +echo "executable modcc: OK." echo "=== Executing a-b-c test ======================================" arbor-build-catalogue --cxx=c++ cat python/example/cat From 2142e5c3fefead1c86df1fb6b89b441d209e392d Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 13:40:12 +0200 Subject: [PATCH 42/49] remove stale code --- spack/package.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spack/package.py b/spack/package.py index 4d9e4634fd..ee760fc447 100644 --- a/spack/package.py +++ b/spack/package.py @@ -5,12 +5,6 @@ from spack.package import * -try: - with open("VERSION", "r") as file: - DEVVERSION = file.readline().strip() -except: - DEVVERSION = None - class Arbor(CMakePackage, CudaPackage): """Arbor is a high-performance library for computational neuroscience From c9a2b72751fdc816555e205dee232353bcc5ef5c Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 15:53:50 +0200 Subject: [PATCH 43/49] Drop Spack unit test for now. --- .github/workflows/spack.yml | 3 +-- spack/package.py | 6 +++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index ca93307abb..3bc4cc4109 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -49,12 +49,11 @@ jobs: spack repo add custom_repo spack reindex cp arbor/spack/package.py custom_repo/packages/arbor - - name: Build Arbor and run unit tests through Spack + - name: Build Arbor run: | source spack/share/spack/setup-env.sh cd arbor spack dev-build arbor@develop +python - ## spack dev-build --test root arbor@develop +python - name: Load Arbor and verify installation, Python Examples. run: | source spack/share/spack/setup-env.sh diff --git a/spack/package.py b/spack/package.py index ee760fc447..cc82e735d4 100644 --- a/spack/package.py +++ b/spack/package.py @@ -92,9 +92,9 @@ class Arbor(CMakePackage, CudaPackage): depends_on("python@3.7:", when="+python", type=("build", "run")) depends_on("py-numpy", when="+python", type=("build", "run")) with when("+python"): - depends_on("py-pybind11@2.6:", type=("build")) - depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type=("build")) - depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type=("build")) + depends_on("py-pybind11@2.6:", type="build") + depends_on("py-pybind11@2.8.1:", when="@0.5.3:", type="build") + depends_on("py-pybind11@2.10.1:", when="@0.7.1:", type="build") # sphinx based documentation depends_on("python@3.7:", when="+doc", type="build") From 93b23f6c1312a552ddb7a77a899581ab0ab472bd Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Fri, 28 Jul 2023 17:22:02 +0200 Subject: [PATCH 44/49] more steps --- .github/workflows/spack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 3bc4cc4109..67c895d734 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -61,6 +61,7 @@ jobs: cd arbor scripts/run_python_examples.sh scripts/test_executables.sh + - name: Remove Arbor spack uninstall -yafR arbor - name: Failure, upload logs uses: actions/upload-artifact@v3 From f0c7c8ae4ed71a2a92a3c15c1eedd1699903a4e0 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Mon, 31 Jul 2023 09:15:39 +0200 Subject: [PATCH 45/49] cibuildwheelhotfix --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7de88c9d18..da47fb4e43 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,7 @@ build-frontend = "build" build = ["cp3*-*linux*","cp3*-macosx*"] skip = ["cp36*", "pp*", "*musllinux*"] test-command = "python -m unittest discover -v -s {project}/python" -before-build = "python -m pip install pip --upgrade" +# before-build = "python -m pip install pip --upgrade" before-test = "python -m pip install pip --upgrade" [tool.cibuildwheel.macos] From 73ae831fc1745ecf440e72efd4bd418a996e2e45 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Mon, 31 Jul 2023 09:22:33 +0200 Subject: [PATCH 46/49] whoops --- .github/workflows/spack.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/spack.yml b/.github/workflows/spack.yml index 67c895d734..21b5305028 100644 --- a/.github/workflows/spack.yml +++ b/.github/workflows/spack.yml @@ -62,6 +62,8 @@ jobs: scripts/run_python_examples.sh scripts/test_executables.sh - name: Remove Arbor + run: | + source spack/share/spack/setup-env.sh spack uninstall -yafR arbor - name: Failure, upload logs uses: actions/upload-artifact@v3 From 84641c979eb6e404d2373b71a5814c22ef92dcdd Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Mon, 31 Jul 2023 11:59:55 +0200 Subject: [PATCH 47/49] Update pip during ciwheelbuild the new way --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index da47fb4e43..9b19d6d825 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,7 @@ build-frontend = "build" build = ["cp3*-*linux*","cp3*-macosx*"] skip = ["cp36*", "pp*", "*musllinux*"] test-command = "python -m unittest discover -v -s {project}/python" -# before-build = "python -m pip install pip --upgrade" -before-test = "python -m pip install pip --upgrade" +dependency-versions = "latest" [tool.cibuildwheel.macos] archs = ["universal2"] From 11358d544d028992e8971c0e231329efb5068836 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Mon, 31 Jul 2023 12:43:17 +0200 Subject: [PATCH 48/49] review --- doc/contrib/release.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contrib/release.rst b/doc/contrib/release.rst index 309ad418a1..658cb7cd4f 100644 --- a/doc/contrib/release.rst +++ b/doc/contrib/release.rst @@ -126,7 +126,7 @@ Post Release #. Update Zenodo with authors and changelog created in previous step and submit. -#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just released ``3.14.15``, change ``VERSION`` to ``3.15.16-dev``. Make the number portion always consists of a 3-tuple. Shorter versions are uninstallable by Spack (``spack install arbor@0.8`` will install v0.8.1, due to anything short than a 3-tuple being interpreted as a version range). +#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just released ``3.14.15``, change ``VERSION`` to ``3.15.16-dev``. Make the number portion always consists of a triple. Shorter versions are uninstallable by Spack (``spack install arbor@0.8`` will install v0.8.1, due to anything shorter than a triple being interpreted as a version range). - Update ``spack/package.py``. The checksum of the targz is the sha256sum. - Include changes such as to ``CITATIONS``, ``doc/index.rst`` in postrel PR. Copy Zenodo BibTex export to ``CITATIONS``. From f9674fc8bbb056ed1795fb44d78d0a80b4d7d864 Mon Sep 17 00:00:00 2001 From: Brent Huisman Date: Mon, 31 Jul 2023 16:19:20 +0200 Subject: [PATCH 49/49] words --- doc/contrib/release.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/contrib/release.rst b/doc/contrib/release.rst index 658cb7cd4f..869d81e27f 100644 --- a/doc/contrib/release.rst +++ b/doc/contrib/release.rst @@ -126,7 +126,7 @@ Post Release #. Update Zenodo with authors and changelog created in previous step and submit. -#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just released ``3.14.15``, change ``VERSION`` to ``3.15.16-dev``. Make the number portion always consists of a triple. Shorter versions are uninstallable by Spack (``spack install arbor@0.8`` will install v0.8.1, due to anything shorter than a triple being interpreted as a version range). +#. Make a new PR setting ``VERSION`` to the next with a trailing ``-dev``. E.g. if you just released ``3.14.15``, change ``VERSION`` to ``3.15.16-dev``. Make sure the number portion always consists of a triple. Shorter versions are uninstallable by Spack (``spack install arbor@0.8`` will install v0.8.1, due to anything shorter than a triple being interpreted as a version range). - Update ``spack/package.py``. The checksum of the targz is the sha256sum. - Include changes such as to ``CITATIONS``, ``doc/index.rst`` in postrel PR. Copy Zenodo BibTex export to ``CITATIONS``.