From 48ed6b303ce08e433a180b8e59333fc146ca8f1c Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Fri, 19 Apr 2024 16:22:10 -0500
Subject: [PATCH 01/15] Bump actions/checkout from 4.1.2 to 4.1.3 (#203)

Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.2 to 4.1.3.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/9bb56186c3b09b4f86b1c65136769dd318469633...1d96c772d19495a3b5c517cd2bc0cb401ea0529f)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/ci.yml      | 2 +-
 .github/workflows/docs.yml    | 2 +-
 .github/workflows/license.yml | 2 +-
 .github/workflows/run.yml     | 2 +-
 .github/workflows/style.yml   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6522a5809..43b845c06 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,7 +23,7 @@ jobs:
       license: ${{ steps.filter.outputs.license }}
 
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # @v2
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # @v2
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 98ba3c468..3411cd7e6 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
+      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
 
     - name: Setup Python
       uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
index 6330ac93d..433f64dce 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/license.yml
@@ -6,7 +6,7 @@ jobs:
   verify-license:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
 
       - name: Set up Python 3.11
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index 907d6e673..7fd82f655 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -11,7 +11,7 @@ jobs:
           sudo apt-get remove -y gcc-13
 
       - name: Checkout Benchpark
-        uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
+        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
 
       - name: Add needed Python libs
         run: |
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index ed17ef2a1..8fca7e0bf 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -6,7 +6,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
+      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
 
       - name: Set up Python 3.11
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d

From 3bf4ae772b97de0d8e92a8265b00e9d1ac2969d7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Apr 2024 12:05:49 -0700
Subject: [PATCH 02/15] Bump sphinx from 7.2.6 to 7.3.7 in
 /.github/workflows/requirements (#205)

Bumps [sphinx](https://github.com/sphinx-doc/sphinx) from 7.2.6 to 7.3.7.
- [Release notes](https://github.com/sphinx-doc/sphinx/releases)
- [Changelog](https://github.com/sphinx-doc/sphinx/blob/master/CHANGES.rst)
- [Commits](https://github.com/sphinx-doc/sphinx/compare/v7.2.6...v7.3.7)

---
updated-dependencies:
- dependency-name: sphinx
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/requirements/docs.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/requirements/docs.txt b/.github/workflows/requirements/docs.txt
index f49db2129..ecc3c198c 100644
--- a/.github/workflows/requirements/docs.txt
+++ b/.github/workflows/requirements/docs.txt
@@ -1,5 +1,5 @@
 # docs
-sphinx==7.2.6
+sphinx==7.3.7
 sphinx-rtd-theme==2.0.0
 codespell==2.2.6
 pandas==2.2.2

From b04069324efb8fe574080ca892b4176dc854f9a2 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 22 Apr 2024 12:17:49 -0700
Subject: [PATCH 03/15] Bump actions/upload-artifact from 4.3.2 to 4.3.3 (#206)

Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 4.3.2 to 4.3.3.
- [Release notes](https://github.com/actions/upload-artifact/releases)
- [Commits](https://github.com/actions/upload-artifact/compare/1746f4ab65b179e0ea60a494b83293b640dd5bba...65462800fd760344b1a7b4382951275a0abb4808)

---
updated-dependencies:
- dependency-name: actions/upload-artifact
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/run.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index 7fd82f655..eaac67979 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -74,7 +74,7 @@ jobs:
 
       - name: Upload Workspace Archive as CI Artifact
         if: always()
-        uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba
+        uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808
         with:
           name: workspace-archive
           path: './workspace/saxpy/openmp/nosite-x86_64/workspace/archive/**'

From 24cda7b1ea16b38e0588fd3c5fb143c86122599d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 24 Apr 2024 11:06:46 -0700
Subject: [PATCH 04/15] Bump actions/checkout from 4.1.3 to 4.1.4 (#208)

Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.3 to 4.1.4.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/1d96c772d19495a3b5c517cd2bc0cb401ea0529f...0ad4b8fadaa221de15dcec353f45205ec38ea70b)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/ci.yml      | 2 +-
 .github/workflows/docs.yml    | 2 +-
 .github/workflows/license.yml | 2 +-
 .github/workflows/run.yml     | 2 +-
 .github/workflows/style.yml   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 43b845c06..254a0dbe3 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,7 +23,7 @@ jobs:
       license: ${{ steps.filter.outputs.license }}
 
     steps:
-      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f # @v2
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # @v2
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 3411cd7e6..4b9e4741b 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
+      uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
 
     - name: Setup Python
       uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
index 433f64dce..63d1ffafb 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/license.yml
@@ -6,7 +6,7 @@ jobs:
   verify-license:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
 
       - name: Set up Python 3.11
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index eaac67979..1c6fdc162 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -11,7 +11,7 @@ jobs:
           sudo apt-get remove -y gcc-13
 
       - name: Checkout Benchpark
-        uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
+        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
 
       - name: Add needed Python libs
         run: |
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index 8fca7e0bf..421bd974d 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -6,7 +6,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f
+      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
 
       - name: Set up Python 3.11
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d

From 15769ecc7cb0fe313aca54657a3afa2fc0df0502 Mon Sep 17 00:00:00 2001
From: pearce8 <pearce8@llnl.gov>
Date: Wed, 24 Apr 2024 12:44:08 -0700
Subject: [PATCH 05/15] Add modifier docs (#190)

* initial commit on caliper modifier

* add links

* updates

* remove extra line

* add following

* reconfigure, separate modifier and caliper modifier sections

- add table for different caliper modifiers

* multi-line list in table

* update table

* Update 3-opt-edit-experiment.rst

* Adding modifiers to the help menu

* List benchmarks and experiments

* Lint

* Flattening list options, removing redundant error output

---------

Co-authored-by: Stephanie Brink <brink2@llnl.gov>
---
 bin/benchpark                  | 93 +++++++++++++++++++++-------------
 docs/3-opt-edit-experiment.rst | 51 ++++++++++++++++++-
 2 files changed, 109 insertions(+), 35 deletions(-)

diff --git a/bin/benchpark b/bin/benchpark
index a5359a2e9..b900b4cb7 100755
--- a/bin/benchpark
+++ b/bin/benchpark
@@ -73,7 +73,7 @@ def source_location():
 
 def benchpark_list(subparsers, actions_dict):
     list_parser = subparsers.add_parser(
-        "list", help="List available benchmarks and systems"
+        "list", help="List available experiments, systems, and modifiers"
     )
     list_parser.add_argument("sublist", nargs="?")
     actions_dict["list"] = benchpark_list_handler
@@ -84,8 +84,7 @@ def benchpark_benchmarks():
     benchmarks = []
     experiments_dir = source_dir / "experiments"
     for x in os.listdir(experiments_dir):
-        for y in os.listdir(experiments_dir / x):
-            benchmarks.append(f"{x}/{y}")
+        benchmarks.append(f"{x}")
     return benchmarks
 
 
@@ -94,7 +93,8 @@ def benchpark_experiments():
     experiments = []
     experiments_dir = source_dir / "experiments"
     for x in os.listdir(experiments_dir):
-        experiments.append(f"{x}")
+        for y in os.listdir(experiments_dir / x):
+            experiments.append(f"{x}/{y}")
     return experiments
 
 
@@ -106,6 +106,14 @@ def benchpark_systems():
     return systems
 
 
+def benchpark_modifiers():
+    source_dir = source_location()
+    modifiers = []
+    for x in os.listdir(source_dir / "modifiers"):
+        modifiers.append(x)
+    return modifiers
+
+
 def benchpark_get_tags():
     f = source_location() / "tags.yaml"
     tags = []
@@ -132,36 +140,44 @@ def benchpark_list_handler(args):
     source_dir = source_location()
     sublist = args.sublist
     benchmarks = benchpark_benchmarks()
+    experiments = benchpark_experiments()
     systems = benchpark_systems()
+    modifiers = benchpark_modifiers()
 
     if sublist == None:
-        print("Benchmarks/ProgrammingModel:")
+        print("Experiments:")
+        for experiment in experiments:
+            print(f"\t{experiment}")
+        print("Systems:")
+        for system in systems:
+            print(f"\t{system}")
+    elif sublist == "benchmarks":
+        print("Benchmarks:")
         for benchmark in benchmarks:
             print(f"\t{benchmark}")
+    elif sublist == "experiments":
+        print("Experiments:")
+        for experiment in experiments:
+            print(f"\t{experiment}")
+    elif sublist == "systems":
         print("Systems:")
         for system in systems:
             print(f"\t{system}")
+    elif sublist == "modifiers":
+        print("Modifiers:")
+        for modifier in modifiers:
+            print(f"\t{modifier}")
     else:
-        if sublist == "benchmarks":
-            print("Benchmarks:")
-            for benchmark in benchmarks:
-                print(f"\t{benchmark}")
-        else:
-            if sublist == "systems":
-                print("Systems:")
-                for system in systems:
-                    print(f"\t{system}")
-            else:
-                raise ValueError(
-                    f'Invalid benchpark list "{sublist}" - must choose [benchmarks], [systems], or leave empty'
-                )
+        raise ValueError(
+            f'Invalid benchpark list "{sublist}" - must choose [experiments], [systems], [modifiers] or leave empty'
+        )
 
 
 def benchpark_check_benchmark(arg_str):
     benchmarks = benchpark_benchmarks()
     found = arg_str in benchmarks
     if not found:
-        out_str = f'Invalid benchmark/experiment "{arg_str}" - must choose one of: '
+        out_str = f'Invalid benchmark "{arg_str}" - must choose one of: '
         for benchmark in benchmarks:
             out_str += f"\n\t{benchmark}"
         raise ValueError(out_str)
@@ -172,7 +188,7 @@ def benchpark_check_experiment(arg_str):
     experiments = benchpark_experiments()
     found = arg_str in experiments
     if not found:
-        out_str = f'Invalid benchmark/experiment "{arg_str}" - must choose one of: '
+        out_str = f'Invalid experiment (benchmark/ProgrammingModel) "{arg_str}" - must choose one of: '
         for experiment in experiments:
             out_str += f"\n\t{experiment}"
         raise ValueError(out_str)
@@ -201,13 +217,26 @@ def benchpark_check_tag(arg_str):
     return found
 
 
+def benchpark_check_modifier(arg_str):
+    modifiers = benchpark_modifiers()
+    found = arg_str in modifiers
+    if not found:
+        out_str = f'Invalid modifier "{arg_str}" - must choose one of: '
+        for modifier in modifiers:
+            out_str += f"\n\t{modifier}"
+        raise ValueError(out_str)
+    return found
+
+
 def benchpark_setup(subparsers, actions_dict):
     create_parser = subparsers.add_parser(
         "setup", help="Set up an experiment and prepare it to build/run"
     )
 
     create_parser.add_argument(
-        "benchmark", type=str, help="The experiment (benchmark/ProgrammingModel) to run"
+        "experiment",
+        type=str,
+        help="The experiment (benchmark/ProgrammingModel) to run",
     )
     create_parser.add_argument(
         "system", type=str, help="The system on which to run the experiment"
@@ -311,32 +340,28 @@ def benchpark_setup_handler(args):
     experiments_root/
         spack/
         ramble/
-        <benchmark>/
+        <experiment>/
             <system>/
                 workspace/
                     configs/
                         (everything from source/configs/<system>)
-                        (everything from source/experiments/<benchmark>)
+                        (everything from source/experiments/<experiment>)
     """
 
-    benchmark = args.benchmark
+    experiment = args.experiment
     system = args.system
     experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
     modifier = args.modifier
     source_dir = source_location()
     debug_print(f"source_dir = {source_dir}")
-    debug_print(f"specified benchmark/ProgrammingModel = {benchmark}")
-    valid_benchmark = benchpark_check_benchmark(benchmark)
+    debug_print(f"specified experiment (benchmark/ProgrammingModel) = {experiment}")
+    valid_experiment = benchpark_check_experiment(experiment)
     debug_print(f"specified system = {system}")
     valid_system = benchpark_check_system(system)
-    if not (valid_benchmark and valid_system):
-        raise ValueError(
-            "Invalid benchmark/experiment and system provided: {0} {1}".format(
-                benchmark, system
-            )
-        )
+    debug_print(f"specified modifier = {modifier}")
+    valid_modifier = benchpark_check_modifier(modifier)
 
-    workspace_dir = experiments_root / str(benchmark) / str(system)
+    workspace_dir = experiments_root / str(experiment) / str(system)
 
     if workspace_dir.exists():
         if workspace_dir.is_dir():
@@ -360,7 +385,7 @@ def benchpark_setup_handler(args):
     print(f"Setting up configs for Ramble workspace {ramble_configs_dir}")
 
     configs_src_dir = source_dir / "configs" / str(system)
-    experiment_src_dir = source_dir / "experiments" / benchmark
+    experiment_src_dir = source_dir / "experiments" / experiment
     modifier_config_dir = source_dir / "modifiers" / modifier / "configs"
     ramble_configs_dir.mkdir(parents=True)
     ramble_logs_dir.mkdir(parents=True)
diff --git a/docs/3-opt-edit-experiment.rst b/docs/3-opt-edit-experiment.rst
index e35ca1758..59b4dd456 100644
--- a/docs/3-opt-edit-experiment.rst
+++ b/docs/3-opt-edit-experiment.rst
@@ -31,7 +31,7 @@ Benchpark configuration files are organized as follows::
      │  └── package.py 
      └── repo.yaml 
 
-You can edit any of them to change the behavior of your experiments.
+You can edit these configuration files to change the behavior of your experiments.
 
 System specification
 --------------------
@@ -54,3 +54,52 @@ If you would like to modify a specification of your benchmark,
 you can do so by upstreaming changes to Spack and/or Ramble,
 or working on your benchmark specification in ``benchpark/repo/${BENCHMARK}`` 
 (see :doc:`add-a-benchmark` for details).
+
+Modifiers
+---------
+In Benchpark, a ``modifier`` follows the `Ramble Modifier
+<https://googlecloudplatform.github.io/ramble/tutorials/10_using_modifiers.html#modifiers>`_
+and is an abstract object that can be applied to a large set of reproducible
+specifications. Modifiers are intended to encasulate reusable patterns that
+perform a specific configuration of an experiment. This may include injecting
+performance analysis or setting up system resources.
+
+Applying the Caliper modifier
+-----------------------------
+We have implemented a Caliper modifier to enable profiling of Caliper-instrumented 
+benchmarks in Benchpark. More documentation on Caliper can be found `here
+<https://software.llnl.gov/Caliper>`_.
+
+To turn on profiling with Caliper, add ``--modifier=<caliper_modifier>`` to the Benchpark
+setup step::
+
+    ./benchpark setup benchmark/programmingmodel system --modifier=<caliper_modifier> <workspace-dir>
+
+Valid values for ``<caliper_modifier>`` are found in the **Caliper Modifier**
+column of the table below.  Benchpark will link the experiment to Caliper,
+and inject appropriate Caliper configuration at runtime.  After the experiments 
+in the workspace have completed running, a ``.cali`` file
+is created which contains the collected performance metrics.
+
+.. list-table:: Available caliper modifiers
+   :widths: 20 20 50
+   :header-rows: 1
+
+   * - Caliper Modifier
+     - Where Applicable
+     - Metrics Collected
+   * - caliper
+     - Platform-independent
+     - | - Min time/rank: Minimum time (in seconds) across all ranks
+       | - Max time/rank: Maximum time (in seconds) across all ranks
+       | - Avg time/rank: Average time (in seconds) across all ranks
+       | - Total time: Aggregated time (in seconds) over all ranks
+   * - caliper-topdown
+     - x86 Intel CPUs
+     - | - Retiring
+       | - Bad speculation
+       | - Front end bound
+       | - Back end bound
+   * - caliper-cuda
+     - NVIDIA GPUs
+     - | - CUDA API functions (e.g., time.gpu)

From 7a444f8656480fa3551dc5d1439c1a3fc0c10e53 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 29 Apr 2024 09:51:26 -0700
Subject: [PATCH 06/15] Bump black from 24.4.0 to 24.4.2 in
 /.github/workflows/requirements (#215)

Bumps [black](https://github.com/psf/black) from 24.4.0 to 24.4.2.
- [Release notes](https://github.com/psf/black/releases)
- [Changelog](https://github.com/psf/black/blob/main/CHANGES.md)
- [Commits](https://github.com/psf/black/compare/24.4.0...24.4.2)

---
updated-dependencies:
- dependency-name: black
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/requirements/style.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/requirements/style.txt b/.github/workflows/requirements/style.txt
index 2c06c97ef..45b9c4bae 100644
--- a/.github/workflows/requirements/style.txt
+++ b/.github/workflows/requirements/style.txt
@@ -1,4 +1,4 @@
-black==24.4.0
+black==24.4.2
 flake8==7.0.0
 isort==5.13.2
 codespell==2.2.6

From 597569c4cfb502c1a4427474691f01ac3a8b2f47 Mon Sep 17 00:00:00 2001
From: Alec Scott <scott112@llnl.gov>
Date: Mon, 29 Apr 2024 16:07:27 -0700
Subject: [PATCH 07/15] Tags command should use benchmarks not experiments
 (#217)

* Benchpark tags cmd should use benchmarks not experiments to generate tags

* Black format code
---
 bin/benchpark                   | 20 +++++++++-----------
 docs/generate-benchmark-list.py |  2 +-
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/bin/benchpark b/bin/benchpark
index b900b4cb7..99c1c6385 100755
--- a/bin/benchpark
+++ b/bin/benchpark
@@ -483,7 +483,7 @@ Further steps are needed to build the experiments (ramble -P -D {ramble_workspac
     print(instructions)
 
 
-def helper_experiments_tags(ramble_exe, experiments):
+def helper_experiments_tags(ramble_exe, benchmarks):
     # find all tags in Ramble applications (both in Ramble built-in and in Benchpark/repo)
     (tags_stdout, tags_stderr) = run_command(f"{ramble_exe} attributes --tags --all")
     ramble_applications_tags = {}
@@ -494,20 +494,20 @@ def helper_experiments_tags(ramble_exe, experiments):
         ramble_applications_tags[key_value[0]] = key_value[1].strip().split(",")
 
     benchpark_experiments_tags = {}
-    for exp in experiments:
-        benchpark_experiments_tags[exp] = ramble_applications_tags[exp]
+    for benchmark in benchmarks:
+        benchpark_experiments_tags[benchmark] = ramble_applications_tags[benchmark]
     return benchpark_experiments_tags
 
 
 def benchpark_tags_handler(args):
     """
-    Filter ramble tags by benchpark experiments
+    Filter ramble tags by benchpark benchmarks
     """
     source_dir = source_location()
     experiments_root = pathlib.Path(os.path.abspath(args.experiments_root))
     ramble_location = experiments_root / "ramble"
     ramble_exe = ramble_location / "bin" / "ramble"
-    experiments = benchpark_experiments()
+    benchmarks = benchpark_benchmarks()
 
     if args.tag:
         if benchpark_check_tag(args.tag):
@@ -516,17 +516,15 @@ def benchpark_tags_handler(args):
             lines = tag_stdout.splitlines()
 
             for line in lines:
-                if line in experiments:
+                if line in benchmarks:
                     print(line)
 
     elif args.application:
-        if benchpark_check_experiment(args.application):
-            benchpark_experiments_tags = helper_experiments_tags(
-                ramble_exe, experiments
-            )
+        if benchpark_check_benchmark(args.application):
+            benchpark_experiments_tags = helper_experiments_tags(ramble_exe, benchmarks)
             print(benchpark_experiments_tags[args.application])
     else:
-        benchpark_experiments_tags = helper_experiments_tags(ramble_exe, experiments)
+        benchpark_experiments_tags = helper_experiments_tags(ramble_exe, benchmarks)
         print("All tags that exist in Benchpark experiments:")
         for k, v in benchpark_experiments_tags.items():
             print(k)
diff --git a/docs/generate-benchmark-list.py b/docs/generate-benchmark-list.py
index 391604a68..209f7eb33 100755
--- a/docs/generate-benchmark-list.py
+++ b/docs/generate-benchmark-list.py
@@ -54,7 +54,7 @@ def main(workspace):
     for bmark in benchmarks:
         # call benchpark tags -a bmark workspace
         cmd = ["../bin/benchpark", "tags", "-a", bmark, workspace]
-        byte_data = subprocess.run(cmd, capture_output=True)
+        byte_data = subprocess.run(cmd, capture_output=True, check=True)
         tags = str(byte_data.stdout, "utf-8")
         tags = (
             tags.replace("[", "")

From f2847e3c0f7df9e432a1f300158ff960712d6350 Mon Sep 17 00:00:00 2001
From: Alec Scott <scott112@llnl.gov>
Date: Mon, 29 Apr 2024 16:08:58 -0700
Subject: [PATCH 08/15] ci: rebuild docs on changes to bin/benchpark (#216)

Co-authored-by: pearce8 <pearce8@llnl.gov>
---
 .github/workflows/ci.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 254a0dbe3..b9b31a25c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -35,6 +35,7 @@ jobs:
           filters: |
             docs:
               - '.github/**'
+              - 'bin/**'
               - 'docs/**'
               - 'README.rst'
             style:

From ea8293e69cf3fd6270274dcef87dbfaddee9515e Mon Sep 17 00:00:00 2001
From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com>
Date: Tue, 30 Apr 2024 15:35:13 -0700
Subject: [PATCH 09/15] Update spack version (#219)

* Update spack version

* Removing specific cmake version requested by saxpy

* Removing specific cmake version requested by raja-perf

* Removing specific cmake version requested by raja-perf

---------

Co-authored-by: Riyaz Haque <your-email@example.com>
Co-authored-by: pearce8 <pearce8@llnl.gov>
---
 checkout-versions.yaml                     | 2 +-
 experiments/raja-perf/mpi-only/ramble.yaml | 2 +-
 experiments/raja-perf/openmp/ramble.yaml   | 2 +-
 experiments/saxpy/openmp/ramble.yaml       | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/checkout-versions.yaml b/checkout-versions.yaml
index 2d7d07ce6..6c3973512 100644
--- a/checkout-versions.yaml
+++ b/checkout-versions.yaml
@@ -5,4 +5,4 @@
 
 versions:
   ramble: bb664f142b2cbdb2b2ea39e70a8535c9f27c1179
-  spack: 31de670bd26beca979ebd75dcb0ce90c535a78c4
+  spack: c2eef8bab26adb00b250992e29d697b4706356a0
diff --git a/experiments/raja-perf/mpi-only/ramble.yaml b/experiments/raja-perf/mpi-only/ramble.yaml
index 6e19371c1..73e58f6d4 100644
--- a/experiments/raja-perf/mpi-only/ramble.yaml
+++ b/experiments/raja-perf/mpi-only/ramble.yaml
@@ -32,7 +32,7 @@ ramble:
         spack_spec: 'blt@0.5.2:'
         compiler: compiler-gcc
       raja-perf:
-        spack_spec: raja-perf@develop ~openmp +mpi ^cmake@3.23.1
+        spack_spec: raja-perf@develop ~openmp +mpi
         compiler: default-compiler
     environments:
       raja-perf:
diff --git a/experiments/raja-perf/openmp/ramble.yaml b/experiments/raja-perf/openmp/ramble.yaml
index 7e734d177..21a531a18 100644
--- a/experiments/raja-perf/openmp/ramble.yaml
+++ b/experiments/raja-perf/openmp/ramble.yaml
@@ -33,7 +33,7 @@ ramble:
     concretized: true
     packages:
       raja-perf:
-        spack_spec: raja-perf@develop +openmp +mpi ^cmake@3.23.1
+        spack_spec: raja-perf@develop +openmp +mpi 
         compiler: default-compiler
     environments:
       raja-perf:
diff --git a/experiments/saxpy/openmp/ramble.yaml b/experiments/saxpy/openmp/ramble.yaml
index cc7bdf7da..f4e601edf 100644
--- a/experiments/saxpy/openmp/ramble.yaml
+++ b/experiments/saxpy/openmp/ramble.yaml
@@ -40,7 +40,7 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: saxpy@1.0.0 +openmp{modifier_spack_variant} ^cmake@3.23.1
+        spack_spec: saxpy@1.0.0 +openmp{modifier_spack_variant}
         compiler: default-compiler
     environments:
       saxpy:

From 7a4f6b69121a59dc10448fa86e098ca1dc887c53 Mon Sep 17 00:00:00 2001
From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com>
Date: Fri, 3 May 2024 12:34:48 -0700
Subject: [PATCH 10/15] Update saxpy spec (#227)

* Disable openmp variant in saxpy cuda spec

* Disable openmp variant in saxpy rocm spec

---------

Co-authored-by: Riyaz Haque <your-email@example.com>
---
 .../auxiliary_software_files/packages.yaml                     | 3 +++
 experiments/saxpy/cuda/ramble.yaml                             | 2 +-
 experiments/saxpy/rocm/ramble.yaml                             | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml
index af604fd29..838cc7542 100644
--- a/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml
+++ b/configs/LLNL-Tioga-HPECray-zen3-MI250X-Slingshot/auxiliary_software_files/packages.yaml
@@ -41,6 +41,9 @@ packages:
       prefix: /usr
     - spec: cmake@3.23.1
       prefix: /usr/tce
+    - spec: cmake@3.24.2
+      prefix: /usr/tce
+    buildable: false
   pkgconf:
     externals:
     - spec: pkgconf@1.4.2
diff --git a/experiments/saxpy/cuda/ramble.yaml b/experiments/saxpy/cuda/ramble.yaml
index 677a096f3..f1260d196 100644
--- a/experiments/saxpy/cuda/ramble.yaml
+++ b/experiments/saxpy/cuda/ramble.yaml
@@ -33,7 +33,7 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: saxpy@1.0.0 +cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: saxpy@1.0.0 +cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ~openmp ^cuda@{default_cuda_version}
         compiler: default-compiler
     environments:
       saxpy:
diff --git a/experiments/saxpy/rocm/ramble.yaml b/experiments/saxpy/rocm/ramble.yaml
index dd80075d0..80f443b21 100644
--- a/experiments/saxpy/rocm/ramble.yaml
+++ b/experiments/saxpy/rocm/ramble.yaml
@@ -33,7 +33,7 @@ ramble:
     concretized: true
     packages:
       saxpy:
-        spack_spec: saxpy@1.0.0 +rocm{modifier_spack_variant} amdgpu_target={rocm_arch}
+        spack_spec: saxpy@1.0.0 +rocm{modifier_spack_variant} ~openmp amdgpu_target={rocm_arch}
         compiler: default-compiler
     environments:
       saxpy:

From 26459096144e983ac8e9b59783f33a55602565f7 Mon Sep 17 00:00:00 2001
From: pearce8 <pearce8@llnl.gov>
Date: Fri, 3 May 2024 16:37:00 -0500
Subject: [PATCH 11/15] Adding Kripke to Benchpark (#83)

* Adding Kripke to Benchpark

* adding license

* adding ramble.yaml for cuda and rocm

* Adding tags

* Adding cuda instead of openmp

* adding Kripke package.py in benchpark repo temporarily

* Update license

* Pulling in new Spack package, updating to point at Robert's PR

* First attempt to align versions of RAJA suite components

* Experiment/kripke new (#214)

* OpenMP implementation of Kripke

* HIP implementation of kripke

* Update spack version. Pin kripke version

* Delete repo/kripke/application.py

* Add kripke application.py again

* Remove redundant file

* CUDA implementation of kripke

* Upgrade cmake externals package

* Update cmake external package config for LLNL-Tioga-HPECray-zen3-MI250X-Slingshot

* Update lassen compiler config

* Removing deprecated cmake from machine config

* Removing deprecated cmake from machine config

* Removing deprecated cmake from machine config

* minor

---------

Co-authored-by: Riyaz Haque <your-email@example.com>
Co-authored-by: pearce8 <pearce8@llnl.gov>

---------

Co-authored-by: Riyaz Haque <5333387+rfhaque@users.noreply.github.com>
Co-authored-by: Riyaz Haque <your-email@example.com>
---
 .../auxiliary_software_files/packages.yaml    |   6 +-
 .../auxiliary_software_files/packages.yaml    |   9 +-
 .../auxiliary_software_files/compilers.yaml   |  20 ++-
 .../auxiliary_software_files/packages.yaml    |  13 +-
 .../spack.yaml                                |   4 +-
 .../kripke/cuda/execute_experiment.tpl        |  13 ++
 experiments/kripke/cuda/ramble.yaml           |  54 +++++++
 .../kripke/openmp/execute_experiment.tpl      |  13 ++
 experiments/kripke/openmp/ramble.yaml         |  57 +++++++
 .../kripke/rocm/execute_experiment.tpl        |  13 ++
 experiments/kripke/rocm/ramble.yaml           |  54 +++++++
 repo/kripke/application.py                    | 118 ++++++++++++++
 repo/kripke/package.py                        | 146 ++++++++++++++++++
 13 files changed, 509 insertions(+), 11 deletions(-)
 create mode 100755 experiments/kripke/cuda/execute_experiment.tpl
 create mode 100644 experiments/kripke/cuda/ramble.yaml
 create mode 100755 experiments/kripke/openmp/execute_experiment.tpl
 create mode 100644 experiments/kripke/openmp/ramble.yaml
 create mode 100755 experiments/kripke/rocm/execute_experiment.tpl
 create mode 100644 experiments/kripke/rocm/ramble.yaml
 create mode 100644 repo/kripke/application.py
 create mode 100644 repo/kripke/package.py

diff --git a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml
index 0cb4fc3b5..7aa44dea9 100644
--- a/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml
+++ b/configs/LLNL-Magma-Penguin-icelake-OmniPath/auxiliary_software_files/packages.yaml
@@ -16,9 +16,9 @@ packages:
       buildable: false
   cmake:
     externals:
-    - spec: cmake@3.23.1
-      prefix: /usr/tce/packages/cmake/cmake-3.23.1
-      buildable: false
+    - spec: cmake@3.26.3
+      prefix: /usr/tce/packages/cmake/cmake-3.26.3
+    buildable: false
   gmake:
     externals:
     - spec: gmake@4.2.1
diff --git a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml
index 1b64a0ee3..129876c20 100644
--- a/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml
+++ b/configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml
@@ -16,8 +16,8 @@ packages:
     buildable: false
   cmake:
     externals:
-    - spec: cmake@3.23.1
-      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+    - spec: cmake@3.26.3
+      prefix: /usr/tce/packages/cmake/cmake-3.26.3
     buildable: false
   gmake:
     externals:
@@ -48,6 +48,11 @@ packages:
       prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-intel-classic-2021.6.0/
     - spec: mvapich2@2.3.7-clang-14.0.6
       prefix: /usr/tce/packages/mvapich2/mvapich2-2.3.7-clang-14.0.6
+  cub:
+    buildable: false
+    externals:
+    - spec: cub@11.8.0
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0/
   cuda:
     buildable: false
     externals:
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
index d27a632c1..4e4c457a1 100644
--- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
+++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
@@ -4,6 +4,22 @@
 # SPDX-License-Identifier: Apache-2.0
 
 compilers:
+- compiler:
+    spec: xl@16.1.1-2023.06.28-cuda-11.8.0-gcc-11.2.1
+    paths:
+      cc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlc
+      cxx: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlC
+      f77: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf
+      fc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++14
+      fflags: -g -O2
+    operating_system: rhel7
+    target: ppc64le
+    modules: [cuda/11.8.0, xl/2023.06.28-cuda-11.8.0-gcc-11.2.1]
+    environment: {}
+    extra_rpaths: []
 - compiler:
     spec: xl@16.1.1-2022.08.19-cuda-11.7.0
     paths:
@@ -29,7 +45,7 @@ compilers:
       fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf
     flags:
       cflags: -g -O2
-      cxxflags: -g -O2 -std=c++11
+      cxxflags: -g -O2 -std=c++14
       fflags: -g -O2
     operating_system: rhel7
     target: ppc64le
@@ -45,7 +61,7 @@ compilers:
       fc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf
     flags: # TODO: Fix spack concretization bug
       cflags: -g -O2
-      cxxflags: -g -O2 -std=c++11
+      cxxflags: -g -O2 -std=c++14
       fflags: -g -O2
     operating_system: rhel7
     target: ppc64le
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
index 61c85f62d..5d26658c1 100644
--- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
+++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
@@ -11,8 +11,8 @@ packages:
     buildable: false
   cmake:
     externals:
-    - spec: cmake@3.23.1
-      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+    - spec: cmake@3.29.2
+      prefix: /usr/tce/packages/cmake/cmake-3.29.2
     buildable: false
   gmake:
     externals:
@@ -41,6 +41,11 @@ packages:
     - spec: cusparse@11.8.0
       prefix: /usr/tce/packages/cuda/cuda-11.8.0
     buildable: false
+  cub:
+    buildable: false
+    externals:
+    - spec: cub@11.8.0
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0/
   cuda:
     externals:
     - spec: cuda@10.1.243
@@ -72,6 +77,10 @@ packages:
       buildable: false
   mpi:
     externals:
+    - spec: spectrum-mpi@2023.06.28-cuda-11.8.0-gcc-11.2.1
+      prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2023.06.28-cuda-11.8.0-gcc-11.2.1
+      extra_attributes:
+        ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
     - spec: spectrum-mpi@2022.08.19-cuda-11.8.0
       prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0
       extra_attributes:
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
index ed1edcb60..7b2f4257b 100644
--- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
+++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
@@ -6,9 +6,9 @@
 spack:
   packages:
     default-compiler:
-      spack_spec: xl@16.1.1-2022.08.19-cuda{default_cuda_version}
+      spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
     default-mpi:
-      spack_spec: spectrum-mpi@2022.08.19
+      spack_spec: spectrum-mpi@2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
     compiler-gcc:
       spack_spec: gcc@8.3.1
     compiler-clang:
diff --git a/experiments/kripke/cuda/execute_experiment.tpl b/experiments/kripke/cuda/execute_experiment.tpl
new file mode 100755
index 000000000..89e73cf49
--- /dev/null
+++ b/experiments/kripke/cuda/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml
new file mode 100644
index 000000000..b4771bdba
--- /dev/null
+++ b/experiments/kripke/cuda/ramble.yaml
@@ -0,0 +1,54 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    kripke:
+      workloads:
+        kripke:
+          env_vars:
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: '2'
+            n_nodes: '2'
+            arch: 'CUDA'
+          experiments:
+            kripke_cuda_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
+              variables:
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '64'
+                nzz: '32'
+                npx: '2'
+                npy: '2'
+                npz: '1'
+  spack:
+    concretized: true
+    packages:
+      kripke:
+        spack_spec: kripke@1.2.7.0 +mpi+cuda{modifier_spack_variant} ^chai@2024.02
+        compiler: default-compiler
+    environments:
+      kripke:
+        packages:
+        - default-mpi
+        - kripke
+        - '{modifier_package_name}'
diff --git a/experiments/kripke/openmp/execute_experiment.tpl b/experiments/kripke/openmp/execute_experiment.tpl
new file mode 100755
index 000000000..89e73cf49
--- /dev/null
+++ b/experiments/kripke/openmp/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/kripke/openmp/ramble.yaml b/experiments/kripke/openmp/ramble.yaml
new file mode 100644
index 000000000..afdbaa127
--- /dev/null
+++ b/experiments/kripke/openmp/ramble.yaml
@@ -0,0 +1,57 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    kripke:
+      workloads:
+        kripke:
+          env_vars:
+            set:
+              OMP_NUM_THREADS: '{omp_num_threads}'
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: ['1', '16']
+            n_nodes: '1'
+            omp_num_threads: ['32', '2']
+            arch: 'OpenMP'
+          experiments:
+            kripke_omp_{n_nodes}_{omp_num_threads}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
+              variables:
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '32'
+                nzz: '32'
+                npx: ['1', '4']
+                npy: ['1', '2']
+                npz: ['1', '2']
+  spack:
+    concretized: true
+    packages:
+      kripke:
+        spack_spec: kripke@1.2.7.0 +mpi+openmp{modifier_spack_variant} ^chai@2024.02
+        compiler: default-compiler
+    environments:
+      kripke:
+        packages:
+        - default-mpi
+        - kripke
+        - '{modifier_package_name}'
diff --git a/experiments/kripke/rocm/execute_experiment.tpl b/experiments/kripke/rocm/execute_experiment.tpl
new file mode 100755
index 000000000..89e73cf49
--- /dev/null
+++ b/experiments/kripke/rocm/execute_experiment.tpl
@@ -0,0 +1,13 @@
+#!/bin/bash
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{batch_nodes}
+{batch_ranks}
+{batch_timeout}
+
+cd {experiment_run_dir}
+
+{command}
diff --git a/experiments/kripke/rocm/ramble.yaml b/experiments/kripke/rocm/ramble.yaml
new file mode 100644
index 000000000..7439b57dc
--- /dev/null
+++ b/experiments/kripke/rocm/ramble.yaml
@@ -0,0 +1,54 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+ramble:
+  include:
+  - ./configs/spack.yaml
+  - ./configs/variables.yaml
+  - ./configs/modifier.yaml
+
+  config:
+    deprecated: true
+    spack_flags:
+      install: '--add --keep-stage'
+      concretize: '-U -f'
+
+  applications:
+    kripke:
+      workloads:
+        kripke:
+          env_vars:
+          variables:
+            n_ranks: '{processes_per_node} * {n_nodes}'
+            experiment_setup: ''
+            processes_per_node: ['8']
+            n_nodes: '1'
+            arch: 'HIP'
+          experiments:
+            kripke_rocm_{n_nodes}_{ngroups}_{gs}_{nquad}_{ds}_{lorder}_{nzx}_{nzy}_{nzz}_{npx}_{npy}_{npz}:
+              variables:
+                ngroups: 64
+                gs: 1
+                nquad: 128
+                ds: 128
+                lorder: 4
+                nzx: '64'
+                nzy: '64'
+                nzz: '64'
+                npx: '2'
+                npy: '2'
+                npz: '2'
+  spack:
+    concretized: true
+    packages:
+      kripke:
+        spack_spec: kripke@1.2.7.0 +mpi+rocm{modifier_spack_variant} ^chai@2024.02
+        compiler: default-compiler
+    environments:
+      kripke:
+        packages:
+        - default-mpi
+        - kripke
+        - '{modifier_package_name}'
diff --git a/repo/kripke/application.py b/repo/kripke/application.py
new file mode 100644
index 000000000..61a79f7f6
--- /dev/null
+++ b/repo/kripke/application.py
@@ -0,0 +1,118 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from ramble.appkit import *
+
+import sys
+
+class Kripke(SpackApplication):
+    """Kripke benchmark uses RAJA Portability Layer"""
+    name = "Kripke"
+
+    tags = ['asc','transport','deterministic','structured-grid',
+            'large-scale','multi-node','single-node','c++','raja',
+            'simd','vectorization','register-pressure','high-fp','atomics','high-branching',
+            'high-memory-bandwidth','large-memory-footprint','regular-memory-access',
+            'mpi','network-latency-bound','network-collectives']
+
+    executable('kripke', 'kripke.exe' +
+                     ' --groups {ngroups}' +
+                     ' --legendre {lorder}' +
+                     ' --quad {nquad}' +
+                     ' --zones {nzx},{nzy},{nzz}' +
+                     ' --sigt {sigt0},{sigt1},{sigt2}' +
+                     ' --sigs {sigs0},{sigs1},{sigs2}' +
+                     ' --arch {arch}' +
+                     ' --layout {layout}' +
+#                     ' --pdist {lout}' +
+                     ' --procs {npx},{npy},{npz}' +
+                     ' --dset {ds}' +
+                     ' --gset {gs}' +
+                     ' --zset {nzsetx},{nzsety},{nzsetz}' +
+                     ' --niter {niter}' +
+                     ' --pmethod {method}', use_mpi=True)
+
+    workload('kripke', executables=['kripke'])
+
+    workload_variable('ngroups', default='32',
+                      description='Number of energy groups. (Default: --groups 32)',
+                      workloads=['kripke'])
+    workload_variable('lorder', default='4',
+                      description='Scattering Legendre Expansion Order (0, 1, ...). (Default: --legendre 4)',
+                      workloads=['kripke'])
+    workload_variable('nquad', default='96',
+                      description='Define the quadrature set to use either a fake S2 with points (ndirs), OR Gauss-Legendre with by points (polar:azim). (Default: --quad 96)',
+                      workloads=['kripke'])
+    workload_variable('nzx', default='16',
+                      description='Number of zones in x. (Default: 16)',
+                      workloads=['kripke'])
+    workload_variable('nzx', default='16',
+                      description='Number of zones in y. (Default: 16)',
+                      workloads=['kripke'])
+    workload_variable('nzz', default='16',
+                      description='Number of zones in z. (Default: 16)',
+                      workloads=['kripke'])
+    workload_variable('sigt0', default='0.1',
+                      description='Total material cross-sections',
+                      workloads=['kripke'])
+    workload_variable('sigt1', default='0.0001',
+                      description='Total material cross-sections',
+                      workloads=['kripke'])
+    workload_variable('sigt2', default='0.1',
+                      description='Total material cross-sections',
+                      workloads=['kripke'])
+    workload_variable('sigs0', default='0.05',
+                      description='Total material cross-sections',
+                      workloads=['kripke'])
+    workload_variable('sigs1', default='0.00005',
+                      description='Total material cross-sections',
+                      workloads=['kripke'])
+    workload_variable('sigs2', default='0.05',
+                      description='Total material cross-sections',
+                      workloads=['kripke'])
+    workload_variable('arch', default='OpenMP',
+                      description='Architecture selection. Selects the back-end used for computation, available are Sequential, OpenMP, CUDA and HIP. The default depends on capabilities selected by the build system and is selected from list of increasing precedence: Sequential, OpenMP, CUDA and HIP.',
+                      workloads=['kripke'])
+    workload_variable('layout', default='DGZ',
+                      description='Data layout selection. This determines the data layout and kernel implementation details (such as loop nesting order). The layouts are determined by the order of unknowns in the angular flux: Direction, Group, and Zone. Available layouts are DGZ, DZG, GDZ, GZD, ZDG, and ZGD. The order is specified left-to-right in longest-to-shortest stride. For example: DGZ means that Directions are the longest stride, and Zones are stride-1. (Default: --nest DGZ)',
+                      workloads=['kripke'])
+#    workload_variable('lout', default='0',
+#                      description='Layout of spatial subdomains over mpi ranks. 0 for "Blocked" where local zone sets represent adjacent regions of space. 1 for "Scattered" where adjacent regions of space are distributed to adjacent MPI ranks. (Default: --layout 0)',
+#                      workloads=['kripke'])
+    workload_variable('npx', default='1',
+                      description='Number of MPI ranks in x dimension',
+                      workloads=['kripke'])
+    workload_variable('npy', default='1',
+                      description='Number of MPI ranks in y dimension',
+                      workloads=['kripke'])
+    workload_variable('npz', default='1',
+                      description='Number of MPI ranks in z dimension',
+                      workloads=['kripke'])
+    workload_variable('ds', default='8',
+                      description='Number of direction-sets. Must be a factor of 8, and divide evenly the number of quadrature points. (Default: --dset 8)',
+                      workloads=['kripke'])
+    workload_variable('gs', default='1',
+                      description='Number of energy group-sets. Must divide evenly the number energy groups. (Default: --gset 1)',
+                      workloads=['kripke'])
+    workload_variable('nzsetx', default='1',
+                      description='Number of zone-sets in x',
+                      workloads=['kripke'])
+    workload_variable('nzsety', default='1',
+                      description='Number of zone-sets in y',
+                      workloads=['kripke'])
+    workload_variable('nzsetz', default='1',
+                      description='Number of zone-sets in z',
+                      workloads=['kripke'])
+    workload_variable('niter', default='10',
+                      description='Number of solver iterations to run. (Default: --niter 10)',
+                      workloads=['kripke'])
+    workload_variable('method', default='sweep',
+                      description='Parallel solver method. "sweep" for full up-wind sweep (wavefront algorithm). "bj" for Block Jacobi. (Default: --pmethod sweep)',
+                      workloads=['kripke'])
+
+    #figure_of_merit('Figure of Merit (FOM)', log_file='{experiment_run_dir}/{experiment_name}.out', fom_regex=r'Figure of Merit \(FOM\):\s+(?P<fom>[0-9]+\.[0-9]*(e^[0-9]*)?)', group_name='fom', units='')
+
+    #TODO: Fix the FOM success_criteria(...)
+    #success_criteria('pass', mode='string', match=r'Figure of Merit \(FOM\)', file='{experiment_run_dir}/{experiment_name}.out')
diff --git a/repo/kripke/package.py b/repo/kripke/package.py
new file mode 100644
index 000000000..7f1269550
--- /dev/null
+++ b/repo/kripke/package.py
@@ -0,0 +1,146 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from spack.package import *
+
+
+class Kripke(CMakePackage, CudaPackage, ROCmPackage):
+    """Kripke is a simple, scalable, 3D Sn deterministic particle
+    transport proxy/mini app.
+    """
+
+    homepage = "https://computing.llnl.gov/projects/co-design/kripke"
+    git = "https://github.com/LLNL/Kripke.git"
+
+    tags = ["proxy-app"]
+
+    maintainers("vsrana01")
+
+    license("BSD-3-Clause")
+
+    version("develop", branch="develop", submodules=False)
+    version(
+        "1.2.7.0", submodules=False, commit="db920c1f5e1dcbb9e949d120e7d86efcdb777635"
+    )
+    version(
+        "1.2.4", submodules=False, tag="v1.2.4", commit="d85c6bc462f17a2382b11ba363059febc487f771"
+    )
+    version(
+        "1.2.3", submodules=True, tag="v1.2.3", commit="66046d8cd51f5bcf8666fd8c810322e253c4ce0e"
+    )
+    version(
+        "1.2.2",
+        submodules=True,
+        tag="v1.2.2-CORAL2",
+        commit="a12bce71e751f8f999009aa2fd0839b908b118a4",
+    )
+    version(
+        "1.2.1",
+        submodules=True,
+        tag="v1.2.1-CORAL2",
+        commit="c36453301ddd684118bb0fb426cfa62764d42398",
+    )
+    version(
+        "1.2.0",
+        submodules=True,
+        tag="v1.2.0-CORAL2",
+        commit="67e4b0a2f092009d61f44b5122111d388a3bec2a",
+    )
+
+    variant("mpi", default=True, description="Build with MPI.")
+    variant("openmp", default=False, description="Build with OpenMP enabled.")
+    variant("caliper", default=False, description="Build with Caliper support enabled.")
+
+    depends_on("mpi", when="+mpi")
+    depends_on("caliper", when="+caliper")
+    depends_on("adiak@0.4:", when="+caliper")
+    conflicts("^blt@:0.3.6", when="+rocm")
+
+    aligned_versions = ["2024.02"]
+    for v in aligned_versions:
+      depends_on(f"raja@{v}~exercises~examples", when=f"^chai@{v}")
+      depends_on(f"umpire@{v}~examples", when=f"^chai@{v}")
+      depends_on(f"chai@{v}~examples+raja", when=f"^chai@{v}")
+      depends_on(f"camp@{v}", when=f"^chai@{v}")
+
+    depends_on("blt@0.6.2:", type="build", when=f"@1.2.7:")
+
+    depends_on("chai+openmp", when="+openmp")
+    depends_on("chai~openmp", when="~openmp")
+    depends_on("chai+cuda", when="+cuda")
+    depends_on("chai~cuda", when="~cuda")
+    depends_on("chai+rocm", when="+rocm")
+    depends_on("chai~rocm", when="~rocm")
+
+    depends_on("umpire+openmp", when="+openmp")
+    depends_on("umpire~openmp", when="~openmp")
+    depends_on("umpire+cuda", when="+cuda")
+    depends_on("umpire~cuda", when="~cuda")
+    depends_on("umpire+rocm", when="+rocm")
+    depends_on("umpire~rocm", when="~rocm")
+
+    def setup_build_environment(self, env):
+        spec = self.spec
+        if "+cuda" in spec:
+            env.set("CUDAHOSTCXX", self.spec["mpi"].mpicxx)
+
+    def cmake_args(self):
+        spec = self.spec
+        args = []
+
+        args.extend(
+            [
+                "-DCAMP_DIR=%s" % self.spec["camp"].prefix,
+                "-DBLT_SOURCE_DIR=%s" % self.spec["blt"].prefix,
+                "-Dumpire_DIR=%s" % self.spec["umpire"].prefix,
+                "-DRAJA_DIR=%s" % self.spec["raja"].prefix,
+                "-Dchai_DIR=%s" % self.spec["chai"].prefix,
+                "-DENABLE_CHAI=ON",
+            ]
+        )
+
+        if "+openmp" in spec:
+            args.append("-DENABLE_OPENMP=ON")
+
+        if "+caliper" in spec:
+            args.append("-DENABLE_CALIPER=ON")
+
+        if "+mpi" in spec:
+            args.append("-DENABLE_MPI=ON")
+            args.append(self.define("CMAKE_CXX_COMPILER", self.spec["mpi"].mpicxx))
+
+        if "+rocm" in spec:
+            # Set up the hip macros needed by the build
+            args.append("-DENABLE_HIP=ON")
+            args.append("-DHIP_ROOT_DIR={0}".format(spec["hip"].prefix))
+            rocm_archs = spec.variants["amdgpu_target"].value
+            if "none" not in rocm_archs:
+                args.append("-DHIP_HIPCC_FLAGS=--amdgpu-target={0}".format(",".join(rocm_archs)))
+                args.append("-DCMAKE_HIP_ARCHITECTURES={0}".format(rocm_archs))
+        else:
+            # Ensure build with hip is disabled
+            args.append("-DENABLE_HIP=OFF")
+
+        if "+cuda" in spec:
+            args.append("-DENABLE_CUDA=ON")
+            args.append(self.define("CMAKE_CUDA_HOST_COMPILER", self.spec["mpi"].mpicxx))
+            if not spec.satisfies("cuda_arch=none"):
+                cuda_arch = spec.variants["cuda_arch"].value
+                args.append("-DCUDA_ARCH={0}".format(cuda_arch[0]))
+                args.append("-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch[0]))
+            args.append(
+                "-DCMAKE_CUDA_FLAGS=--extended-lambda -I%s -I=%s"
+                % (self.spec["cub"].prefix.include, self.spec["mpi"].prefix.include)
+            )
+        else:
+            args.append("-DENABLE_CUDA=OFF")
+
+        return args
+
+    def install(self, spec, prefix):
+        # Kripke does not provide install target, so we have to copy
+        # things into place.
+        mkdirp(prefix.bin)
+        install(join_path(self.build_directory, "kripke.exe"), prefix.bin)

From 5db2c9644145684472b2f754a31417feec3b14ce Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Tue, 7 May 2024 12:08:20 -0500
Subject: [PATCH 12/15] Bump actions/checkout from 4.1.4 to 4.1.5 (#228)

Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.4 to 4.1.5.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/0ad4b8fadaa221de15dcec353f45205ec38ea70b...44c2b7a8a4ea60a981eaca3cf939b5f4305c123b)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 .github/workflows/ci.yml      | 2 +-
 .github/workflows/docs.yml    | 2 +-
 .github/workflows/license.yml | 2 +-
 .github/workflows/run.yml     | 2 +-
 .github/workflows/style.yml   | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index b9b31a25c..6e4fdb54b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -23,7 +23,7 @@ jobs:
       license: ${{ steps.filter.outputs.license }}
 
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # @v2
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b # @v2
         if: ${{ github.event_name == 'push' }}
         with:
           fetch-depth: 0
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 4b9e4741b..1518e6ec3 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -7,7 +7,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
 
     - name: Setup Python
       uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
diff --git a/.github/workflows/license.yml b/.github/workflows/license.yml
index 63d1ffafb..a0f91ecaa 100644
--- a/.github/workflows/license.yml
+++ b/.github/workflows/license.yml
@@ -6,7 +6,7 @@ jobs:
   verify-license:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
 
       - name: Set up Python 3.11
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d
diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml
index 1c6fdc162..5cf898ab8 100644
--- a/.github/workflows/run.yml
+++ b/.github/workflows/run.yml
@@ -11,7 +11,7 @@ jobs:
           sudo apt-get remove -y gcc-13
 
       - name: Checkout Benchpark
-        uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+        uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
 
       - name: Add needed Python libs
         run: |
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
index 421bd974d..0b5fa700e 100644
--- a/.github/workflows/style.yml
+++ b/.github/workflows/style.yml
@@ -6,7 +6,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b
+      - uses: actions/checkout@44c2b7a8a4ea60a981eaca3cf939b5f4305c123b
 
       - name: Set up Python 3.11
         uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d

From eec41da4ab1c2043747dd8b764399cb4f18e5a6f Mon Sep 17 00:00:00 2001
From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com>
Date: Wed, 8 May 2024 10:57:15 -0700
Subject: [PATCH 13/15] Update ramble version (#236)

Co-authored-by: Riyaz Haque <your-email@example.com>
---
 checkout-versions.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/checkout-versions.yaml b/checkout-versions.yaml
index 6c3973512..fa20e8338 100644
--- a/checkout-versions.yaml
+++ b/checkout-versions.yaml
@@ -4,5 +4,5 @@
 # SPDX-License-Identifier: Apache-2.0
 
 versions:
-  ramble: bb664f142b2cbdb2b2ea39e70a8535c9f27c1179
+  ramble: d33af765be716ae27ebbe65a4cc5553661fa92b1
   spack: c2eef8bab26adb00b250992e29d697b4706356a0

From 564df51c93e25ba0a92fc082bbdae43bfc8035ee Mon Sep 17 00:00:00 2001
From: Riyaz Haque <5333387+rfhaque@users.noreply.github.com>
Date: Mon, 13 May 2024 12:09:02 -0700
Subject: [PATCH 14/15] Change default compiler on lassen from xl to clang-ibm
 (#243)

Co-authored-by: Riyaz Haque <your-email@example.com>
---
 .../auxiliary_software_files/compilers.yaml      | 16 ++++++++++++++++
 .../auxiliary_software_files/packages.yaml       |  4 ++++
 .../spack.yaml                                   | 12 ++++++++++--
 experiments/amg2023/cuda/ramble.yaml             |  8 ++++++--
 experiments/gromacs/cuda/ramble.yaml             | 10 +++++-----
 experiments/kripke/cuda/ramble.yaml              |  4 ++++
 experiments/raja-perf/cuda/ramble.yaml           |  6 +++++-
 experiments/saxpy/cuda/ramble.yaml               |  6 +++++-
 8 files changed, 55 insertions(+), 11 deletions(-)

diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
index 4e4c457a1..bcfcd8000 100644
--- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
+++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/compilers.yaml
@@ -4,6 +4,22 @@
 # SPDX-License-Identifier: Apache-2.0
 
 compilers:
+- compiler:
+    spec: clang@16.0.6-ibm-cuda-11.8.0-gcc-11.2.1
+    paths:
+      cc: /usr/tce/packages/clang/clang-ibm-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang
+      cxx: /usr/tce/packages/clang/clang-ibm-16.0.6-cuda-11.8.0-gcc-11.2.1/bin/clang++
+      f77: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf_r
+      fc: /usr/tce/packages/xl/xl-2023.06.28-cuda-11.8.0-gcc-11.2.1/bin/xlf_r
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++17
+      fflags: -g -O2
+    operating_system: rhel7
+    target: ppc64le
+    modules: [cuda/11.8.0, clang/ibm-16.0.6-cuda-11.8.0-gcc-11.2.1]
+    environment: {}
+    extra_rpaths: []
 - compiler:
     spec: xl@16.1.1-2023.06.28-cuda-11.8.0-gcc-11.2.1
     paths:
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
index 5d26658c1..e3877a10a 100644
--- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
+++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/auxiliary_software_files/packages.yaml
@@ -77,6 +77,10 @@ packages:
       buildable: false
   mpi:
     externals:
+    - spec: spectrum-mpi@2023.06.28-clang-ibm-16.0.6-cuda-11.8.0-gcc-11.2.1
+      prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-clang-ibm-16.0.6-cuda-11.8.0-gcc-11.2.1
+      extra_attributes:
+        ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
     - spec: spectrum-mpi@2023.06.28-cuda-11.8.0-gcc-11.2.1
       prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2023.06.28-cuda-11.8.0-gcc-11.2.1
       extra_attributes:
diff --git a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
index 7b2f4257b..3e1cce76f 100644
--- a/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
+++ b/configs/LLNL-Sierra-IBM-power9-V100-Infiniband/spack.yaml
@@ -6,8 +6,12 @@
 spack:
   packages:
     default-compiler:
-      spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
+      spack_spec: clang@16.0.6-ibm-cuda-{default_cuda_version}-gcc-11.2.1
     default-mpi:
+      spack_spec: spectrum-mpi@2023.06.28-clang-ibm-16.0.6-cuda-{default_cuda_version}-gcc-11.2.1
+    compiler-xl:
+      spack_spec: xl@16.1.1-2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
+    mpi-xl:
       spack_spec: spectrum-mpi@2023.06.28-cuda-{default_cuda_version}-gcc-11.2.1
     compiler-gcc:
       spack_spec: gcc@8.3.1
@@ -15,8 +19,12 @@ spack:
       spack_spec: clang@16.0.6-cuda{default_cuda_version}
     mpi-clang:
       spack_spec: spectrum-mpi@2022.08.19-clang16.0.6-cuda-{default_cuda_version}
+    compiler-clang-ibm:
+      spack_spec: clang@16.0.6-ibm-cuda-{default_cuda_version}-gcc-11.2.1
+    mpi-clang-ibm:
+      spack_spec: spectrum-mpi@2023.06.28-clang-ibm-16.0.6-cuda-{default_cuda_version}-gcc-11.2.1
     blas:
-      spack_spec: cublas@11.8.0
+      spack_spec: cublas@{default_cuda_version}
     cublas-cuda:
       spack_spec: cublas@{default_cuda_version}
     lapack:
diff --git a/experiments/amg2023/cuda/ramble.yaml b/experiments/amg2023/cuda/ramble.yaml
index 0b9587f5b..0a5d7f76d 100644
--- a/experiments/amg2023/cuda/ramble.yaml
+++ b/experiments/amg2023/cuda/ramble.yaml
@@ -48,15 +48,19 @@ ramble:
   spack:
     concretized: true
     packages:
+      cuda:
+        spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
+        compiler: default-compiler
       hypre:
-        spack_spec: hypre@2.28.0 +mpi+cuda+mixedint{modifier_spack_variant} cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: hypre@2.28.0 +mpi+cuda+mixedint{modifier_spack_variant} cuda_arch=={cuda_arch}
         compiler: default-compiler
       amg2023:
-        spack_spec: amg2023@develop +mpi+cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: amg2023@develop +mpi+cuda{modifier_spack_variant} cuda_arch=={cuda_arch}
         compiler: default-compiler
     environments:
       amg2023:
         packages:
+        - cuda
         - cublas-cuda
         - lapack
         - default-mpi
diff --git a/experiments/gromacs/cuda/ramble.yaml b/experiments/gromacs/cuda/ramble.yaml
index 6591db87b..0dc188d0f 100644
--- a/experiments/gromacs/cuda/ramble.yaml
+++ b/experiments/gromacs/cuda/ramble.yaml
@@ -44,18 +44,18 @@ ramble:
   spack:
     concretized: true
     packages:
-      gromacs:
-        spack_spec: gromacs@2023.3 +mpi+openmp+cuda~hwloc~double cuda_arch=={cuda_arch}
-        compiler: compiler-clang
       cuda:
         spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
-        compiler: compiler-clang
+        compiler: default-compiler
+      gromacs:
+        spack_spec: gromacs@2023.3 +mpi+openmp+cuda~hwloc~double cuda_arch=={cuda_arch}
+        compiler: default-compiler
     environments:
       gromacs:
         packages:
         - cuda
         - blas
         - lapack
-        - mpi-clang
+        - default-mpi
         - fftw
         - gromacs
diff --git a/experiments/kripke/cuda/ramble.yaml b/experiments/kripke/cuda/ramble.yaml
index b4771bdba..e33617839 100644
--- a/experiments/kripke/cuda/ramble.yaml
+++ b/experiments/kripke/cuda/ramble.yaml
@@ -43,12 +43,16 @@ ramble:
   spack:
     concretized: true
     packages:
+      cuda:
+        spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
+        compiler: default-compiler
       kripke:
         spack_spec: kripke@1.2.7.0 +mpi+cuda{modifier_spack_variant} ^chai@2024.02
         compiler: default-compiler
     environments:
       kripke:
         packages:
+        - cuda
         - default-mpi
         - kripke
         - '{modifier_package_name}'
diff --git a/experiments/raja-perf/cuda/ramble.yaml b/experiments/raja-perf/cuda/ramble.yaml
index 070d4f1c1..607b10fea 100644
--- a/experiments/raja-perf/cuda/ramble.yaml
+++ b/experiments/raja-perf/cuda/ramble.yaml
@@ -26,11 +26,15 @@ ramble:
   spack:
     concretized: true
     packages:
+      cuda:
+        spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
+        compiler: default-compiler
       raja-perf:
-        spack_spec: raja-perf@develop +cuda~openmp+mpi cuda_arch=={cuda_arch} ^cuda@{default_cuda_version}
+        spack_spec: raja-perf@develop +cuda~openmp+mpi cuda_arch=={cuda_arch}
         compiler: default-compiler
     environments:
       raja-perf:
         packages:
+        - cuda
         - default-mpi
         - raja-perf
diff --git a/experiments/saxpy/cuda/ramble.yaml b/experiments/saxpy/cuda/ramble.yaml
index f1260d196..a819742b0 100644
--- a/experiments/saxpy/cuda/ramble.yaml
+++ b/experiments/saxpy/cuda/ramble.yaml
@@ -32,12 +32,16 @@ ramble:
   spack:
     concretized: true
     packages:
+      cuda:
+        spack_spec: cuda@{default_cuda_version}+allow-unsupported-compilers
+        compiler: default-compiler
       saxpy:
-        spack_spec: saxpy@1.0.0 +cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ~openmp ^cuda@{default_cuda_version}
+        spack_spec: saxpy@1.0.0 +cuda{modifier_spack_variant} cuda_arch=={cuda_arch} ~openmp
         compiler: default-compiler
     environments:
       saxpy:
         packages:
+        - cuda
         - default-mpi
         - saxpy
         - '{modifier_package_name}'

From 4770efbfa1b3332eae7f9c7e0b90ed102d9f881d Mon Sep 17 00:00:00 2001
From: pearce8 <pearce8@llnl.gov>
Date: Tue, 14 May 2024 12:17:31 -0500
Subject: [PATCH 15/15] Adding docs on using Ramble workspaces (#209)

---
 bin/benchpark                  |  5 ++-
 docs/3-opt-edit-experiment.rst | 56 +++++++++++++++++-----------------
 docs/4-benchpark-setup.rst     | 27 ++++++++--------
 docs/5-build-experiment.rst    | 25 ++++++++-------
 docs/FAQ-what-to-rerun.rst     |  4 +--
 docs/add-a-benchmark.rst       | 44 +++++++++++++++-----------
 docs/add-an-experiment.rst     | 18 +++++------
 docs/set-of-experiments.rst    | 24 +++++++--------
 8 files changed, 107 insertions(+), 96 deletions(-)

diff --git a/bin/benchpark b/bin/benchpark
index 99c1c6385..e2466159d 100755
--- a/bin/benchpark
+++ b/bin/benchpark
@@ -102,7 +102,10 @@ def benchpark_systems():
     source_dir = source_location()
     systems = []
     for x in os.listdir(source_dir / "configs"):
-        systems.append(x)
+        if not (
+            os.path.isfile(os.path.join(source_dir / "configs", x)) or x == "common"
+        ):
+            systems.append(x)
     return systems
 
 
diff --git a/docs/3-opt-edit-experiment.rst b/docs/3-opt-edit-experiment.rst
index 59b4dd456..fd9838117 100644
--- a/docs/3-opt-edit-experiment.rst
+++ b/docs/3-opt-edit-experiment.rst
@@ -7,29 +7,29 @@
 (optional) Edit experiment
 ==========================
 
-Benchpark configuration files are organized as follows:: 
+Benchpark configuration files are organized as follows::
 
-  $benchpark 
-  ├── configs 
-  │  ├── ${SYSTEM1} 
-  │  │  ├── auxiliary_software_files 
-  │  │  │  ├── compilers.yaml 
-  │  │  │  └── packages.yaml 
-  │  │  ├── spack.yaml 
-  │  │  └── variables.yaml 
-  ├── experiments 
-  │  ├── ${BENCHMARK1} 
-  │  │  ├── ${ProgrammingModel1} 
-  │  │  │  ├── execute_experiment.tpl 
-  │  │  │  └── ramble.yaml 
-  │  │  ├── ${ProgrammingModel1} 
-  │  │  │  ├── execute_experiment.tpl 
-  │  │  │  └── ramble.yaml 
-  └── repo 
-     ├── ${BENCHMARK1} 
-     │  ├── application.py 
-     │  └── package.py 
-     └── repo.yaml 
+  $benchpark
+  ├── configs
+  │  ├── ${SYSTEM1}
+  │  │  ├── auxiliary_software_files
+  │  │  │  ├── compilers.yaml
+  │  │  │  └── packages.yaml
+  │  │  ├── spack.yaml
+  │  │  └── variables.yaml
+  ├── experiments
+  │  ├── ${BENCHMARK1}
+  │  │  ├── ${ProgrammingModel1}
+  │  │  │  ├── execute_experiment.tpl
+  │  │  │  └── ramble.yaml
+  │  │  ├── ${ProgrammingModel1}
+  │  │  │  ├── execute_experiment.tpl
+  │  │  │  └── ramble.yaml
+  └── repo
+     ├── ${BENCHMARK1}
+     │  ├── application.py
+     │  └── package.py
+     └── repo.yaml
 
 You can edit these configuration files to change the behavior of your experiments.
 
@@ -43,8 +43,8 @@ Experiment specification
 ------------------------
 Files under ``benchpark/experiments/${BENCHMARK}/${ProgrammingModel}``
 provide the specifications for the experiments.
-If you would like to make changes to your experiments,  such as enabling 
-specific tools to measure the performance of your experiments, 
+If you would like to make changes to your experiments,  such as enabling
+specific tools to measure the performance of your experiments,
 you can manually edit the specifications in ``ramble.yaml``
 (see :doc:`add-an-experiment` for details).
 
@@ -52,7 +52,7 @@ Benchmark specification
 -----------------------
 If you would like to modify a specification of your benchmark,
 you can do so by upstreaming changes to Spack and/or Ramble,
-or working on your benchmark specification in ``benchpark/repo/${BENCHMARK}`` 
+or working on your benchmark specification in ``benchpark/repo/${BENCHMARK}``
 (see :doc:`add-a-benchmark` for details).
 
 Modifiers
@@ -66,18 +66,18 @@ performance analysis or setting up system resources.
 
 Applying the Caliper modifier
 -----------------------------
-We have implemented a Caliper modifier to enable profiling of Caliper-instrumented 
+We have implemented a Caliper modifier to enable profiling of Caliper-instrumented
 benchmarks in Benchpark. More documentation on Caliper can be found `here
 <https://software.llnl.gov/Caliper>`_.
 
 To turn on profiling with Caliper, add ``--modifier=<caliper_modifier>`` to the Benchpark
 setup step::
 
-    ./benchpark setup benchmark/programmingmodel system --modifier=<caliper_modifier> <workspace-dir>
+    ./benchpark setup <Benchmark/ProgrammingModel> <System> --modifier=<caliper_modifier> <workspace-dir>
 
 Valid values for ``<caliper_modifier>`` are found in the **Caliper Modifier**
 column of the table below.  Benchpark will link the experiment to Caliper,
-and inject appropriate Caliper configuration at runtime.  After the experiments 
+and inject appropriate Caliper configuration at runtime.  After the experiments
 in the workspace have completed running, a ``.cali`` file
 is created which contains the collected performance metrics.
 
diff --git a/docs/4-benchpark-setup.rst b/docs/4-benchpark-setup.rst
index 80b0c58ca..a67c8c199 100644
--- a/docs/4-benchpark-setup.rst
+++ b/docs/4-benchpark-setup.rst
@@ -10,12 +10,12 @@ Benchpark Setup
 Select a benchmark experiment to run, along with the programming model to use, and a system to run them on.
 Also choose a directory for your experiment::
 
-    benchpark setup benchmark/ProgrammingModel system /output/path/to/experiments_root
+    benchpark setup <Benchmark/ProgrammingModel> <System> </output/path/to/experiments_root>
 
 where:
 
-- ``benchmark/ProgrammingModel``: amg2023/openmp | amg2023/cuda | saxpy/openmp (available choices in ``benchpark/experiments``)
-- ``system``: x86_64 | HPECray-zen3-MI250X-Slingshot | AWS_PCluster_Hpc7a-zen4-EFA (available choices in :doc:`available-system-specs`)
+- ``<Benchmark/ProgrammingModel>``: amg2023/openmp | amg2023/cuda | saxpy/openmp (available choices in ``benchpark/experiments``)
+- ``<System>``: nosite-x86_64 | LLNL-Sierra-IBM-power9-V100-Infiniband | RCCS-Fugaku-Fujitsu-A64FX-TofuD | nosite-AWS_PCluster_Hpc7a-zen4-EFA (available choices in :doc:`available-system-specs`)
 
 This command will assemble a Ramble workspace per experiment
 with a configuration for the specified benchmark and system
@@ -24,19 +24,20 @@ with the following directory structure::
     experiments_root/
         ramble/
         spack/
-        <benchmark>/
-            <ProgrammingModel>/
-                <system>/
-                    workspace/
-                        configs/
-                            (everything from source/configs/<system>)
-                            (everything from source/experiments/<benchmark>)
+        <Benchmark/ProgrammingModel>/
+            <System>/
+                workspace/
+                    configs/
+                        (everything from source/configs/<System>)
+                        (everything from source/experiments/<Benchmark/ProgrammingModel>)
 
 ``benchpark setup`` will output instructions to follow::
 
-   . <experiments_root>/spack/share/spack/setup-env.sh
-   . <experiments_root>/ramble/share/ramble/setup-env.sh
+   . <experiments_root>/setup.sh
 
-   export SPACK_DISABLE_LOCAL_CONFIG=1
+The ``setup.sh`` script calls the Spack and Ramble setup scripts.  It optionally accepts
+parameters to ``ramble workspace setup`` as `documented in Ramble
+<https://googlecloudplatform.github.io/ramble/workspace.html#setting-up-a-workspace>`_,
+including ``--dry-run`` and ``--phases make_experiments``.
 
 Now you are ready to compile your experiments as described in :doc:`5-build-experiment`.
diff --git a/docs/5-build-experiment.rst b/docs/5-build-experiment.rst
index c8ecfd010..928256214 100644
--- a/docs/5-build-experiment.rst
+++ b/docs/5-build-experiment.rst
@@ -10,7 +10,7 @@ Build experiment
 ``benchpark setup`` has set up the directory structure for your experiment.
 The next step is setting up the Ramble workspace and building the code::
 
-   cd <experiments_root>/<benchmark/ProgrammingModel>/<system>/workspace
+   cd <experiments_root>/<Benchmark/ProgrammingModel>/<System>/workspace
    ramble -P -D . workspace setup
 
 
@@ -19,18 +19,17 @@ Ramble will build the source code and set up the following workspace directory s
     experiments_root/
         ramble/
         spack/
-        <benchmark>/
-            <ProgrammingModel>/
-                <system>/
-                    workspace/
-                        configs/
-                            (everything from source/configs/<system>)
-                            (everything from source/experiments/<benchmark>)
-                        experiments/
-                            <benchmark>/
-                                <problem>/
-                                    <benchmark>_<ProgrammingModel>_<problem>
-                                        execute_experiment
+        <Benchmark/ProgrammingModel>/
+            <System>/
+                workspace/
+                    configs/
+                        (everything from source/configs/<System>)
+                        (everything from source/experiments/<Benchmark/ProgrammingModel>)
+                    experiments/
+                        <Benchmark>/
+                           <Problem>/
+                              <Benchmark>_<ProgrammingModel>_<Problem>
+                                    execute_experiment
 
 If you edit any of the files, see :doc:`FAQ-what-to-rerun` to determine
 whether you need to re-do any of the previous steps.
diff --git a/docs/FAQ-what-to-rerun.rst b/docs/FAQ-what-to-rerun.rst
index a9c387f37..e5c8bcf63 100644
--- a/docs/FAQ-what-to-rerun.rst
+++ b/docs/FAQ-what-to-rerun.rst
@@ -4,7 +4,7 @@
    SPDX-License-Identifier: Apache-2.0
 
 ==========================================
-FAQ: I made changes.  What should I rerun?
+FAQ
 ==========================================
 
 .. list-table:: I made changes.  What should I rerun?
@@ -15,7 +15,7 @@ FAQ: I made changes.  What should I rerun?
      - Commands to rerun
    * - configs
      - ``ramble -P -D . workspace setup``
-   * - benchmark's package.py 
+   * - benchmark's package.py
      - ``ramble -P -D . workspace setup``
    * - dependency of package.py
      - ``ramble -P -D . workspace setup``
diff --git a/docs/add-a-benchmark.rst b/docs/add-a-benchmark.rst
index fbd466beb..2bccc9ab5 100644
--- a/docs/add-a-benchmark.rst
+++ b/docs/add-a-benchmark.rst
@@ -7,27 +7,35 @@
 Adding a Benchmark
 ==================
 
-The following system-independent specification is required for each benchmark: 
+The following system-independent specification is required for each ${Benchmark1}:
 
-- ``package.py`` is a Spack specification that defines how to build and install the benchmark.
-- ``application.py`` is a Ramble specification that defines the benchmark input and parameters.
+- ``package.py`` is a Spack specification that defines how to build and install ${Benchmark1}.
+- ``application.py`` is a Ramble specification that defines the ${Benchmark1} input and parameters.
 
-By default, Benchpark will use the benchmark specifications provided in the Spack and Ramble repos.
+During ``benchpark setup`` the user selects ${Benchmark1} to run as the following::
 
+     benchpark setup ${Benchmark1}/${ProgrammingModel1} ${System1} </output/path/to/experiments_root>
+
+By default, Benchpark will use ${Benchmark1} specifications (``package.py`` and ``application.py``)
+provided in the Spack and Ramble repos.
+
+==============
 benchpark/repo
---------------
-If  you are working on a benchmark that is not (yet) available in Spack and/or Ramble,
-or you would like to override the benchmark specifications available in Spack and/or Ramble,
-you can add the following specification in the ``benchpark/repo``::
+==============
+It is possible to overwrite the benchmark specifications provided in the Spack and Ramble repos.
+To do so, add ``package.py`` and/or ``application.py`` for ${Benchmark1} in 
+``benchpark/repo/${Benchmark1}``::
 
   benchpark
-  └── repo 
-     ├── ${BENCHMARK1} 
-     │  ├── application.py 
-     │  └── package.py 
-     └── repo.yaml 
-
-where ``application.py`` is a Ramble specification for the benchmark,
-``package.py`` is a Spack specification for the benchmark, and
-``repo.yaml`` points at those two files, indicating to Benchpark
-to use them instead of the specifications that may be available in Spack and/or Ramble repos.
+  └── repo
+     ├── ${Benchmark1}
+     │  ├── application.py
+     │  └── package.py
+     └── repo.yaml
+
+``benchpark/repo`` can be used as a staging area for the build and run specification for 
+${Benchmark1} so that you can test them together with your experiment specification 
+on a given system.  Once the experiment is working and merged into the develop branch of Benchpark,
+we would like to encourage the contributors to upstream the ``package.py`` to Spack,
+and the ``application.py`` to Ramble.  When those PRs are approved and merged,
+please submit a PR to remove the duplicated specification from ``benchpark/repo``.
diff --git a/docs/add-an-experiment.rst b/docs/add-an-experiment.rst
index 878833c12..c1c9a3b20 100644
--- a/docs/add-an-experiment.rst
+++ b/docs/add-an-experiment.rst
@@ -6,15 +6,15 @@
 =================
 Add an Experiment
 =================
-  
-Experiment specifications are located in ``$benchpark/experiments``. 
-They are organized by the *ProgrammingModel* used for on-node parallelization for the experiment, 
-e.g., ``$benchpark/experiments/amg2023/cuda`` for an AMG2023 experiment using CUDA (on an NVIDIA GPU),
-and ``$benchpark/experiments/amg2023/openmp`` for an AMG2023 experiment using OpenMP (on a CPU). 
-These files, in conjunction with the system configuration files and package/application repositories, 
-are used to generate a set of concrete Ramble experiments for the target system and programming model. 
 
-- ``ramble.yaml`` defines the `Ramble specs <https://googlecloudplatform.github.io/ramble/workspace_config.html#workspace-config>`_ for building, running, analyzing and archiving experiments. 
-- ``execution_template.tpl`` serves as a template for the final experiment script that will be concretized and executed. 
+Experiment specifications are located in ``benchpark/experiments``.
+They are organized by the *ProgrammingModel* used for on-node parallelization for the experiment,
+e.g., ``benchpark/experiments/amg2023/cuda`` for an AMG2023 experiment using CUDA (on an NVIDIA GPU),
+and ``benchpark/experiments/amg2023/openmp`` for an AMG2023 experiment using OpenMP (on a CPU).
+These files, in conjunction with the system configuration files and package/application repositories,
+are used to generate a set of concrete Ramble experiments for the target system and programming model.
+
+- ``ramble.yaml`` defines the `Ramble specs <https://googlecloudplatform.github.io/ramble/workspace_config.html#workspace-config>`_ for building, running, analyzing and archiving experiments.
+- ``execution_template.tpl`` serves as a template for the final experiment script that will be concretized and executed.
 
 A detailed description of Ramble configuration files is available at `Ramble workspace_config <https://googlecloudplatform.github.io/ramble/workspace_config.html>`_.
diff --git a/docs/set-of-experiments.rst b/docs/set-of-experiments.rst
index a4e01437c..4a6dd982d 100644
--- a/docs/set-of-experiments.rst
+++ b/docs/set-of-experiments.rst
@@ -18,29 +18,29 @@ You may want to use the same experiments_root directory when setting up multiple
 To use the same experiments directory when setting up multiple experiments,
 instruct benchpark to set them up in the same experiments_root::
 
-    benchpark setup benchmark1/ProgrammingModel1 system1 /output/path/to/experiments_root
-    benchpark setup benchmark1/ProgrammingModel2 system2 /output/path/to/experiments_root
-    benchpark setup benchmark2/ProgrammingModel2 system1 /output/path/to/experiments_root
+    benchpark setup ${Benchmark1}/${ProgrammingModel1} ${System1} /output/path/to/experiments_root
+    benchpark setup ${Benchmark1}/${ProgrammingModel2} ${System2} /output/path/to/experiments_root
+    benchpark setup ${Benchmark2}/${ProgrammingModel2} ${System1} /output/path/to/experiments_root
 
 This will result in the following directory structure::
 
     experiments_root/
         ramble/
         spack/
-        benchmark1/
-            ProgrammingModel1/
-                system1/
+        ${Benchmark1}/
+            ${ProgrammingModel1}/
+                ${System1}/
                     workspace/
-            ProgrammingModel2/
-                system2/
+            ${ProgrammingModel2}/
+                ${System2}/
                     workspace/
-        benchmark2/
-            ProgrammingModel2/
-                system1/
+        ${Benchmark2}/
+            ${ProgrammingModel2}/
+                ${System1}/
                     workspace/
 
 Note that there is a single clone of Ramble, and a single clone of Spack,
 which all of the ``experiments`` use.
-Each ``experiment`` (``benchmark/ProgrammingModel`` x ``system`` combination)
+Each ``experiment`` (``Benchmark/ProgrammingModel`` x ``system`` combination)
 has its own ``Ramble workspace``, where this specific ``experiment``
 will be compiled and run.