From daa975a71337e444d2ded40f99383306f0406b3f Mon Sep 17 00:00:00 2001
From: Valentin Seitz <valentin.seitz@tum.de>
Date: Thu, 14 Dec 2023 22:23:29 +0100
Subject: [PATCH 01/13] Added a first rudimentary test workflow

---
 .github/workflows/run-tests-on-pr.yaml | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100644 .github/workflows/run-tests-on-pr.yaml

diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/run-tests-on-pr.yaml
new file mode 100644
index 00000000..6662590d
--- /dev/null
+++ b/.github/workflows/run-tests-on-pr.yaml
@@ -0,0 +1,13 @@
+on:
+  pull_request_review:
+    types: [submitted]
+
+jobs:
+  testing-approved-pr:
+    if: github.event.review.state == 'APPROVED'
+    uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
+    with:
+      suites: openfoam_adapter_pr
+      build_args: TUTORIALS_REF:develop,PRECICE_REF:develop,OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
+      systests_branch: develop
+      loglevel: "DEBUG"
\ No newline at end of file

From 1bd83d3fd774a494b70cdce5ba0f5ae310c76f59 Mon Sep 17 00:00:00 2001
From: valentin-seitz <valentin.seitz@gmail.com>
Date: Mon, 8 Jan 2024 15:30:30 +0100
Subject: [PATCH 02/13] Update run-tests-on-pr.yaml

---
 .github/workflows/run-tests-on-pr.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/run-tests-on-pr.yaml
index 6662590d..fef9df91 100644
--- a/.github/workflows/run-tests-on-pr.yaml
+++ b/.github/workflows/run-tests-on-pr.yaml
@@ -10,4 +10,4 @@ jobs:
       suites: openfoam_adapter_pr
       build_args: TUTORIALS_REF:develop,PRECICE_REF:develop,OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
       systests_branch: develop
-      loglevel: "DEBUG"
\ No newline at end of file
+      loglevel: "DEBUG"

From 9db6146cf5aaf9ccfbbdc83000a2fb9aca225ba7 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Tue, 12 Mar 2024 10:20:38 +0100
Subject: [PATCH 03/13] Make workflow get triggered by the trigger-system-tests
 label

---
 .github/workflows/run-tests-on-pr.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/run-tests-on-pr.yaml
index fef9df91..60c64f77 100644
--- a/.github/workflows/run-tests-on-pr.yaml
+++ b/.github/workflows/run-tests-on-pr.yaml
@@ -1,10 +1,10 @@
 on:
   pull_request_review:
-    types: [submitted]
+    types: [labeled]
 
 jobs:
   testing-approved-pr:
-    if: github.event.review.state == 'APPROVED'
+    if: ${{ github.event.label.name == 'trigger-system-tests' }}
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
     with:
       suites: openfoam_adapter_pr

From 7d19f75264028f575e44d980f7e91715f3d21222 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 09:19:25 +0100
Subject: [PATCH 04/13] Fix event specification

---
 .github/workflows/run-tests-on-pr.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/run-tests-on-pr.yaml
index 60c64f77..62d1f64e 100644
--- a/.github/workflows/run-tests-on-pr.yaml
+++ b/.github/workflows/run-tests-on-pr.yaml
@@ -1,5 +1,5 @@
 on:
-  pull_request_review:
+  pull_request:
     types: [labeled]
 
 jobs:

From 912f1d51f57e5aee306a1d670d0c1694e0d4cf50 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 09:21:36 +0100
Subject: [PATCH 05/13] Rename test job

---
 .github/workflows/run-tests-on-pr.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/run-tests-on-pr.yaml
index 62d1f64e..4b32703f 100644
--- a/.github/workflows/run-tests-on-pr.yaml
+++ b/.github/workflows/run-tests-on-pr.yaml
@@ -3,7 +3,7 @@ on:
     types: [labeled]
 
 jobs:
-  testing-approved-pr:
+  system-tests:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
     with:

From 57d2ee9360558eee3752528853a7bda27d3ffbea Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 09:22:14 +0100
Subject: [PATCH 06/13] Rename workflow

---
 .github/workflows/{run-tests-on-pr.yaml => system-tests.yaml} | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/workflows/{run-tests-on-pr.yaml => system-tests.yaml} (100%)

diff --git a/.github/workflows/run-tests-on-pr.yaml b/.github/workflows/system-tests.yaml
similarity index 100%
rename from .github/workflows/run-tests-on-pr.yaml
rename to .github/workflows/system-tests.yaml

From f6dc76a7ce5eaf59872f81f0916a3f73d7bc5881 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 09:32:00 +0100
Subject: [PATCH 07/13] Give a human-readable name

---
 .github/workflows/system-tests.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/system-tests.yaml b/.github/workflows/system-tests.yaml
index 4b32703f..f8c57a99 100644
--- a/.github/workflows/system-tests.yaml
+++ b/.github/workflows/system-tests.yaml
@@ -1,3 +1,5 @@
+name: System tests
+
 on:
   pull_request:
     types: [labeled]

From 37b1785230e4ed21ff830c31861aeccef6ae9878 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 12:08:07 +0100
Subject: [PATCH 08/13] Add a job to gather git references

debug
---
 .github/workflows/system-tests.yaml | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/system-tests.yaml b/.github/workflows/system-tests.yaml
index f8c57a99..739b65ad 100644
--- a/.github/workflows/system-tests.yaml
+++ b/.github/workflows/system-tests.yaml
@@ -5,11 +5,25 @@ on:
     types: [labeled]
 
 jobs:
-  system-tests:
+  gather-refs:
+    runs-on: ubuntu-latest
+    outputs:
+      refprecice: ${{ steps.refprecice.outputs.shorthash }}
+    steps:
+      - id: refprecice
+        uses: nmbgeek/github-action-get-latest-commit@main
+        with:
+          owner: precice
+          repo: precice
+          branch: develop
+      - run: |
+          echo ${{ steps.refprecice.outputs.shorthash }}
+
+  trigger-system-tests:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
     with:
       suites: openfoam_adapter_pr
-      build_args: TUTORIALS_REF:develop,PRECICE_REF:develop,OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
+      build_args: TUTORIALS_REF:develop,PRECICE_REF:${{ needs.gather-refs.refprecice }},OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
       systests_branch: develop
       loglevel: "DEBUG"

From ef02c2df234649075e556a40193b051d446de932 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 12:18:02 +0100
Subject: [PATCH 09/13] Specify dependency

Do name conflicts matter here?

Debugging

Debugging

Debugging
---
 .github/workflows/system-tests.yaml | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/system-tests.yaml b/.github/workflows/system-tests.yaml
index 739b65ad..c10915d8 100644
--- a/.github/workflows/system-tests.yaml
+++ b/.github/workflows/system-tests.yaml
@@ -6,9 +6,10 @@ on:
 
 jobs:
   gather-refs:
+    if: ${{ github.event.label.name == 'trigger-system-tests' }}
     runs-on: ubuntu-latest
     outputs:
-      refprecice: ${{ steps.refprecice.outputs.shorthash }}
+      output_refprecice: ${{ steps.refprecice.outputs.shorthash }}
     steps:
       - id: refprecice
         uses: nmbgeek/github-action-get-latest-commit@main
@@ -16,14 +17,22 @@ jobs:
           owner: precice
           repo: precice
           branch: develop
+
+
+  test-job:
+    if: ${{ github.event.label.name == 'trigger-system-tests' }}
+    needs: gather-refs
+    runs-on: ubuntu-latest
+    steps:
       - run: |
-          echo ${{ steps.refprecice.outputs.shorthash }}
+          echo "${{ needs.gather-refs.output_refprecice }}" >> $GITHUB_OUTPUT
 
   trigger-system-tests:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
+    needs: gather-refs
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
     with:
       suites: openfoam_adapter_pr
-      build_args: TUTORIALS_REF:develop,PRECICE_REF:${{ needs.gather-refs.refprecice }},OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
+      build_args: TUTORIALS_REF:develop,PRECICE_REF:${{ needs.gather-refs.output_refprecice }},OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
       systests_branch: develop
       loglevel: "DEBUG"

From 30064e66df3604ded8ea0d41ab6498a65e71f273 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 12:37:52 +0100
Subject: [PATCH 10/13] Fix levels

---
 .github/workflows/system-tests.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/system-tests.yaml b/.github/workflows/system-tests.yaml
index c10915d8..dca2f5c5 100644
--- a/.github/workflows/system-tests.yaml
+++ b/.github/workflows/system-tests.yaml
@@ -9,7 +9,7 @@ jobs:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
     runs-on: ubuntu-latest
     outputs:
-      output_refprecice: ${{ steps.refprecice.outputs.shorthash }}
+      refprecice: ${{ steps.refprecice.outputs.shorthash }}
     steps:
       - id: refprecice
         uses: nmbgeek/github-action-get-latest-commit@main
@@ -25,7 +25,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - run: |
-          echo "${{ needs.gather-refs.output_refprecice }}" >> $GITHUB_OUTPUT
+          echo "${{ needs.gather-refs.outputs.refprecice }}" >> $GITHUB_OUTPUT
 
   trigger-system-tests:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
@@ -33,6 +33,6 @@ jobs:
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
     with:
       suites: openfoam_adapter_pr
-      build_args: TUTORIALS_REF:develop,PRECICE_REF:${{ needs.gather-refs.output_refprecice }},OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
+      build_args: TUTORIALS_REF:develop,PRECICE_REF:${{ needs.gather-refs.outputs.refprecice }},OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
       systests_branch: develop
       loglevel: "DEBUG"

From 110474944557d27de4bc735c3c98c29eb1bcd69d Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 12:42:26 +0100
Subject: [PATCH 11/13] Use preCICE from v3.0.0 tag, tutorials from latest
 develop

---
 .github/workflows/system-tests.yaml | 17 ++++-------------
 1 file changed, 4 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/system-tests.yaml b/.github/workflows/system-tests.yaml
index dca2f5c5..0c2f5ecd 100644
--- a/.github/workflows/system-tests.yaml
+++ b/.github/workflows/system-tests.yaml
@@ -9,30 +9,21 @@ jobs:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
     runs-on: ubuntu-latest
     outputs:
-      refprecice: ${{ steps.refprecice.outputs.shorthash }}
+      reftutorials: ${{ steps.reftutorials.outputs.shorthash }}
     steps:
-      - id: refprecice
+      - id: reftutorials
         uses: nmbgeek/github-action-get-latest-commit@main
         with:
           owner: precice
-          repo: precice
+          repo: tutorials
           branch: develop
 
-
-  test-job:
-    if: ${{ github.event.label.name == 'trigger-system-tests' }}
-    needs: gather-refs
-    runs-on: ubuntu-latest
-    steps:
-      - run: |
-          echo "${{ needs.gather-refs.outputs.refprecice }}" >> $GITHUB_OUTPUT
-
   trigger-system-tests:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
     needs: gather-refs
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop
     with:
       suites: openfoam_adapter_pr
-      build_args: TUTORIALS_REF:develop,PRECICE_REF:${{ needs.gather-refs.outputs.refprecice }},OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
+      build_args: TUTORIALS_REF:${{ needs.gather-refs.outputs.reftutorials }},PRECICE_REF:v3.0.0,OPENFOAM_EXECUTABLE:openfoam2306,OPENFOAM_ADAPTER_REF:${{ github.event.pull_request.head.sha }}
       systests_branch: develop
       loglevel: "DEBUG"

From 0e915424c2cf036987d66ed2cbe62444063cf702 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 12:51:34 +0100
Subject: [PATCH 12/13] Rename job

---
 .github/workflows/system-tests.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/system-tests.yaml b/.github/workflows/system-tests.yaml
index 0c2f5ecd..d72794aa 100644
--- a/.github/workflows/system-tests.yaml
+++ b/.github/workflows/system-tests.yaml
@@ -18,7 +18,7 @@ jobs:
           repo: tutorials
           branch: develop
 
-  trigger-system-tests:
+  run-system-tests:
     if: ${{ github.event.label.name == 'trigger-system-tests' }}
     needs: gather-refs
     uses: precice/tutorials/.github/workflows/run_testsuite_workflow.yml@develop

From 1a1db0cc968b902a92d61d13669490e4e17aaf17 Mon Sep 17 00:00:00 2001
From: Gerasimos Chourdakis <chourdak@in.tum.de>
Date: Wed, 13 Mar 2024 13:08:41 +0100
Subject: [PATCH 13/13] Add note about system tests in contributing guidelines

---
 CONTRIBUTING.md | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index f48c65ce..b09fa65b 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -27,3 +27,13 @@ There are also a few additional workflows that can be triggered manually:
 - `Check links`: checks the links in all markdown files to verify if they are still reachable.
 
 Members of the repository can trigger these workflows in the "Actions" tab.
+
+## System tests
+
+For non-trivial pull requests, we also need to execute [system regression tests](https://precice.org/dev-docs-system-tests.html),
+to ensure that complete simulations still run and give the same results.
+Because these take long, run on an external system, and consume significant resources,
+we are only triggering these on demand. Add (or ask a maintainer to add) the
+`trigger-system-tests` label to the pull request to trigger them.
+The tests will only run once, so that further commits don't consume additional
+resources: In case you want to re-trigger them, remove and add the label again.