From 9369af5a990e6e2596e1dd49756bfc43fff8d29e Mon Sep 17 00:00:00 2001
From: itowlson <ivan.towlson@fermyon.com>
Date: Tue, 1 Oct 2024 15:35:53 +1300
Subject: [PATCH] Fix for failing runtime tests not failing the test suite

Signed-off-by: itowlson <ivan.towlson@fermyon.com>
---
 Cargo.lock                          |  4 ++--
 Cargo.toml                          |  4 ++--
 tests/conformance-tests/src/main.rs |  6 ++++--
 tests/runtime.rs                    | 12 ++++++++----
 4 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index bb84e5a0d9..e6aee06e40 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1351,7 +1351,7 @@ dependencies = [
 [[package]]
 name = "conformance-tests"
 version = "0.1.0"
-source = "git+https://github.com/fermyon/conformance-tests?rev=387b7f375df59e6254a7c29cf4a53507a9f46d32#387b7f375df59e6254a7c29cf4a53507a9f46d32"
+source = "git+https://github.com/fermyon/conformance-tests?rev=ecd22a45bcc5c775a56c67689a89aa4005866ac0#ecd22a45bcc5c775a56c67689a89aa4005866ac0"
 dependencies = [
  "anyhow",
  "flate2",
@@ -8228,7 +8228,7 @@ dependencies = [
 [[package]]
 name = "test-environment"
 version = "0.1.0"
-source = "git+https://github.com/fermyon/conformance-tests?rev=387b7f375df59e6254a7c29cf4a53507a9f46d32#387b7f375df59e6254a7c29cf4a53507a9f46d32"
+source = "git+https://github.com/fermyon/conformance-tests?rev=ecd22a45bcc5c775a56c67689a89aa4005866ac0#ecd22a45bcc5c775a56c67689a89aa4005866ac0"
 dependencies = [
  "anyhow",
  "fslock",
diff --git a/Cargo.toml b/Cargo.toml
index f6b19eae5f..79e6f89d21 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -126,7 +126,7 @@ members = [
 anyhow = "1"
 async-trait = "0.1"
 bytes = "1"
-conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "387b7f375df59e6254a7c29cf4a53507a9f46d32" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" }
 dirs = "5.0"
 futures = "0.3"
 glob = "0.3"
@@ -146,7 +146,7 @@ serde = { version = "1", features = ["derive", "rc"] }
 serde_json = "1.0"
 sha2 = "0.10"
 tempfile = "3"
-test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "387b7f375df59e6254a7c29cf4a53507a9f46d32" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" }
 thiserror = "1"
 tokio = "1"
 toml = "0.8"
diff --git a/tests/conformance-tests/src/main.rs b/tests/conformance-tests/src/main.rs
index c297b2d07c..b0c5667eee 100644
--- a/tests/conformance-tests/src/main.rs
+++ b/tests/conformance-tests/src/main.rs
@@ -3,8 +3,10 @@ fn main() {
         .nth(1)
         .expect("expected first argument to be path to spin binary")
         .into();
-    conformance_tests::run_tests("canary", move |test| {
+    let config = conformance_tests::Config::new("canary");
+    conformance_tests::run_tests(config, move |test| {
         conformance::run_test(test, &spin_binary)
     })
-    .unwrap();
+    .unwrap()
+    .exit();
 }
diff --git a/tests/runtime.rs b/tests/runtime.rs
index ba3b6b41cd..434475bec9 100644
--- a/tests/runtime.rs
+++ b/tests/runtime.rs
@@ -27,11 +27,15 @@ mod runtime_tests {
     }
 
     #[test]
-    fn conformance_tests() {
-        conformance_tests::run_tests("canary", move |test| {
+    fn conformance_tests() -> anyhow::Result<()> {
+        let config = conformance_tests::Config::new("canary");
+        let conclusion = conformance_tests::run_tests(config, move |test| {
             conformance::run_test(test, &spin_binary())
-        })
-        .unwrap();
+        })?;
+        if conclusion.has_failed() {
+            anyhow::bail!("One or more errors occurred in the conformance tests");
+        }
+        Ok(())
     }
 
     fn spin_binary() -> PathBuf {