diff --git a/Cargo.lock b/Cargo.lock
index 8aa4213e3e..9b504d440e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1351,7 +1351,7 @@ dependencies = [
 [[package]]
 name = "conformance-tests"
 version = "0.1.0"
-source = "git+https://github.com/fermyon/conformance-tests?rev=387b7f375df59e6254a7c29cf4a53507a9f46d32#387b7f375df59e6254a7c29cf4a53507a9f46d32"
+source = "git+https://github.com/fermyon/conformance-tests?rev=ecd22a45bcc5c775a56c67689a89aa4005866ac0#ecd22a45bcc5c775a56c67689a89aa4005866ac0"
 dependencies = [
  "anyhow",
  "flate2",
@@ -8118,7 +8118,7 @@ dependencies = [
 [[package]]
 name = "test-environment"
 version = "0.1.0"
-source = "git+https://github.com/fermyon/conformance-tests?rev=387b7f375df59e6254a7c29cf4a53507a9f46d32#387b7f375df59e6254a7c29cf4a53507a9f46d32"
+source = "git+https://github.com/fermyon/conformance-tests?rev=ecd22a45bcc5c775a56c67689a89aa4005866ac0#ecd22a45bcc5c775a56c67689a89aa4005866ac0"
 dependencies = [
  "anyhow",
  "fslock",
diff --git a/Cargo.toml b/Cargo.toml
index ee0226fbc4..0b62a44137 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -119,7 +119,7 @@ members = [
 anyhow = "1"
 async-trait = "0.1"
 bytes = "1"
-conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "387b7f375df59e6254a7c29cf4a53507a9f46d32" }
+conformance-tests = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" }
 dirs = "5.0"
 futures = "0.3"
 glob = "0.3"
@@ -139,7 +139,7 @@ serde = { version = "1", features = ["derive", "rc"] }
 serde_json = "1.0"
 sha2 = "0.10"
 tempfile = "3"
-test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "387b7f375df59e6254a7c29cf4a53507a9f46d32" }
+test-environment = { git = "https://github.com/fermyon/conformance-tests", rev = "ecd22a45bcc5c775a56c67689a89aa4005866ac0" }
 thiserror = "1"
 tokio = "1"
 toml = "0.8"
diff --git a/tests/conformance-tests/src/main.rs b/tests/conformance-tests/src/main.rs
index c297b2d07c..b0c5667eee 100644
--- a/tests/conformance-tests/src/main.rs
+++ b/tests/conformance-tests/src/main.rs
@@ -3,8 +3,10 @@ fn main() {
         .nth(1)
         .expect("expected first argument to be path to spin binary")
         .into();
-    conformance_tests::run_tests("canary", move |test| {
+    let config = conformance_tests::Config::new("canary");
+    conformance_tests::run_tests(config, move |test| {
         conformance::run_test(test, &spin_binary)
     })
-    .unwrap();
+    .unwrap()
+    .exit();
 }
diff --git a/tests/runtime.rs b/tests/runtime.rs
index ba3b6b41cd..434475bec9 100644
--- a/tests/runtime.rs
+++ b/tests/runtime.rs
@@ -27,11 +27,15 @@ mod runtime_tests {
     }
 
     #[test]
-    fn conformance_tests() {
-        conformance_tests::run_tests("canary", move |test| {
+    fn conformance_tests() -> anyhow::Result<()> {
+        let config = conformance_tests::Config::new("canary");
+        let conclusion = conformance_tests::run_tests(config, move |test| {
             conformance::run_test(test, &spin_binary())
-        })
-        .unwrap();
+        })?;
+        if conclusion.has_failed() {
+            anyhow::bail!("One or more errors occurred in the conformance tests");
+        }
+        Ok(())
     }
 
     fn spin_binary() -> PathBuf {