ci: fix WPT failures for missing tests (#107)

* ci: fix WPT failures for missing tests * fixup * fixup * exclude currently missing tests * fixup backtrace logging
bytecodealliance · Aug 8, 2024 · cab7ffb · cab7ffb
1 parent c83e718
commit cab7ffb
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 8 deletions.
diff --git a/tests/wpt-harness/run-wpt.mjs b/tests/wpt-harness/run-wpt.mjs
@@ -222,8 +222,8 @@ async function run() {
     if (config.tests.updateExpectations) {
       console.log(`Expectations updated: ${expectationsUpdated}`);
       shutdown();
-    } else if (stats.unexpectedFail + stats.unexpectedPass != 0 || unexpectedFailure) {
-      shutdown('Unexpected failures or passes. Verify that these new results match your expectations and run with --update-expectations to update the recorded expectations.');
+    } else if (stats.unexpectedFail + stats.unexpectedPass + stats.missing != 0 || unexpectedFailure) {
+      shutdown('Unexpected failures, passes or missing results. Verify that these new results match your expectations and run with --update-expectations to update the recorded expectations.');
     } else {
       shutdown();
     }
@@ -331,7 +331,7 @@ async function startWasmtime(runtime, addr, logLevel) {
   });
 
   let backtrace = "";
-  let backtrace_re = /Error \{\s+context: "error while executing at wasm backtrace:(.+?)",\s+source: "(.+?)"/s;
+  let backtrace_re = /Error \{\s+context: "error while executing at wasm backtrace:(.+?)",\s+source: (.+?)/s;
   wasmtime.stderr.on("data", data => {
     if (backtrace.length > 0 || data.includes("error while executing at wasm backtrace:")) {
       backtrace += data;
@@ -489,7 +489,8 @@ async function runTests(testPaths, wasmtime, resultCallback, errorCallback) {
       await resultCallback(path, results, stats);
     } catch (e) {
       if (!results) {
-        e = new Error(`Parsing test results as JSON failed. Output was:\n  ${body}`);
+        e = new Error(`\nMISSING TEST RESULTS: ${path}\nParsing test results as JSON failed. Output was:\n  ${body}`);
+        totalStats.missing += Math.max(Object.keys(expectations).length, 1);
       }
       if (config.logLevel >= LogLevel.Verbose) {
         console.log(`Error running file ${path}: ${e.message}, stack:\n${e.stack}`);

diff --git a/tests/wpt-harness/tests.json b/tests/wpt-harness/tests.json
@@ -54,7 +54,7 @@
   "fetch/api/abort/request.any.js",
   "fetch/api/basic/accept-header.any.js",
   "fetch/api/basic/conditional-get.any.js",
-  "fetch/api/basic/error-after-response.any.js",
+  "SKIP fetch/api/basic/error-after-response.any.js",
   "fetch/api/basic/header-value-combining.any.js",
   "fetch/api/basic/header-value-null-byte.any.js",
   "fetch/api/basic/historical.any.js",
@@ -73,7 +73,7 @@
   "SKIP fetch/api/basic/request-private-network-headers.tentative.any.js",
   "fetch/api/basic/request-referrer.any.js",
   "fetch/api/basic/request-upload.any.js",
-  "fetch/api/basic/request-upload.h2.any.js",
+  "SKIP fetch/api/basic/request-upload.h2.any.js",
   "fetch/api/basic/response-null-body.any.js",
   "fetch/api/basic/response-url.sub.any.js",
   "fetch/api/basic/scheme-about.any.js",
@@ -98,7 +98,7 @@
   "fetch/api/headers/headers-normalize.any.js",
   "fetch/api/headers/headers-record.any.js",
   "fetch/api/headers/headers-structure.any.js",
-  "fetch/api/idlharness.any.js",
+  "SKIP fetch/api/idlharness.any.js",
   "fetch/api/redirect/redirect-back-to-original-origin.any.js",
   "fetch/api/redirect/redirect-count.any.js",
   "fetch/api/redirect/redirect-empty-location.any.js",
@@ -235,7 +235,7 @@
   "streams/writable-streams/start.any.js",
   "streams/writable-streams/write.any.js",
   "url/historical.any.js",
-  "url/idlharness.any.js",
+  "SKIP url/idlharness.any.js",
   "SLOW url/url-constructor.any.js",
   "url/url-origin.any.js",
   "url/url-searchparams.any.js",