From 30b74515802125f7e4f71a245e7968101a2268e8 Mon Sep 17 00:00:00 2001
From: Ingvar Stepanyan <me@rreverser.com>
Date: Wed, 17 Jan 2024 00:16:43 +0000
Subject: [PATCH] Switch to fetching tests directly from proposals

testsuite repo is usually severely outdated, so it's better to fetch tests directly from the spec and proposal repos for more control. We already have deduplication mechanism anyway, so this shouldn't cause any issues.
---
 .gitmodules     | 23 +++++++++++++--
 tests/spec.rs   | 74 +++++++++++++++----------------------------------
 tests/testsuite |  1 -
 3 files changed, 42 insertions(+), 56 deletions(-)
 delete mode 160000 tests/testsuite

diff --git a/.gitmodules b/.gitmodules
index a779dd4..69fbed4 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,3 +1,20 @@
-[submodule "tests/testsuite"]
-	path = tests/testsuite
-	url = https://github.com/WebAssembly/testsuite.git
+[submodule "tests/spec"]
+	path = tests/spec
+	url = https://github.com/WebAssembly/spec
+	shallow = true
+[submodule "tests/exception-handling"]
+	path = tests/exception-handling
+	url = https://github.com/WebAssembly/exception-handling
+	shallow = true
+[submodule "tests/extended-name-section"]
+	path = tests/extended-name-section
+	url = https://github.com/WebAssembly/extended-name-section
+	shallow = true
+[submodule "tests/tail-call"]
+	path = tests/tail-call
+	url = https://github.com/WebAssembly/tail-call
+	shallow = true
+[submodule "tests/threads"]
+	path = tests/threads
+	url = https://github.com/WebAssembly/threads
+	shallow = true
diff --git a/tests/spec.rs b/tests/spec.rs
index 5201f89..c00d754 100644
--- a/tests/spec.rs
+++ b/tests/spec.rs
@@ -18,7 +18,7 @@ use fs_err::{read_dir, read_to_string};
 use indexmap::IndexMap;
 use libtest_mimic::{run as run_tests, Arguments, Failed, Trial};
 use rayon::prelude::*;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use wasmbin::io::DecodeError;
 use wasmbin::visit::{Visit, VisitError};
@@ -39,24 +39,6 @@ const IGNORED_ERRORS: &[&str] = &[
 ];
 
 const IGNORED_MODULES: &[&[u8]] = &[
-    // These are outdated tests in WebAssembly/testsuite itself.
-    // It needs updating, but see https://github.com/WebAssembly/testsuite/pull/39#issuecomment-863496809.
-    &[
-        0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x05, 0x03, 0x01, 0x00, 0x00, 0x0B, 0x07,
-        0x01, 0x80, 0x00, 0x41, 0x00, 0x0B, 0x00,
-    ],
-    &[
-        0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x04, 0x04, 0x01, 0x70, 0x00, 0x00, 0x09,
-        0x07, 0x01, 0x80, 0x00, 0x41, 0x00, 0x0B, 0x00,
-    ],
-    &[
-        0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x05, 0x03, 0x01, 0x00, 0x00, 0x0B, 0x06,
-        0x01, 0x01, 0x41, 0x00, 0x0B, 0x00,
-    ],
-    &[
-        0x00, 0x61, 0x73, 0x6D, 0x01, 0x00, 0x00, 0x00, 0x0B, 0x06, 0x01, 0x01, 0x41, 0x00, 0x0B,
-        0x00,
-    ],
     // Upstream malformed test that becomes well-formed if threads are enabled.
     #[cfg(feature = "threads")]
     &[
@@ -149,46 +131,34 @@ impl Tests {
     }
 
     #[throws]
-    fn read_all_tests(path: &Path) -> Vec<Trial> {
-        let mut test_files = Vec::new();
-
-        let mut add_test_files_in_dir = |path: &Path| -> anyhow::Result<()> {
-            for file in read_dir(path)? {
+    fn read_all_tests() -> Vec<Trial> {
+        #[throws]
+        fn add_test_files_in_dir(test_files: &mut Vec<PathBuf>, dir: PathBuf) {
+            for file in read_dir(dir)? {
                 let path = file?.path();
                 if path.extension().map_or(false, |ext| ext == "wast") {
                     test_files.push(path);
+                } else if path.is_dir() {
+                    add_test_files_in_dir(test_files, path)?;
                 }
             }
-            Ok(())
-        };
-
-        add_test_files_in_dir(path)?;
-
-        let proposals_dir = path.join("proposals");
-
-        macro_rules! read_proposal_tests {
-            ($name:literal) => {
-                add_test_files_in_dir(&proposals_dir.join($name))?;
-            };
-
-            (? $name:literal) => {
-                if cfg!(feature = $name) {
-                    read_proposal_tests!($name);
-                }
-            };
         }
 
-        read_proposal_tests!("bulk-memory-operations");
-        read_proposal_tests!(? "exception-handling");
-        read_proposal_tests!("reference-types");
-        read_proposal_tests!("simd");
-        read_proposal_tests!(? "tail-call");
-        read_proposal_tests!(? "threads");
+        let mut test_files = Vec::new();
 
-        ensure!(
-            !test_files.is_empty(),
-            "Couldn't find any tests. Did you run `git submodule update --init`?"
-        );
+        let root = Path::new("tests");
+
+        for dir in [
+            "spec",
+            #[cfg(feature = "exception-handling")]
+            "exception-handling",
+            #[cfg(feature = "tail-call")]
+            "tail-call",
+            #[cfg(feature = "threads")]
+            "threads",
+        ] {
+            add_test_files_in_dir(&mut test_files, root.join(dir).join("test").join("core"))?;
+        }
 
         test_files
             .into_par_iter()
@@ -244,7 +214,7 @@ fn run_test(mut test_module: &[u8], expect_result: Result<(), String>) {
 
 #[throws]
 fn main() {
-    let tests = Tests::read_all_tests(&Path::new("tests").join("testsuite"))?;
+    let tests = Tests::read_all_tests()?;
 
     let mut args = Arguments::from_args();
     // Default to terse output as we have *a lot* of tests.
diff --git a/tests/testsuite b/tests/testsuite
deleted file mode 160000
index 5fddf13..0000000
--- a/tests/testsuite
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 5fddf13f296cb08a7a8055f4f6d63632485cab14