test(e2e): add e2e runner and harness (#16)

test/harness.zig

@@ -0,0 +1,7 @@
+pub const TestSuite = @import("harness/TestSuite.zig");
+
+const runner = @import("harness/runner.zig");
+pub const getRunner = runner.getRunner;
+pub const addTest = runner.addTest;
+pub const globalShutdown = runner.globalShutdown;
+pub const TestFile = runner.TestFile;

test/TestSuite.zig → test/harness/TestSuite.zig

@@ -141,7 +141,7 @@ const Allocator = std.mem.Allocator;
 const ThreadPool = std.Thread.Pool;
 const panic = std.debug.panic;
 
-const utils = @import("utils.zig");
+const utils = @import("../utils.zig");
 const string = utils.string;
 const Source = zlint.Source;
 

test/harness/runner.zig

@@ -0,0 +1,90 @@
+const std = @import("std");
+const utils = @import("../utils.zig");
+
+const Allocator = std.mem.Allocator;
+const assert = std.debug.assert;
+const panic = std.debug.panic;
+const print = std.debug.print;
+
+const TestAllocator = std.heap.GeneralPurposeAllocator(.{
+    .never_unmap = true,
+    .retain_metadata = true,
+});
+
+var gpa = TestAllocator{};
+var global_runner_instance = TestRunner.new(gpa.allocator());
+
+pub fn getRunner() *TestRunner {
+    return &global_runner_instance;
+}
+
+pub fn addTest(test_file: TestRunner.TestFile) *TestRunner {
+    assert(global_runner_instance != null);
+    return global_runner_instance.?.addTest(test_file);
+}
+
+pub fn globalShutdown() void {
+    getRunner().deinit();
+
+    _ = gpa.detectLeaks();
+    const status = gpa.deinit();
+    if (status == .leak) {
+        panic("Memory leak detected\n", .{});
+    }
+}
+
+pub const TestRunner = struct {
+    tests: std.ArrayListUnmanaged(TestFile) = .{},
+    alloc: Allocator,
+
+    pub inline fn new(alloc: Allocator) TestRunner {
+        return TestRunner{ .alloc = alloc };
+    }
+
+    pub inline fn deinit(self: *TestRunner) void {
+        for (self.tests.items) |test_file| {
+            if (test_file.deinit) |deinit_fn| {
+                deinit_fn(self.alloc);
+            }
+        }
+        self.tests.deinit(self.alloc);
+    }
+
+    pub inline fn addTest(self: *TestRunner, test_file: TestFile) *TestRunner {
+        self.tests.append(self.alloc, test_file) catch |e| panic("Failed to add test {s}: {any}\n", .{ test_file.name, e });
+        return self;
+    }
+
+    pub inline fn runAll(self: *TestRunner) !void {
+        try utils.TestFolders.globalInit();
+
+        var last_error: ?anyerror = null;
+        for (self.tests.items) |test_file| {
+            if (test_file.globalSetup) |global_setup| {
+                try global_setup(self.alloc);
+            }
+            print("Running test {s}...\n", .{test_file.name});
+            test_file.run(self.alloc) catch |e| {
+                print("Failed to run test {s}: {any}\n", .{ test_file.name, e });
+                last_error = e;
+            };
+        }
+
+        if (last_error) |e| {
+            return e;
+        }
+    }
+};
+
+pub const TestFile = struct {
+    /// Inlined string (`&'static str`). Never deallocated.
+    name: []const u8,
+    globalSetup: ?*const GlobalSetupFn = null,
+    deinit: ?*const GlobalTeardownFn = null,
+    run: *const RunFn,
+
+    pub const GlobalSetupFn = fn (alloc: Allocator) anyerror!void;
+    pub const GlobalTeardownFn = fn (alloc: Allocator) void;
+    pub const RunFn = fn (alloc: Allocator) anyerror!void;
+};
+

test/semantic/ecosystem_coverage.zig

@@ -1,3 +1,5 @@
+const test_runner = @import("../harness.zig");
+
 const std = @import("std");
 const fs = std.fs;
 const path = fs.path;
@@ -8,7 +10,6 @@ const print = std.debug.print;
 const zlint = @import("zlint");
 const Source = zlint.Source;
 
-const TestSuite = @import("../TestSuite.zig");
 const utils = @import("../utils.zig");
 const string = utils.string;
 const Repo = utils.Repo;
@@ -33,7 +34,7 @@ pub fn run(alloc: Allocator) !void {
     defer repos.deinit();
     for (repos.value) |repo| {
         const repo_dir = try utils.TestFolders.openRepo(alloc, repo.name);
-        var suite = try TestSuite.init(alloc, repo_dir, "semantic-coverage", repo.name, &testSemantic);
+        var suite = try test_runner.TestSuite.init(alloc, repo_dir, "semantic-coverage", repo.name, &testSemantic);
         defer suite.deinit();
 
         try suite.run();
@@ -49,3 +50,9 @@ fn testSemantic(alloc: Allocator, source: *const Source) !void {
 const SemanticError = error{
     analysis_failed,
 };
+
+pub const SUITE = test_runner.TestFile{
+    .name = "semantic_coverage",
+    .globalSetup = globalSetup,
+    .run = run,
+};

test/test_e2e.zig

@@ -1,27 +1,19 @@
 const std = @import("std");
-const print = std.debug.print;
 const utils = @import("utils.zig");
 
+const print = std.debug.print;
+const panic = std.debug.panic;
+
+const test_runner = @import("harness/runner.zig");
+const TestRunner = test_runner.TestRunner;
+
 // test suites
 const semantic_coverage = @import("semantic/ecosystem_coverage.zig");
 
 pub fn main() !void {
-    var gpa = std.heap.GeneralPurposeAllocator(.{
-        .never_unmap = true,
-        .retain_metadata = true,
-    }){};
-    defer {
-        _ = gpa.detectLeaks();
-        const status = gpa.deinit();
-        if (status == .leak) {
-            std.debug.panic("Memory leak detected\n", .{});
-        }
-    }
-    const alloc = gpa.allocator();
-
-    try utils.TestFolders.globalInit();
-
-    print("running semantic coverage tests\n", .{});
-    try semantic_coverage.globalSetup(alloc);
-    try semantic_coverage.run(alloc);
+    const runner = test_runner.getRunner();
+    defer runner.deinit();
+    try runner
+        .addTest(semantic_coverage.SUITE)
+        .runAll();
 }