From bdec112bd54f9f1d1abb54dd7819426284c1552d Mon Sep 17 00:00:00 2001 From: Makosai Date: Fri, 16 Aug 2024 18:42:37 -0400 Subject: [PATCH] Start implementing a test runner. --- build.zig | 69 +++++--- src/Transport/BaseClient.zig | 23 +++ src/main.zig | 8 +- src/root.zig | 10 -- test_runner.zig | 299 +++++++++++++++++++++++++++++++++++ 5 files changed, 374 insertions(+), 35 deletions(-) delete mode 100644 src/root.zig create mode 100644 test_runner.zig diff --git a/build.zig b/build.zig index e1d39ab..766f5e9 100644 --- a/build.zig +++ b/build.zig @@ -15,19 +15,29 @@ pub fn build(b: *std.Build) void { // set a preferred release mode, allowing the user to decide how to optimize. const optimize = b.standardOptimizeOption(.{}); - const lib = b.addStaticLibrary(.{ - .name = "Sustenet", - // In this case the main source file is merely a path, however, in more - // complicated build scripts, this could be a generated file. - .root_source_file = b.path("src/root.zig"), - .target = target, - .optimize = optimize, + const main_module = b.addModule("main", .{ + .root_source_file = b.path("src/main.zig"), }); - // This declares intent for the library to be installed into the standard - // location when the user invokes the "install" step (the default step when - // running `zig build`). - b.installArtifact(lib); + { + const options = b.addOptions(); + options.addOption(bool, "main_blocking", false); + main_module.addOptions("build", options); + } + + // const lib = b.addStaticLibrary(.{ + // .name = "Sustenet", + // // In this case the main source file is merely a path, however, in more + // // complicated build scripts, this could be a generated file. + // .root_source_file = b.path("src/root.zig"), + // .target = target, + // .optimize = optimize, + // }); + + // // This declares intent for the library to be installed into the standard + // // location when the user invokes the "install" step (the default step when + // // running `zig build`). + // b.installArtifact(lib); const exe = b.addExecutable(.{ .name = "Sustenet", @@ -66,26 +76,39 @@ pub fn build(b: *std.Build) void { // Creates a step for unit testing. This only builds the test executable // but does not run it. - const lib_unit_tests = b.addTest(.{ - .root_source_file = b.path("src/root.zig"), - .target = target, - .optimize = optimize, - }); + // const lib_unit_tests = b.addTest(.{ + // .root_source_file = b.path("src/root.zig"), + // .target = target, + // .optimize = optimize, + // }); - const run_lib_unit_tests = b.addRunArtifact(lib_unit_tests); + // const run_lib_unit_tests = b.addRunArtifact(lib_unit_tests); - const exe_unit_tests = b.addTest(.{ - .root_source_file = b.path("src/main.zig"), + // const exe_unit_tests = b.addTest(.{ + // .root_source_file = b.path("src/main.zig"), + // .target = target, + // .optimize = optimize, + // }); + + const tests = b.addTest(.{ .target = target, .optimize = optimize, + .root_source_file = b.path("src/main.zig"), + .test_runner = b.path("test_runner.zig"), }); - const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests); + const run_tests = b.addRunArtifact(tests); + run_tests.has_side_effects = true; + + const test_step = b.step("test", "Run tests"); + test_step.dependOn(&tests.step); + + // const run_exe_unit_tests = b.addRunArtifact(exe_unit_tests); // Similar to creating the run step earlier, this exposes a `test` step to // the `zig build --help` menu, providing a way for the user to request // running the unit tests. - const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&run_lib_unit_tests.step); - test_step.dependOn(&run_exe_unit_tests.step); + // const test_step = b.step("test", "Run unit tests"); + // test_step.dependOn(&run_lib_unit_tests.step); + // test_step.dependOn(&run_exe_unit_tests.step); } diff --git a/src/Transport/BaseClient.zig b/src/Transport/BaseClient.zig index 6197ef8..5fdef35 100644 --- a/src/Transport/BaseClient.zig +++ b/src/Transport/BaseClient.zig @@ -4,6 +4,7 @@ const std = @import("std"); const net = std.net; const print = std.debug.print; +const testing = std.testing; // Add import for testing package const BaseClient = @This(); const bufferSize = 4028; @@ -39,3 +40,25 @@ pub fn send(self: *BaseClient, data: []const u8) !void { const size = try writer.write(data); print("Sending '{s}' to peer, total written: {d} bytes\n", .{ data, size }); } + +test "setup client" { + const client = BaseClient.new(4337); + try client.connect(); + + try testing.expect(client.id == 1); // Use expect from testing package +} + +test "send data" { + const client = BaseClient.new(4337); + try client.connect(); + try client.send("hello ziggy!"); + + try testing.expect(client.stream != null); +} + +test "send data with null stream" { + const client = BaseClient.new(4337); + try client.send("hello ziggy!"); + + try testing.expect(client.stream == null); +} diff --git a/src/main.zig b/src/main.zig index 9300b3c..53b24b4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,8 +9,8 @@ const clients = sustenet.clients; const BaseServer = transport.BaseServer; pub var client_list: std.ArrayList(clients.Client) = undefined; -pub var cluster = undefined; -pub var master = undefined; +// pub var cluster = undefined; +// pub var master = undefined; fn entrypoint() !void { // Get allocator @@ -71,3 +71,7 @@ test "simple test" { try list.append(42); try std.testing.expectEqual(@as(i32, 42), list.pop()); } + +test "all" { + std.testing.refAllDecls(@This()); +} diff --git a/src/root.zig b/src/root.zig deleted file mode 100644 index ecfeade..0000000 --- a/src/root.zig +++ /dev/null @@ -1,10 +0,0 @@ -const std = @import("std"); -const testing = std.testing; - -export fn add(a: i32, b: i32) i32 { - return a + b; -} - -test "basic add functionality" { - try testing.expect(add(3, 7) == 10); -} diff --git a/test_runner.zig b/test_runner.zig new file mode 100644 index 0000000..b7c1712 --- /dev/null +++ b/test_runner.zig @@ -0,0 +1,299 @@ +const std = @import("std"); +const builtin = @import("builtin"); +pub const sustenet = @import("src/main.zig").sustenet; + +const Allocator = std.mem.Allocator; + +const BORDER = "=" ** 80; + +// use in custom panic handler +var current_test: ?[]const u8 = null; + +pub fn main() !void { + var mem: [8192]u8 = undefined; + var fba = std.heap.FixedBufferAllocator.init(&mem); + + const allocator = fba.allocator(); + + const env = Env.init(allocator); + defer env.deinit(allocator); + + var slowest = SlowTracker.init(allocator, 5); + defer slowest.deinit(); + + var pass: usize = 0; + var fail: usize = 0; + var skip: usize = 0; + var leak: usize = 0; + + const printer = Printer.init(); + printer.fmt("\r\x1b[0K", .{}); // beginning of line and clear to end of line + + for (builtin.test_functions) |t| { + if (isSetup(t)) { + t.func() catch |err| { + printer.status(.fail, "\nsetup \"{s}\" failed: {}\n", .{ t.name, err }); + return err; + }; + } + } + + for (builtin.test_functions) |t| { + if (isSetup(t) or isTeardown(t)) { + continue; + } + + var status = Status.pass; + slowest.startTiming(); + + const is_unnamed_test = isUnnamed(t); + if (env.filter) |f| { + if (!is_unnamed_test and std.mem.indexOf(u8, t.name, f) == null) { + continue; + } + } + + const friendly_name = blk: { + const name = t.name; + var it = std.mem.splitScalar(u8, name, '.'); + while (it.next()) |value| { + if (std.mem.eql(u8, value, "test")) { + const rest = it.rest(); + break :blk if (rest.len > 0) rest else name; + } + } + break :blk name; + }; + + current_test = friendly_name; + std.testing.allocator_instance = .{}; + const result = t.func(); + current_test = null; + + const ns_taken = slowest.endTiming(friendly_name); + + if (std.testing.allocator_instance.deinit() == .leak) { + leak += 1; + printer.status(.fail, "\n{s}\n\"{s}\" - Memory Leak\n{s}\n", .{ BORDER, friendly_name, BORDER }); + } + + if (result) |_| { + pass += 1; + } else |err| switch (err) { + error.SkipZigTest => { + skip += 1; + status = .skip; + }, + else => { + status = .fail; + fail += 1; + printer.status(.fail, "\n{s}\n\"{s}\" - {s}\n{s}\n", .{ BORDER, friendly_name, @errorName(err), BORDER }); + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + if (env.fail_first) { + break; + } + }, + } + + if (env.verbose) { + const ms = @as(f64, @floatFromInt(ns_taken)) / 1_000_000.0; + printer.status(status, "{s} ({d:.2}ms)\n", .{ friendly_name, ms }); + } else { + printer.status(status, ".", .{}); + } + } + + for (builtin.test_functions) |t| { + if (isTeardown(t)) { + t.func() catch |err| { + printer.status(.fail, "\nteardown \"{s}\" failed: {}\n", .{ t.name, err }); + return err; + }; + } + } + + const total_tests = pass + fail; + const status = if (fail == 0) Status.pass else Status.fail; + printer.status(status, "\n{d} of {d} test{s} passed\n", .{ pass, total_tests, if (total_tests != 1) "s" else "" }); + if (skip > 0) { + printer.status(.skip, "{d} test{s} skipped\n", .{ skip, if (skip != 1) "s" else "" }); + } + if (leak > 0) { + printer.status(.fail, "{d} test{s} leaked\n", .{ leak, if (leak != 1) "s" else "" }); + } + printer.fmt("\n", .{}); + try slowest.display(printer); + printer.fmt("\n", .{}); + std.posix.exit(if (fail == 0) 0 else 1); +} + +const Printer = struct { + out: std.fs.File.Writer, + + fn init() Printer { + return .{ + .out = std.io.getStdErr().writer(), + }; + } + + fn fmt(self: Printer, comptime format: []const u8, args: anytype) void { + std.fmt.format(self.out, format, args) catch unreachable; + } + + fn status(self: Printer, s: Status, comptime format: []const u8, args: anytype) void { + const color = switch (s) { + .pass => "\x1b[32m", + .fail => "\x1b[31m", + .skip => "\x1b[33m", + else => "", + }; + const out = self.out; + out.writeAll(color) catch @panic("writeAll failed?!"); + std.fmt.format(out, format, args) catch @panic("std.fmt.format failed?!"); + self.fmt("\x1b[0m", .{}); + } +}; + +const Status = enum { + pass, + fail, + skip, + text, +}; + +const SlowTracker = struct { + const SlowestQueue = std.PriorityDequeue(TestInfo, void, compareTiming); + max: usize, + slowest: SlowestQueue, + timer: std.time.Timer, + + fn init(allocator: Allocator, count: u32) SlowTracker { + const timer = std.time.Timer.start() catch @panic("failed to start timer"); + var slowest = SlowestQueue.init(allocator, {}); + slowest.ensureTotalCapacity(count) catch @panic("OOM"); + return .{ + .max = count, + .timer = timer, + .slowest = slowest, + }; + } + + const TestInfo = struct { + ns: u64, + name: []const u8, + }; + + fn deinit(self: SlowTracker) void { + self.slowest.deinit(); + } + + fn startTiming(self: *SlowTracker) void { + self.timer.reset(); + } + + fn endTiming(self: *SlowTracker, test_name: []const u8) u64 { + var timer = self.timer; + const ns = timer.lap(); + + var slowest = &self.slowest; + + if (slowest.count() < self.max) { + // Capacity is fixed to the # of slow tests we want to track + // If we've tracked fewer tests than this capacity, than always add + slowest.add(TestInfo{ .ns = ns, .name = test_name }) catch @panic("failed to track test timing"); + return ns; + } + + { + // Optimization to avoid shifting the dequeue for the common case + // where the test isn't one of our slowest. + const fastest_of_the_slow = slowest.peekMin() orelse unreachable; + if (fastest_of_the_slow.ns > ns) { + // the test was faster than our fastest slow test, don't add + return ns; + } + } + + // the previous fastest of our slow tests, has been pushed off. + _ = slowest.removeMin(); + slowest.add(TestInfo{ .ns = ns, .name = test_name }) catch @panic("failed to track test timing"); + return ns; + } + + fn display(self: *SlowTracker, printer: Printer) !void { + var slowest = self.slowest; + const count = slowest.count(); + printer.fmt("Slowest {d} test{s}: \n", .{ count, if (count != 1) "s" else "" }); + while (slowest.removeMinOrNull()) |info| { + const ms = @as(f64, @floatFromInt(info.ns)) / 1_000_000.0; + printer.fmt(" {d:.2}ms\t{s}\n", .{ ms, info.name }); + } + } + + fn compareTiming(context: void, a: TestInfo, b: TestInfo) std.math.Order { + _ = context; + return std.math.order(a.ns, b.ns); + } +}; + +const Env = struct { + verbose: bool, + fail_first: bool, + filter: ?[]const u8, + + fn init(allocator: Allocator) Env { + return .{ + .verbose = readEnvBool(allocator, "TEST_VERBOSE", true), + .fail_first = readEnvBool(allocator, "TEST_FAIL_FIRST", false), + .filter = readEnv(allocator, "TEST_FILTER"), + }; + } + + fn deinit(self: Env, allocator: Allocator) void { + if (self.filter) |f| { + allocator.free(f); + } + } + + fn readEnv(allocator: Allocator, key: []const u8) ?[]const u8 { + const v = std.process.getEnvVarOwned(allocator, key) catch |err| { + if (err == error.EnvironmentVariableNotFound) { + return null; + } + std.log.warn("failed to get env var {s} due to err {}", .{ key, err }); + return null; + }; + return v; + } + + fn readEnvBool(allocator: Allocator, key: []const u8, deflt: bool) bool { + const value = readEnv(allocator, key) orelse return deflt; + defer allocator.free(value); + return std.ascii.eqlIgnoreCase(value, "true"); + } +}; + +pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, ret_addr: ?usize) noreturn { + if (current_test) |ct| { + std.debug.print("\x1b[31m{s}\npanic running \"{s}\"\n{s}\x1b[0m\n", .{ BORDER, ct, BORDER }); + } + std.builtin.default_panic(msg, error_return_trace, ret_addr); +} + +fn isUnnamed(t: std.builtin.TestFn) bool { + const marker = ".test_"; + const test_name = t.name; + const index = std.mem.indexOf(u8, test_name, marker) orelse return false; + _ = std.fmt.parseInt(u32, test_name[index + marker.len ..], 10) catch return false; + return true; +} + +fn isSetup(t: std.builtin.TestFn) bool { + return std.mem.endsWith(u8, t.name, "tests:beforeAll"); +} + +fn isTeardown(t: std.builtin.TestFn) bool { + return std.mem.endsWith(u8, t.name, "tests:afterAll"); +}