diff --git a/Tests/FuzzilliTests/LiveTests.swift b/Tests/FuzzilliTests/LiveTests.swift index cde90f719..9a1291a26 100644 --- a/Tests/FuzzilliTests/LiveTests.swift +++ b/Tests/FuzzilliTests/LiveTests.swift @@ -15,39 +15,12 @@ import XCTest @testable import Fuzzilli -func executeAndParseResults(program: Program, fuzzer: Fuzzer, runner: JavaScriptExecutor, failures: inout Int, failureMessages: inout [String: Int]) { - - let jsProgram = fuzzer.lifter.lift(program, withOptions: .includeComments) - - do { - let result = try runner.executeScript(jsProgram, withTimeout: 5 * Seconds) - if result.isFailure { - failures += 1 - - for line in result.output.split(separator: "\n") { - if line.contains("Error:") { - // Remove anything after a potential 2nd ":", which is usually testcase dependent content, e.g. "SyntaxError: Invalid regular expression: /ep{}[]Z7/: Incomplete quantifier" - let signature = line.split(separator: ":")[0...1].joined(separator: ":") - failureMessages[signature] = (failureMessages[signature] ?? 0) + 1 - } - } - - if LiveTests.VERBOSE { - let fuzzilProgram = FuzzILLifter().lift(program) - print("Program is invalid:") - print(jsProgram) - print("Out:") - print(result.output) - print("FuzzILCode:") - print(fuzzilProgram) - } - } - } catch { - XCTFail("Could not execute script: \(error)") +class LiveTests: XCTestCase { + enum ExecutionResult { + case failed(failureMessage: String?) + case succeeded } -} -class LiveTests: XCTestCase { // Set to true to log failing programs static let VERBOSE = false @@ -56,37 +29,104 @@ class LiveTests: XCTestCase { throw XCTSkip("Could not find js shell executable.") } + let results = try Self.runLiveTest(withRunner: runner) { b in + b.buildPrefix() + } + + // We expect a maximum of 10% of ValueGeneration to fail + checkFailureRate(testResults: results, maxFailureRate: 0.10) + } + + // The closure can use the ProgramBuilder to emit a program of a specific + // shape that is then executed with the given runner. We then check that + // we stay below the maximum failure rate over the given number of iterations. + static func runLiveTest(iterations n: Int = 250, withRunner runner: JavaScriptExecutor, body: (inout ProgramBuilder) -> Void) throws -> (failureRate: Double, failureMessages: [String: Int]) { let liveTestConfig = Configuration(logLevel: .warning, enableInspection: true) // We have to use the proper JavaScriptEnvironment here. // This ensures that we use the available builtins. let fuzzer = makeMockFuzzer(config: liveTestConfig, environment: JavaScriptEnvironment()) - let N = 250 var failures = 0 var failureMessages = [String: Int]() - // TODO: consider running these in parallel. - for _ in 0..= maxFailureRate { - var message = "Failure rate for value generators is too high. Should be below \(String(format: "%.2f", maxFailureRate * 100))% but we observed \(String(format: "%.2f", failureRate * 100))%\n" + let failureRate = Double(failures) / Double(n) + + return (failureRate, failureMessages) + } + + func checkFailureRate(testResults: (failureRate: Double, failureMessages: [String: Int]), maxFailureRate: Double) { + if testResults.failureRate >= maxFailureRate { + var message = "Failure rate is too high. Should be below \(String(format: "%.2f", maxFailureRate * 100))% but we observed \(String(format: "%.2f", testResults.failureRate * 100))%\n" message += "Observed failures:\n" - for (signature, count) in failureMessages.sorted(by: { $0.value > $1.value }) { + for (signature, count) in testResults.failureMessages.sorted(by: { $0.value > $1.value }) { message += " \(count)x \(signature)\n" } XCTFail(message) } } + + static func executeAndParseResults(program: (program: Program, jsProgram: String), runner: JavaScriptExecutor) -> ExecutionResult { + + do { + let result = try runner.executeScript(program.jsProgram, withTimeout: 5 * Seconds) + if result.isFailure { + var signature: String? = nil + + for line in result.output.split(separator: "\n") { + if line.contains("Error:") { + // Remove anything after a potential 2nd ":", which is usually testcase dependent content, e.g. "SyntaxError: Invalid regular expression: /ep{}[]Z7/: Incomplete quantifier" + signature = line.split(separator: ":")[0...1].joined(separator: ":") + } + } + + if Self.VERBOSE { + let fuzzilProgram = FuzzILLifter().lift(program.program) + print("Program is invalid:") + print(program.jsProgram) + print("Out:") + print(result.output) + print("FuzzILCode:") + print(fuzzilProgram) + } + + return .failed(failureMessage: signature) + } + } catch { + XCTFail("Could not execute script: \(error)") + } + + return .succeeded + } }