From 03944a2be514012dbb5f0bbc7e969c78fb4f7550 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Samuel=20Gro=C3=9F?= Date: Tue, 20 Feb 2024 13:35:13 +0100 Subject: [PATCH] Extend startup test mechanism Instead of only supporting crash tests, a profile can now specify the expected test outcome which, besides .shouldCrash, can also be .shouldSucceed (for a test that should exit successfully, for example to check that expected features/builtins are available) or .shouldNotCrash (for a test that should not cause a crash, for example to check that known-safe crashes such as OOMs are ignored). --- Sources/Fuzzilli/Configuration.swift | 20 +++++++++---- Sources/Fuzzilli/Fuzzer.swift | 29 +++++++++++++++---- .../FuzzilliCli/Profiles/DuktapeProfile.swift | 9 +++++- Sources/FuzzilliCli/Profiles/JSCProfile.swift | 12 +++++++- .../Profiles/JerryscriptProfile.swift | 9 +++++- Sources/FuzzilliCli/Profiles/Profile.swift | 5 ++-- Sources/FuzzilliCli/Profiles/QjsProfile.swift | 10 ++++++- .../FuzzilliCli/Profiles/QtjsProfile.swift | 10 +++++-- Sources/FuzzilliCli/Profiles/Serenity.swift | 9 +++++- .../Profiles/SpidermonkeyProfile.swift | 12 +++++++- .../Profiles/V8HoleFuzzingProfile.swift | 15 +++++++++- Sources/FuzzilliCli/Profiles/V8Profile.swift | 18 +++++++++++- Sources/FuzzilliCli/Profiles/XSProfile.swift | 10 ++++++- Sources/FuzzilliCli/main.swift | 4 +-- 14 files changed, 144 insertions(+), 28 deletions(-) diff --git a/Sources/Fuzzilli/Configuration.swift b/Sources/Fuzzilli/Configuration.swift index 64295368..f4600e1f 100644 --- a/Sources/Fuzzilli/Configuration.swift +++ b/Sources/Fuzzilli/Configuration.swift @@ -22,9 +22,13 @@ public struct Configuration { /// Log level to use. public let logLevel: LogLevel - /// Code snippets that cause an observable crash in the target engine. - /// Used to verify that crashes can be detected. - public let crashTests: [String] + /// Code snippets that are be executed during startup and then checked to lead to the expected result. + /// + /// These can for example be used to: + /// - Check that (dummy) crashes are detected correctly (with `.shouldCrash`) + /// - Check that certain features or builtins exist (with `.shouldSucceed`) + /// - Check that known-safe crashes are ignored (with `.shouldNotCrash`) + public let startupTests: [(String, ExpectedStartupTestResult)] /// The fraction of instruction to keep from the original program when minimizing. /// This setting is useful to avoid "over-minimization", which can negatively impact the fuzzer's @@ -61,7 +65,7 @@ public struct Configuration { timeout: UInt32 = 250, skipStartupTests: Bool = false, logLevel: LogLevel = .info, - crashTests: [String] = [], + startupTests: [(String, ExpectedStartupTestResult)] = [], minimizationLimit: Double = 0.0, dropoutRate: Double = 0, collectRuntimeTypes: Bool = false, @@ -72,7 +76,7 @@ public struct Configuration { self.arguments = arguments self.timeout = timeout self.logLevel = logLevel - self.crashTests = crashTests + self.startupTests = startupTests self.dropoutRate = dropoutRate self.minimizationLimit = minimizationLimit self.enableDiagnostics = enableDiagnostics @@ -82,6 +86,12 @@ public struct Configuration { } } +public enum ExpectedStartupTestResult { + case shouldSucceed + case shouldCrash + case shouldNotCrash +} + public struct InspectionOptions: OptionSet { public let rawValue: Int public init(rawValue: Int) { diff --git a/Sources/Fuzzilli/Fuzzer.swift b/Sources/Fuzzilli/Fuzzer.swift index 9608f7ec..309c9c28 100644 --- a/Sources/Fuzzilli/Fuzzer.swift +++ b/Sources/Fuzzilli/Fuzzer.swift @@ -726,18 +726,35 @@ public class Fuzzer { maxExecutionTime = max(maxExecutionTime, execution.execTime) } - // Check if we can detect crashes and measure their execution time - for test in config.crashTests { + // Check if the profile's startup tests pass. + var hasAnyCrashTests = false + for (test, expectedResult) in config.startupTests { b = makeBuilder() b.eval(test) execution = execute(b.finalize(), purpose: .startup) - guard case .crashed = execution.outcome else { + + switch expectedResult { + case .shouldSucceed where execution.outcome != .succeeded: + logger.fatal("Testcase \"\(test)\" did not execute successfully") + case .shouldCrash where !execution.outcome.isCrash(): logger.fatal("Testcase \"\(test)\" did not crash") + case .shouldNotCrash where execution.outcome.isCrash(): + logger.fatal("Testcase \"\(test)\" unexpectedly crashed") + default: + // Test passed + break + } + + if expectedResult == .shouldCrash { + // In this case, also measure the execution time here to make sure that + // we don't set our timeout too low to detect crashes. + maxExecutionTime = max(maxExecutionTime, execution.execTime) + hasAnyCrashTests = true } - maxExecutionTime = max(maxExecutionTime, execution.execTime) } - if config.crashTests.isEmpty { - logger.warning("Cannot check if crashes are detected") + + if !hasAnyCrashTests { + logger.warning("Cannot check if crashes are detected as there are no startup tests that should cause a crash") } // Determine recommended timeout value (rounded up to nearest multiple of 10ms) diff --git a/Sources/FuzzilliCli/Profiles/DuktapeProfile.swift b/Sources/FuzzilliCli/Profiles/DuktapeProfile.swift index 8d38b9be..bfa7f0d2 100644 --- a/Sources/FuzzilliCli/Profiles/DuktapeProfile.swift +++ b/Sources/FuzzilliCli/Profiles/DuktapeProfile.swift @@ -33,7 +33,14 @@ let duktapeProfile = Profile( ecmaVersion: ECMAScriptVersion.es5, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ], additionalCodeGenerators: [], diff --git a/Sources/FuzzilliCli/Profiles/JSCProfile.swift b/Sources/FuzzilliCli/Profiles/JSCProfile.swift index ee6a2f37..c629f2a3 100644 --- a/Sources/FuzzilliCli/Profiles/JSCProfile.swift +++ b/Sources/FuzzilliCli/Profiles/JSCProfile.swift @@ -84,7 +84,17 @@ let jscProfile = Profile( ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash), + + // TODO we could try to check that OOM crashes are ignored here ( with.shouldNotCrash). + ], additionalCodeGenerators: [ (ForceDFGCompilationGenerator, 5), diff --git a/Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift b/Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift index 3a41c648..2fdcab29 100644 --- a/Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift +++ b/Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift @@ -33,7 +33,14 @@ let jerryscriptProfile = Profile( ecmaVersion: ECMAScriptVersion.es5, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ], additionalCodeGenerators: [], diff --git a/Sources/FuzzilliCli/Profiles/Profile.swift b/Sources/FuzzilliCli/Profiles/Profile.swift index 7dc2a8eb..8b370c93 100644 --- a/Sources/FuzzilliCli/Profiles/Profile.swift +++ b/Sources/FuzzilliCli/Profiles/Profile.swift @@ -24,9 +24,8 @@ struct Profile { let codeSuffix: String let ecmaVersion: ECMAScriptVersion - // JavaScript code snippets that cause a crash in the target engine. - // Used to verify that crashes can be detected. - let crashTests: [String] + // JavaScript code snippets that are executed at startup time to ensure that Fuzzilli and the target engine are configured correctly. + let startupTests: [(String, ExpectedStartupTestResult)] let additionalCodeGenerators: [(CodeGenerator, Int)] let additionalProgramTemplates: WeightedList diff --git a/Sources/FuzzilliCli/Profiles/QjsProfile.swift b/Sources/FuzzilliCli/Profiles/QjsProfile.swift index 90bcbf3d..1b8f038e 100644 --- a/Sources/FuzzilliCli/Profiles/QjsProfile.swift +++ b/Sources/FuzzilliCli/Profiles/QjsProfile.swift @@ -33,7 +33,15 @@ let qjsProfile = Profile( ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash), + ], additionalCodeGenerators: [], diff --git a/Sources/FuzzilliCli/Profiles/QtjsProfile.swift b/Sources/FuzzilliCli/Profiles/QtjsProfile.swift index 75955164..09c795bb 100644 --- a/Sources/FuzzilliCli/Profiles/QtjsProfile.swift +++ b/Sources/FuzzilliCli/Profiles/QtjsProfile.swift @@ -41,9 +41,13 @@ let qtjsProfile = Profile( ecmaVersion: ECMAScriptVersion.es6, - // JavaScript code snippets that cause a crash in the target engine. - // Used to verify that crashes can be detected. - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ], additionalCodeGenerators: [ (ForceQV4JITGenerator, 20), diff --git a/Sources/FuzzilliCli/Profiles/Serenity.swift b/Sources/FuzzilliCli/Profiles/Serenity.swift index 8312b51b..32ccef2f 100644 --- a/Sources/FuzzilliCli/Profiles/Serenity.swift +++ b/Sources/FuzzilliCli/Profiles/Serenity.swift @@ -31,7 +31,14 @@ let serenityProfile = Profile( """, ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ], additionalCodeGenerators: [], additionalProgramTemplates: WeightedList([]), diff --git a/Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift b/Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift index 081cbd39..fa1f0186 100644 --- a/Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift +++ b/Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift @@ -85,7 +85,17 @@ let spidermonkeyProfile = Profile( ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash), + + // TODO we could try to check that OOM crashes are ignored here ( with.shouldNotCrash). + ], additionalCodeGenerators: [ (ForceSpidermonkeyIonGenerator, 10), diff --git a/Sources/FuzzilliCli/Profiles/V8HoleFuzzingProfile.swift b/Sources/FuzzilliCli/Profiles/V8HoleFuzzingProfile.swift index 1fcf1e35..3548d387 100644 --- a/Sources/FuzzilliCli/Profiles/V8HoleFuzzingProfile.swift +++ b/Sources/FuzzilliCli/Profiles/V8HoleFuzzingProfile.swift @@ -85,7 +85,20 @@ let v8HoleFuzzingProfile = Profile( codeSuffix: """ """, ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 7)"], + + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that "hard" crashes are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 7)", .shouldCrash), + + // DCHECK and CHECK failures should be ignored. + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldNotCrash), + ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldNotCrash), + ], + additionalCodeGenerators: [ (ForceJITCompilationThroughLoopGenerator, 5), (ForceTurboFanCompilationGenerator, 5), diff --git a/Sources/FuzzilliCli/Profiles/V8Profile.swift b/Sources/FuzzilliCli/Profiles/V8Profile.swift index 70da6d7c..14f40901 100644 --- a/Sources/FuzzilliCli/Profiles/V8Profile.swift +++ b/Sources/FuzzilliCli/Profiles/V8Profile.swift @@ -556,6 +556,7 @@ let v8Profile = Profile( return args }, + // We typically fuzz without any sanitizer instrumentation, but if any sanitizers are active, "abort_on_error=1" must probably be set so that sanitizer errors can be detected. processEnv: [:], maxExecsBeforeRespawn: 1000, @@ -570,7 +571,22 @@ let v8Profile = Profile( ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)", "fuzzilli('FUZZILLI_CRASH', 3)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + // IMMEDIATE_CRASH() + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + // CHECK failure + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + // DCHECK failure + ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash), + // Wild-write + ("fuzzilli('FUZZILLI_CRASH', 3)", .shouldCrash), + + // TODO we could try to check that OOM crashes are ignored here ( with.shouldNotCrash). + ], additionalCodeGenerators: [ (ForceJITCompilationThroughLoopGenerator, 5), diff --git a/Sources/FuzzilliCli/Profiles/XSProfile.swift b/Sources/FuzzilliCli/Profiles/XSProfile.swift index 422ecc42..779942bd 100644 --- a/Sources/FuzzilliCli/Profiles/XSProfile.swift +++ b/Sources/FuzzilliCli/Profiles/XSProfile.swift @@ -34,7 +34,15 @@ let xsProfile = Profile( ecmaVersion: ECMAScriptVersion.es6, - crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"], + startupTests: [ + // Check that the fuzzilli integration is available. + ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed), + + // Check that common crash types are detected. + ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash), + ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash), + ], additionalCodeGenerators: [], diff --git a/Sources/FuzzilliCli/main.swift b/Sources/FuzzilliCli/main.swift index 3ae057e9..ac9880c6 100644 --- a/Sources/FuzzilliCli/main.swift +++ b/Sources/FuzzilliCli/main.swift @@ -480,7 +480,7 @@ func makeFuzzer(with configuration: Configuration) -> Fuzzer { let mainConfig = Configuration(arguments: CommandLine.arguments, timeout: UInt32(timeout), logLevel: logLevel, - crashTests: profile.crashTests, + startupTests: profile.startupTests, minimizationLimit: minimizationLimit, enableDiagnostics: diagnostics, enableInspection: inspect, @@ -613,7 +613,7 @@ fuzzer.sync { let workerConfig = Configuration(arguments: CommandLine.arguments, timeout: UInt32(timeout), logLevel: .warning, - crashTests: profile.crashTests, + startupTests: profile.startupTests, minimizationLimit: minimizationLimit, enableDiagnostics: false, enableInspection: inspect,