Extend startup test mechanism

Instead of only supporting crash tests, a profile can now specify the expected test outcome which, besides .shouldCrash, can also be .shouldSucceed (for a test that should exit successfully, for example to check that expected features/builtins are available) or .shouldNotCrash (for a test that should not cause a crash, for example to check that known-safe crashes such as OOMs are ignored).
googleprojectzero · Feb 23, 2024 · 03944a2 · 03944a2
1 parent a3b420d
commit 03944a2
Show file tree

Hide file tree

Showing 14 changed files with 144 additions and 28 deletions.
diff --git a/Sources/Fuzzilli/Configuration.swift b/Sources/Fuzzilli/Configuration.swift
@@ -22,9 +22,13 @@ public struct Configuration {
     /// Log level to use.
     public let logLevel: LogLevel
 
-    /// Code snippets that cause an observable crash in the target engine.
-    /// Used to verify that crashes can be detected.
-    public let crashTests: [String]
+    /// Code snippets that are be executed during startup and then checked to lead to the expected result.
+    ///
+    /// These can for example be used to:
+    ///   - Check that (dummy) crashes are detected correctly (with `.shouldCrash`)
+    ///   - Check that certain features or builtins exist (with `.shouldSucceed`)
+    ///   - Check that known-safe crashes are ignored (with `.shouldNotCrash`)
+    public let startupTests: [(String, ExpectedStartupTestResult)]
 
     /// The fraction of instruction to keep from the original program when minimizing.
     /// This setting is useful to avoid "over-minimization", which can negatively impact the fuzzer's
@@ -61,7 +65,7 @@ public struct Configuration {
                 timeout: UInt32 = 250,
                 skipStartupTests: Bool = false,
                 logLevel: LogLevel = .info,
-                crashTests: [String] = [],
+                startupTests: [(String, ExpectedStartupTestResult)] = [],
                 minimizationLimit: Double = 0.0,
                 dropoutRate: Double = 0,
                 collectRuntimeTypes: Bool = false,
@@ -72,7 +76,7 @@ public struct Configuration {
         self.arguments = arguments
         self.timeout = timeout
         self.logLevel = logLevel
-        self.crashTests = crashTests
+        self.startupTests = startupTests
         self.dropoutRate = dropoutRate
         self.minimizationLimit = minimizationLimit
         self.enableDiagnostics = enableDiagnostics
@@ -82,6 +86,12 @@ public struct Configuration {
     }
 }
 
+public enum ExpectedStartupTestResult {
+    case shouldSucceed
+    case shouldCrash
+    case shouldNotCrash
+}
+
 public struct InspectionOptions: OptionSet {
     public let rawValue: Int
     public init(rawValue: Int) {

diff --git a/Sources/Fuzzilli/Fuzzer.swift b/Sources/Fuzzilli/Fuzzer.swift
@@ -726,18 +726,35 @@ public class Fuzzer {
             maxExecutionTime = max(maxExecutionTime, execution.execTime)
         }
 
-        // Check if we can detect crashes and measure their execution time
-        for test in config.crashTests {
+        // Check if the profile's startup tests pass.
+        var hasAnyCrashTests = false
+        for (test, expectedResult) in config.startupTests {
             b = makeBuilder()
             b.eval(test)
             execution = execute(b.finalize(), purpose: .startup)
-            guard case .crashed = execution.outcome else {
+
+            switch expectedResult {
+            case .shouldSucceed where execution.outcome != .succeeded:
+                logger.fatal("Testcase \"\(test)\" did not execute successfully")
+            case .shouldCrash where !execution.outcome.isCrash():
                 logger.fatal("Testcase \"\(test)\" did not crash")
+            case .shouldNotCrash where execution.outcome.isCrash():
+                logger.fatal("Testcase \"\(test)\" unexpectedly crashed")
+            default:
+                // Test passed
+                break
+            }
+
+            if expectedResult == .shouldCrash {
+                // In this case, also measure the execution time here to make sure that
+                // we don't set our timeout too low to detect crashes.
+                maxExecutionTime = max(maxExecutionTime, execution.execTime)
+                hasAnyCrashTests = true
             }
-            maxExecutionTime = max(maxExecutionTime, execution.execTime)
         }
-        if config.crashTests.isEmpty {
-            logger.warning("Cannot check if crashes are detected")
+
+        if !hasAnyCrashTests {
+            logger.warning("Cannot check if crashes are detected as there are no startup tests that should cause a crash")
         }
 
         // Determine recommended timeout value (rounded up to nearest multiple of 10ms)

diff --git a/Sources/FuzzilliCli/Profiles/DuktapeProfile.swift b/Sources/FuzzilliCli/Profiles/DuktapeProfile.swift
@@ -33,7 +33,14 @@ let duktapeProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es5,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+    ],
 
     additionalCodeGenerators: [],
 

diff --git a/Sources/FuzzilliCli/Profiles/JSCProfile.swift b/Sources/FuzzilliCli/Profiles/JSCProfile.swift
@@ -84,7 +84,17 @@ let jscProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es6,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash),
+
+        // TODO we could try to check that OOM crashes are ignored here ( with.shouldNotCrash).
+    ],
 
     additionalCodeGenerators: [
         (ForceDFGCompilationGenerator, 5),

diff --git a/Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift b/Sources/FuzzilliCli/Profiles/JerryscriptProfile.swift
@@ -33,7 +33,14 @@ let jerryscriptProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es5,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+    ],
 
     additionalCodeGenerators: [],
 

diff --git a/Sources/FuzzilliCli/Profiles/Profile.swift b/Sources/FuzzilliCli/Profiles/Profile.swift
@@ -24,9 +24,8 @@ struct Profile {
     let codeSuffix: String
     let ecmaVersion: ECMAScriptVersion
 
-    // JavaScript code snippets that cause a crash in the target engine.
-    // Used to verify that crashes can be detected.
-    let crashTests: [String]
+    // JavaScript code snippets that are executed at startup time to ensure that Fuzzilli and the target engine are configured correctly.
+    let startupTests: [(String, ExpectedStartupTestResult)]
 
     let additionalCodeGenerators: [(CodeGenerator, Int)]
     let additionalProgramTemplates: WeightedList<ProgramTemplate>

diff --git a/Sources/FuzzilliCli/Profiles/QjsProfile.swift b/Sources/FuzzilliCli/Profiles/QjsProfile.swift
@@ -33,7 +33,15 @@ let qjsProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es6,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash),
+    ],
 
     additionalCodeGenerators: [],
 

diff --git a/Sources/FuzzilliCli/Profiles/QtjsProfile.swift b/Sources/FuzzilliCli/Profiles/QtjsProfile.swift
@@ -41,9 +41,13 @@ let qtjsProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es6,
 
-    // JavaScript code snippets that cause a crash in the target engine.
-    // Used to verify that crashes can be detected.
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+    ],
 
     additionalCodeGenerators: [
         (ForceQV4JITGenerator,    20),

diff --git a/Sources/FuzzilliCli/Profiles/Serenity.swift b/Sources/FuzzilliCli/Profiles/Serenity.swift
@@ -31,7 +31,14 @@ let serenityProfile = Profile(
                  """,
     ecmaVersion: ECMAScriptVersion.es6,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+    ],
 
     additionalCodeGenerators: [],
     additionalProgramTemplates: WeightedList<ProgramTemplate>([]),

diff --git a/Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift b/Sources/FuzzilliCli/Profiles/SpidermonkeyProfile.swift
@@ -85,7 +85,17 @@ let spidermonkeyProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es6,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash),
+
+        // TODO we could try to check that OOM crashes are ignored here ( with.shouldNotCrash).
+    ],
 
     additionalCodeGenerators: [
         (ForceSpidermonkeyIonGenerator, 10),

diff --git a/Sources/FuzzilliCli/Profiles/V8HoleFuzzingProfile.swift b/Sources/FuzzilliCli/Profiles/V8HoleFuzzingProfile.swift
@@ -85,7 +85,20 @@ let v8HoleFuzzingProfile = Profile(
     codeSuffix: """
                 """,
     ecmaVersion: ECMAScriptVersion.es6,
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 7)"],
+
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that "hard" crashes are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 7)", .shouldCrash),
+
+        // DCHECK and CHECK failures should be ignored.
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldNotCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldNotCrash),
+    ],
+
     additionalCodeGenerators: [
         (ForceJITCompilationThroughLoopGenerator,  5),
         (ForceTurboFanCompilationGenerator,        5),

diff --git a/Sources/FuzzilliCli/Profiles/V8Profile.swift b/Sources/FuzzilliCli/Profiles/V8Profile.swift
@@ -556,6 +556,7 @@ let v8Profile = Profile(
         return args
     },
 
+    // We typically fuzz without any sanitizer instrumentation, but if any sanitizers are active, "abort_on_error=1" must probably be set so that sanitizer errors can be detected.
     processEnv: [:],
 
     maxExecsBeforeRespawn: 1000,
@@ -570,7 +571,22 @@ let v8Profile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es6,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)", "fuzzilli('FUZZILLI_CRASH', 3)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        // IMMEDIATE_CRASH()
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        // CHECK failure
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+        // DCHECK failure
+        ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash),
+        // Wild-write
+        ("fuzzilli('FUZZILLI_CRASH', 3)", .shouldCrash),
+
+        // TODO we could try to check that OOM crashes are ignored here ( with.shouldNotCrash).
+    ],
 
     additionalCodeGenerators: [
         (ForceJITCompilationThroughLoopGenerator,  5),

diff --git a/Sources/FuzzilliCli/Profiles/XSProfile.swift b/Sources/FuzzilliCli/Profiles/XSProfile.swift
@@ -34,7 +34,15 @@ let xsProfile = Profile(
 
     ecmaVersion: ECMAScriptVersion.es6,
 
-    crashTests: ["fuzzilli('FUZZILLI_CRASH', 0)", "fuzzilli('FUZZILLI_CRASH', 1)", "fuzzilli('FUZZILLI_CRASH', 2)"],
+    startupTests: [
+        // Check that the fuzzilli integration is available.
+        ("fuzzilli('FUZZILLI_PRINT', 'test')", .shouldSucceed),
+
+        // Check that common crash types are detected.
+        ("fuzzilli('FUZZILLI_CRASH', 0)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 1)", .shouldCrash),
+        ("fuzzilli('FUZZILLI_CRASH', 2)", .shouldCrash),
+    ],
 
     additionalCodeGenerators: [],
 

diff --git a/Sources/FuzzilliCli/main.swift b/Sources/FuzzilliCli/main.swift
@@ -480,7 +480,7 @@ func makeFuzzer(with configuration: Configuration) -> Fuzzer {
 let mainConfig = Configuration(arguments: CommandLine.arguments,
                                timeout: UInt32(timeout),
                                logLevel: logLevel,
-                               crashTests: profile.crashTests,
+                               startupTests: profile.startupTests,
                                minimizationLimit: minimizationLimit,
                                enableDiagnostics: diagnostics,
                                enableInspection: inspect,
@@ -613,7 +613,7 @@ fuzzer.sync {
 let workerConfig = Configuration(arguments: CommandLine.arguments,
                                  timeout: UInt32(timeout),
                                  logLevel: .warning,
-                                 crashTests: profile.crashTests,
+                                 startupTests: profile.startupTests,
                                  minimizationLimit: minimizationLimit,
                                  enableDiagnostics: false,
                                  enableInspection: inspect,