Switch to a JavaScriptExecutor class

Switch from a specific NodeJS class used for tests and
the FuzzIL compiler to a more generic JavaScriptExecutor
that uses the binary given through the FUZZILLI_TEST_SHELL
environment variable to execute tests.
The FuzzIL compiler also uses this class although it still needs
npm modules and therefore still tries to find the NodeJS binary
in the users path.

Sources/FuzzILTool/main.swift

@@ -165,7 +165,8 @@ else if args.has("--checkCorpus") {
 
 // Compile a JavaScript program to a FuzzIL program. Requires node.js
 else if args.has("--compile") {
-    guard let nodejs = NodeJS() else {
+    // We require a NodeJS executor here as we need certain node modules.
+    guard let nodejs = JavaScriptExecutor(type: .nodejs) else {
         print("Could not find the NodeJS executable.")
         exit(-1)
     }

Sources/Fuzzilli/Compiler/JavaScriptParser.swift

@@ -20,8 +20,8 @@ import Foundation
 public class JavaScriptParser {
     public typealias AST = Compiler_Protobuf_AST
 
-    /// The nodejs executable wrapper that we are using to run the parse.js script.
-    public let executor: NodeJS
+    /// The JavaScriptExecutor executable wrapper that we are using to run the parse.js script.
+    public let executor: JavaScriptExecutor
 
     // Simple error enum for errors that are displayed to the user.
     public enum ParserError: Error {
@@ -31,9 +31,11 @@ public class JavaScriptParser {
     /// The path to the parse.js script that implements the actual parsing using babel.js.
     private let parserScriptPath: String
 
-    public init?(executor: NodeJS) {
+    public init?(executor: JavaScriptExecutor) {
         self.executor = executor
 
+        // This will only work if the executor is node as we will need to use node modules.
+
         // The Parser/ subdirectory is copied verbatim into the module bundle, see Package.swift.
         self.parserScriptPath = Bundle.module.path(forResource: "parser", ofType: "js", inDirectory: "Parser")!
 
@@ -63,7 +65,7 @@ public class JavaScriptParser {
         task.standardError = output
         task.arguments = [parserScriptPath] + arguments
         // TODO: move this method into the NodeJS class instead of manually invoking the node.js binary here
-        task.executableURL = URL(fileURLWithPath: executor.nodejsExecutablePath)
+        task.executableURL = URL(fileURLWithPath: executor.executablePath)
         try task.run()
         task.waitUntilExit()
 

Sources/Fuzzilli/Util/NodeJS.swift → Sources/Fuzzilli/Util/JavaScriptExecutor.swift

@@ -1,4 +1,4 @@
-// Copyright 2023 Google LLC
+// Copyright 2024 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,69 +14,77 @@
 
 import Foundation
 
-/// This class wraps a NodeJS executable and allows executing JavaScript code with it.
-public class NodeJS {
-    /// Path to the node.js binary.
-    let nodejsExecutablePath: String
+public class JavaScriptExecutor {
+    /// Path to the js shell binary.
+    let executablePath: String
 
     /// Prefix to execute before every JavaScript testcase. Its main task is to define the `output` function.
     let prefix = Data("const output = console.log;\n".utf8)
 
-    public init?() {
-        if let path = NodeJS.findNodeJsExecutable() {
-            self.nodejsExecutablePath = path
-        } else {
-            return nil
-        }
+    /// The js shell mode for this JavaScriptExecutor
+    public enum ExecutorType {
+        // The default behavior, we will try to use the user supplied binary first.
+        // And fall back to node if we don't find anything supplied through FUZZILLI_TEST_SHELL
+        case any
+        // Try to find the node binary (useful if node modules are required) or fail.
+        case nodejs
+        // Try to find the user supplied binary or fail
+        case user
     }
 
-    /// The result of executing a Script.
-    public struct Result {
-        enum Outcome {
-            case terminated(status: Int32)
-            case timedOut
-        }
+    let arguments: [String]
 
-        let outcome: Outcome
-        let output: String
+    /// Depending on the type this constructor will try to find the requested shell or fail
+    public init?(type: ExecutorType = .any, withArguments maybeArguments: [String]? = nil) {
+        self.arguments = maybeArguments ?? []
+        let path: String?
 
-        var isSuccess: Bool {
-            switch outcome {
-            case .terminated(status: let status):
-                return status == 0
-            case .timedOut:
-                return false
-            }
+        switch type {
+            case .any:
+                path = JavaScriptExecutor.findJsShellExecutable() ?? JavaScriptExecutor.findNodeJsExecutable()
+            case .nodejs:
+                path = JavaScriptExecutor.findNodeJsExecutable()
+            case .user:
+                path = JavaScriptExecutor.findJsShellExecutable()
         }
-        var isFailure: Bool {
-            return !isSuccess
+
+        if path == nil {
+            return nil
         }
+
+        self.executablePath = path!
     }
 
     /// Executes the JavaScript script using the configured engine and returns the stdout.
     public func executeScript(_ script: String, withTimeout timeout: TimeInterval? = nil) throws -> Result {
-        return try execute(nodejsExecutablePath, withInput: prefix + script.data(using: .utf8)!, withArguments: ["--allow-natives-syntax"], timeout: timeout)
+        return try execute(executablePath, withInput: prefix + script.data(using: .utf8)!, withArguments: self.arguments, timeout: timeout)
     }
 
     /// Executes the JavaScript script at the specified path using the configured engine and returns the stdout.
     public func executeScript(at url: URL, withTimeout timeout: TimeInterval? = nil) throws -> Result {
         let script = try Data(contentsOf: url)
-        return try execute(nodejsExecutablePath, withInput: prefix + script, withArguments: ["--allow-natives-syntax"], timeout: timeout)
+        return try execute(executablePath, withInput: prefix + script, withArguments: self.arguments, timeout: timeout)
     }
 
     func execute(_ path: String, withInput input: Data = Data(), withArguments arguments: [String] = [], timeout maybeTimeout: TimeInterval? = nil) throws -> Result {
         let inputPipe = Pipe()
         let outputPipe = Pipe()
+        let errorPipe = Pipe()
+
+        // Write input into file.
+        let url = FileManager.default.temporaryDirectory
+               .appendingPathComponent(UUID().uuidString)
+               .appendingPathExtension("js")
 
-        // Write input into input pipe, then close it.
-        try inputPipe.fileHandleForWriting.write(contentsOf: input)
+        try input.write(to: url)
+        // Close stdin
         try inputPipe.fileHandleForWriting.close()
 
         // Execute the subprocess.
         let task = Process()
         task.standardOutput = outputPipe
-        task.standardError = outputPipe
-        task.arguments = arguments
+        task.standardError = errorPipe
+        task.arguments = arguments + [url.path]
         task.executableURL = URL(fileURLWithPath: path)
         task.standardInput = inputPipe
         try task.run()
@@ -98,6 +106,9 @@ public class NodeJS {
 
         task.waitUntilExit()
 
+        // Delete the temporary file
+        try FileManager.default.removeItem(at: url)
+
         // Fetch and return the output.
         var output = ""
         if let data = String(data: outputPipe.fileHandleForReading.readDataToEndOfFile(), encoding: .utf8) {
@@ -130,5 +141,33 @@ public class NodeJS {
         }
         return nil
     }
-}
 
+    /// Tries to find a JS shell that is usable for testing.
+    private static func findJsShellExecutable() -> String? {
+        if let path = ProcessInfo.processInfo.environment["FUZZILLI_TEST_SHELL"] {
+            return path
+        }
+        return nil
+    }
+
+    /// The Result of a JavaScript Execution, the exit code and any associated output.
+    public struct Result {
+        enum Outcome: Equatable {
+            case terminated(status: Int32)
+            case timedOut
+        }
+
+        let outcome: Outcome
+        let output: String
+
+        var isSuccess: Bool {
+            return outcome == .terminated(status: 0)
+        }
+        var isFailure: Bool {
+            return !isSuccess
+        }
+        var isTimeOut: Bool {
+            return outcome == .timedOut
+        }
+    }
+}

Tests/FuzzilliTests/CompilerTests.swift

@@ -27,7 +27,7 @@ import Foundation
 ///  - The test passes if there are no errors along the way and if the output of both executions is identical
 class CompilerTests: XCTestCase {
     func testFuzzILCompiler() throws {
-        guard let nodejs = NodeJS() else {
+        guard let nodejs = JavaScriptExecutor(type: .nodejs, withArguments: ["--allow-natives-syntax"]) else {
             throw XCTSkip("Could not find NodeJS executable. See Sources/Fuzzilli/Compiler/Parser/README.md for details on how to set up the parser.")
         }
 

Tests/FuzzilliTests/LiveTests.swift

@@ -15,16 +15,48 @@
 import XCTest
 @testable import Fuzzilli
 
+func executeAndParseResults(program: Program, fuzzer: Fuzzer, runner: JavaScriptExecutor, failures: inout Int, failureMessages: inout [String: Int]) {
+
+    let jsProgram = fuzzer.lifter.lift(program, withOptions: .includeComments)
+
+    do {
+        let result = try runner.executeScript(jsProgram, withTimeout: 5 * Seconds)
+        if result.isFailure {
+            failures += 1
+
+            for line in result.output.split(separator: "\n") {
+                if line.contains("Error:") {
+                    // Remove anything after a potential 2nd ":", which is usually testcase dependent content, e.g. "SyntaxError: Invalid regular expression: /ep{}[]Z7/: Incomplete quantifier"
+                    let signature = line.split(separator: ":")[0...1].joined(separator: ":")
+                    failureMessages[signature] = (failureMessages[signature] ?? 0) + 1
+                }
+            }
+
+            if LiveTests.VERBOSE {
+                let fuzzilProgram = FuzzILLifter().lift(program)
+                print("Program is invalid:")
+                print(jsProgram)
+                print("Out:")
+                print(result.output)
+                print("FuzzILCode:")
+                print(fuzzilProgram)
+            }
+        }
+    } catch {
+        XCTFail("Could not execute script: \(error)")
+    }
+}
+
 class LiveTests: XCTestCase {
     // Set to true to log failing programs
     static let VERBOSE = false
 
     func testValueGeneration() throws {
-        guard let nodejs = NodeJS() else {
-            throw XCTSkip("Could not find NodeJS executable.")
+        guard let runner = JavaScriptExecutor() else {
+            throw XCTSkip("Could not find js shell executable.")
         }
 
-        let liveTestConfig = Configuration(enableInspection: true)
+        let liveTestConfig = Configuration(logLevel: .warning, enableInspection: true)
 
         // We have to use the proper JavaScriptEnvironment here.
         // This ensures that we use the available builtins.
@@ -36,40 +68,14 @@ class LiveTests: XCTestCase {
         // TODO: consider running these in parallel.
         for _ in 0..<N {
             let b = fuzzer.makeBuilder()
+
             // Prefix building will run a handful of value generators. We use it instead of directly
             // calling buildValues() since we're mostly interested in emitting valid program prefixes.
             b.buildPrefix()
 
             let program = b.finalize()
-            let jsProgram = fuzzer.lifter.lift(program, withOptions: .includeComments)
 
-            // TODO: consider moving this code into a shared function once other tests need it as well.
-            do {
-                let result = try nodejs.executeScript(jsProgram, withTimeout: 5 * Seconds)
-                if result.isFailure {
-                    failures += 1
-
-                    for line in result.output.split(separator: "\n") {
-                        if line.contains("Error:") {
-                            // Remove anything after a potential 2nd ":", which is usually testcase dependent content, e.g. "SyntaxError: Invalid regular expression: /ep{}[]Z7/: Incomplete quantifier"
-                            let signature = line.split(separator: ":")[0...1].joined(separator: ":")
-                            failureMessages[signature] = (failureMessages[signature] ?? 0) + 1
-                        }
-                    }
-
-                    if LiveTests.VERBOSE {
-                        let fuzzilProgram = FuzzILLifter().lift(program)
-                        print("Program is invalid:")
-                        print(jsProgram)
-                        print("Out:")
-                        print(result.output)
-                        print("FuzzILCode:")
-                        print(fuzzilProgram)
-                    }
-                }
-            } catch {
-                XCTFail("Could not execute script: \(error)")
-            }
+            executeAndParseResults(program: program, fuzzer: fuzzer, runner: runner, failures: &failures, failureMessages: &failureMessages)
         }
 
         let failureRate = Double(failures) / Double(N)