Add JavaScript support

Merge pull request #32 from ferus-web/javascript-support

README.md

@@ -1,10 +1,13 @@
 # The Ferus Web Engine
-The Ferus web engine is a tiny web engine written in Nim which aims to be a full replacement for something like Chromium's Blink, Safari's WebKit or Firefox's Gecko.
+The Ferus web engine is a tiny web engine written in Nim which aims to be a full replacement for something like Chromium's Blink, Safari's WebKit or Firefox's Gecko. \
+It has rudimentary support for layout and JavaScript.
 ![A screenshot of Ferus rendering nim-lang.org](screenshots/ferus_nimlang.jpg)
 
 # Features
 - When possible, we write our own solution to a component of the overall modern web stack. This has resulted in projects like Stylus (our CSS3 compliant parser derived from Servo) and Bali (our JavaScript engine based on Mirage, our bytecode interpreter) and Sanchar (our HTTP client and URL parser) that are beneficial for the overall Nim community.
 
+- Ferus has some initial JavaScript support. Our JavaScript engine is hovering around 2% of passing tests in Test262, so don't expect to be able to daily drive Ferus!
+
 - Maximizing security will be our number one priority once we're past the toy engine stage. We have a sandbox that (mostly) works, but it has been disabled by default for the sake of everyone's (mostly our) sanity. If you want to try it out (and improve it), uncomment `-d:ferusInJail` in `nim.cfg`.
 
 - Everything is multiprocessed. This ensures that an error in one component (say, the JavaScript engine) won't bring down the entire session*
@@ -29,7 +32,7 @@ So far,
 - Getting Mirage up to speed [X]
 - Finalizing Stylus' API [X]
 - Layout compliance
-- Implementing the JavaScript runtime process component
+- Implementing the JavaScript runtime process component [X]
 
 ## Reality check on the progress
 I started this project when I was 14 to learn how the web works. It took me a while to learn a lot of things about the web (and I appreciate that!). I'm not some ex-WebKit/Blink/Gecko contributor or anything like that. My experience is currently fairly limited in web engine development. \

ferus.nimble

@@ -11,16 +11,18 @@ bin = @["ferus", "ferus_process"]
 # Dependencies
 
 requires "nim >= 2.0.2"
-requires "ferusgfx >= 1.1"
+requires "ferusgfx >= 1.1.1"
 requires "colored_logger >= 0.1.0"
 requires "stylus >= 0.1.0"
-requires "https://github.com/ferus-web/ferus_ipc"
+requires "https://github.com/ferus-web/ferus_ipc >= 0.1.3"
 requires "https://github.com/ferus-web/sanchar"
 requires "https://git.sr.ht/~bptato/chame >= 0.14.5"
 requires "seccomp >= 0.2.1"
 requires "simdutf >= 5.5.0"
+requires "https://github.com/ferus-web/bali#master"
 requires "results"
 requires "pretty"
+requires "jsony >= 1.1.5"
 requires "chagashi >= 0.5.4"
 
 when defined(ferusUseGlfw):

flake.nix

@@ -31,11 +31,11 @@
             ];
 
             buildInputs = with pkgs; [
+              pkg-config
               libxkbcommon
               libseccomp
               libGL
               glfw
-              simdutf
 
               xorg.libX11
               openssl.dev
@@ -46,8 +46,14 @@
 
             LD_LIBRARY_PATH = with pkgs; lib.makeLibraryPath [
               libGL
+              simdutf
             ];
 
+            env = {
+              # pkg-config cannot find simdutf without this, weird.
+              PKG_CONFIG_PATH = "${with pkgs; lib.makeLibraryPath [pkgs.simdutf]}/pkgconfig";
+            };
+
             wrapFerus =
               let
                 makeWrapperArgs = "--prefix LD_LIBRARY_PATH : ${LD_LIBRARY_PATH}";

nim.cfg

@@ -12,6 +12,7 @@
 # -d:ferusAddMangohudToRendererPrefix
 # -d:ferusJustWaitForConnection
 -d:ferusUseGlfw
+# -d:ferusIpcLogSendsToStdout
 
 # glfw flags
 # -d:glfwStaticLib
@@ -20,4 +21,4 @@
 # -d:ferusgfxDrawDamagedRegions
 
 # Enable SIMD support
---passC: "-march=native -mtune=native -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpclmul -mavx -mavx2"
+--passC: "-march=znver3 -mtune=znver3 -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpclmul -mavx -mavx2"

samples/003.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Hello BaliJS!</title>
+  </head>
+  <body>
+    <script>
+      console.log("Hello Ferus!")
+      console.warn("This is a warning")
+      console.info("Very important information")
+      console.error("Failed to send your very important data to Google")
+    </script>
+    <h1>Check your console, there must be a few log messages there :^)</h1>
+  </body>
+</html>

src/components/js/ipc.nim

@@ -0,0 +1,15 @@
+import ferus_ipc/shared
+import bali/stdlib/console
+
+type
+  JSExecPacket* = object
+    name*: string
+    kind: FerusMagic = feJSExec
+    buffer*: string
+
+  JSConsoleMessage* = object
+    kind: FerusMagic = feJSConsoleMessage
+    message*: string
+    level*: ConsoleLevel
+
+export ConsoleLevel

src/components/js/process.nim

@@ -0,0 +1,72 @@
+import std/[logging, json, base64, net]
+import ferus_ipc/client/prelude
+import bali/grammar/prelude
+import bali/runtime/prelude
+import bali/stdlib/console
+import jsony
+import ../../components/shared/[nix, sugar]
+import ./ipc
+
+type
+  JSProcess* = object
+    ipc*: IPCClient
+    parser*: Parser
+    runtime*: Runtime
+
+proc initConsoleIPC*(js: var JSProcess) =
+  var pJs = addr(js)
+
+  attachConsoleDelegate(
+    proc(level: ConsoleLevel, msg: string) =
+      pJs[].ipc.send(
+        JSConsoleMessage(message: msg, level: level)
+      )
+  )
+
+proc talk(js: var JSProcess, process: FerusProcess) =
+  var count: cint
+
+  discard nix.ioctl(js.ipc.socket.getFd().cint, nix.FIONREAD, addr count)
+
+  if count < 1:
+    return
+
+  let
+    data = js.ipc.receive()
+    jdata = tryParseJson(data, JsonNode)
+
+  if not *jdata:
+    warn "Did not get any valid JSON data."
+    warn data
+    return
+
+  let kind = (&jdata).getOrDefault("kind").getStr().magicFromStr()
+
+  if not *kind:
+    warn "No `kind` field inside JSON data provided."
+    return
+
+  case &kind
+  of feJSExec:
+    info "Executing JavaScript buffer"
+    js.ipc.setState(Processing)
+    let data = &tryParseJson(data, JSExecPacket)
+
+    js.parser = newParser(data.buffer.decode())
+    js.runtime = newRuntime(data.name.decode(), js.parser.parse())
+    js.runtime.run()
+
+    js.ipc.setState(Idling)
+  else:
+    discard
+
+proc jsProcessLogic*(client: var IPCClient, process: FerusProcess) {.inline.} =
+  info "Entering JavaScript runtime process logic."
+  var js = JSProcess(ipc: client)
+  client.setState(Idling)
+  client.poll()
+  initConsoleIPC(js)
+  setLogFilter(lvlNone)
+
+  while true:
+    js.talk(process)

src/components/layout/processor.nim

@@ -226,6 +226,7 @@ proc constructFromElem*(layout: var Layout, elem: HTMLElement) =
     if *image:
       info (&image).data
       layout.addImage((&image).data)
+  of TAG_SCRIPT: discard
   else:
     warn "layout: unhandled tag: " & $elem.tag
 

src/components/master/master.nim

@@ -1,4 +1,4 @@
-import std/[os, logging, osproc, strutils, options, base64, sets]
+import std/[os, logging, osproc, strutils, options, base64, net, sets, terminal]
 import ferus_ipc/server/prelude
 import jsony
 import ./summon
@@ -10,8 +10,8 @@ import pretty
 import sanchar/parse/url
 import sanchar/proto/http/shared
 
-import ../../components/shared/sugar
-import ../../components/[network/ipc, renderer/ipc, cookie_worker/ipc, parsers/html/ipc, parsers/html/document]
+import ../../components/shared/[nix, sugar]
+import ../../components/[network/ipc, renderer/ipc, cookie_worker/ipc, parsers/html/ipc, parsers/html/document, js/ipc]
 import ../../components/web/cookie/parsed_cookie
 
 when defined(unix):
@@ -27,6 +27,16 @@ proc initialize*(master: MasterProcess) {.inline.} =
 proc poll*(master: MasterProcess) {.inline.} =
   master.server.poll()
 
+  for group, _ in master.server.groups:
+    for i, _ in master.server.groups[group]:
+      let socket = master.server.groups[group][i].socket
+      var count: cint
+
+      discard nix.ioctl(socket.getFd().cint, nix.FIONREAD, addr count)
+
+      if count > 0:
+        master.server.receiveFrom(group.uint, i.uint)
+
 proc launchAndWait(master: MasterProcess, summoned: string) =
   info "launchAndWait(\"" & summoned & "\"): starting execution of process."
   when defined(ferusJustWaitForConnection):
@@ -142,13 +152,24 @@ proc summonCookieWorker*(master: MasterProcess) {.inline.} =
   let summoned = summon(CookieWorker, ipcPath = master.server.path).dispatch()
   master.launchAndWait(summoned)
 
+proc summonJSRuntime*(master: MasterProcess, group: uint) {.inline.} =
+  info "Summoning JS runtime process for group " & $group
+  let summoned = summon(JSRuntime, ipcPath = master.server.path).dispatch()
+  master.launchAndWait(summoned)
+
 proc summonRendererProcess*(master: MasterProcess) {.inline.} =
   info "Summoning renderer process."
   let summoned = summon(Renderer, ipcPath = master.server.path).dispatch()
   master.launchAndWait(summoned)
 
+
+  let oproc = master.server.groups[0].findProcess(Renderer, workers = false)
+  if !oproc:
+    error "Failed to spawn renderer process!"
+    quit(1)
+
   # FIXME: investigate why this happens
-  var process = &master.server.groups[0].findProcess(Renderer, workers = false)
+  var process = &oproc
   let idx = master.server.groups[0].processes.find(process)
 
   process.state = Initialized
@@ -174,6 +195,27 @@ proc renderDocument*(master: MasterProcess, document: HTMLDocument) =
 
   info "Dispatched document to renderer process."
 
+proc executeJS*(master: MasterProcess, group: uint, name: string = "<inline script>", code: string) =
+  info "Dispatching execution of JS code to runtime process"
+  var process = master.server.groups[group.int].findProcess(JSRuntime, workers = false)
+
+  if not *process:
+    master.summonJSRuntime(group)
+    master.executeJS(group, name = name, code =code)
+    return
+
+  var prc = &process
+  assert prc.kind == JSRuntime
+  master.server.send(
+    prc.socket,
+    JSExecPacket(
+      name: name.encode(),
+      buffer: code.encode()
+    )
+  )
+
+  info "Dispatching JS code to runtime process."
+
 proc setWindowTitle*(master: MasterProcess, title: string) {.inline.} =
   var process = master.server.groups[0].findProcess(Renderer, workers = false)
 
@@ -317,10 +359,50 @@ proc dataTransfer*(master: MasterProcess, process: FerusProcess, request: DataTr
   else:
     warn "Unimplemented data transfer: FileRequest"
 
+proc onConsoleLog*(master: MasterProcess, process: FerusProcess, data: JSConsoleMessage) =
+  styledWriteLine(
+    stdout,
+    "(", fgYellow, "JS Console", resetStyle, ") ",
+    (
+      case data.level
+      of ConsoleLevel.Log:
+        fgGreen
+      of ConsoleLevel.Error:
+        fgRed
+      of ConsoleLevel.Debug:
+        fgMagenta
+      of ConsoleLevel.Info:
+        fgBlue
+      of ConsoleLevel.Trace:
+        fgMagenta
+      of ConsoleLevel.Warn:
+        fgYellow
+    ),
+    data.message,
+    resetStyle
+  )
+
 proc newMasterProcess*(): MasterProcess {.inline.} =
   var master = MasterProcess(server: newIPCServer())
 
   master.server.onDataTransfer = proc(process: FerusProcess, request: DataTransferRequest) =
     master.dataTransfer(process, request)
+
+  master.server.handler = proc(process: FerusProcess, kind: FerusMagic, data: string) =
+    case kind
+    of feJSConsoleMessage:
+      let data = tryParseJson(data, JSConsoleMessage)
+
+      if !data:
+        master.server.reportBadMessage(process,
+          "Cannot reinterpret data for kind `feJSConsoleLog` as `JSConsoleMessage`",
+          Low
+        )
+        return
+
+      master.onConsoleLog(process, &data)
+    else:
+      warn "Unhandled IPC protocol magic: " & $kind
+      return
 
   master

src/components/network/process.nim

@@ -52,6 +52,7 @@ proc talk(client: var IPCClient, process: FerusProcess) {.inline.} =
 
   if not *jdata:
     warn "Did not get any valid JSON data."
+    warn data
     return
 
   let kind = (&jdata).getOrDefault("kind").getStr().magicFromStr()

src/components/parsers/html/process.nim

@@ -1,8 +1,8 @@
-import std/[base64, options, json, logging, monotimes]
+import std/[base64, options, json, logging, monotimes, net]
 import ./ipc
 import ./document
 import ../../web/dom
-import ../../shared/sugar
+import ../../shared/[nix, sugar]
 import ferus_ipc/client/prelude
 import jsony
 
@@ -29,12 +29,19 @@ proc htmlParse*(
   )
 
 proc talk(client: var IPCClient, process: FerusProcess) {.inline.} =
+  var count: cint
+  discard nix.ioctl(client.socket.getFd().cint, nix.FIONREAD, addr(count))
+
+  if count < 1:
+    return
+
   let
     data = client.receive()
     jdata = tryParseJson(data, JsonNode)
 
   if not *jdata:
     warn "Did not get any valid JSON data."
+    warn data
     return
 
   let kind = (&jdata).getOrDefault("kind").getStr().magicFromStr()