diff --git a/README.md b/README.md index 0cec052..bdc2e9b 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,13 @@ # The Ferus Web Engine -The Ferus web engine is a tiny web engine written in Nim which aims to be a full replacement for something like Chromium's Blink, Safari's WebKit or Firefox's Gecko. +The Ferus web engine is a tiny web engine written in Nim which aims to be a full replacement for something like Chromium's Blink, Safari's WebKit or Firefox's Gecko. \ +It has rudimentary support for layout and JavaScript. ![A screenshot of Ferus rendering nim-lang.org](screenshots/ferus_nimlang.jpg) # Features - When possible, we write our own solution to a component of the overall modern web stack. This has resulted in projects like Stylus (our CSS3 compliant parser derived from Servo) and Bali (our JavaScript engine based on Mirage, our bytecode interpreter) and Sanchar (our HTTP client and URL parser) that are beneficial for the overall Nim community. +- Ferus has some initial JavaScript support. Our JavaScript engine is hovering around 2% of passing tests in Test262, so don't expect to be able to daily drive Ferus! + - Maximizing security will be our number one priority once we're past the toy engine stage. We have a sandbox that (mostly) works, but it has been disabled by default for the sake of everyone's (mostly our) sanity. If you want to try it out (and improve it), uncomment `-d:ferusInJail` in `nim.cfg`. - Everything is multiprocessed. This ensures that an error in one component (say, the JavaScript engine) won't bring down the entire session* @@ -29,7 +32,7 @@ So far, - Getting Mirage up to speed [X] - Finalizing Stylus' API [X] - Layout compliance -- Implementing the JavaScript runtime process component +- Implementing the JavaScript runtime process component [X] ## Reality check on the progress I started this project when I was 14 to learn how the web works. It took me a while to learn a lot of things about the web (and I appreciate that!). I'm not some ex-WebKit/Blink/Gecko contributor or anything like that. My experience is currently fairly limited in web engine development. \ diff --git a/ferus.nimble b/ferus.nimble index 36cb7f1..efae8a4 100644 --- a/ferus.nimble +++ b/ferus.nimble @@ -11,16 +11,18 @@ bin = @["ferus", "ferus_process"] # Dependencies requires "nim >= 2.0.2" -requires "ferusgfx >= 1.1" +requires "ferusgfx >= 1.1.1" requires "colored_logger >= 0.1.0" requires "stylus >= 0.1.0" -requires "https://github.com/ferus-web/ferus_ipc" +requires "https://github.com/ferus-web/ferus_ipc >= 0.1.3" requires "https://github.com/ferus-web/sanchar" requires "https://git.sr.ht/~bptato/chame >= 0.14.5" requires "seccomp >= 0.2.1" requires "simdutf >= 5.5.0" +requires "https://github.com/ferus-web/bali#master" requires "results" requires "pretty" +requires "jsony >= 1.1.5" requires "chagashi >= 0.5.4" when defined(ferusUseGlfw): diff --git a/flake.lock b/flake.lock index 72f2242..e3d8e8c 100644 --- a/flake.lock +++ b/flake.lock @@ -2,11 +2,11 @@ "nodes": { "nixpkgs": { "locked": { - "lastModified": 1725194671, - "narHash": "sha256-tLGCFEFTB5TaOKkpfw3iYT9dnk4awTP/q4w+ROpMfuw=", + "lastModified": 1729951556, + "narHash": "sha256-bpb6r3GjzhNW8l+mWtRtLNg5PhJIae041sPyqcFNGb4=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b833ff01a0d694b910daca6e2ff4a3f26dee478c", + "rev": "4e0eec54db79d4d0909f45a88037210ff8eaffee", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 00b3e7c..8936e5b 100644 --- a/flake.nix +++ b/flake.nix @@ -31,11 +31,11 @@ ]; buildInputs = with pkgs; [ + pkg-config libxkbcommon libseccomp libGL glfw - simdutf xorg.libX11 openssl.dev @@ -46,8 +46,14 @@ LD_LIBRARY_PATH = with pkgs; lib.makeLibraryPath [ libGL + simdutf ]; + env = { + # pkg-config cannot find simdutf without this, weird. + PKG_CONFIG_PATH = "${with pkgs; lib.makeLibraryPath [pkgs.simdutf]}/pkgconfig"; + }; + wrapFerus = let makeWrapperArgs = "--prefix LD_LIBRARY_PATH : ${LD_LIBRARY_PATH}"; diff --git a/nim.cfg b/nim.cfg index 371181d..aeef1d2 100644 --- a/nim.cfg +++ b/nim.cfg @@ -12,6 +12,7 @@ # -d:ferusAddMangohudToRendererPrefix # -d:ferusJustWaitForConnection -d:ferusUseGlfw +# -d:ferusIpcLogSendsToStdout # glfw flags # -d:glfwStaticLib @@ -20,4 +21,4 @@ # -d:ferusgfxDrawDamagedRegions # Enable SIMD support ---passC: "-march=native -mtune=native -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpclmul -mavx -mavx2" +--passC: "-march=znver3 -mtune=znver3 -msse -msse2 -msse3 -mssse3 -msse4.1 -msse4.2 -mpclmul -mavx -mavx2" diff --git a/samples/003.html b/samples/003.html new file mode 100644 index 0000000..b7de373 --- /dev/null +++ b/samples/003.html @@ -0,0 +1,15 @@ + + + + Hello BaliJS! + + + +

Check your console, there must be a few log messages there :^)

+ + diff --git a/src/components/js/ipc.nim b/src/components/js/ipc.nim new file mode 100644 index 0000000..628bd46 --- /dev/null +++ b/src/components/js/ipc.nim @@ -0,0 +1,15 @@ +import ferus_ipc/shared +import bali/stdlib/console + +type + JSExecPacket* = object + name*: string + kind: FerusMagic = feJSExec + buffer*: string + + JSConsoleMessage* = object + kind: FerusMagic = feJSConsoleMessage + message*: string + level*: ConsoleLevel + +export ConsoleLevel diff --git a/src/components/js/process.nim b/src/components/js/process.nim new file mode 100644 index 0000000..87cf3c5 --- /dev/null +++ b/src/components/js/process.nim @@ -0,0 +1,72 @@ +import std/[logging, json, base64, net] +import ferus_ipc/client/prelude +import bali/grammar/prelude +import bali/runtime/prelude +import bali/stdlib/console +import jsony +import ../../components/shared/[nix, sugar] +import ./ipc + +type + JSProcess* = object + ipc*: IPCClient + parser*: Parser + runtime*: Runtime + +proc initConsoleIPC*(js: var JSProcess) = + var pJs = addr(js) + + attachConsoleDelegate( + proc(level: ConsoleLevel, msg: string) = + pJs[].ipc.send( + JSConsoleMessage(message: msg, level: level) + ) + ) + +proc talk(js: var JSProcess, process: FerusProcess) = + var count: cint + + discard nix.ioctl(js.ipc.socket.getFd().cint, nix.FIONREAD, addr count) + + if count < 1: + return + + let + data = js.ipc.receive() + jdata = tryParseJson(data, JsonNode) + + if not *jdata: + warn "Did not get any valid JSON data." + warn data + return + + let kind = (&jdata).getOrDefault("kind").getStr().magicFromStr() + + if not *kind: + warn "No `kind` field inside JSON data provided." + return + + case &kind + of feJSExec: + info "Executing JavaScript buffer" + js.ipc.setState(Processing) + let data = &tryParseJson(data, JSExecPacket) + + js.parser = newParser(data.buffer.decode()) + js.runtime = newRuntime(data.name.decode(), js.parser.parse()) + js.runtime.run() + + js.ipc.setState(Idling) + else: + discard + +proc jsProcessLogic*(client: var IPCClient, process: FerusProcess) {.inline.} = + info "Entering JavaScript runtime process logic." + var js = JSProcess(ipc: client) + client.setState(Idling) + client.poll() + initConsoleIPC(js) + setLogFilter(lvlNone) + + while true: + js.talk(process) diff --git a/src/components/layout/processor.nim b/src/components/layout/processor.nim index 14ab10d..101f482 100644 --- a/src/components/layout/processor.nim +++ b/src/components/layout/processor.nim @@ -226,6 +226,7 @@ proc constructFromElem*(layout: var Layout, elem: HTMLElement) = if *image: info (&image).data layout.addImage((&image).data) + of TAG_SCRIPT: discard else: warn "layout: unhandled tag: " & $elem.tag diff --git a/src/components/master/master.nim b/src/components/master/master.nim index 8d6ee07..27ef139 100644 --- a/src/components/master/master.nim +++ b/src/components/master/master.nim @@ -1,4 +1,4 @@ -import std/[os, logging, osproc, strutils, options, base64, sets] +import std/[os, logging, osproc, strutils, options, base64, net, sets, terminal] import ferus_ipc/server/prelude import jsony import ./summon @@ -10,8 +10,8 @@ import pretty import sanchar/parse/url import sanchar/proto/http/shared -import ../../components/shared/sugar -import ../../components/[network/ipc, renderer/ipc, cookie_worker/ipc, parsers/html/ipc, parsers/html/document] +import ../../components/shared/[nix, sugar] +import ../../components/[network/ipc, renderer/ipc, cookie_worker/ipc, parsers/html/ipc, parsers/html/document, js/ipc] import ../../components/web/cookie/parsed_cookie when defined(unix): @@ -27,6 +27,16 @@ proc initialize*(master: MasterProcess) {.inline.} = proc poll*(master: MasterProcess) {.inline.} = master.server.poll() + for group, _ in master.server.groups: + for i, _ in master.server.groups[group]: + let socket = master.server.groups[group][i].socket + var count: cint + + discard nix.ioctl(socket.getFd().cint, nix.FIONREAD, addr count) + + if count > 0: + master.server.receiveFrom(group.uint, i.uint) + proc launchAndWait(master: MasterProcess, summoned: string) = info "launchAndWait(\"" & summoned & "\"): starting execution of process." when defined(ferusJustWaitForConnection): @@ -142,13 +152,24 @@ proc summonCookieWorker*(master: MasterProcess) {.inline.} = let summoned = summon(CookieWorker, ipcPath = master.server.path).dispatch() master.launchAndWait(summoned) +proc summonJSRuntime*(master: MasterProcess, group: uint) {.inline.} = + info "Summoning JS runtime process for group " & $group + let summoned = summon(JSRuntime, ipcPath = master.server.path).dispatch() + master.launchAndWait(summoned) + proc summonRendererProcess*(master: MasterProcess) {.inline.} = info "Summoning renderer process." let summoned = summon(Renderer, ipcPath = master.server.path).dispatch() master.launchAndWait(summoned) + + let oproc = master.server.groups[0].findProcess(Renderer, workers = false) + if !oproc: + error "Failed to spawn renderer process!" + quit(1) + # FIXME: investigate why this happens - var process = &master.server.groups[0].findProcess(Renderer, workers = false) + var process = &oproc let idx = master.server.groups[0].processes.find(process) process.state = Initialized @@ -174,6 +195,27 @@ proc renderDocument*(master: MasterProcess, document: HTMLDocument) = info "Dispatched document to renderer process." +proc executeJS*(master: MasterProcess, group: uint, name: string = "", code: string) = + info "Dispatching execution of JS code to runtime process" + var process = master.server.groups[group.int].findProcess(JSRuntime, workers = false) + + if not *process: + master.summonJSRuntime(group) + master.executeJS(group, name = name, code =code) + return + + var prc = &process + assert prc.kind == JSRuntime + master.server.send( + prc.socket, + JSExecPacket( + name: name.encode(), + buffer: code.encode() + ) + ) + + info "Dispatching JS code to runtime process." + proc setWindowTitle*(master: MasterProcess, title: string) {.inline.} = var process = master.server.groups[0].findProcess(Renderer, workers = false) @@ -317,10 +359,50 @@ proc dataTransfer*(master: MasterProcess, process: FerusProcess, request: DataTr else: warn "Unimplemented data transfer: FileRequest" +proc onConsoleLog*(master: MasterProcess, process: FerusProcess, data: JSConsoleMessage) = + styledWriteLine( + stdout, + "(", fgYellow, "JS Console", resetStyle, ") ", + ( + case data.level + of ConsoleLevel.Log: + fgGreen + of ConsoleLevel.Error: + fgRed + of ConsoleLevel.Debug: + fgMagenta + of ConsoleLevel.Info: + fgBlue + of ConsoleLevel.Trace: + fgMagenta + of ConsoleLevel.Warn: + fgYellow + ), + data.message, + resetStyle + ) + proc newMasterProcess*(): MasterProcess {.inline.} = var master = MasterProcess(server: newIPCServer()) master.server.onDataTransfer = proc(process: FerusProcess, request: DataTransferRequest) = master.dataTransfer(process, request) + + master.server.handler = proc(process: FerusProcess, kind: FerusMagic, data: string) = + case kind + of feJSConsoleMessage: + let data = tryParseJson(data, JSConsoleMessage) + + if !data: + master.server.reportBadMessage(process, + "Cannot reinterpret data for kind `feJSConsoleLog` as `JSConsoleMessage`", + Low + ) + return + + master.onConsoleLog(process, &data) + else: + warn "Unhandled IPC protocol magic: " & $kind + return master diff --git a/src/components/network/process.nim b/src/components/network/process.nim index bdc16fb..4a13770 100644 --- a/src/components/network/process.nim +++ b/src/components/network/process.nim @@ -52,6 +52,7 @@ proc talk(client: var IPCClient, process: FerusProcess) {.inline.} = if not *jdata: warn "Did not get any valid JSON data." + warn data return let kind = (&jdata).getOrDefault("kind").getStr().magicFromStr() diff --git a/src/components/parsers/html/process.nim b/src/components/parsers/html/process.nim index 76c52fa..7db5b23 100644 --- a/src/components/parsers/html/process.nim +++ b/src/components/parsers/html/process.nim @@ -1,8 +1,8 @@ -import std/[base64, options, json, logging, monotimes] +import std/[base64, options, json, logging, monotimes, net] import ./ipc import ./document import ../../web/dom -import ../../shared/sugar +import ../../shared/[nix, sugar] import ferus_ipc/client/prelude import jsony @@ -29,12 +29,19 @@ proc htmlParse*( ) proc talk(client: var IPCClient, process: FerusProcess) {.inline.} = + var count: cint + discard nix.ioctl(client.socket.getFd().cint, nix.FIONREAD, addr(count)) + + if count < 1: + return + let data = client.receive() jdata = tryParseJson(data, JsonNode) if not *jdata: warn "Did not get any valid JSON data." + warn data return let kind = (&jdata).getOrDefault("kind").getStr().magicFromStr() diff --git a/src/components/renderer/process.nim b/src/components/renderer/process.nim index 254fe94..80f9ea6 100644 --- a/src/components/renderer/process.nim +++ b/src/components/renderer/process.nim @@ -6,14 +6,11 @@ when defined(linux): import ../../components/sandbox/linux import ../../components/renderer/[core] -import ../../components/shared/sugar +import ../../components/shared/[nix, sugar] import ../../components/renderer/ipc except newDisplayList {.passL: "-lwayland-client -lwayland-cursor -lwayland-egl -lxkbcommon -lGL".} -var FIONREAD {.importc, header: "".}: cint -proc ioctl(fd: cint, op: cint, argp: pointer): cint {.importc, header: "".} - proc readTypeface*(data, format: string): Typeface {.raises: [PixieError].} = ## Loads a typeface from data. try: @@ -116,7 +113,7 @@ proc talk( ) {.inline.} = var count: cint - discard ioctl(client.socket.getFd().cint, FIONREAD, addr count) + discard nix.ioctl(client.socket.getFd().cint, nix.FIONREAD, addr count) if count < 1: return diff --git a/src/components/shared/nix.nim b/src/components/shared/nix.nim new file mode 100644 index 0000000..a79417b --- /dev/null +++ b/src/components/shared/nix.nim @@ -0,0 +1,4 @@ +## Shared *NIX utilities + +var FIONREAD* {.importc, header: "".}: cint +proc ioctl*(fd: cint, op: cint, argp: pointer): cint {.importc, header: "".} diff --git a/src/ferus.nim b/src/ferus.nim index 88a0ba7..d7b1ba6 100644 --- a/src/ferus.nim +++ b/src/ferus.nim @@ -4,6 +4,7 @@ import components/[ build_utils, master/master, network/ipc, renderer/ipc, shared/sugar ] +import components/parsers/html/document import sanchar/parse/url import pretty @@ -23,6 +24,7 @@ proc main() {.inline.} = var master = newMasterProcess() initialize master + master.summonJSRuntime(0) master.summonRendererProcess() master.loadFont("assets/fonts/IBMPlexSans-Regular.ttf", "Default") @@ -55,7 +57,18 @@ proc main() {.inline.} = quit(1) let document = &(&parsedHtml).document # i love unwrapping: electric boogaloo - print document + var scriptNodes: seq[HTMLElement] + + for node in document.elems: + scriptNodes &= node.findAll(TAG_SCRIPT, descend = true) + + for node in scriptNodes: + let text = node.text() + + if !text: continue + + master.executeJS(0, code = &text) + break master.renderDocument(document) diff --git a/src/ferus_process.nim b/src/ferus_process.nim index 1c0e14b..403a951 100644 --- a/src/ferus_process.nim +++ b/src/ferus_process.nim @@ -1,7 +1,7 @@ import std/[os, options, strutils, parseopt, logging] import colored_logger import ferus_ipc/client/[prelude, logger] -import components/[network/process, renderer/process, parsers/html/process] +import components/[network/process, renderer/process, parsers/html/process, js/process] when defined(linux): import components/sandbox/linux @@ -59,7 +59,7 @@ proc main() {.inline.} = discard client.connect(path) client.handshake() - addHandler newIPCLogger(lvlAll, client) + # addHandler newIPCLogger(lvlAll, client) setLogFilter(lvlInfo) if process.kind != Renderer: @@ -72,6 +72,8 @@ proc main() {.inline.} = renderProcessLogic(client, process) of Parser: htmlParserProcessLogic(client, process) + of JSRuntime: + jsProcessLogic(client, process) else: discard