feat: Vortex WebAssembly bindings

.github/workflows/wasm.yml

@@ -0,0 +1,38 @@
+name: CI
+
+on:
+  push:
+    branches: [ "develop" ]
+  pull_request: { }
+  workflow_dispatch: { }
+
+permissions:
+  actions: read
+  contents: read
+  checks: write # audit-check creates checks
+  issues: write # audit-check creates issues
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+
+jobs:
+  build:
+    name: "build for wasm32-wasip1"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./.github/actions/cleanup
+      - uses: rui314/setup-mold@v1
+      - uses: ./.github/actions/setup-rust
+      - uses: jetli/[email protected]
+        with:
+          # Optional version of wasm-pack to install(eg. 'v0.9.1', 'latest')
+          version: 'latest'
+      - name: build for wasm32-wasip1 target
+        working-directory: ./vortex-wasm
+        run: |
+          echo 'building Rust + TS, making sure it completes with success'
+          npm run build-dev
+
+

Cargo.toml

@@ -19,6 +19,7 @@ members = [
     "vortex-proto",
     "vortex-sampling-compressor",
     "vortex-scalar",
+    "vortex-wasm",
     "xtask",
 ]
 resolver = "2"
@@ -81,6 +82,7 @@ flexbuffers = "2.0.0"
 flume = "0.11"
 fsst-rs = "0.4.1"
 futures = { version = "0.3", default-features = false }
+futures-channel = "0.3"
 futures-executor = "0.3"
 futures-util = "0.3"
 getrandom = "0.2.14"
@@ -91,6 +93,7 @@ humansize = "2.1.3"
 indicatif = "0.17.8"
 itertools = "0.13.0"
 jiff = "0.1.8"
+js-sys = "0.3"
 libfuzzer-sys = "0.4"
 log = "0.4.21"
 mimalloc = "0.1.42"
@@ -125,7 +128,9 @@ tokio = "1.37.0"
 tracing = "0.1"
 url = "2"
 uuid = "1.8.0"
+wasm-bindgen = "0.2"
 wasm-bindgen-futures = "0.4"
+web-sys = { version = "0.3" }
 
 # BEGIN crates published by this project
 vortex = { version = "0.21.0", path = "./vortex" }

vortex-io/src/dispatcher/mod.rs

@@ -8,7 +8,7 @@ mod wasm;
 use std::future::Future;
 
 use futures::channel::oneshot;
-#[cfg(not(any(feature = "compio", feature = "tokio")))]
+#[cfg(not(any(feature = "compio", feature = "tokio", target_arch = "wasm32")))]
 use vortex_error::vortex_panic;
 use vortex_error::VortexResult;
 

vortex-wasm/.appveyor.yml

@@ -0,0 +1,11 @@
+install:
+  - appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe
+  - if not defined RUSTFLAGS rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain nightly
+  - set PATH=%PATH%;C:\Users\appveyor\.cargo\bin
+  - rustc -V
+  - cargo -V
+
+build: false
+
+test_script:
+  - cargo test --locked

vortex-wasm/.gitignore

@@ -0,0 +1,11 @@
+/target
+/.idea
+**/*.rs.bk
+Cargo.lock
+bin/
+pkg/
+wasm-pack.log
+/vortex.js
+/vortex.d.ts
+tsconfig.tsbuildinfo
+/node_modules

vortex-wasm/Cargo.toml

@@ -0,0 +1,53 @@
+[package]
+name = "vortex-wasm"
+description = "Vortex WebAssembly bindings"
+publish = false
+version = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+keywords = { workspace = true }
+include = { workspace = true }
+edition = { workspace = true }
+rust-version = { workspace = true }
+categories = { workspace = true }
+readme = { workspace = true }
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = ["console_error_panic_hook"]
+
+[dependencies]
+
+# The `console_error_panic_hook` crate provides better debugging of panics by
+# logging them with `console.error`. This is great for development, but requires
+# all the `std::fmt` and `std::panicking` infrastructure, so isn't great for
+# code size when deploying.
+console_error_panic_hook = { version = "0.1.7", optional = true }
+bytes = { workspace = true }
+vortex = { workspace = true }
+js-sys = { workspace = true }
+futures-channel = { workspace = true }
+futures-util = { workspace = true }
+wasm-bindgen = { workspace = true }
+wasm-bindgen-futures = { workspace = true }
+web-sys = { workspace = true, features = [
+    "console",
+    "Blob",
+    "File",
+    "FileReader",
+    'ReadableStream',
+    "ReadableStreamByobReader",
+    "ReadableStreamByobRequest",
+    "ReadableStreamDefaultReader",
+    "ReadableStreamReadResult",
+    "ReadableStreamReaderMode",
+    "ReadableStreamGetReaderOptions",
+] }
+
+[profile.release]
+# Tell `rustc` to optimize for small code size.
+opt-level = "s"

vortex-wasm/index.html

@@ -0,0 +1,40 @@
+<html>
+<head>
+    <meta content="text/html;charset=utf-8" http-equiv="Content-Type"/>
+</head>
+<body>
+<input type="file" id="file-picker" accept=".vortex"" />
+<p>Upload a Vortex file</p>
+
+<!-- Note the usage of `type=module` here as this is an ES6 module -->
+<script type="module">
+    // Use ES module import syntax to import functionality from the module
+    // that we have compiled.
+    //
+    // Note that the `default` import is an initialization function which
+    // will "boot" the module and make it ready to use. Currently browsers
+    // don't support natively imported WebAssembly as an ES module, but
+    // eventually the manual initialization won't be required!
+    import init, { VortexFile } from "./pkg/vortex_wasm.js";
+
+    async function run() {
+        await init();
+
+        const processFile = async (blob) => {
+            const file = await VortexFile.fromBlob(blob);
+            console.log("loaded the file");
+            await file.printSchema();
+        }
+
+        const filePicker = document.getElementById("file-picker");
+        filePicker.addEventListener("change", () => {
+            console.log("file uploaded");
+            const file = filePicker.files[0];
+            processFile(file);
+        });
+    }
+
+    run()
+</script>
+</body>
+</html>

vortex-wasm/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@vortex-dev/vortex",
+  "version": "0.2.0",
+  "description": "Vortex WASM bindings (WIP)",
+  "type": "module",
+  "module": "./vortex.js",
+  "types": "./vortex.d.ts",
+  "directories": {
+    "test": "tests"
+  },
+  "scripts": {
+    "build": "wasm-pack build --target web && tsc",
+    "build-dev": "wasm-pack build --dev --target web && tsc"
+  },
+  "author": "Vortex Developers <[email protected]>",
+  "license": "ISC",
+  "files": [
+    "vortex.ts",
+    "vortex.js",
+    "vortex.d.ts",
+    "pkg/vortex_wasm.js",
+    "pkg/vortex_wasm_bg.wasm",
+    "pkg/vortex_wasm_bg.wasm.d.ts"
+  ],
+  "devDependencies": {
+    "typescript": "~5.6.2"
+  }
+}

vortex-wasm/src/blob.rs

@@ -0,0 +1,69 @@
+use std::cell::RefCell;
+use std::future::Future;
+use std::rc::Rc;
+
+use bytes::{Bytes, BytesMut};
+use futures_channel::oneshot;
+use futures_util::FutureExt;
+use js_sys::Uint8Array;
+use vortex::io::VortexReadAt;
+use wasm_bindgen::closure::Closure;
+use wasm_bindgen::JsCast;
+use web_sys::{Blob, FileReader};
+
+#[derive(Clone)]
+pub struct BlobReader(pub Rc<RefCell<Blob>>);
+
+// (•_•)           it's time to get
+// ( •_•)>⌐■-■    ...
+// (⌐■_■)          Send + Sync
+//
+// This is safe because the browser runs single-threaded.
+// TODO(aduffy): is this actually safe?
+unsafe impl Send for BlobReader {}
+unsafe impl Sync for BlobReader {}
+
+impl VortexReadAt for BlobReader {
+    fn read_byte_range(
+        &self,
+        pos: u64,
+        len: u64,
+    ) -> impl Future<Output = std::io::Result<Bytes>> + 'static {
+        let this = self.clone();
+        web_sys::console::log_1(&format!("read_byte_range({pos}, {len})").into());
+
+        let (tx, rx) = oneshot::channel();
+
+        let start: i32 = pos.try_into().unwrap();
+        let end: i32 = (pos + len).try_into().unwrap();
+        let sliced = this.0.borrow().slice_with_i32_and_i32(start, end).unwrap();
+
+        let file_reader = FileReader::new().unwrap();
+        let file_reader_cb = file_reader.clone();
+
+        // Send the onload handler
+        let loadend = Closure::once_into_js(move || {
+            let array_buf = file_reader_cb.result().unwrap();
+            let array = Uint8Array::new(array_buf.as_ref());
+            let mut result = BytesMut::with_capacity(len.try_into().unwrap());
+            unsafe {
+                result.set_len(result.capacity());
+            }
+            array.copy_to(&mut result);
+
+            // Send the result to the main thread.
+            tx.send(result).unwrap();
+        });
+        file_reader.set_onloadend(loadend.dyn_ref());
+
+        // Trigger the streaming read.
+        file_reader.read_as_array_buffer(&sliced).unwrap();
+
+        // Return the reader which will be awaited.
+        rx.map(|res| Ok(res.unwrap().freeze()))
+    }
+
+    fn size(&self) -> impl Future<Output = std::io::Result<u64>> + 'static {
+        std::future::ready(Ok(self.0.borrow().size() as u64))
+    }
+}