-
Notifications
You must be signed in to change notification settings - Fork 32
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
14 changed files
with
606 additions
and
1 deletion.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
install: | ||
- appveyor-retry appveyor DownloadFile https://win.rustup.rs/ -FileName rustup-init.exe | ||
- if not defined RUSTFLAGS rustup-init.exe -y --default-host x86_64-pc-windows-msvc --default-toolchain nightly | ||
- set PATH=%PATH%;C:\Users\appveyor\.cargo\bin | ||
- rustc -V | ||
- cargo -V | ||
|
||
build: false | ||
|
||
test_script: | ||
- cargo test --locked |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,11 @@ | ||
/target | ||
/.idea | ||
**/*.rs.bk | ||
Cargo.lock | ||
bin/ | ||
pkg/ | ||
wasm-pack.log | ||
/vortex.js | ||
/vortex.d.ts | ||
tsconfig.tsbuildinfo | ||
/node_modules |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,53 @@ | ||
[package] | ||
name = "vortex-wasm" | ||
description = "Vortex WebAssembly bindings" | ||
publish = false | ||
version = { workspace = true } | ||
homepage = { workspace = true } | ||
repository = { workspace = true } | ||
authors = { workspace = true } | ||
license = { workspace = true } | ||
keywords = { workspace = true } | ||
include = { workspace = true } | ||
edition = { workspace = true } | ||
rust-version = { workspace = true } | ||
categories = { workspace = true } | ||
readme = { workspace = true } | ||
|
||
[lib] | ||
crate-type = ["cdylib", "rlib"] | ||
|
||
[features] | ||
default = ["console_error_panic_hook"] | ||
|
||
[dependencies] | ||
|
||
# The `console_error_panic_hook` crate provides better debugging of panics by | ||
# logging them with `console.error`. This is great for development, but requires | ||
# all the `std::fmt` and `std::panicking` infrastructure, so isn't great for | ||
# code size when deploying. | ||
console_error_panic_hook = { version = "0.1.7", optional = true } | ||
bytes = { workspace = true } | ||
vortex = { workspace = true } | ||
js-sys = { workspace = true } | ||
futures-channel = { workspace = true } | ||
futures-util = { workspace = true } | ||
wasm-bindgen = { workspace = true } | ||
wasm-bindgen-futures = { workspace = true } | ||
web-sys = { workspace = true, features = [ | ||
"console", | ||
"Blob", | ||
"File", | ||
"FileReader", | ||
'ReadableStream', | ||
"ReadableStreamByobReader", | ||
"ReadableStreamByobRequest", | ||
"ReadableStreamDefaultReader", | ||
"ReadableStreamReadResult", | ||
"ReadableStreamReaderMode", | ||
"ReadableStreamGetReaderOptions", | ||
] } | ||
|
||
[profile.release] | ||
# Tell `rustc` to optimize for small code size. | ||
opt-level = "s" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
<html> | ||
<head> | ||
<meta content="text/html;charset=utf-8" http-equiv="Content-Type"/> | ||
</head> | ||
<body> | ||
<input type="file" id="file-picker" accept=".vortex"" /> | ||
<p>Upload a Vortex file</p> | ||
|
||
<!-- Note the usage of `type=module` here as this is an ES6 module --> | ||
<script type="module"> | ||
// Use ES module import syntax to import functionality from the module | ||
// that we have compiled. | ||
// | ||
// Note that the `default` import is an initialization function which | ||
// will "boot" the module and make it ready to use. Currently browsers | ||
// don't support natively imported WebAssembly as an ES module, but | ||
// eventually the manual initialization won't be required! | ||
import init, { VortexFile } from "./pkg/vortex_wasm.js"; | ||
|
||
async function run() { | ||
await init(); | ||
|
||
const processFile = async (blob) => { | ||
const file = await VortexFile.fromBlob(blob); | ||
console.log("loaded the file"); | ||
await file.printSchema(); | ||
} | ||
|
||
const filePicker = document.getElementById("file-picker"); | ||
filePicker.addEventListener("change", () => { | ||
console.log("file uploaded"); | ||
const file = filePicker.files[0]; | ||
processFile(file); | ||
}); | ||
} | ||
|
||
run() | ||
</script> | ||
</body> | ||
</html> |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
{ | ||
"name": "@vortex-dev/vortex", | ||
"version": "0.2.0", | ||
"description": "Vortex WASM bindings (WIP)", | ||
"type": "module", | ||
"module": "./vortex.js", | ||
"types": "./vortex.d.ts", | ||
"directories": { | ||
"test": "tests" | ||
}, | ||
"scripts": { | ||
"build": "wasm-pack build --target web && tsc" | ||
}, | ||
"author": "", | ||
"license": "ISC", | ||
"files": [ | ||
"vortex.ts", | ||
"vortex.js", | ||
"vortex.d.ts", | ||
"pkg/vortex_wasm.js", | ||
"pkg/vortex_wasm_bg.wasm", | ||
"pkg/vortex_wasm_bg.wasm.d.ts" | ||
], | ||
"devDependencies": { | ||
"typescript": "~5.6.2" | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,69 @@ | ||
use std::cell::RefCell; | ||
use std::future::Future; | ||
use std::rc::Rc; | ||
|
||
use bytes::{Bytes, BytesMut}; | ||
use futures_channel::oneshot; | ||
use futures_util::FutureExt; | ||
use js_sys::Uint8Array; | ||
use vortex::io::VortexReadAt; | ||
use wasm_bindgen::closure::Closure; | ||
use wasm_bindgen::JsCast; | ||
use web_sys::{Blob, FileReader}; | ||
|
||
#[derive(Clone)] | ||
pub struct BlobReader(pub Rc<RefCell<Blob>>); | ||
|
||
// (•_•) it's time to get | ||
// ( •_•)>⌐■-■ ... | ||
// (⌐■_■) Send + Sync | ||
// | ||
// This is safe because the browser runs single-threaded. | ||
// TODO(aduffy): is this actually safe? | ||
unsafe impl Send for BlobReader {} | ||
unsafe impl Sync for BlobReader {} | ||
|
||
impl VortexReadAt for BlobReader { | ||
fn read_byte_range( | ||
&self, | ||
pos: u64, | ||
len: u64, | ||
) -> impl Future<Output = std::io::Result<Bytes>> + 'static { | ||
let this = self.clone(); | ||
web_sys::console::log_1(&format!("read_byte_range({pos}, {len})").into()); | ||
|
||
let (tx, rx) = oneshot::channel(); | ||
|
||
let start: i32 = pos.try_into().unwrap(); | ||
let end: i32 = (pos + len).try_into().unwrap(); | ||
let sliced = this.0.borrow().slice_with_i32_and_i32(start, end).unwrap(); | ||
|
||
let file_reader = FileReader::new().unwrap(); | ||
let file_reader_cb = file_reader.clone(); | ||
|
||
// Send the onload handler | ||
let loadend = Closure::once_into_js(move || { | ||
let array_buf = file_reader_cb.result().unwrap(); | ||
let array = Uint8Array::new(array_buf.as_ref()); | ||
let mut result = BytesMut::with_capacity(len.try_into().unwrap()); | ||
unsafe { | ||
result.set_len(result.capacity()); | ||
} | ||
array.copy_to(&mut result); | ||
|
||
// Send the result to the main thread. | ||
tx.send(result).unwrap(); | ||
}); | ||
file_reader.set_onloadend(loadend.dyn_ref()); | ||
|
||
// Trigger the streaming read. | ||
file_reader.read_as_array_buffer(&sliced).unwrap(); | ||
|
||
// Return the reader which will be awaited. | ||
rx.map(|res| Ok(res.unwrap().freeze())) | ||
} | ||
|
||
fn size(&self) -> impl Future<Output = std::io::Result<u64>> + 'static { | ||
std::future::ready(Ok(self.0.borrow().size() as u64)) | ||
} | ||
} |
Oops, something went wrong.