Skip to content

Commit

Permalink
Add support for async/streams/futures
Browse files Browse the repository at this point in the history
This adds support for loading, compiling, linking, and running components which
use the [Async
ABI](https://github.com/WebAssembly/component-model/blob/main/design/mvp/Async.md)
along with the [`stream`, `future`, and
`error-context`](WebAssembly/component-model#405) types.
It also adds support for generating host bindings such that multiple host
functions can be run concurrently with guest tasks -- without monopolizing the
`Store`.

See the [implementation RFC](bytecodealliance/rfcs#38)
for details, as well as [this
repo](https://github.com/dicej/component-async-demo) containing end-to-end smoke
tests.

This is very much a work-in progress, with a number of tasks remaining:

- [ ] Avoid exposing global task IDs to guests and use per-instance IDs instead
- [ ] Track `task.return` type during compilation and assert the actual and expected types match at runtime
- [ ] Ensure all guest pointers are bounds-checked when lifting, lowering, or copying values
- [ ] Reduce code duplication in `wasmtime_cranelift::compiler::component`
- [ ] Add support for `(Typed)Func::call_concurrent` per the RFC
- [ ] Add support for multiplexing stream/future reads/writes and concurrent calls to guest exports per the RFC
- [ ] Refactor, clean up, and unify handling of backpressure, yields, and even polling
- [ ] Guard against reentrance where required (e.g. in certain fused adapter calls)
- [ ] Add integration test cases covering new functionality to tests/all/component_model (starting by porting over the tests in https://github.com/dicej/component-async-demo)
- [ ] Add binding generation test cases to crates/component-macro/tests
- [ ] Add WAST tests to tests/misc_testsuite/component-model
- [ ] Add support and test coverage for callback-less async functions (e.g. goroutines)
- [ ] Switch to back to upstream `wasm-tools` once bytecodealliance/wasm-tools#1895 has been merged and released

Signed-off-by: Joel Dice <[email protected]>
  • Loading branch information
dicej committed Nov 7, 2024
1 parent a126152 commit 05f0998
Show file tree
Hide file tree
Showing 55 changed files with 9,554 additions and 428 deletions.
167 changes: 120 additions & 47 deletions Cargo.lock

Large diffs are not rendered by default.

18 changes: 9 additions & 9 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -274,15 +274,15 @@ wit-bindgen = { version = "0.34.0", default-features = false }
wit-bindgen-rust-macro = { version = "0.34.0", default-features = false }

# wasm-tools family:
wasmparser = { version = "0.219.1", default-features = false }
wat = "1.219.1"
wast = "219.0.1"
wasmprinter = "0.219.1"
wasm-encoder = "0.219.1"
wasm-smith = "0.219.1"
wasm-mutate = "0.219.1"
wit-parser = "0.219.1"
wit-component = "0.219.1"
wasmparser = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wat = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wast = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wasmprinter = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wasm-encoder = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wasm-smith = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wasm-mutate = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wit-parser = { git = "https://github.com/dicej/wasm-tools", branch = "async" }
wit-component = { git = "https://github.com/dicej/wasm-tools", branch = "async" }

# Non-Bytecode Alliance maintained dependencies:
# --------------------------
Expand Down
1 change: 1 addition & 0 deletions crates/component-macro/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,3 +41,4 @@ similar = { workspace = true }
[features]
async = []
std = ['wasmtime-wit-bindgen/std']
component-model-async = ['std', 'async', 'wasmtime-wit-bindgen/component-model-async']
43 changes: 35 additions & 8 deletions crates/component-macro/src/bindgen.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,15 @@
use proc_macro2::{Span, TokenStream};
use quote::ToTokens;
use std::collections::HashMap;
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::env;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicUsize, Ordering::Relaxed};
use syn::parse::{Error, Parse, ParseStream, Result};
use syn::punctuated::Punctuated;
use syn::{braced, token, Token};
use wasmtime_wit_bindgen::{AsyncConfig, Opts, Ownership, TrappableError, TrappableImports};
use wasmtime_wit_bindgen::{
AsyncConfig, CallStyle, Opts, Ownership, TrappableError, TrappableImports,
};
use wit_parser::{PackageId, Resolve, UnresolvedPackageGroup, WorldId};

pub struct Config {
Expand All @@ -20,13 +21,22 @@ pub struct Config {
}

pub fn expand(input: &Config) -> Result<TokenStream> {
if !cfg!(feature = "async") && input.opts.async_.maybe_async() {
if let (CallStyle::Async | CallStyle::Concurrent, false) =
(input.opts.call_style(), cfg!(feature = "async"))
{
return Err(Error::new(
Span::call_site(),
"cannot enable async bindings unless `async` crate feature is active",
));
}

if input.opts.concurrent_imports && !cfg!(feature = "component-model-async") {
return Err(Error::new(
Span::call_site(),
"cannot enable `concurrent_imports` option unless `component-model-async` crate feature is active",
));
}

let mut src = match input.opts.generate(&input.resolve, input.world) {
Ok(s) => s,
Err(e) => return Err(Error::new(Span::call_site(), e.to_string())),
Expand All @@ -40,7 +50,10 @@ pub fn expand(input: &Config) -> Result<TokenStream> {
// place a formatted version of the expanded code into a file. This file
// will then show up in rustc error messages for any codegen issues and can
// be inspected manually.
if input.include_generated_code_from_file || std::env::var("WASMTIME_DEBUG_BINDGEN").is_ok() {
if input.include_generated_code_from_file
|| input.opts.debug
|| std::env::var("WASMTIME_DEBUG_BINDGEN").is_ok()
{
static INVOCATION: AtomicUsize = AtomicUsize::new(0);
let root = Path::new(env!("DEBUG_OUTPUT_DIR"));
let world_name = &input.resolve.worlds[input.world].name;
Expand Down Expand Up @@ -107,13 +120,15 @@ impl Parse for Config {
}
Opt::Tracing(val) => opts.tracing = val,
Opt::VerboseTracing(val) => opts.verbose_tracing = val,
Opt::Debug(val) => opts.debug = val,
Opt::Async(val, span) => {
if async_configured {
return Err(Error::new(span, "cannot specify second async config"));
}
async_configured = true;
opts.async_ = val;
}
Opt::ConcurrentImports(val) => opts.concurrent_imports = val,
Opt::TrappableErrorType(val) => opts.trappable_error_type = val,
Opt::TrappableImports(val) => opts.trappable_imports = val,
Opt::Ownership(val) => opts.ownership = val,
Expand All @@ -138,7 +153,7 @@ impl Parse for Config {
"cannot specify a world with `interfaces`",
));
}
world = Some("interfaces".to_string());
world = Some("wasmtime:component-macro-synthesized/interfaces".to_string());

opts.only_interfaces = true;
}
Expand Down Expand Up @@ -205,7 +220,7 @@ fn parse_source(
};
let (pkg, sources) = resolve.push_path(normalized_path)?;
pkgs.push(pkg);
files.extend(sources);
files.extend(sources.package_paths(pkg).unwrap().map(|v| v.to_owned()));
}
Ok(())
};
Expand Down Expand Up @@ -281,6 +296,8 @@ mod kw {
syn::custom_keyword!(require_store_data_send);
syn::custom_keyword!(wasmtime_crate);
syn::custom_keyword!(include_generated_code_from_file);
syn::custom_keyword!(concurrent_imports);
syn::custom_keyword!(debug);
}

enum Opt {
Expand All @@ -301,12 +318,18 @@ enum Opt {
RequireStoreDataSend(bool),
WasmtimeCrate(syn::Path),
IncludeGeneratedCodeFromFile(bool),
ConcurrentImports(bool),
Debug(bool),
}

impl Parse for Opt {
fn parse(input: ParseStream<'_>) -> Result<Self> {
let l = input.lookahead1();
if l.peek(kw::path) {
if l.peek(kw::debug) {
input.parse::<kw::debug>()?;
input.parse::<Token![:]>()?;
Ok(Opt::Debug(input.parse::<syn::LitBool>()?.value))
} else if l.peek(kw::path) {
input.parse::<kw::path>()?;
input.parse::<Token![:]>()?;

Expand Down Expand Up @@ -380,6 +403,10 @@ impl Parse for Opt {
span,
))
}
} else if l.peek(kw::concurrent_imports) {
input.parse::<kw::concurrent_imports>()?;
input.parse::<Token![:]>()?;
Ok(Opt::ConcurrentImports(input.parse::<syn::LitBool>()?.value))
} else if l.peek(kw::ownership) {
input.parse::<kw::ownership>()?;
input.parse::<Token![:]>()?;
Expand Down
1 change: 1 addition & 0 deletions crates/cranelift/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,3 +45,4 @@ gc = ["wasmtime-environ/gc"]
gc-drc = ["gc", "wasmtime-environ/gc-drc"]
gc-null = ["gc", "wasmtime-environ/gc-null"]
threads = ["wasmtime-environ/threads"]

Loading

0 comments on commit 05f0998

Please sign in to comment.