diff --git a/Cargo.toml b/Cargo.toml index 2e520bf..14e437a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,7 @@ keywords = ["deep-learning", "language", "model", "rwkv"] license = "MIT OR Apache-2.0" name = "web-rwkv" repository = "https://github.com/cryscan/web-rwkv" -version = "0.6.21" +version = "0.6.22" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -18,6 +18,7 @@ ahash = "0.8" anyhow = "1" bitflags = "2.3" bytemuck = { version = "1.13", features = ["extern_crate_alloc"] } +derive-getters = "0.3" flume = "0.11.0" futures = "0.3" gpp = "0.6.2" diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 38c973c..0b16fb6 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1,4 +1,5 @@ use ahash::{AHashMap as HashMap, AHashSet as HashSet}; +use derive_getters::Getters; use std::collections::BTreeMap; use wasm_bindgen::prelude::wasm_bindgen; use web_rwkv_derive::JsError; @@ -31,7 +32,7 @@ impl std::fmt::Display for TokenizerError { impl std::error::Error for TokenizerError {} #[wasm_bindgen] -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Getters)] pub struct Tokenizer { first_bytes_to_lengths: Vec>, bytes_to_token_index: HashMap, u16>,