Getters for tokenizer.

Cargo.toml

@@ -9,7 +9,7 @@ keywords = ["deep-learning", "language", "model", "rwkv"]
 license = "MIT OR Apache-2.0"
 name = "web-rwkv"
 repository = "https://github.com/cryscan/web-rwkv"
-version = "0.6.21"
+version = "0.6.22"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
@@ -18,6 +18,7 @@ ahash = "0.8"
 anyhow = "1"
 bitflags = "2.3"
 bytemuck = { version = "1.13", features = ["extern_crate_alloc"] }
+derive-getters = "0.3"
 flume = "0.11.0"
 futures = "0.3"
 gpp = "0.6.2"

src/tokenizer.rs

@@ -1,4 +1,5 @@
 use ahash::{AHashMap as HashMap, AHashSet as HashSet};
+use derive_getters::Getters;
 use std::collections::BTreeMap;
 use wasm_bindgen::prelude::wasm_bindgen;
 use web_rwkv_derive::JsError;
@@ -31,7 +32,7 @@ impl std::fmt::Display for TokenizerError {
 impl std::error::Error for TokenizerError {}
 
 #[wasm_bindgen]
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Getters)]
 pub struct Tokenizer {
     first_bytes_to_lengths: Vec<Box<[u16]>>,
     bytes_to_token_index: HashMap<Vec<u8>, u16>,