diff --git a/Cargo.toml b/Cargo.toml index d76dc97..34a26d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "tiktoken-node" -version = "0.0.6" +version = "0.0.8" edition = "2021" [lib] @@ -10,7 +10,7 @@ crate-type = ["cdylib"] rayon = "1.7.0" napi = "2" napi-derive = "2" -tiktoken-rs = "0.5.8" +tiktoken-rs = "0.5.9" [build-dependencies] napi-build = "2" diff --git a/index.d.ts b/index.d.ts index 0f1ef9d..ab01209 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,4 +1,4 @@ -export function getEncoding(encoding: 'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base'): Encoding +export function getEncoding(encoding: 'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base' | 'o200k_base'): Encoding export function encodingForModel(modelName: string): Encoding export class Encoding { encode(text: string): Array diff --git a/package-lock.json b/package-lock.json index 2dc9c7e..f76e323 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,25 +1,25 @@ { "name": "tiktoken-node", - "version": "0.0.7", + "version": "0.0.8", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "tiktoken-node", - "version": "0.0.7", + "version": "0.0.8", "license": "MIT", "devDependencies": { - "@napi-rs/cli": "2.18.0", - "typescript": "5.3.3" + "@napi-rs/cli": "2.18.3", + "typescript": "5.4.5" }, "engines": { "node": ">= 14" } }, "node_modules/@napi-rs/cli": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.0.tgz", - "integrity": "sha512-lfSRT7cs3iC4L+kv9suGYQEezn5Nii7Kpu+THsYVI0tA1Vh59LH45p4QADaD7hvIkmOz79eEGtoKQ9nAkAPkzA==", + "version": "2.18.3", + "resolved": "https://registry.npmjs.org/@napi-rs/cli/-/cli-2.18.3.tgz", + "integrity": "sha512-L0f4kP0dyG8W5Qtc7MtP73VvLLrOLyRcUEBzknIfu8Jk4Jfhrsx1ItMHgyalYqMSslWdY3ojEfAaU5sx1VyeQQ==", "dev": true, "bin": { "napi": "scripts/index.js" @@ -33,9 +33,9 @@ } }, "node_modules/typescript": { - "version": "5.3.3", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "version": "5.4.5", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz", + "integrity": "sha512-vcI4UpRgg81oIRUFwR0WSIHKt11nJ7SAVlYNIu+QpqeyXP+gpQJy/Z4+F0aGxSE4MqwjyXvW/TzgkLAx2AGHwQ==", "dev": true, "bin": { "tsc": "bin/tsc", diff --git a/package.json b/package.json index c25dda8..0bcc8e2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "tiktoken-node", - "version": "0.0.7", + "version": "0.0.8", "types": "index.d.ts", "main": "index.cjs", "napi": { @@ -29,8 +29,8 @@ ], "license": "MIT", "devDependencies": { - "@napi-rs/cli": "2.18.0", - "typescript": "5.3.3" + "@napi-rs/cli": "2.18.3", + "typescript": "5.4.5" }, "engines": { "node": ">= 14" diff --git a/src/lib.rs b/src/lib.rs index 801a129..584b439 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -46,7 +46,7 @@ impl Encoding { #[napi] pub fn get_encoding( - #[napi(ts_arg_type = "'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base'")] + #[napi(ts_arg_type = "'gpt2' | 'r50k_base' | 'p50k_base' | 'p50k_edit' | 'cl100k_base' | 'o200k_base'")] encoding: String, ) -> Result { let encoding: Result = match encoding.as_str() { @@ -55,6 +55,7 @@ pub fn get_encoding( "p50k_base" => Ok(tiktoken_rs::p50k_base().unwrap()), "p50k_edit" => Ok(tiktoken_rs::p50k_edit().unwrap()), "cl100k_base" => Ok(tiktoken_rs::cl100k_base().unwrap()), + "o200k_base" => Ok(tiktoken_rs::o200k_base().unwrap()), _ => Err(Error::from_reason("Invalid encoding")), };