chore: (dependencies) bump tiktoken version to 0.8.0 and update compa…

…tibility in token.py
yufeikang · Dec 11, 2024 · 0ba1e94 · 0ba1e94
1 parent 78fda0b
commit 0ba1e94
Show file tree

Hide file tree

Showing 2 changed files with 35 additions and 31 deletions.
diff --git a/pdm.lock b/pdm.lock
diff --git a/src/ai_cli/bot/token.py b/src/ai_cli/bot/token.py
@@ -2,6 +2,8 @@
 
 
 def get_token_count(text: str, model: str = "gpt-3.5-turbo") -> int:
+    if model.startswith("o1"):
+        model = "gpt-4o"
     encoding = tiktoken.encoding_for_model(model)
     tokens = encoding.encode(text)
     return len(tokens)