Merge pull request #42 from mistralai/patrickvonplaten-patch-1

Update from_model with new mistral-large
mistralai · Aug 15, 2024 · 9313953 · 9313953
2 parents 0748675 + 524a08d
commit 9313953
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py
@@ -81,8 +81,10 @@ def from_model(cls, model: str) -> "MistralTokenizer":
             "open-mistral-7b": MistralTokenizer.v1,
             "open-mixtral-8x7b": MistralTokenizer.v1,
             "mistral-embed": MistralTokenizer.v1,
-            "mistral-small": MistralTokenizer.v2,
-            "mistral-large": MistralTokenizer.v2,
+            "mistral-small-v1": MistralTokenizer.v2,
+            "mistral-large-v1": MistralTokenizer.v2,
+            "mistral-small": MistralTokenizer.v3,
+            "mistral-large": MistralTokenizer.v3,
             "open-mixtral-8x22b": MistralTokenizer.v3,
             "codestral-22b": MistralTokenizer.v3,
             "mistral-nemo": lambda: MistralTokenizer.v3(is_tekken=True),