From 524a08debef5797122092219273a7fb502ef0588 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 15 Aug 2024 11:26:41 +0200 Subject: [PATCH] Update mistral.py --- src/mistral_common/tokens/tokenizers/mistral.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index 1878e42..b8c6dea 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -81,8 +81,10 @@ def from_model(cls, model: str) -> "MistralTokenizer": "open-mistral-7b": MistralTokenizer.v1, "open-mixtral-8x7b": MistralTokenizer.v1, "mistral-embed": MistralTokenizer.v1, - "mistral-small": MistralTokenizer.v2, - "mistral-large": MistralTokenizer.v2, + "mistral-small-v1": MistralTokenizer.v2, + "mistral-large-v1": MistralTokenizer.v2, + "mistral-small": MistralTokenizer.v3, + "mistral-large": MistralTokenizer.v3, "open-mixtral-8x22b": MistralTokenizer.v3, "codestral-22b": MistralTokenizer.v3, "mistral-nemo": lambda: MistralTokenizer.v3(is_tekken=True),