From 49582b303bfa6efc2ba86950605e4a45fbda03b3 Mon Sep 17 00:00:00 2001 From: Patrick von Platen Date: Thu, 18 Jul 2024 13:40:05 +0200 Subject: [PATCH] Update src/mistral_common/tokens/tokenizers/mistral.py --- src/mistral_common/tokens/tokenizers/mistral.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mistral_common/tokens/tokenizers/mistral.py b/src/mistral_common/tokens/tokenizers/mistral.py index 51b7743..1878e42 100644 --- a/src/mistral_common/tokens/tokenizers/mistral.py +++ b/src/mistral_common/tokens/tokenizers/mistral.py @@ -85,7 +85,7 @@ def from_model(cls, model: str) -> "MistralTokenizer": "mistral-large": MistralTokenizer.v2, "open-mixtral-8x22b": MistralTokenizer.v3, "codestral-22b": MistralTokenizer.v3, - "nemostral": lambda: MistralTokenizer.v3(is_tekken=True), + "mistral-nemo": lambda: MistralTokenizer.v3(is_tekken=True), } # Prefix search the model name mapping