Update tests/test_tokenize_v3.py

mistralai · Jul 20, 2024 · 8434ee6 · 8434ee6
1 parent 2e41e7a
commit 8434ee6
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/tests/test_tokenize_v3.py b/tests/test_tokenize_v3.py
@@ -30,7 +30,7 @@ def tokenizer() -> InstructTokenizer:
 
 def tekken_tokenizer() -> InstructTokenizer:
     tekken = MistralTokenizer.v3(is_tekken=True).instruct_tokenizer
-    tekken.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE   # type: ignore
+    tekken.tokenizer.special_token_policy = SpecialTokenPolicy.IGNORE  # type: ignore
     return tekken