fix: tokenizer with no whitespace prefix will no longer fail (#1195)

Co-authored-by: niklas.finken <[email protected]>
Aleph-Alpha · Jan 7, 2025 · 0945f10 · 0945f10
1 parent b669860
commit 0945f10
Showing 1 changed file with 1 addition and 3 deletions.
diff --git a/src/intelligence_layer/core/model.py b/src/intelligence_layer/core/model.py
@@ -380,9 +380,7 @@ def _tokenizer_no_whitespace_prefix(
         copied_tokenizer.pre_tokenizer.add_prefix_space = False
         return copied_tokenizer
 
-    raise ValueError(
-        "Tokenizer does not support `.pre_tokenizer` and thus `.add_prefix_space` option."
-    )
+    return tokenizer
 
 
 @lru_cache(maxsize=10)