diff --git a/src/intelligence_layer/core/model.py b/src/intelligence_layer/core/model.py index c16121746..ba0b4b3a5 100644 --- a/src/intelligence_layer/core/model.py +++ b/src/intelligence_layer/core/model.py @@ -380,9 +380,7 @@ def _tokenizer_no_whitespace_prefix( copied_tokenizer.pre_tokenizer.add_prefix_space = False return copied_tokenizer - raise ValueError( - "Tokenizer does not support `.pre_tokenizer` and thus `.add_prefix_space` option." - ) + return tokenizer @lru_cache(maxsize=10)