From 0945f104741441a616b725f536a9ab784244d32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20K=C3=B6hnecke?= <155443293+NiklasKoehneckeAA@users.noreply.github.com> Date: Tue, 7 Jan 2025 17:16:32 +0100 Subject: [PATCH] fix: tokenizer with no whitespace prefix will no longer fail (#1195) Co-authored-by: niklas.finken --- src/intelligence_layer/core/model.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/intelligence_layer/core/model.py b/src/intelligence_layer/core/model.py index c16121746..ba0b4b3a5 100644 --- a/src/intelligence_layer/core/model.py +++ b/src/intelligence_layer/core/model.py @@ -380,9 +380,7 @@ def _tokenizer_no_whitespace_prefix( copied_tokenizer.pre_tokenizer.add_prefix_space = False return copied_tokenizer - raise ValueError( - "Tokenizer does not support `.pre_tokenizer` and thus `.add_prefix_space` option." - ) + return tokenizer @lru_cache(maxsize=10)