oob check

microsoft · Jan 16, 2025 · 921562a · 921562a
1 parent 7540330
commit 921562a
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/operators/tokenizer/bpe_utils.hpp b/operators/tokenizer/bpe_utils.hpp
@@ -360,7 +360,7 @@ class PreTokenizerWithRegEx {
   }
 
   void CategoryMatch(size_t& index, std::set<ufal::unilib::unicode::category_t>& categories){
-    while (categories.find(ufal::unilib::unicode::category(m_text[index])) != categories.end()){
+    while (index < m_text.size() && categories.find(ufal::unilib::unicode::category(m_text[index])) != categories.end()){
       index++;
     }
   }