fix tokenizer

ztxz16 · May 20, 2024 · f28b119 · f28b119
1 parent 5d366e1
commit f28b119
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/fastllm.cpp b/src/fastllm.cpp
@@ -897,7 +897,7 @@ namespace fastllm {
             charByteDict[special_token] = L'\x100' + n;
         }
         byteCharDict[L'\x100' + n++] = L'\xAD';
-        charByteDict[L'\xAD'] = L'\x100' + n++;
+        charByteDict[L'\xAD'] = L'\x100' + (n - 1);
     }
 
     Tokenizer::~Tokenizer() {