tokenizer没有chat_template的时候用fastllm生成prompt

ztxz16 · Jun 27, 2024 · 193fd06 · 193fd06
1 parent 38296be
commit 193fd06
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/src/model.cpp b/src/model.cpp
@@ -398,7 +398,9 @@ namespace fastllm {
         auto tokenizerConfig = json11::Json::parse(ReadAllFile(tokenizerConfigFile), error);
         model->weight.tokenizer.SetTokenizerConfig(tokenizerConfig);
         std::string tokenizerClass = tokenizerConfig["tokenizer_class"].string_value();
-        if (tokenizerClass == "PreTrainedTokenizerFast" || tokenizerClass == "Qwen2Tokenizer") {
+        if (tokenizerClass == "PreTrainedTokenizerFast" 
+            || tokenizerClass == "Qwen2Tokenizer"
+            || tokenizerClass == "BloomTokenizer") {
             // PreTrainedTokenizerFast
             std::string tokenizerFile = path + "tokenizer.json";
             auto tokenizer = json11::Json::parse(ReadAllFile(tokenizerFile), error);

diff --git a/tools/fastllm_pytools/llm.py b/tools/fastllm_pytools/llm.py
@@ -446,7 +446,7 @@ def get_prompt(self,
             history = [];
         messages = []
 
-        if (self.hf_tokenizer != None):
+        if (self.hf_tokenizer != None and hasattr(self.hf_tokenizer, "chat_template") and self.hf_tokenizer.chat_template != ""):
             if (self.system_prompt != ""):
                 messages.append({"role": "system", "content": self.system_prompt})
             for his in history: