修改库名

README.md

@@ -1,5 +1,7 @@
 # fastllm
 
+[English Document](README_EN.md)
+
 ## 介绍
 
 fastllm是纯c++实现，无第三方依赖的多平台高性能大模型推理库

docs/faq.md

@@ -108,7 +108,7 @@ GPU编译时，根据使用的CUDA版本，将cudart cublas的相关dll文件复
   * %CUDA_PATH%\bin\cublas64_12.dll
   * %CUDA_PATH%\bin\cublasLt64_12.dll
 
-## fastllm_pytools
+## ftllm
 
 ### 释放内存报错： CUDA error when release memory
 

example/openai_server/README.md

@@ -16,7 +16,7 @@
 
 ## 依赖
 以下依赖在python 3.12.2上没有问题
-1. 需要先安装fastllm_pytools工具包
+1. 需要先安装ftllm工具包
 2. 需要安装以下依赖
 ```bash
 cd example/openai_server

example/openai_server/fastllm_completion.py

@@ -11,7 +11,7 @@
                                ChatCompletionRole)
 
 from protocal.openai_protocol import *
-from fastllm_pytools import llm
+from ftllm import llm
 
 class ConversationMessage:
     def __init__(self, role:str, content:str):

test/basic/forward_check.py

@@ -7,7 +7,7 @@
 import torch
 import tqdm
 from transformers import AutoModelForCausalLM, AutoTokenizer
-from fastllm_pytools import llm
+from ftllm import llm
 
 def args_parser():
     parser = argparse.ArgumentParser(description = 'fastllm_test')

test/basic/tokenizer_check.py

@@ -4,7 +4,7 @@
 import logging
 import os
 from transformers import AutoTokenizer
-from fastllm_pytools import llm
+from ftllm import llm
 
 def args_parser():
     parser = argparse.ArgumentParser(description = 'fastllm_test')

test/cmmlu/baichuan.py

@@ -72,7 +72,7 @@ def eval(model, tokenizer, subject, dev_df, test_df, num_few_shot, max_length, c
             torch_dtype=torch.float16,
         )
 
-    from fastllm_pytools import llm;
+    from ftllm import llm;
     model = llm.from_hf(model, tokenizer, dtype = args.dtype);
     model.direct_query = True;
 

test/cmmlu/chatglm.py

@@ -63,7 +63,7 @@ def eval_chat_multithread(model, tokenizer, subject, dev_df, test_df, num_few_sh
     tokenizer = AutoTokenizer.from_pretrained(args.model_name_or_path, trust_remote_code=True,)
     model = AutoModel.from_pretrained(args.model_name_or_path, trust_remote_code=True).cpu()
 
-    from fastllm_pytools import llm;
+    from ftllm import llm;
     model = llm.from_hf(model, tokenizer, dtype = args.dtype);
 
     # model.save("/root/test.flm");

test/cmmlu/qwen.py

@@ -77,7 +77,7 @@ def eval_chat_multithread(model, tokenizer, subject, dev_df, test_df, num_few_sh
             torch_dtype=torch.float16,
         )
 
-    from fastllm_pytools import llm;
+    from ftllm import llm;
     model = llm.from_hf(model, tokenizer, dtype = args.dtype)
     model.direct_query = True
 

tools/fastllm_pytools/hf_model.py

@@ -1,4 +1,4 @@
-from fastllm_pytools import llm;
+from ftllm import llm;
 import ctypes;
 import builtins, os, json
 import numpy as np

tools/fastllm_pytools/llm.py

@@ -127,7 +127,7 @@ def from_hf(model,
             bot_role = None,
             history_sep = None,
             dtype = "float16"):
-    from fastllm_pytools import hf_model;
+    from ftllm import hf_model;
     return hf_model.create(model, tokenizer, pre_prompt = pre_prompt, user_role = user_role,
                            bot_role = bot_role, history_sep = history_sep, dtype = dtype);
 

tools/scripts/alpaca2flm.py

@@ -1,7 +1,7 @@
 import sys
 import torch
 from transformers import AutoTokenizer, LlamaForCausalLM
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     model_name = sys.argv[3] if len(sys.argv) >= 4 else 'minlik/chinese-alpaca-33b-merged'

tools/scripts/baichuan2_2flm.py

@@ -2,7 +2,7 @@
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 from transformers.generation.utils import GenerationConfig
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     modelpath = "baichuan-inc/Baichuan2-7B-Chat"

tools/scripts/baichuan2flm.py

@@ -2,7 +2,7 @@
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer
 from transformers.generation.utils import GenerationConfig
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     modelpath = "baichuan-inc/baichuan-13B-Chat"

tools/scripts/bert2flm.py

@@ -1,6 +1,6 @@
 import sys
 from transformers import AutoTokenizer, AutoModel
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     modelpath = sys.argv[3] if len(sys.argv) >= 4 else 'BAAI/bge-small-zh-v1.5'

tools/scripts/chatglm_export.py

@@ -1,6 +1,6 @@
 import sys
 from transformers import AutoTokenizer, AutoModel
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm2-6b", trust_remote_code=True)

tools/scripts/cli_demo.py

@@ -1,5 +1,5 @@
 import argparse
-from fastllm_pytools import llm
+from ftllm import llm
 import readline
 
 def args_parser():

tools/scripts/glm_export.py

@@ -4,7 +4,7 @@
 import torch
 import binascii
 from transformers import AutoTokenizer, AutoModel
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 def glmtofile(exportPath,
            model,

tools/scripts/llama3_to_flm.py

@@ -1,7 +1,7 @@
 import sys
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     modelNameOrPath = sys.argv[3] if len(sys.argv) >= 4 else 'meta-llama/Meta-Llama-3-8B'

tools/scripts/llamalike2flm.py

@@ -1,7 +1,7 @@
 import sys
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     modelNameOrPath = sys.argv[3] if len(sys.argv) >= 4 else 'qwen/Qwen1.5-7B-Chat'

tools/scripts/minicpm2flm.py

@@ -1,7 +1,7 @@
 import sys
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     modelNameOrPath = sys.argv[3] if len(sys.argv) >= 4 else "openbmb/MiniCPM-2B-dpo-fp16"

tools/scripts/moss_export.py

@@ -1,6 +1,6 @@
 import sys
 from transformers import AutoTokenizer, AutoModelForCausalLM
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 tokenizer = AutoTokenizer.from_pretrained("fnlp/moss-moon-003-sft", trust_remote_code=True);
 model = AutoModelForCausalLM.from_pretrained("fnlp/moss-moon-003-sft", trust_remote_code=True).float();

tools/scripts/qwen2flm.py

@@ -1,7 +1,7 @@
 import sys
 from transformers import AutoModelForCausalLM, AutoTokenizer
 from transformers.generation import GenerationConfig
-from fastllm_pytools import torch2flm
+from ftllm import torch2flm
 
 if __name__ == "__main__":
     tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen-7B-Chat", trust_remote_code=True)

tools/scripts/web_demo.py

@@ -1,6 +1,6 @@
 import streamlit as st
 from streamlit_chat import message
-from fastllm_pytools import llm
+from ftllm import llm
 import sys
 
 st.set_page_config(