Merge branch 'master' of https://www.github.com/ztxz16/fastllm

ztxz16 · Dec 9, 2024 · a51dea3 · a51dea3
2 parents 5e27fb7 + d22dd1d
commit a51dea3
Show file tree

Hide file tree

Showing 15 changed files with 269 additions and 64 deletions.
diff --git a/docs/models.md b/docs/models.md
@@ -50,10 +50,10 @@
 
 |              模型  | 加载后转换 |  离线转换  |  直接读取  |
 |-------------------: |------------|------------|------------|
-| Qwen/Qwen-7B-Chat   | [✔](#其它模型) | [✔](#qwen模型导出) |  |
-| Qwen/Qwen-14B-Chat  | [✔](#其它模型) | [✔](#qwen模型导出) |  |
-| Qwen/Qwen-72B-Chat  | [✔](#其它模型) | [✔](#qwen模型导出) |  |
-| Qwen/Qwen-1_8B-Chat | [✔](#其它模型) | [✔](#qwen模型导出) |  |
+| Qwen/Qwen-7B-Chat   | [✔](#其它模型) | [✔](#qwen模型导出) | ✔ |
+| Qwen/Qwen-14B-Chat  | [✔](#其它模型) | [✔](#qwen模型导出) | ✔ |
+| Qwen/Qwen-72B-Chat  | [✔](#其它模型) | [✔](#qwen模型导出) | √ |
+| Qwen/Qwen-1_8B-Chat | [✔](#其它模型) | [✔](#qwen模型导出) | ✔ |
 | Qwen/Qwen1.5-0.5B-Chat | [✔](#其它模型) | [✔](#qwen模型导出) | ✔<sup>3</sup> |
 | Qwen/Qwen1.5-1.8B-Chat | [✔](#其它模型) | [✔](#qwen模型导出) | ✔<sup>3</sup> |
 | Qwen/Qwen1.5-4B-Chat   | [✔](#其它模型) | [✔](#qwen模型导出) | ✔<sup>3</sup> |
@@ -99,8 +99,8 @@
 |-----------------: |------------|------------|------------|
 | meta-llama/Llama-2-7b-chat-hf | [✔](llama_cookbook.md#llama2-chat) | [✔](llama_cookbook.md#llama2-chat) |  |
 | meta-llama/Llama-2-13b-chat-hf | [✔](llama_cookbook.md#llama2-chat) | [✔](llama_cookbook.md#llama2-chat) |  |
-| codellama/CodeLlama-7b-Instruct-hf | [✔](llama_cookbook.md#llama2-chat) | [✔](llama_cookbook.md#llama2-chat) |  |
-| codellama/CodeLlama-13b-Instruct-hf | [✔](llama_cookbook.md#llama2-chat) | [✔](llama_cookbook.md#llama2-chat) |  |
+| codellama/CodeLlama-7b-Instruct-hf | [✔](llama_cookbook.md#llama2-chat) | [✔](llama_cookbook.md#llama2-chat) | ✔ |
+| codellama/CodeLlama-13b-Instruct-hf | [✔](llama_cookbook.md#llama2-chat) | [✔](llama_cookbook.md#llama2-chat) | ✔ |
 | xverse/XVERSE-13B-Chat | [✔](llama_cookbook.md#xverse) | [✔](llama_cookbook.md#xverse) |  |
 | xverse/XVERSE-7B-Chat | [✔](llama_cookbook.md#xverse) | [✔](llama_cookbook.md#xverse) |  |
 |  |  |  |  |
@@ -130,6 +130,9 @@
 |  |  |  |  |
 | openbmb/MiniCPM-2B-sft-fp16 | [✔](#其它模型) | [✔](#minicpm模型导出) |  |
 | openbmb/MiniCPM-2B-dpo-fp16 | [✔](#其它模型) | [✔](#minicpm模型导出) |  |
+| openbmb/MiniCPM3-4B | [✔](#其它模型) | [✔](#minicpm模型导出) |  |
+|  |  |  |  |
+| microsoft/Phi-3-mini-4k-instruct |  |  | ✔ |
 
 
 ### 加载后转换（两行加速模式）(convert on-the-fly)
@@ -265,6 +268,7 @@ python3 tools/llamalike2flm.py qwen1.5-7b-int4.flm int4 "qwen/Qwen1.5-14B-Chat"
 # 需要先安装MiniCPM环境（transformers >= 4.36.0） 
 # 默认脚本导出iniCPM-2B-dpo-fp16模型
 cd build 
-python tools/minicpm2flm.py minicpm-2b-float16.flm #导出dpo-float16模型
+python tools/minicpm2flm.py minicpm-2b-fp16.flm #导出dpo-float16模型
+python tools/minicpm2flm.py minicpm3-4b-fp16.flm openbmb/MiniCPM3-4B #导出minicpm3-float16模型
 ./main -p minicpm-2b-float16.flm # 执行模型
 ```
diff --git a/example/Android/LLMAssistant/app/src/main/cpp/CMakeLists.txt b/example/Android/LLMAssistant/app/src/main/cpp/CMakeLists.txt
@@ -33,17 +33,23 @@ set(PROJECT_SOURCE
         ../../../../../../../src/template.cpp
         ../../../../../../../src/devices/cpu/cpudevice.cpp
         ../../../../../../../src/devices/cpu/cpudevicebatch.cpp
+        ../../../../../../../src/devices/cpu/linear.cpp
         ../../../../../../../src/models/basellm.cpp
         ../../../../../../../src/models/bert.cpp
         ../../../../../../../src/models/chatglm.cpp
+        ../../../../../../../src/models/cogvlm.cpp
         ../../../../../../../src/models/deepseekv2.cpp
-        ../../../../../../../src/models/moss.cpp
-        ../../../../../../../src/models/llama.cpp
-        ../../../../../../../src/models/qwen.cpp
         ../../../../../../../src/models/glm.cpp
+        ../../../../../../../src/models/graphllm.cpp
+        ../../../../../../../src/models/llama.cpp
         ../../../../../../../src/models/internlm2.cpp
         ../../../../../../../src/models/minicpm.cpp
+        ../../../../../../../src/models/minicpm3.cpp
         ../../../../../../../src/models/moe.cpp
+        ../../../../../../../src/models/moss.cpp
+        ../../../../../../../src/models/phi3.cpp
+        ../../../../../../../src/models/qwen.cpp
+        ../../../../../../../src/models/xlmroberta.cpp
         )
 
 include_directories(

diff --git a/example/Win32Demo/fastllm-gpu.vcxproj b/example/Win32Demo/fastllm-gpu.vcxproj
@@ -196,26 +196,32 @@
   <ItemGroup>
     <ClInclude Include="..\..\include\device.h" />
     <ClInclude Include="..\..\include\devices\cpu\alivethreadpool.h" />
+    <ClInclude Include="..\..\include\devices\cpu\computeutils.h" />
     <ClInclude Include="..\..\include\devices\cpu\cpudevice.h" />
     <ClInclude Include="..\..\include\devices\cpu\cputhreadpool.h" />
     <ClInclude Include="..\..\include\devices\cuda\cudadevice.h" />
+    <ClInclude Include="..\..\include\devices\multicuda\multicudadevice.h" />
     <ClInclude Include="..\..\include\executor.h" />
     <ClInclude Include="..\..\include\fastllm.h" />
     <ClInclude Include="..\..\include\graph.h" />
     <ClInclude Include="..\..\include\model.h" />
     <ClInclude Include="..\..\include\models\basellm.h" />
     <ClInclude Include="..\..\include\models\bert.h" />
     <ClInclude Include="..\..\include\models\chatglm.h" />
+    <ClInclude Include="..\..\include\models\cogvlm.h" />
     <ClInclude Include="..\..\include\models\deepseekv2.h" />
     <ClInclude Include="..\..\include\models\factoryllm.h" />
     <ClInclude Include="..\..\include\models\glm.h" />
     <ClInclude Include="..\..\include\models\graphllm.h" />
     <ClInclude Include="..\..\include\models\internlm2.h" />
     <ClInclude Include="..\..\include\models\llama.h" />
     <ClInclude Include="..\..\include\models\minicpm.h" />
+    <ClInclude Include="..\..\include\models\minicpm3.h" />
     <ClInclude Include="..\..\include\models\moe.h" />
     <ClInclude Include="..\..\include\models\moss.h" />
+    <ClInclude Include="..\..\include\models\phi3.h" />
     <ClInclude Include="..\..\include\models\qwen.h" />
+    <ClInclude Include="..\..\include\models\xlmroberta.h" />
     <ClInclude Include="..\..\include\template.h" />
     <ClInclude Include="..\..\include\utils\armMath.h" />
     <ClInclude Include="..\..\include\utils\utils.h" />
@@ -225,37 +231,50 @@
     <ClCompile Include="..\..\src\device.cpp" />
     <ClCompile Include="..\..\src\devices\cpu\cpudevice.cpp" />
     <ClCompile Include="..\..\src\devices\cpu\cpudevicebatch.cpp" />
+    <ClCompile Include="..\..\src\devices\cpu\linear.cpp" />
     <ClCompile Include="..\..\src\devices\cuda\cudadevice.cpp" />
     <ClCompile Include="..\..\src\devices\cuda\cudadevicebatch.cpp" />
+    <ClCompile Include="..\..\src\devices\multicuda\multicudadevice.cpp" />
     <ClCompile Include="..\..\src\executor.cpp" />
     <ClCompile Include="..\..\src\fastllm.cpp" />
     <ClCompile Include="..\..\src\graph.cpp" />
     <ClCompile Include="..\..\src\model.cpp" />
     <ClCompile Include="..\..\src\models\basellm.cpp" />
     <ClCompile Include="..\..\src\models\bert.cpp" />
     <ClCompile Include="..\..\src\models\chatglm.cpp" />
+    <ClCompile Include="..\..\src\models\cogvlm.cpp" />
     <ClCompile Include="..\..\src\models\deepseekv2.cpp" />
     <ClCompile Include="..\..\src\models\glm.cpp" />
     <ClCompile Include="..\..\src\models\graphllm.cpp" />
     <ClCompile Include="..\..\src\models\graph\fastllmjson.cpp" />
+    <ClCompile Include="..\..\src\models\graph\phi3.cpp" />
     <ClCompile Include="..\..\src\models\graph\qwen2.cpp" />
     <ClCompile Include="..\..\src\models\graph\telechat.cpp" />
     <ClCompile Include="..\..\src\models\internlm2.cpp" />
     <ClCompile Include="..\..\src\models\llama.cpp" />
     <ClCompile Include="..\..\src\models\minicpm.cpp" />
+    <ClCompile Include="..\..\src\models\minicpm3.cpp" />
     <ClCompile Include="..\..\src\models\moe.cpp" />
     <ClCompile Include="..\..\src\models\moss.cpp" />
+    <ClCompile Include="..\..\src\models\phi3.cpp" />
     <ClCompile Include="..\..\src\models\qwen.cpp" />
+    <ClCompile Include="..\..\src\models\xlmroberta.cpp" />
     <ClCompile Include="..\..\src\template.cpp" />
     <ClCompile Include="..\..\third_party\json11\json11.cpp" />
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\include\devices\cuda\fastllm-cuda.cuh">
       <FileType>Document</FileType>
     </ClInclude>
+    <ClInclude Include="..\..\include\devices\multicuda\fastllm-multicuda.cuh">
+      <FileType>Document</FileType>
+    </ClInclude>
     <CudaCompile Include="..\..\src\devices\cuda\fastllm-cuda.cu">
       <FileType>Document</FileType>
     </CudaCompile>
+    <CudaCompile Include="..\..\src\devices\multicuda\fastllm-multicuda.cu">
+      <FileType>Document</FileType>
+    </CudaCompile>
   </ItemGroup>
   <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
 </Project>
diff --git a/example/Win32Demo/fastllm-gpu.vcxproj.filters b/example/Win32Demo/fastllm-gpu.vcxproj.filters
@@ -49,6 +49,12 @@
     <Filter Include="源文件\third_party">
       <UniqueIdentifier>{385ba64f-d978-469c-af0c-498ec33a1bb7}</UniqueIdentifier>
     </Filter>
+    <Filter Include="头文件\devices\multicuda">
+      <UniqueIdentifier>{917a368b-3a0e-476b-9f35-6af4433aca16}</UniqueIdentifier>
+    </Filter>
+    <Filter Include="源文件\devices\multicuda">
+      <UniqueIdentifier>{9fbf8989-df11-4dc7-9697-d69d10b7f0bf}</UniqueIdentifier>
+    </Filter>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\include\device.h">
@@ -78,6 +84,9 @@
     <ClInclude Include="..\..\include\models\chatglm.h">
       <Filter>头文件\models</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\models\cogvlm.h">
+      <Filter>头文件\models</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\include\models\deepseekv2.h">
       <Filter>头文件\models</Filter>
     </ClInclude>
@@ -99,21 +108,33 @@
     <ClInclude Include="..\..\include\models\minicpm.h">
       <Filter>头文件\models</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\models\minicpm3.h">
+      <Filter>头文件\models</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\include\models\moe.h">
       <Filter>头文件\models</Filter>
     </ClInclude>
     <ClInclude Include="..\..\include\models\moss.h">
       <Filter>头文件\models</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\models\phi3.h">
+      <Filter>头文件\models</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\include\models\qwen.h">
       <Filter>头文件\models</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\include\devices\cpu\cpudevice.h">
-      <Filter>头文件\devices\cpu</Filter>
+    <ClInclude Include="..\..\include\models\xlmroberta.h">
+      <Filter>头文件\models</Filter>
     </ClInclude>
     <ClInclude Include="..\..\include\devices\cpu\alivethreadpool.h">
       <Filter>头文件\devices\cpu</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\devices\cpu\computeutils.h">
+      <Filter>头文件\devices\cpu</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\include\devices\cpu\cpudevice.h">
+      <Filter>头文件\devices\cpu</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\include\devices\cpu\cputhreadpool.h">
       <Filter>头文件\devices\cpu</Filter>
     </ClInclude>
@@ -129,6 +150,12 @@
     <ClInclude Include="..\..\include\devices\cuda\fastllm-cuda.cuh">
       <Filter>头文件\devices\cuda</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\include\devices\multicuda\multicudadevice.h">
+      <Filter>头文件\devices\multicuda</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\include\devices\multicuda\fastllm-multicuda.cuh">
+      <Filter>头文件\devices\multicuda</Filter>
+    </ClInclude>
     <ClInclude Include="..\..\third_party\json11\json11.hpp">
       <Filter>头文件\third_party</Filter>
     </ClInclude>
@@ -161,6 +188,9 @@
     <ClCompile Include="..\..\src\models\chatglm.cpp">
       <Filter>源文件\models</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\models\cogvlm.cpp">
+      <Filter>源文件\models</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\models\deepseekv2.cpp">
       <Filter>源文件\models</Filter>
     </ClCompile>
@@ -179,18 +209,30 @@
     <ClCompile Include="..\..\src\models\minicpm.cpp">
       <Filter>源文件\models</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\models\minicpm3.cpp">
+      <Filter>源文件\models</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\models\moe.cpp">
       <Filter>源文件\models</Filter>
     </ClCompile>
     <ClCompile Include="..\..\src\models\moss.cpp">
       <Filter>源文件\models</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\models\phi3.cpp">
+      <Filter>源文件\models</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\models\qwen.cpp">
       <Filter>源文件\models</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\models\xlmroberta.cpp">
+      <Filter>源文件\models</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\models\graph\fastllmjson.cpp">
       <Filter>源文件\models\graph</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\models\graph\phi3.cpp">
+      <Filter>源文件\models\graph</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\models\graph\qwen2.cpp">
       <Filter>源文件\models\graph</Filter>
     </ClCompile>
@@ -203,12 +245,18 @@
     <ClCompile Include="..\..\src\devices\cpu\cpudevicebatch.cpp">
       <Filter>源文件\devices\cpu</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\devices\cpu\linear.cpp">
+      <Filter>源文件\devices\cpu</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\src\devices\cuda\cudadevice.cpp">
       <Filter>源文件\devices\cuda</Filter>
     </ClCompile>
     <ClCompile Include="..\..\src\devices\cuda\cudadevicebatch.cpp">
       <Filter>源文件\devices\cuda</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\src\devices\multicuda\multicudadevice.cpp">
+      <Filter>源文件\devices\multicuda</Filter>
+    </ClCompile>
     <ClCompile Include="..\..\third_party\json11\json11.cpp">
       <Filter>源文件\third_party</Filter>
     </ClCompile>
@@ -217,5 +265,8 @@
     <CudaCompile Include="..\..\src\devices\cuda\fastllm-cuda.cu">
       <Filter>源文件\devices\cuda</Filter>
     </CudaCompile>
+    <CudaCompile Include="..\..\src\devices\multicuda\fastllm-multicuda.cu">
+      <Filter>源文件\devices\multicuda</Filter>
+    </CudaCompile>
   </ItemGroup>
 </Project>
diff --git a/example/Win32Demo/fastllm.vcxproj b/example/Win32Demo/fastllm.vcxproj
@@ -173,6 +173,7 @@
   <ItemGroup>
     <ClInclude Include="..\..\include\device.h" />
     <ClInclude Include="..\..\include\devices\cpu\alivethreadpool.h" />
+    <ClInclude Include="..\..\include\devices\cpu\computeutils.h" />
     <ClInclude Include="..\..\include\devices\cpu\cpudevice.h" />
     <ClInclude Include="..\..\include\devices\cpu\cputhreadpool.h" />
     <ClInclude Include="..\..\include\executor.h" />
@@ -182,16 +183,20 @@
     <ClInclude Include="..\..\include\models\basellm.h" />
     <ClInclude Include="..\..\include\models\bert.h" />
     <ClInclude Include="..\..\include\models\chatglm.h" />
+    <ClInclude Include="..\..\include\models\cogvlm.h" />
     <ClInclude Include="..\..\include\models\deepseekv2.h" />
     <ClInclude Include="..\..\include\models\factoryllm.h" />
     <ClInclude Include="..\..\include\models\glm.h" />
     <ClInclude Include="..\..\include\models\graphllm.h" />
     <ClInclude Include="..\..\include\models\internlm2.h" />
     <ClInclude Include="..\..\include\models\llama.h" />
     <ClInclude Include="..\..\include\models\minicpm.h" />
+    <ClInclude Include="..\..\include\models\minicpm3.h" />
     <ClInclude Include="..\..\include\models\moe.h" />
     <ClInclude Include="..\..\include\models\moss.h" />
+    <ClInclude Include="..\..\include\models\phi3.h" />
     <ClInclude Include="..\..\include\models\qwen.h" />
+    <ClInclude Include="..\..\include\models\xlmroberta.h" />
     <ClInclude Include="..\..\include\template.h" />
     <ClInclude Include="..\..\include\utils\armMath.h" />
     <ClInclude Include="..\..\include\utils\utils.h" />
@@ -201,25 +206,31 @@
     <ClCompile Include="..\..\src\device.cpp" />
     <ClCompile Include="..\..\src\devices\cpu\cpudevice.cpp" />
     <ClCompile Include="..\..\src\devices\cpu\cpudevicebatch.cpp" />
+    <ClCompile Include="..\..\src\devices\cpu\linear.cpp" />
     <ClCompile Include="..\..\src\executor.cpp" />
     <ClCompile Include="..\..\src\fastllm.cpp" />
     <ClCompile Include="..\..\src\graph.cpp" />
     <ClCompile Include="..\..\src\model.cpp" />
     <ClCompile Include="..\..\src\models\basellm.cpp" />
     <ClCompile Include="..\..\src\models\bert.cpp" />
     <ClCompile Include="..\..\src\models\chatglm.cpp" />
+    <ClCompile Include="..\..\src\models\cogvlm.cpp" />
     <ClCompile Include="..\..\src\models\deepseekv2.cpp" />
     <ClCompile Include="..\..\src\models\glm.cpp" />
     <ClCompile Include="..\..\src\models\graphllm.cpp" />
     <ClCompile Include="..\..\src\models\graph\fastllmjson.cpp" />
+    <ClCompile Include="..\..\src\models\graph\phi3.cpp" />
     <ClCompile Include="..\..\src\models\graph\qwen2.cpp" />
     <ClCompile Include="..\..\src\models\graph\telechat.cpp" />
     <ClCompile Include="..\..\src\models\internlm2.cpp" />
     <ClCompile Include="..\..\src\models\llama.cpp" />
     <ClCompile Include="..\..\src\models\minicpm.cpp" />
+    <ClCompile Include="..\..\src\models\minicpm3.cpp" />
     <ClCompile Include="..\..\src\models\moe.cpp" />
     <ClCompile Include="..\..\src\models\moss.cpp" />
+    <ClCompile Include="..\..\src\models\phi3.cpp" />
     <ClCompile Include="..\..\src\models\qwen.cpp" />
+    <ClCompile Include="..\..\src\models\xlmroberta.cpp" />
     <ClCompile Include="..\..\src\template.cpp" />
     <ClCompile Include="..\..\third_party\json11\json11.cpp" />
   </ItemGroup>