fix(llama.cpp): bump upstream fix for starcoder model on cuda

TabbyML · Oct 28, 2023 · 4442226 · 4442226
1 parent 3151d91
commit 4442226
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/crates/llama-cpp-bindings/llama.cpp b/crates/llama-cpp-bindings/llama.cpp
diff --git a/crates/llama-cpp-bindings/src/engine.cc b/crates/llama-cpp-bindings/src/engine.cc
@@ -106,7 +106,7 @@ std::unique_ptr<TextInferenceEngine> create_engine(bool use_gpu, rust::Str model
   static BackendInitializer initializer;
 
   llama_model_params model_params = llama_model_default_params();
-  model_params.n_gpu_layers = use_gpu ? 1 : 0;
+  model_params.n_gpu_layers = use_gpu ? 9999 : 0;
   llama_model* model = llama_load_model_from_file(std::string(model_path).c_str(), model_params);
 
   if (!model) {
+7 −0		CMakeLists.txt
+3 −0		Makefile
+1 −1		examples/server/server.cpp
+4 −14		examples/simple/simple.cpp
+32 −1		examples/speculative/speculative.cpp
+114 −16		ggml-cuda.cu
+88 −24		llama.cpp
+1 −1		llama.h