Add LowVRAM option parameter (#642)

mudler · Jun 20, 2023 · 2f5feb4 · 2f5feb4
1 parent 4e3c319
commit 2f5feb4
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 0 deletions.
diff --git a/api/config.go b/api/config.go
@@ -35,6 +35,7 @@ type Config struct {
 	NGPULayers     int               `yaml:"gpu_layers"`
 	MMap           bool              `yaml:"mmap"`
 	MMlock         bool              `yaml:"mmlock"`
+	LowVRAM        bool              `yaml:"low_vram"`
 
 	TensorSplit           string `yaml:"tensor_split"`
 	MainGPU               string `yaml:"main_gpu"`

diff --git a/api/prediction.go b/api/prediction.go
@@ -48,6 +48,10 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
 		llamaOpts = append(llamaOpts, llama.SetNBatch(512))
 	}
 
+	if c.LowVRAM {
+		llamaOpts = append(llamaOpts, llama.EnabelLowVRAM)
+	}
+
 	return llamaOpts
 }