Add LowVRAM option parameter ()

This commit is contained in:
Ettore Di Giacinto 2023-06-20 20:33:47 +02:00 committed by GitHub
parent 4e3c319e83
commit 2f5feb4841
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 0 deletions

@ -35,6 +35,7 @@ type Config struct {
NGPULayers int `yaml:"gpu_layers"` NGPULayers int `yaml:"gpu_layers"`
MMap bool `yaml:"mmap"` MMap bool `yaml:"mmap"`
MMlock bool `yaml:"mmlock"` MMlock bool `yaml:"mmlock"`
LowVRAM bool `yaml:"low_vram"`
TensorSplit string `yaml:"tensor_split"` TensorSplit string `yaml:"tensor_split"`
MainGPU string `yaml:"main_gpu"` MainGPU string `yaml:"main_gpu"`

@ -48,6 +48,10 @@ func defaultLLamaOpts(c Config) []llama.ModelOption {
llamaOpts = append(llamaOpts, llama.SetNBatch(512)) llamaOpts = append(llamaOpts, llama.SetNBatch(512))
} }
if c.LowVRAM {
llamaOpts = append(llamaOpts, llama.EnabelLowVRAM)
}
return llamaOpts return llamaOpts
} }