ExternalVendorCode/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-04 00:10:49 +00:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

fix: change initialization order of llama-cpp-avx512 to go before avx2 variant (#4837)

Browse Source 
changed to initialization order of the avx512 version of llama.cpp, now tries before avx2

Signed-off-by: Bas Hulsken <bhulsken@hotmail.com>
This commit is contained in:
Bas Hulsken 2025-02-17 09:32:21 +01:00 committed by GitHub
parent 27617a1b06
commit bbf30d416d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
pkg/model/initializers.go
View File

@ -262,18 +262,18 @@ func selectGRPCProcessByHostCapabilities(backend, assetDir string, f16 bool) str
} }
// IF we find any optimized binary, we use that // IF we find any optimized binary, we use that
if xsysinfo.HasCPUCaps(cpuid.AVX2) { if xsysinfo.HasCPUCaps(cpuid.AVX512F) {
p := backendPath(assetDir, LLamaCPPAVX2)
if _, err := os.Stat(p); err == nil {
log.Info().Msgf("[%s] attempting to load with AVX2 variant", backend)
selectedProcess = p
}
} else if xsysinfo.HasCPUCaps(cpuid.AVX512F) {
p := backendPath(assetDir, LLamaCPPAVX512) p := backendPath(assetDir, LLamaCPPAVX512)
if _, err := os.Stat(p); err == nil { if _, err := os.Stat(p); err == nil {
log.Info().Msgf("[%s] attempting to load with AVX512 variant", backend) log.Info().Msgf("[%s] attempting to load with AVX512 variant", backend)
selectedProcess = p selectedProcess = p
} }
} else if xsysinfo.HasCPUCaps(cpuid.AVX2) {
p := backendPath(assetDir, LLamaCPPAVX2)
if _, err := os.Stat(p); err == nil {
log.Info().Msgf("[%s] attempting to load with AVX2 variant", backend)
selectedProcess = p
}
} else if xsysinfo.HasCPUCaps(cpuid.AVX) { } else if xsysinfo.HasCPUCaps(cpuid.AVX) {
p := backendPath(assetDir, LLamaCPPAVX) p := backendPath(assetDir, LLamaCPPAVX)
if _, err := os.Stat(p); err == nil { if _, err := os.Stat(p); err == nil {