feat(libpath): refactor and expose functions for external library paths (#2578)

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/cli/run.go

@@ -43,6 +43,7 @@ type RunCMD struct {
 	Address              string   `env:"LOCALAI_ADDRESS,ADDRESS" default:":8080" help:"Bind address for the API server" group:"api"`
 	CORS                 bool     `env:"LOCALAI_CORS,CORS" help:"" group:"api"`
 	CORSAllowOrigins     string   `env:"LOCALAI_CORS_ALLOW_ORIGINS,CORS_ALLOW_ORIGINS" group:"api"`
+	LibraryPath          string   `env:"LOCALAI_LIBRARY_PATH,LIBRARY_PATH" help:"Path to the library directory (for e.g. external libraries used by backends)" default:"/usr/share/local-ai/libs" group:"backends"`
 	CSRF                 bool     `env:"LOCALAI_CSRF" help:"Enables fiber CSRF middleware" group:"api"`
 	UploadLimit          int      `env:"LOCALAI_UPLOAD_LIMIT,UPLOAD_LIMIT" default:"15" help:"Default upload-limit in MB" group:"api"`
 	APIKeys              []string `env:"LOCALAI_API_KEY,API_KEY" help:"List of API Keys to enable API authentication. When this is set, all the requests must be authenticated with one of these API keys" group:"api"`
@@ -80,6 +81,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithCors(r.CORS),
 		config.WithCorsAllowOrigins(r.CORSAllowOrigins),
 		config.WithCsrf(r.CSRF),
+		config.WithLibPath(r.LibraryPath),
 		config.WithThreads(r.Threads),
 		config.WithBackendAssets(ctx.BackendAssets),
 		config.WithBackendAssetsOutput(r.BackendAssetsPath),

core/config/application_config.go

@@ -15,6 +15,7 @@ type ApplicationConfig struct {
 	Context                             context.Context
 	ConfigFile                          string
 	ModelPath                           string
+	LibPath                             string
 	UploadLimitMB, Threads, ContextSize int
 	DisableWebUI                        bool
 	F16                                 bool
@@ -101,6 +102,12 @@ func WithModelLibraryURL(url string) AppOption {
 	}
 }
 
+func WithLibPath(path string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.LibPath = path
+	}
+}
+
 var EnableWatchDog = func(o *ApplicationConfig) {
 	o.WatchDog = true
 }

core/startup/startup.go

@@ -9,6 +9,7 @@ import (
 	"github.com/go-skynet/LocalAI/core/services"
 	"github.com/go-skynet/LocalAI/internal"
 	"github.com/go-skynet/LocalAI/pkg/assets"
+	"github.com/go-skynet/LocalAI/pkg/library"
 	"github.com/go-skynet/LocalAI/pkg/model"
 	pkgStartup "github.com/go-skynet/LocalAI/pkg/startup"
 	"github.com/go-skynet/LocalAI/pkg/xsysinfo"
@@ -109,6 +110,11 @@ func Startup(opts ...config.AppOption) (*config.BackendConfigLoader, *model.Mode
 		}
 	}
 
+	if options.LibPath != "" {
+		// If there is a lib directory, set LD_LIBRARY_PATH to include it
+		library.LoadExternal(options.LibPath)
+	}
+
 	// turn off any process that was started by GRPC if the context is canceled
 	go func() {
 		<-options.Context.Done()

pkg/assets/extract.go

@@ -6,7 +6,8 @@ import (
 	"io/fs"
 	"os"
 	"path/filepath"
-	"runtime"
+
+	"github.com/go-skynet/LocalAI/pkg/library"
 )
 
 func ResolvePath(dir string, paths ...string) string {
@@ -55,28 +56,7 @@ func ExtractFiles(content embed.FS, extractDir string) error {
 	// If there is a lib directory, set LD_LIBRARY_PATH to include it
 	// we might use this mechanism to carry over e.g. Nvidia CUDA libraries
 	// from the embedded FS to the target directory
+	library.LoadExtractedLibs(extractDir)
 
-	// Skip this if LOCALAI_SKIP_LIBRARY_PATH is set
-	if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" {
-		return err
-	}
-
-
-	lpathVar := "LD_LIBRARY_PATH"
-	if runtime.GOOS == "darwin" {
-		lpathVar = "DYLD_FALLBACK_LIBRARY_PATH" // should it be DYLD_LIBRARY_PATH ?
-	}
-
-	for _, libDir := range []string{filepath.Join(extractDir, "backend-assets", "lib"), filepath.Join(extractDir, "lib")} {
-		if _, err := os.Stat(libDir); err == nil {
-			ldLibraryPath := os.Getenv(lpathVar)
-			if ldLibraryPath == "" {
-				ldLibraryPath = libDir
-			} else {
-				ldLibraryPath = fmt.Sprintf("%s:%s", ldLibraryPath, libDir)
-			}
-			os.Setenv(lpathVar, ldLibraryPath)
-		}
-	}
 	return err
 }

pkg/library/dynaload.go

@@ -0,0 +1,41 @@
+package library
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+func LoadExtractedLibs(dir string) {
+	// Skip this if LOCALAI_SKIP_LIBRARY_PATH is set
+	if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" {
+		return
+	}
+
+	for _, libDir := range []string{filepath.Join(dir, "backend-assets", "lib"), filepath.Join(dir, "lib")} {
+		LoadExternal(libDir)
+	}
+}
+
+func LoadExternal(dir string) {
+	// Skip this if LOCALAI_SKIP_LIBRARY_PATH is set
+	if os.Getenv("LOCALAI_SKIP_LIBRARY_PATH") != "" {
+		return
+	}
+
+	lpathVar := "LD_LIBRARY_PATH"
+	if runtime.GOOS == "darwin" {
+		lpathVar = "DYLD_FALLBACK_LIBRARY_PATH" // should it be DYLD_LIBRARY_PATH ?
+	}
+
+	if _, err := os.Stat(dir); err == nil {
+		ldLibraryPath := os.Getenv(lpathVar)
+		if ldLibraryPath == "" {
+			ldLibraryPath = dir
+		} else {
+			ldLibraryPath = fmt.Sprintf("%s:%s", ldLibraryPath, dir)
+		}
+		os.Setenv(lpathVar, ldLibraryPath)
+	}
+}