From 66fa4f1767e71740d6b5e33f8bee3c77ce64f962 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto <mudler@users.noreply.github.com> Date: Mon, 1 Jan 2024 04:31:03 -0500 Subject: [PATCH] feat: share models by url (#1522) * feat: allow to pass by models via args * expose it also as an env/arg * docs: enhancements to build/requirements * do not display status always * print download status * not all mesages are debug --- api/api.go | 22 ++++++++++ api/config/config.go | 2 +- api/options/options.go | 15 +++++-- docs/content/advanced/_index.en.md | 8 ---- docs/content/build/_index.en.md | 65 +++++++++++++++++++++++++----- main.go | 6 +++ pkg/model/initializers.go | 10 ++--- pkg/utils/logging.go | 4 +- pkg/utils/uri.go | 62 ++++++++++++++++++---------- 9 files changed, 145 insertions(+), 49 deletions(-) diff --git a/api/api.go b/api/api.go index b89f47e1..365346bd 100644 --- a/api/api.go +++ b/api/api.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "os" + "path/filepath" "strings" config "github.com/go-skynet/LocalAI/api/config" @@ -16,6 +17,7 @@ import ( "github.com/go-skynet/LocalAI/metrics" "github.com/go-skynet/LocalAI/pkg/assets" "github.com/go-skynet/LocalAI/pkg/model" + "github.com/go-skynet/LocalAI/pkg/utils" "github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2/middleware/cors" @@ -36,6 +38,26 @@ func Startup(opts ...options.AppOption) (*options.Option, *config.ConfigLoader, log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.Loader.ModelPath) log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion()) + modelPath := options.Loader.ModelPath + if len(options.ModelsURL) > 0 { + for _, url := range options.ModelsURL { + if utils.LooksLikeURL(url) { + // md5 of model name + md5Name := utils.MD5(url) + + // check if file exists + if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) { + err := utils.DownloadFile(url, filepath.Join(modelPath, md5Name)+".yaml", "", func(fileName, current, total string, percent float64) { + utils.DisplayDownloadFunction(fileName, current, total, percent) + }) + if err != nil { + log.Error().Msgf("error loading model: %s", err.Error()) + } + } + } + } + } + cl := config.NewConfigLoader() if err := cl.LoadConfigs(options.Loader.ModelPath); err != nil { log.Error().Msgf("error loading config files: %s", err.Error()) diff --git a/api/config/config.go b/api/config/config.go index 350d2c65..bfcc7a6b 100644 --- a/api/config/config.go +++ b/api/config/config.go @@ -286,7 +286,7 @@ func (cm *ConfigLoader) Preload(modelPath string) error { // check if file exists if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) { err := utils.DownloadFile(modelURL, filepath.Join(modelPath, md5Name), "", func(fileName, current, total string, percent float64) { - log.Info().Msgf("Downloading %s: %s/%s (%.2f%%)", fileName, current, total, percent) + utils.DisplayDownloadFunction(fileName, current, total, percent) }) if err != nil { return err diff --git a/api/options/options.go b/api/options/options.go index 127d06f0..e83eaaad 100644 --- a/api/options/options.go +++ b/api/options/options.go @@ -40,9 +40,12 @@ type Option struct { SingleBackend bool ParallelBackendRequests bool - WatchDogIdle bool - WatchDogBusy bool - WatchDog bool + WatchDogIdle bool + WatchDogBusy bool + WatchDog bool + + ModelsURL []string + WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration } @@ -63,6 +66,12 @@ func NewOptions(o ...AppOption) *Option { return opt } +func WithModelsURL(urls ...string) AppOption { + return func(o *Option) { + o.ModelsURL = urls + } +} + func WithCors(b bool) AppOption { return func(o *Option) { o.CORS = b diff --git a/docs/content/advanced/_index.en.md b/docs/content/advanced/_index.en.md index dd5e722e..3b00d24e 100644 --- a/docs/content/advanced/_index.en.md +++ b/docs/content/advanced/_index.en.md @@ -359,15 +359,7 @@ docker run --env REBUILD=true localai docker run --env-file .env localai ``` -### Build only a single backend -You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only: - -```bash -make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build -``` - -By default, all the backends are built. ### Extra backends diff --git a/docs/content/build/_index.en.md b/docs/content/build/_index.en.md index 3acce8e5..2697468f 100644 --- a/docs/content/build/_index.en.md +++ b/docs/content/build/_index.en.md @@ -7,16 +7,15 @@ url = '/basics/build/' +++ -### Build locally +### Build + +#### Container image Requirements: -Either Docker/podman, or -- Golang >= 1.21 -- Cmake/make -- GCC +- Docker or podman, or a container engine -In order to build the `LocalAI` container image locally you can use `docker`: +In order to build the `LocalAI` container image locally you can use `docker`, for example: ``` # build the image @@ -24,7 +23,45 @@ docker build -t localai . docker run localai ``` -Or you can build the manually binary with `make`: +#### Locally + +In order to build LocalAI locally, you need the following requirements: + +- Golang >= 1.21 +- Cmake/make +- GCC +- GRPC + +To install the dependencies follow the instructions below: + +{{< tabs >}} +{{% tab name="Apple" %}} + +```bash +brew install abseil cmake go grpc protobuf wget +``` + +{{% /tab %}} +{{% tab name="Debian" %}} + +```bash +apt install protobuf-compiler-grpc libgrpc-dev make cmake +``` + +{{% /tab %}} +{{% tab name="From source" %}} + +Specify `BUILD_GRPC_FOR_BACKEND_LLAMA=true` to build automatically the gRPC dependencies + +```bash +make ... BUILD_GRPC_FOR_BACKEND_LLAMA=true build +``` + +{{% /tab %}} +{{< /tabs >}} + + +To build LocalAI with `make`: ``` git clone https://github.com/go-skynet/LocalAI @@ -32,7 +69,7 @@ cd LocalAI make build ``` -To run: `./local-ai` +This should produce the binary `local-ai` {{% notice note %}} @@ -54,7 +91,7 @@ docker run --rm -ti -p 8080:8080 -e DEBUG=true -e MODELS_PATH=/models -e THREADS {{% /notice %}} -### Build on mac +### Example: Build on mac Building on Mac (M1 or M2) works, but you may need to install some prerequisites using `brew`. @@ -188,6 +225,16 @@ make BUILD_TYPE=metal build # Note: only models quantized with q4_0 are supported! ``` +### Build only a single backend + +You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only: + +```bash +make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build +``` + +By default, all the backends are built. + ### Windows compatibility Make sure to give enough resources to the running container. See https://github.com/go-skynet/LocalAI/issues/2 diff --git a/main.go b/main.go index 97b258c0..be4e4ed8 100644 --- a/main.go +++ b/main.go @@ -99,6 +99,11 @@ func main() { Usage: "A List of models to apply in JSON at start", EnvVars: []string{"PRELOAD_MODELS"}, }, + &cli.StringFlag{ + Name: "models", + Usage: "A List of models URLs configurations.", + EnvVars: []string{"MODELS"}, + }, &cli.StringFlag{ Name: "preload-models-config", Usage: "A List of models to apply at startup. Path to a YAML config file", @@ -222,6 +227,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit options.WithBackendAssetsOutput(ctx.String("backend-assets-path")), options.WithUploadLimitMB(ctx.Int("upload-limit")), options.WithApiKeys(ctx.StringSlice("api-keys")), + options.WithModelsURL(append(ctx.StringSlice("models"), ctx.Args().Slice()...)...), } idleWatchDog := ctx.Bool("enable-watchdog-idle") diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index 3195fac9..c2182918 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -239,10 +239,10 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) { for _, b := range o.externalBackends { allBackendsToAutoLoad = append(allBackendsToAutoLoad, b) } - log.Debug().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", ")) + log.Info().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", ")) for _, b := range allBackendsToAutoLoad { - log.Debug().Msgf("[%s] Attempting to load", b) + log.Info().Msgf("[%s] Attempting to load", b) options := []Option{ WithBackendString(b), WithModel(o.model), @@ -257,14 +257,14 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) { model, modelerr := ml.BackendLoader(options...) if modelerr == nil && model != nil { - log.Debug().Msgf("[%s] Loads OK", b) + log.Info().Msgf("[%s] Loads OK", b) return model, nil } else if modelerr != nil { err = multierror.Append(err, modelerr) - log.Debug().Msgf("[%s] Fails: %s", b, modelerr.Error()) + log.Info().Msgf("[%s] Fails: %s", b, modelerr.Error()) } else if model == nil { err = multierror.Append(err, fmt.Errorf("backend returned no usable model")) - log.Debug().Msgf("[%s] Fails: %s", b, "backend returned no usable model") + log.Info().Msgf("[%s] Fails: %s", b, "backend returned no usable model") } } diff --git a/pkg/utils/logging.go b/pkg/utils/logging.go index d69cbf8e..0f71358e 100644 --- a/pkg/utils/logging.go +++ b/pkg/utils/logging.go @@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc } if total != "" { - log.Debug().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta) + log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta) } else { - log.Debug().Msgf("Downloading: %s", current) + log.Info().Msgf("Downloading: %s", current) } } } diff --git a/pkg/utils/uri.go b/pkg/utils/uri.go index 5dde047d..185e44b9 100644 --- a/pkg/utils/uri.go +++ b/pkg/utils/uri.go @@ -15,27 +15,8 @@ import ( "github.com/rs/zerolog/log" ) -const ( - githubURI = "github:" -) - func GetURI(url string, f func(url string, i []byte) error) error { - if strings.HasPrefix(url, githubURI) { - parts := strings.Split(url, ":") - repoParts := strings.Split(parts[1], "@") - branch := "main" - - if len(repoParts) > 1 { - branch = repoParts[1] - } - - repoPath := strings.Split(repoParts[0], "/") - org := repoPath[0] - project := repoPath[1] - projectPath := strings.Join(repoPath[2:], "/") - - url = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath) - } + url = ConvertURL(url) if strings.HasPrefix(url, "file://") { rawURL := strings.TrimPrefix(url, "file://") @@ -73,14 +54,53 @@ func GetURI(url string, f func(url string, i []byte) error) error { const ( HuggingFacePrefix = "huggingface://" + HTTPPrefix = "http://" + HTTPSPrefix = "https://" + GithubURI = "github:" + GithubURI2 = "github://" ) func LooksLikeURL(s string) bool { - return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, HuggingFacePrefix) + return strings.HasPrefix(s, HTTPPrefix) || + strings.HasPrefix(s, HTTPSPrefix) || + strings.HasPrefix(s, HuggingFacePrefix) || + strings.HasPrefix(s, GithubURI) || + strings.HasPrefix(s, GithubURI2) } func ConvertURL(s string) string { switch { + case strings.HasPrefix(s, GithubURI2): + repository := strings.Replace(s, GithubURI2, "", 1) + + repoParts := strings.Split(repository, "@") + branch := "main" + + if len(repoParts) > 1 { + branch = repoParts[1] + } + + repoPath := strings.Split(repoParts[0], "/") + org := repoPath[0] + project := repoPath[1] + projectPath := strings.Join(repoPath[2:], "/") + + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath) + case strings.HasPrefix(s, GithubURI): + parts := strings.Split(s, ":") + repoParts := strings.Split(parts[1], "@") + branch := "main" + + if len(repoParts) > 1 { + branch = repoParts[1] + } + + repoPath := strings.Split(repoParts[0], "/") + org := repoPath[0] + project := repoPath[1] + projectPath := strings.Join(repoPath[2:], "/") + + return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath) case strings.HasPrefix(s, HuggingFacePrefix): repository := strings.Replace(s, HuggingFacePrefix, "", 1) // convert repository to a full URL.