feat: share models by url (#1522)

* feat: allow to pass by models via args

* expose it also as an env/arg

* docs: enhancements to build/requirements

* do not display status always

* print download status

* not all mesages are debug
This commit is contained in:
Ettore Di Giacinto 2024-01-01 04:31:03 -05:00 committed by GitHub
parent d6565f3b99
commit 66fa4f1767
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 145 additions and 49 deletions

View File

@ -5,6 +5,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"os" "os"
"path/filepath"
"strings" "strings"
config "github.com/go-skynet/LocalAI/api/config" config "github.com/go-skynet/LocalAI/api/config"
@ -16,6 +17,7 @@ import (
"github.com/go-skynet/LocalAI/metrics" "github.com/go-skynet/LocalAI/metrics"
"github.com/go-skynet/LocalAI/pkg/assets" "github.com/go-skynet/LocalAI/pkg/assets"
"github.com/go-skynet/LocalAI/pkg/model" "github.com/go-skynet/LocalAI/pkg/model"
"github.com/go-skynet/LocalAI/pkg/utils"
"github.com/gofiber/fiber/v2" "github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors" "github.com/gofiber/fiber/v2/middleware/cors"
@ -36,6 +38,26 @@ func Startup(opts ...options.AppOption) (*options.Option, *config.ConfigLoader,
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.Loader.ModelPath) log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.Loader.ModelPath)
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion()) log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
modelPath := options.Loader.ModelPath
if len(options.ModelsURL) > 0 {
for _, url := range options.ModelsURL {
if utils.LooksLikeURL(url) {
// md5 of model name
md5Name := utils.MD5(url)
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
err := utils.DownloadFile(url, filepath.Join(modelPath, md5Name)+".yaml", "", func(fileName, current, total string, percent float64) {
utils.DisplayDownloadFunction(fileName, current, total, percent)
})
if err != nil {
log.Error().Msgf("error loading model: %s", err.Error())
}
}
}
}
}
cl := config.NewConfigLoader() cl := config.NewConfigLoader()
if err := cl.LoadConfigs(options.Loader.ModelPath); err != nil { if err := cl.LoadConfigs(options.Loader.ModelPath); err != nil {
log.Error().Msgf("error loading config files: %s", err.Error()) log.Error().Msgf("error loading config files: %s", err.Error())

View File

@ -286,7 +286,7 @@ func (cm *ConfigLoader) Preload(modelPath string) error {
// check if file exists // check if file exists
if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) { if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
err := utils.DownloadFile(modelURL, filepath.Join(modelPath, md5Name), "", func(fileName, current, total string, percent float64) { err := utils.DownloadFile(modelURL, filepath.Join(modelPath, md5Name), "", func(fileName, current, total string, percent float64) {
log.Info().Msgf("Downloading %s: %s/%s (%.2f%%)", fileName, current, total, percent) utils.DisplayDownloadFunction(fileName, current, total, percent)
}) })
if err != nil { if err != nil {
return err return err

View File

@ -43,6 +43,9 @@ type Option struct {
WatchDogIdle bool WatchDogIdle bool
WatchDogBusy bool WatchDogBusy bool
WatchDog bool WatchDog bool
ModelsURL []string
WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration
} }
@ -63,6 +66,12 @@ func NewOptions(o ...AppOption) *Option {
return opt return opt
} }
func WithModelsURL(urls ...string) AppOption {
return func(o *Option) {
o.ModelsURL = urls
}
}
func WithCors(b bool) AppOption { func WithCors(b bool) AppOption {
return func(o *Option) { return func(o *Option) {
o.CORS = b o.CORS = b

View File

@ -359,15 +359,7 @@ docker run --env REBUILD=true localai
docker run --env-file .env localai docker run --env-file .env localai
``` ```
### Build only a single backend
You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only:
```bash
make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build
```
By default, all the backends are built.
### Extra backends ### Extra backends

View File

@ -7,16 +7,15 @@ url = '/basics/build/'
+++ +++
### Build locally ### Build
#### Container image
Requirements: Requirements:
Either Docker/podman, or - Docker or podman, or a container engine
- Golang >= 1.21
- Cmake/make
- GCC
In order to build the `LocalAI` container image locally you can use `docker`: In order to build the `LocalAI` container image locally you can use `docker`, for example:
``` ```
# build the image # build the image
@ -24,7 +23,45 @@ docker build -t localai .
docker run localai docker run localai
``` ```
Or you can build the manually binary with `make`: #### Locally
In order to build LocalAI locally, you need the following requirements:
- Golang >= 1.21
- Cmake/make
- GCC
- GRPC
To install the dependencies follow the instructions below:
{{< tabs >}}
{{% tab name="Apple" %}}
```bash
brew install abseil cmake go grpc protobuf wget
```
{{% /tab %}}
{{% tab name="Debian" %}}
```bash
apt install protobuf-compiler-grpc libgrpc-dev make cmake
```
{{% /tab %}}
{{% tab name="From source" %}}
Specify `BUILD_GRPC_FOR_BACKEND_LLAMA=true` to build automatically the gRPC dependencies
```bash
make ... BUILD_GRPC_FOR_BACKEND_LLAMA=true build
```
{{% /tab %}}
{{< /tabs >}}
To build LocalAI with `make`:
``` ```
git clone https://github.com/go-skynet/LocalAI git clone https://github.com/go-skynet/LocalAI
@ -32,7 +69,7 @@ cd LocalAI
make build make build
``` ```
To run: `./local-ai` This should produce the binary `local-ai`
{{% notice note %}} {{% notice note %}}
@ -54,7 +91,7 @@ docker run --rm -ti -p 8080:8080 -e DEBUG=true -e MODELS_PATH=/models -e THREADS
{{% /notice %}} {{% /notice %}}
### Build on mac ### Example: Build on mac
Building on Mac (M1 or M2) works, but you may need to install some prerequisites using `brew`. Building on Mac (M1 or M2) works, but you may need to install some prerequisites using `brew`.
@ -188,6 +225,16 @@ make BUILD_TYPE=metal build
# Note: only models quantized with q4_0 are supported! # Note: only models quantized with q4_0 are supported!
``` ```
### Build only a single backend
You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only:
```bash
make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build
```
By default, all the backends are built.
### Windows compatibility ### Windows compatibility
Make sure to give enough resources to the running container. See https://github.com/go-skynet/LocalAI/issues/2 Make sure to give enough resources to the running container. See https://github.com/go-skynet/LocalAI/issues/2

View File

@ -99,6 +99,11 @@ func main() {
Usage: "A List of models to apply in JSON at start", Usage: "A List of models to apply in JSON at start",
EnvVars: []string{"PRELOAD_MODELS"}, EnvVars: []string{"PRELOAD_MODELS"},
}, },
&cli.StringFlag{
Name: "models",
Usage: "A List of models URLs configurations.",
EnvVars: []string{"MODELS"},
},
&cli.StringFlag{ &cli.StringFlag{
Name: "preload-models-config", Name: "preload-models-config",
Usage: "A List of models to apply at startup. Path to a YAML config file", Usage: "A List of models to apply at startup. Path to a YAML config file",
@ -222,6 +227,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
options.WithBackendAssetsOutput(ctx.String("backend-assets-path")), options.WithBackendAssetsOutput(ctx.String("backend-assets-path")),
options.WithUploadLimitMB(ctx.Int("upload-limit")), options.WithUploadLimitMB(ctx.Int("upload-limit")),
options.WithApiKeys(ctx.StringSlice("api-keys")), options.WithApiKeys(ctx.StringSlice("api-keys")),
options.WithModelsURL(append(ctx.StringSlice("models"), ctx.Args().Slice()...)...),
} }
idleWatchDog := ctx.Bool("enable-watchdog-idle") idleWatchDog := ctx.Bool("enable-watchdog-idle")

View File

@ -239,10 +239,10 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) {
for _, b := range o.externalBackends { for _, b := range o.externalBackends {
allBackendsToAutoLoad = append(allBackendsToAutoLoad, b) allBackendsToAutoLoad = append(allBackendsToAutoLoad, b)
} }
log.Debug().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", ")) log.Info().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", "))
for _, b := range allBackendsToAutoLoad { for _, b := range allBackendsToAutoLoad {
log.Debug().Msgf("[%s] Attempting to load", b) log.Info().Msgf("[%s] Attempting to load", b)
options := []Option{ options := []Option{
WithBackendString(b), WithBackendString(b),
WithModel(o.model), WithModel(o.model),
@ -257,14 +257,14 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) {
model, modelerr := ml.BackendLoader(options...) model, modelerr := ml.BackendLoader(options...)
if modelerr == nil && model != nil { if modelerr == nil && model != nil {
log.Debug().Msgf("[%s] Loads OK", b) log.Info().Msgf("[%s] Loads OK", b)
return model, nil return model, nil
} else if modelerr != nil { } else if modelerr != nil {
err = multierror.Append(err, modelerr) err = multierror.Append(err, modelerr)
log.Debug().Msgf("[%s] Fails: %s", b, modelerr.Error()) log.Info().Msgf("[%s] Fails: %s", b, modelerr.Error())
} else if model == nil { } else if model == nil {
err = multierror.Append(err, fmt.Errorf("backend returned no usable model")) err = multierror.Append(err, fmt.Errorf("backend returned no usable model"))
log.Debug().Msgf("[%s] Fails: %s", b, "backend returned no usable model") log.Info().Msgf("[%s] Fails: %s", b, "backend returned no usable model")
} }
} }

View File

@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc
} }
if total != "" { if total != "" {
log.Debug().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta) log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
} else { } else {
log.Debug().Msgf("Downloading: %s", current) log.Info().Msgf("Downloading: %s", current)
} }
} }
} }

View File

@ -15,27 +15,8 @@ import (
"github.com/rs/zerolog/log" "github.com/rs/zerolog/log"
) )
const (
githubURI = "github:"
)
func GetURI(url string, f func(url string, i []byte) error) error { func GetURI(url string, f func(url string, i []byte) error) error {
if strings.HasPrefix(url, githubURI) { url = ConvertURL(url)
parts := strings.Split(url, ":")
repoParts := strings.Split(parts[1], "@")
branch := "main"
if len(repoParts) > 1 {
branch = repoParts[1]
}
repoPath := strings.Split(repoParts[0], "/")
org := repoPath[0]
project := repoPath[1]
projectPath := strings.Join(repoPath[2:], "/")
url = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
}
if strings.HasPrefix(url, "file://") { if strings.HasPrefix(url, "file://") {
rawURL := strings.TrimPrefix(url, "file://") rawURL := strings.TrimPrefix(url, "file://")
@ -73,14 +54,53 @@ func GetURI(url string, f func(url string, i []byte) error) error {
const ( const (
HuggingFacePrefix = "huggingface://" HuggingFacePrefix = "huggingface://"
HTTPPrefix = "http://"
HTTPSPrefix = "https://"
GithubURI = "github:"
GithubURI2 = "github://"
) )
func LooksLikeURL(s string) bool { func LooksLikeURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, HuggingFacePrefix) return strings.HasPrefix(s, HTTPPrefix) ||
strings.HasPrefix(s, HTTPSPrefix) ||
strings.HasPrefix(s, HuggingFacePrefix) ||
strings.HasPrefix(s, GithubURI) ||
strings.HasPrefix(s, GithubURI2)
} }
func ConvertURL(s string) string { func ConvertURL(s string) string {
switch { switch {
case strings.HasPrefix(s, GithubURI2):
repository := strings.Replace(s, GithubURI2, "", 1)
repoParts := strings.Split(repository, "@")
branch := "main"
if len(repoParts) > 1 {
branch = repoParts[1]
}
repoPath := strings.Split(repoParts[0], "/")
org := repoPath[0]
project := repoPath[1]
projectPath := strings.Join(repoPath[2:], "/")
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
case strings.HasPrefix(s, GithubURI):
parts := strings.Split(s, ":")
repoParts := strings.Split(parts[1], "@")
branch := "main"
if len(repoParts) > 1 {
branch = repoParts[1]
}
repoPath := strings.Split(repoParts[0], "/")
org := repoPath[0]
project := repoPath[1]
projectPath := strings.Join(repoPath[2:], "/")
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
case strings.HasPrefix(s, HuggingFacePrefix): case strings.HasPrefix(s, HuggingFacePrefix):
repository := strings.Replace(s, HuggingFacePrefix, "", 1) repository := strings.Replace(s, HuggingFacePrefix, "", 1)
// convert repository to a full URL. // convert repository to a full URL.