ExternalVendorCode/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2024-12-19 20:57:54 +00:00
Code Issues Actions 13 Packages Projects Releases Wiki Activity

feat: share models by url (#1522)

Browse Source 
* feat: allow to pass by models via args

* expose it also as an env/arg

* docs: enhancements to build/requirements

* do not display status always

* print download status

* not all mesages are debug
This commit is contained in:
Ettore Di Giacinto 2024-01-01 04:31:03 -05:00 committed by GitHub
parent d6565f3b99
commit 66fa4f1767
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 145 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
api/api.go
View File

@ -5,6 +5,7 @@ import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
config "github.com/go-skynet/LocalAI/api/config"
@ -16,6 +17,7 @@ import (
"github.com/go-skynet/LocalAI/metrics"
"github.com/go-skynet/LocalAI/pkg/assets"
"github.com/go-skynet/LocalAI/pkg/model"
"github.com/go-skynet/LocalAI/pkg/utils"
"github.com/gofiber/fiber/v2"
"github.com/gofiber/fiber/v2/middleware/cors"
@ -36,6 +38,26 @@ func Startup(opts ...options.AppOption) (*options.Option, *config.ConfigLoader,
log.Info().Msgf("Starting LocalAI using %d threads, with models path: %s", options.Threads, options.Loader.ModelPath)
log.Info().Msgf("LocalAI version: %s", internal.PrintableVersion())
modelPath := options.Loader.ModelPath
if len(options.ModelsURL) > 0 {
for _, url := range options.ModelsURL {
if utils.LooksLikeURL(url) {
// md5 of model name
md5Name := utils.MD5(url)
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
err := utils.DownloadFile(url, filepath.Join(modelPath, md5Name)+".yaml", "", func(fileName, current, total string, percent float64) {
utils.DisplayDownloadFunction(fileName, current, total, percent)
})
if err != nil {
log.Error().Msgf("error loading model: %s", err.Error())
}
}
}
}
}
cl := config.NewConfigLoader()
if err := cl.LoadConfigs(options.Loader.ModelPath); err != nil {
log.Error().Msgf("error loading config files: %s", err.Error())

2
api/config/config.go
View File

@ -286,7 +286,7 @@ func (cm *ConfigLoader) Preload(modelPath string) error {
// check if file exists
if _, err := os.Stat(filepath.Join(modelPath, md5Name)); errors.Is(err, os.ErrNotExist) {
err := utils.DownloadFile(modelURL, filepath.Join(modelPath, md5Name), "", func(fileName, current, total string, percent float64) {
log.Info().Msgf("Downloading %s: %s/%s (%.2f%%)", fileName, current, total, percent)
utils.DisplayDownloadFunction(fileName, current, total, percent)
})
if err != nil {
return err

9
api/options/options.go
View File

@ -43,6 +43,9 @@ type Option struct {
WatchDogIdle bool
WatchDogBusy bool
WatchDog bool
ModelsURL []string
WatchDogBusyTimeout, WatchDogIdleTimeout time.Duration
}
@ -63,6 +66,12 @@ func NewOptions(o ...AppOption) *Option {
return opt
}
func WithModelsURL(urls ...string) AppOption {
return func(o *Option) {
o.ModelsURL = urls
}
}
func WithCors(b bool) AppOption {
return func(o *Option) {
o.CORS = b

8
docs/content/advanced/_index.en.md
View File

@ -359,15 +359,7 @@ docker run --env REBUILD=true localai
docker run --env-file .env localai
```
### Build only a single backend
You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only:
```bash
make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build
```
By default, all the backends are built.
### Extra backends

65
docs/content/build/_index.en.md
View File

@ -7,16 +7,15 @@ url = '/basics/build/'
+++
### Build locally
### Build
#### Container image
Requirements:
Either Docker/podman, or
- Golang >= 1.21
- Cmake/make
- GCC
- Docker or podman, or a container engine
In order to build the `LocalAI` container image locally you can use `docker`:
In order to build the `LocalAI` container image locally you can use `docker`, for example:
```
# build the image
@ -24,7 +23,45 @@ docker build -t localai .
docker run localai
```
Or you can build the manually binary with `make`:
#### Locally
In order to build LocalAI locally, you need the following requirements:
- Golang >= 1.21
- Cmake/make
- GCC
- GRPC
To install the dependencies follow the instructions below:
{{< tabs >}}
{{% tab name="Apple" %}}
```bash
brew install abseil cmake go grpc protobuf wget
```
{{% /tab %}}
{{% tab name="Debian" %}}
```bash
apt install protobuf-compiler-grpc libgrpc-dev make cmake
```
{{% /tab %}}
{{% tab name="From source" %}}
Specify `BUILD_GRPC_FOR_BACKEND_LLAMA=true` to build automatically the gRPC dependencies
```bash
make ... BUILD_GRPC_FOR_BACKEND_LLAMA=true build
```
{{% /tab %}}
{{< /tabs >}}
To build LocalAI with `make`:
```
git clone https://github.com/go-skynet/LocalAI
@ -32,7 +69,7 @@ cd LocalAI
make build
```
To run: `./local-ai`
This should produce the binary `local-ai`
{{% notice note %}}
@ -54,7 +91,7 @@ docker run --rm -ti -p 8080:8080 -e DEBUG=true -e MODELS_PATH=/models -e THREADS
{{% /notice %}}
### Build on mac
### Example: Build on mac
Building on Mac (M1 or M2) works, but you may need to install some prerequisites using `brew`.
@ -188,6 +225,16 @@ make BUILD_TYPE=metal build
# Note: only models quantized with q4_0 are supported!
```
### Build only a single backend
You can control the backends that are built by setting the `GRPC_BACKENDS` environment variable. For instance, to build only the `llama-cpp` backend only:
```bash
make GRPC_BACKENDS=backend-assets/grpc/llama-cpp build
```
By default, all the backends are built.
### Windows compatibility
Make sure to give enough resources to the running container. See https://github.com/go-skynet/LocalAI/issues/2

6
main.go
View File

@ -99,6 +99,11 @@ func main() {
Usage: "A List of models to apply in JSON at start",
EnvVars: []string{"PRELOAD_MODELS"},
},
&cli.StringFlag{
Name: "models",
Usage: "A List of models URLs configurations.",
EnvVars: []string{"MODELS"},
},
&cli.StringFlag{
Name: "preload-models-config",
Usage: "A List of models to apply at startup. Path to a YAML config file",
@ -222,6 +227,7 @@ For a list of compatible model, check out: https://localai.io/model-compatibilit
options.WithBackendAssetsOutput(ctx.String("backend-assets-path")),
options.WithUploadLimitMB(ctx.Int("upload-limit")),
options.WithApiKeys(ctx.StringSlice("api-keys")),
options.WithModelsURL(append(ctx.StringSlice("models"), ctx.Args().Slice()...)...),
}
idleWatchDog := ctx.Bool("enable-watchdog-idle")

10
pkg/model/initializers.go
View File

@ -239,10 +239,10 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) {
for _, b := range o.externalBackends {
allBackendsToAutoLoad = append(allBackendsToAutoLoad, b)
}
log.Debug().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", "))
log.Info().Msgf("Loading model '%s' greedly from all the available backends: %s", o.model, strings.Join(allBackendsToAutoLoad, ", "))
for _, b := range allBackendsToAutoLoad {
log.Debug().Msgf("[%s] Attempting to load", b)
log.Info().Msgf("[%s] Attempting to load", b)
options := []Option{
WithBackendString(b),
WithModel(o.model),
@ -257,14 +257,14 @@ func (ml *ModelLoader) GreedyLoader(opts ...Option) (*grpc.Client, error) {
model, modelerr := ml.BackendLoader(options...)
if modelerr == nil && model != nil {
log.Debug().Msgf("[%s] Loads OK", b)
log.Info().Msgf("[%s] Loads OK", b)
return model, nil
} else if modelerr != nil {
err = multierror.Append(err, modelerr)
log.Debug().Msgf("[%s] Fails: %s", b, modelerr.Error())
log.Info().Msgf("[%s] Fails: %s", b, modelerr.Error())
} else if model == nil {
err = multierror.Append(err, fmt.Errorf("backend returned no usable model"))
log.Debug().Msgf("[%s] Fails: %s", b, "backend returned no usable model")
log.Info().Msgf("[%s] Fails: %s", b, "backend returned no usable model")
}
}

4
pkg/utils/logging.go
View File

@ -29,9 +29,9 @@ func DisplayDownloadFunction(fileName string, current string, total string, perc
}
if total != "" {
log.Debug().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
log.Info().Msgf("Downloading %s: %s/%s (%.2f%%) ETA: %s", fileName, current, total, percentage, eta)
} else {
log.Debug().Msgf("Downloading: %s", current)
log.Info().Msgf("Downloading: %s", current)
}
}
}

62
pkg/utils/uri.go
View File

@ -15,27 +15,8 @@ import (
"github.com/rs/zerolog/log"
)
const (
githubURI = "github:"
)
func GetURI(url string, f func(url string, i []byte) error) error {
if strings.HasPrefix(url, githubURI) {
parts := strings.Split(url, ":")
repoParts := strings.Split(parts[1], "@")
branch := "main"
if len(repoParts) > 1 {
branch = repoParts[1]
}
repoPath := strings.Split(repoParts[0], "/")
org := repoPath[0]
project := repoPath[1]
projectPath := strings.Join(repoPath[2:], "/")
url = fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
}
url = ConvertURL(url)
if strings.HasPrefix(url, "file://") {
rawURL := strings.TrimPrefix(url, "file://")
@ -73,14 +54,53 @@ func GetURI(url string, f func(url string, i []byte) error) error {
const (
HuggingFacePrefix = "huggingface://"
HTTPPrefix = "http://"
HTTPSPrefix = "https://"
GithubURI = "github:"
GithubURI2 = "github://"
)
func LooksLikeURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") || strings.HasPrefix(s, HuggingFacePrefix)
return strings.HasPrefix(s, HTTPPrefix) ||
strings.HasPrefix(s, HTTPSPrefix) ||
strings.HasPrefix(s, HuggingFacePrefix) ||
strings.HasPrefix(s, GithubURI) ||
strings.HasPrefix(s, GithubURI2)
}
func ConvertURL(s string) string {
switch {
case strings.HasPrefix(s, GithubURI2):
repository := strings.Replace(s, GithubURI2, "", 1)
repoParts := strings.Split(repository, "@")
branch := "main"
if len(repoParts) > 1 {
branch = repoParts[1]
}
repoPath := strings.Split(repoParts[0], "/")
org := repoPath[0]
project := repoPath[1]
projectPath := strings.Join(repoPath[2:], "/")
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
case strings.HasPrefix(s, GithubURI):
parts := strings.Split(s, ":")
repoParts := strings.Split(parts[1], "@")
branch := "main"
if len(repoParts) > 1 {
branch = repoParts[1]
}
repoPath := strings.Split(repoParts[0], "/")
org := repoPath[0]
project := repoPath[1]
projectPath := strings.Join(repoPath[2:], "/")
return fmt.Sprintf("https://raw.githubusercontent.com/%s/%s/%s/%s", org, project, branch, projectPath)
case strings.HasPrefix(s, HuggingFacePrefix):
repository := strings.Replace(s, HuggingFacePrefix, "", 1)
// convert repository to a full URL.