2023-03-18 22:59:06 +00:00
|
|
|
package main
|
|
|
|
|
|
|
|
import (
|
2023-05-03 21:03:31 +00:00
|
|
|
"fmt"
|
2023-03-18 22:59:06 +00:00
|
|
|
"os"
|
2023-05-03 21:03:31 +00:00
|
|
|
"path/filepath"
|
|
|
|
|
|
|
|
|
2023-03-18 22:59:06 +00:00
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
api "github.com/go-skynet/LocalAI/api"
|
|
|
|
model "github.com/go-skynet/LocalAI/pkg/model"
|
2023-04-20 22:55:19 +00:00
|
|
|
"github.com/jaypipes/ghw"
|
2023-04-20 16:33:02 +00:00
|
|
|
"github.com/rs/zerolog"
|
|
|
|
"github.com/rs/zerolog/log"
|
2023-03-18 22:59:06 +00:00
|
|
|
"github.com/urfave/cli/v2"
|
|
|
|
)
|
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
func main() {
|
2023-04-20 16:33:02 +00:00
|
|
|
log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
|
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
path, err := os.Getwd()
|
2023-03-18 22:59:06 +00:00
|
|
|
if err != nil {
|
2023-04-20 16:33:02 +00:00
|
|
|
log.Error().Msgf("error: %s", err.Error())
|
2023-04-19 16:43:10 +00:00
|
|
|
os.Exit(1)
|
2023-03-18 22:59:06 +00:00
|
|
|
}
|
|
|
|
|
2023-04-20 22:55:19 +00:00
|
|
|
threads := 4
|
|
|
|
cpu, err := ghw.CPU()
|
|
|
|
if err == nil {
|
|
|
|
threads = int(cpu.TotalCores)
|
|
|
|
}
|
|
|
|
|
2023-03-18 22:59:06 +00:00
|
|
|
app := &cli.App{
|
2023-04-19 16:43:10 +00:00
|
|
|
Name: "LocalAI",
|
|
|
|
Usage: "OpenAI compatible API for running LLaMA/GPT models locally on CPU with consumer grade hardware.",
|
|
|
|
Flags: []cli.Flag{
|
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "f16",
|
|
|
|
EnvVars: []string{"F16"},
|
|
|
|
},
|
2023-04-20 16:33:02 +00:00
|
|
|
&cli.BoolFlag{
|
|
|
|
Name: "debug",
|
|
|
|
EnvVars: []string{"DEBUG"},
|
|
|
|
},
|
2023-04-19 16:43:10 +00:00
|
|
|
&cli.IntFlag{
|
|
|
|
Name: "threads",
|
|
|
|
DefaultText: "Number of threads used for parallel computation. Usage of the number of physical cores in the system is suggested.",
|
|
|
|
EnvVars: []string{"THREADS"},
|
2023-04-20 22:55:19 +00:00
|
|
|
Value: threads,
|
2023-03-18 22:59:06 +00:00
|
|
|
},
|
|
|
|
&cli.StringFlag{
|
2023-04-19 16:43:10 +00:00
|
|
|
Name: "models-path",
|
|
|
|
DefaultText: "Path containing models used for inferencing",
|
|
|
|
EnvVars: []string{"MODELS_PATH"},
|
2023-05-03 21:03:31 +00:00
|
|
|
Value: filepath.Join(path, "models"),
|
2023-03-18 22:59:06 +00:00
|
|
|
},
|
2023-04-27 04:18:18 +00:00
|
|
|
&cli.StringFlag{
|
|
|
|
Name: "config-file",
|
|
|
|
DefaultText: "Config file",
|
|
|
|
EnvVars: []string{"CONFIG_FILE"},
|
|
|
|
},
|
2023-03-18 22:59:06 +00:00
|
|
|
&cli.StringFlag{
|
2023-04-19 16:43:10 +00:00
|
|
|
Name: "address",
|
|
|
|
DefaultText: "Bind address for the API server.",
|
|
|
|
EnvVars: []string{"ADDRESS"},
|
|
|
|
Value: ":8080",
|
|
|
|
},
|
|
|
|
&cli.IntFlag{
|
|
|
|
Name: "context-size",
|
|
|
|
DefaultText: "Default context size of the model",
|
|
|
|
EnvVars: []string{"CONTEXT_SIZE"},
|
|
|
|
Value: 512,
|
|
|
|
},
|
|
|
|
},
|
|
|
|
Description: `
|
|
|
|
LocalAI is a drop-in replacement OpenAI API which runs inference locally.
|
2023-03-18 22:59:06 +00:00
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
Some of the models compatible are:
|
|
|
|
- Vicuna
|
|
|
|
- Koala
|
|
|
|
- GPT4ALL
|
|
|
|
- GPT4ALL-J
|
2023-04-20 22:06:55 +00:00
|
|
|
- Cerebras
|
2023-04-19 16:43:10 +00:00
|
|
|
- Alpaca
|
2023-04-20 22:06:55 +00:00
|
|
|
- StableLM (ggml quantized)
|
2023-03-18 22:59:06 +00:00
|
|
|
|
2023-04-20 22:06:55 +00:00
|
|
|
It uses llama.cpp, ggml and gpt4all as backend with golang c bindings.
|
2023-03-18 22:59:06 +00:00
|
|
|
`,
|
2023-04-19 16:43:10 +00:00
|
|
|
UsageText: `local-ai [options]`,
|
2023-03-18 22:59:06 +00:00
|
|
|
Copyright: "go-skynet authors",
|
|
|
|
Action: func(ctx *cli.Context) error {
|
2023-05-03 21:03:31 +00:00
|
|
|
fmt.Printf("Starting LocalAI using %d threads, with models path: %s\n", ctx.Int("threads"), ctx.String("models-path"))
|
2023-04-27 04:18:18 +00:00
|
|
|
return api.App(ctx.String("config-file"), model.NewModelLoader(ctx.String("models-path")), ctx.Int("threads"), ctx.Int("context-size"), ctx.Bool("f16"), ctx.Bool("debug"), false).Listen(ctx.String("address"))
|
2023-03-18 22:59:06 +00:00
|
|
|
},
|
|
|
|
}
|
|
|
|
|
2023-04-19 16:43:10 +00:00
|
|
|
err = app.Run(os.Args)
|
2023-03-18 22:59:06 +00:00
|
|
|
if err != nil {
|
2023-04-20 16:33:02 +00:00
|
|
|
log.Error().Msgf("error: %s", err.Error())
|
2023-03-18 22:59:06 +00:00
|
|
|
os.Exit(1)
|
|
|
|
}
|
|
|
|
}
|