mirror of
https://github.com/mudler/LocalAI.git
synced 2024-12-21 05:33:09 +00:00
fdb45153fe
* feat(llama.cpp): Enable decentralized, distributed inference As https://github.com/mudler/LocalAI/pull/2324 introduced distributed inferencing thanks to @rgerganov implementation in https://github.com/ggerganov/llama.cpp/pull/6829 in upstream llama.cpp, now it is possible to distribute the workload to remote llama.cpp gRPC server. This changeset now uses mudler/edgevpn to establish a secure, distributed network between the nodes using a shared token. The token is generated automatically when starting the server with the `--p2p` flag, and can be used by starting the workers with `local-ai worker p2p-llama-cpp-rpc` by passing the token via environment variable (TOKEN) or with args (--token). As per how mudler/edgevpn works, a network is established between the server and the workers with dht and mdns discovery protocols, the llama.cpp rpc server is automatically started and exposed to the underlying p2p network so the API server can connect on. When the HTTP server is started, it will discover the workers in the network and automatically create the port-forwards to the service locally. Then llama.cpp is configured to use the services. This feature is behind the "p2p" GO_FLAGS Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * go mod tidy Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: add p2p tag Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * better message Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
81 lines
2.3 KiB
Go
81 lines
2.3 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
|
|
cliContext "github.com/go-skynet/LocalAI/core/cli/context"
|
|
|
|
"github.com/go-skynet/LocalAI/pkg/gallery"
|
|
"github.com/rs/zerolog/log"
|
|
"github.com/schollz/progressbar/v3"
|
|
)
|
|
|
|
type ModelsCMDFlags struct {
|
|
Galleries string `env:"LOCALAI_GALLERIES,GALLERIES" help:"JSON list of galleries" group:"models"`
|
|
ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"`
|
|
}
|
|
|
|
type ModelsList struct {
|
|
ModelsCMDFlags `embed:""`
|
|
}
|
|
|
|
type ModelsInstall struct {
|
|
ModelArgs []string `arg:"" optional:"" name:"models" help:"Model configuration URLs to load"`
|
|
|
|
ModelsCMDFlags `embed:""`
|
|
}
|
|
|
|
type ModelsCMD struct {
|
|
List ModelsList `cmd:"" help:"List the models available in your galleries" default:"withargs"`
|
|
Install ModelsInstall `cmd:"" help:"Install a model from the gallery"`
|
|
}
|
|
|
|
func (ml *ModelsList) Run(ctx *cliContext.Context) error {
|
|
var galleries []gallery.Gallery
|
|
if err := json.Unmarshal([]byte(ml.Galleries), &galleries); err != nil {
|
|
log.Error().Err(err).Msg("unable to load galleries")
|
|
}
|
|
|
|
models, err := gallery.AvailableGalleryModels(galleries, ml.ModelsPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for _, model := range models {
|
|
if model.Installed {
|
|
fmt.Printf(" * %s@%s (installed)\n", model.Gallery.Name, model.Name)
|
|
} else {
|
|
fmt.Printf(" - %s@%s\n", model.Gallery.Name, model.Name)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (mi *ModelsInstall) Run(ctx *cliContext.Context) error {
|
|
modelName := mi.ModelArgs[0]
|
|
|
|
var galleries []gallery.Gallery
|
|
if err := json.Unmarshal([]byte(mi.Galleries), &galleries); err != nil {
|
|
log.Error().Err(err).Msg("unable to load galleries")
|
|
}
|
|
|
|
progressBar := progressbar.NewOptions(
|
|
1000,
|
|
progressbar.OptionSetDescription(fmt.Sprintf("downloading model %s", modelName)),
|
|
progressbar.OptionShowBytes(false),
|
|
progressbar.OptionClearOnFinish(),
|
|
)
|
|
progressCallback := func(fileName string, current string, total string, percentage float64) {
|
|
v := int(percentage * 10)
|
|
err := progressBar.Set(v)
|
|
if err != nil {
|
|
log.Error().Err(err).Str("filename", fileName).Int("value", v).Msg("error while updating progress bar")
|
|
}
|
|
}
|
|
err := gallery.InstallModelFromGallery(galleries, modelName, mi.ModelsPath, gallery.GalleryModel{}, progressCallback)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return nil
|
|
}
|