mirror of
https://github.com/mudler/LocalAI.git
synced 2024-12-18 20:27:57 +00:00
be6c4e6061
* fix(llama-cpp): consistently select fallback We didn't took in consideration the case where the host has the CPU flagset, but the binaries were not actually present in the asset dir. This made possible for instance for models that specified the llama-cpp backend directly in the config to not eventually pick-up the fallback binary in case the optimized binaries were not present. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * chore: adjust and simplify selection Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix: move failure recovery to BackendLoader() Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * comments Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * minor fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
107 lines
1.9 KiB
Go
107 lines
1.9 KiB
Go
package model
|
|
|
|
import (
|
|
"context"
|
|
|
|
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
|
)
|
|
|
|
type Options struct {
|
|
backendString string
|
|
model string
|
|
modelID string
|
|
assetDir string
|
|
context context.Context
|
|
|
|
gRPCOptions *pb.ModelOptions
|
|
|
|
externalBackends map[string]string
|
|
|
|
grpcAttempts int
|
|
grpcAttemptsDelay int
|
|
singleActiveBackend bool
|
|
parallelRequests bool
|
|
}
|
|
|
|
type Option func(*Options)
|
|
|
|
var EnableParallelRequests = func(o *Options) {
|
|
o.parallelRequests = true
|
|
}
|
|
|
|
func WithExternalBackend(name string, uri string) Option {
|
|
return func(o *Options) {
|
|
if o.externalBackends == nil {
|
|
o.externalBackends = make(map[string]string)
|
|
}
|
|
o.externalBackends[name] = uri
|
|
}
|
|
}
|
|
|
|
func WithGRPCAttempts(attempts int) Option {
|
|
return func(o *Options) {
|
|
o.grpcAttempts = attempts
|
|
}
|
|
}
|
|
|
|
func WithGRPCAttemptsDelay(delay int) Option {
|
|
return func(o *Options) {
|
|
o.grpcAttemptsDelay = delay
|
|
}
|
|
}
|
|
|
|
func WithBackendString(backend string) Option {
|
|
return func(o *Options) {
|
|
o.backendString = backend
|
|
}
|
|
}
|
|
|
|
func WithModel(modelFile string) Option {
|
|
return func(o *Options) {
|
|
o.model = modelFile
|
|
}
|
|
}
|
|
|
|
func WithLoadGRPCLoadModelOpts(opts *pb.ModelOptions) Option {
|
|
return func(o *Options) {
|
|
o.gRPCOptions = opts
|
|
}
|
|
}
|
|
|
|
func WithAssetDir(assetDir string) Option {
|
|
return func(o *Options) {
|
|
o.assetDir = assetDir
|
|
}
|
|
}
|
|
|
|
func WithContext(ctx context.Context) Option {
|
|
return func(o *Options) {
|
|
o.context = ctx
|
|
}
|
|
}
|
|
|
|
func WithSingleActiveBackend() Option {
|
|
return func(o *Options) {
|
|
o.singleActiveBackend = true
|
|
}
|
|
}
|
|
|
|
func WithModelID(id string) Option {
|
|
return func(o *Options) {
|
|
o.modelID = id
|
|
}
|
|
}
|
|
|
|
func NewOptions(opts ...Option) *Options {
|
|
o := &Options{
|
|
gRPCOptions: &pb.ModelOptions{},
|
|
context: context.Background(),
|
|
grpcAttempts: 20,
|
|
grpcAttemptsDelay: 2,
|
|
}
|
|
for _, opt := range opts {
|
|
opt(o)
|
|
}
|
|
return o
|
|
}
|