mirror of
https://github.com/mudler/LocalAI.git
synced 2025-02-11 05:11:14 +00:00
Load wrapper clients
Testing with: ```yaml name: gpt-4o pipeline: tts: voice-it-riccardo_fasol-x-low transcription: whisper-base-q5_1 llm: llama-3.2-1b-instruct:q4_k_m ``` Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
parent
9e965033bb
commit
b4fea58076
@ -81,7 +81,7 @@ type BackendConfig struct {
|
|||||||
type Pipeline struct {
|
type Pipeline struct {
|
||||||
TTS string `yaml:"tts"`
|
TTS string `yaml:"tts"`
|
||||||
LLM string `yaml:"llm"`
|
LLM string `yaml:"llm"`
|
||||||
Transcription string `yaml:"sst"`
|
Transcription string `yaml:"transcription"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type File struct {
|
type File struct {
|
||||||
|
@ -10,7 +10,9 @@ import (
|
|||||||
"github.com/gofiber/websocket/v2"
|
"github.com/gofiber/websocket/v2"
|
||||||
"github.com/mudler/LocalAI/core/backend"
|
"github.com/mudler/LocalAI/core/backend"
|
||||||
"github.com/mudler/LocalAI/core/config"
|
"github.com/mudler/LocalAI/core/config"
|
||||||
|
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
||||||
model "github.com/mudler/LocalAI/pkg/model"
|
model "github.com/mudler/LocalAI/pkg/model"
|
||||||
|
|
||||||
"github.com/rs/zerolog/log"
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -111,13 +113,17 @@ type Model interface {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type wrappedModel struct {
|
type wrappedModel struct {
|
||||||
TTS *config.BackendConfig
|
TTSConfig *config.BackendConfig
|
||||||
SST *config.BackendConfig
|
TranscriptionConfig *config.BackendConfig
|
||||||
LLM *config.BackendConfig
|
LLMConfig *config.BackendConfig
|
||||||
|
TTSClient grpc.Backend
|
||||||
|
TranscriptionClient grpc.Backend
|
||||||
|
LLMClient grpc.Backend
|
||||||
}
|
}
|
||||||
|
|
||||||
// returns and loads either a wrapped model or a model that support audio-to-audio
|
// returns and loads either a wrapped model or a model that support audio-to-audio
|
||||||
func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, modelName string) (Model, error) {
|
func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, modelName string) (Model, error) {
|
||||||
|
|
||||||
cfg, err := cl.LoadBackendConfigFileByName(modelName, ml.ModelPath)
|
cfg, err := cl.LoadBackendConfigFileByName(modelName, ml.ModelPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to load backend config: %w", err)
|
return nil, fmt.Errorf("failed to load backend config: %w", err)
|
||||||
@ -134,6 +140,8 @@ func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *
|
|||||||
return ml.BackendLoader(opts...)
|
return ml.BackendLoader(opts...)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Debug().Msg("Loading a wrapped model")
|
||||||
|
|
||||||
// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
|
// Otherwise we want to return a wrapped model, which is a "virtual" model that re-uses other models to perform operations
|
||||||
cfgLLM, err := cl.LoadBackendConfigFileByName(cfg.Pipeline.LLM, ml.ModelPath)
|
cfgLLM, err := cl.LoadBackendConfigFileByName(cfg.Pipeline.LLM, ml.ModelPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -165,10 +173,31 @@ func newModel(cl *config.BackendConfigLoader, ml *model.ModelLoader, appConfig *
|
|||||||
return nil, fmt.Errorf("failed to validate config: %w", err)
|
return nil, fmt.Errorf("failed to validate config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
opts := backend.ModelOptions(*cfgTTS, appConfig)
|
||||||
|
ttsClient, err := ml.BackendLoader(opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to load tts model: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts = backend.ModelOptions(*cfgSST, appConfig)
|
||||||
|
transcriptionClient, err := ml.BackendLoader(opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to load SST model: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
opts = backend.ModelOptions(*cfgLLM, appConfig)
|
||||||
|
llmClient, err := ml.BackendLoader(opts...)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to load LLM model: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
return &wrappedModel{
|
return &wrappedModel{
|
||||||
TTS: cfgTTS,
|
TTSConfig: cfgTTS,
|
||||||
SST: cfgSST,
|
TranscriptionConfig: cfgSST,
|
||||||
LLM: cfgLLM,
|
LLMConfig: cfgLLM,
|
||||||
|
TTSClient: ttsClient,
|
||||||
|
TranscriptionClient: transcriptionClient,
|
||||||
|
LLMClient: llmClient,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user