From 307a835199d0f10b8095ca5665accb9f25468073 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 1 Oct 2024 14:55:46 -0400 Subject: [PATCH] groundwork: ListModels Filtering Upgrade (#2773) * seperate the filtering from the middleware changes --------- Signed-off-by: Dave Lee --- core/cli/util.go | 38 +++++- core/config/backend_config.go | 147 ++++++++++++++++++++++-- core/config/backend_config_filter.go | 35 ++++++ core/config/backend_config_loader.go | 20 ++++ core/config/backend_config_test.go | 102 +++++++++++++++- core/http/ctx/fiber.go | 4 +- core/http/endpoints/localai/welcome.go | 12 +- core/http/endpoints/openai/assistant.go | 2 +- core/http/endpoints/openai/list.go | 38 +++--- core/http/routes/ui.go | 6 +- core/services/list_models.go | 64 +++++------ 11 files changed, 387 insertions(+), 81 deletions(-) create mode 100644 core/config/backend_config_filter.go diff --git a/core/cli/util.go b/core/cli/util.go index b3e545d8..57b8ad9e 100644 --- a/core/cli/util.go +++ b/core/cli/util.go @@ -15,8 +15,9 @@ import ( ) type UtilCMD struct { - GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"` - HFScan HFScanCMD `cmd:"" name:"hf-scan" help:"Checks installed models for known security issues. WARNING: this is a best-effort feature and may not catch everything!"` + GGUFInfo GGUFInfoCMD `cmd:"" name:"gguf-info" help:"Get information about a GGUF file"` + HFScan HFScanCMD `cmd:"" name:"hf-scan" help:"Checks installed models for known security issues. WARNING: this is a best-effort feature and may not catch everything!"` + UsecaseHeuristic UsecaseHeuristicCMD `cmd:"" name:"usecase-heuristic" help:"Checks a specific model config and prints what usecase LocalAI will offer for it."` } type GGUFInfoCMD struct { @@ -30,6 +31,11 @@ type HFScanCMD struct { ToScan []string `arg:""` } +type UsecaseHeuristicCMD struct { + ConfigName string `name:"The config file to check"` + ModelsPath string `env:"LOCALAI_MODELS_PATH,MODELS_PATH" type:"path" default:"${basepath}/models" help:"Path containing models used for inferencing" group:"storage"` +} + func (u *GGUFInfoCMD) Run(ctx *cliContext.Context) error { if u.Args == nil || len(u.Args) == 0 { return fmt.Errorf("no GGUF file provided") @@ -99,3 +105,31 @@ func (hfscmd *HFScanCMD) Run(ctx *cliContext.Context) error { return nil } } + +func (uhcmd *UsecaseHeuristicCMD) Run(ctx *cliContext.Context) error { + if len(uhcmd.ConfigName) == 0 { + log.Error().Msg("ConfigName is a required parameter") + return fmt.Errorf("config name is a required parameter") + } + if len(uhcmd.ModelsPath) == 0 { + log.Error().Msg("ModelsPath is a required parameter") + return fmt.Errorf("model path is a required parameter") + } + bcl := config.NewBackendConfigLoader(uhcmd.ModelsPath) + err := bcl.LoadBackendConfig(uhcmd.ConfigName) + if err != nil { + log.Error().Err(err).Str("ConfigName", uhcmd.ConfigName).Msg("error while loading backend") + return err + } + bc, exists := bcl.GetBackendConfig(uhcmd.ConfigName) + if !exists { + log.Error().Str("ConfigName", uhcmd.ConfigName).Msg("ConfigName not found") + } + for name, uc := range config.GetAllBackendConfigUsecases() { + if bc.HasUsecases(uc) { + log.Info().Str("Usecase", name) + } + } + log.Info().Msg("---") + return nil +} diff --git a/core/config/backend_config.go b/core/config/backend_config.go index 027e18a4..8db94f7c 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -3,11 +3,13 @@ package config import ( "os" "regexp" + "slices" "strings" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/functions" + "gopkg.in/yaml.v3" ) const ( @@ -27,13 +29,15 @@ type BackendConfig struct { schema.PredictionOptions `yaml:"parameters"` Name string `yaml:"name"` - F16 *bool `yaml:"f16"` - Threads *int `yaml:"threads"` - Debug *bool `yaml:"debug"` - Roles map[string]string `yaml:"roles"` - Embeddings *bool `yaml:"embeddings"` - Backend string `yaml:"backend"` - TemplateConfig TemplateConfig `yaml:"template"` + F16 *bool `yaml:"f16"` + Threads *int `yaml:"threads"` + Debug *bool `yaml:"debug"` + Roles map[string]string `yaml:"roles"` + Embeddings *bool `yaml:"embeddings"` + Backend string `yaml:"backend"` + TemplateConfig TemplateConfig `yaml:"template"` + KnownUsecaseStrings []string `yaml:"known_usecases"` + KnownUsecases *BackendConfigUsecases `yaml:"-"` PromptStrings, InputStrings []string `yaml:"-"` InputToken [][]int `yaml:"-"` @@ -194,6 +198,17 @@ type TemplateConfig struct { JoinChatMessagesByCharacter *string `yaml:"join_chat_messages_by_character"` } +func (c *BackendConfig) UnmarshalYAML(value *yaml.Node) error { + type BCAlias BackendConfig + var aux BCAlias + if err := value.Decode(&aux); err != nil { + return err + } + *c = BackendConfig(aux) + c.KnownUsecases = GetUsecasesFromYAML(c.KnownUsecaseStrings) + return nil +} + func (c *BackendConfig) SetFunctionCallString(s string) { c.functionCallString = s } @@ -410,3 +425,121 @@ func (c *BackendConfig) Validate() bool { func (c *BackendConfig) HasTemplate() bool { return c.TemplateConfig.Completion != "" || c.TemplateConfig.Edit != "" || c.TemplateConfig.Chat != "" || c.TemplateConfig.ChatMessage != "" } + +type BackendConfigUsecases int + +const ( + FLAG_ANY BackendConfigUsecases = 0b000000000 + FLAG_CHAT BackendConfigUsecases = 0b000000001 + FLAG_COMPLETION BackendConfigUsecases = 0b000000010 + FLAG_EDIT BackendConfigUsecases = 0b000000100 + FLAG_EMBEDDINGS BackendConfigUsecases = 0b000001000 + FLAG_RERANK BackendConfigUsecases = 0b000010000 + FLAG_IMAGE BackendConfigUsecases = 0b000100000 + FLAG_TRANSCRIPT BackendConfigUsecases = 0b001000000 + FLAG_TTS BackendConfigUsecases = 0b010000000 + FLAG_SOUND_GENERATION BackendConfigUsecases = 0b100000000 + + // Common Subsets + FLAG_LLM BackendConfigUsecases = FLAG_CHAT & FLAG_COMPLETION & FLAG_EDIT +) + +func GetAllBackendConfigUsecases() map[string]BackendConfigUsecases { + return map[string]BackendConfigUsecases{ + "FLAG_ANY": FLAG_ANY, + "FLAG_CHAT": FLAG_CHAT, + "FLAG_COMPLETION": FLAG_COMPLETION, + "FLAG_EDIT": FLAG_EDIT, + "FLAG_EMBEDDINGS": FLAG_EMBEDDINGS, + "FLAG_RERANK": FLAG_RERANK, + "FLAG_IMAGE": FLAG_IMAGE, + "FLAG_TRANSCRIPT": FLAG_TRANSCRIPT, + "FLAG_TTS": FLAG_TTS, + "FLAG_SOUND_GENERATION": FLAG_SOUND_GENERATION, + "FLAG_LLM": FLAG_LLM, + } +} + +func GetUsecasesFromYAML(input []string) *BackendConfigUsecases { + if len(input) == 0 { + return nil + } + result := FLAG_ANY + flags := GetAllBackendConfigUsecases() + for _, str := range input { + flag, exists := flags["FLAG_"+strings.ToUpper(str)] + if exists { + result |= flag + } + } + return &result +} + +// HasUsecases examines a BackendConfig and determines which endpoints have a chance of success. +func (c *BackendConfig) HasUsecases(u BackendConfigUsecases) bool { + if (c.KnownUsecases != nil) && ((u & *c.KnownUsecases) == u) { + return true + } + return c.GuessUsecases(u) +} + +// GuessUsecases is a **heuristic based** function, as the backend in question may not be loaded yet, and the config may not record what it's useful at. +// In its current state, this function should ideally check for properties of the config like templates, rather than the direct backend name checks for the lower half. +// This avoids the maintenance burden of updating this list for each new backend - but unfortunately, that's the best option for some services currently. +func (c *BackendConfig) GuessUsecases(u BackendConfigUsecases) bool { + if (u & FLAG_CHAT) == FLAG_CHAT { + if c.TemplateConfig.Chat == "" && c.TemplateConfig.ChatMessage == "" { + return false + } + } + if (u & FLAG_COMPLETION) == FLAG_COMPLETION { + if c.TemplateConfig.Completion == "" { + return false + } + } + if (u & FLAG_EDIT) == FLAG_EDIT { + if c.TemplateConfig.Edit == "" { + return false + } + } + if (u & FLAG_EMBEDDINGS) == FLAG_EMBEDDINGS { + if c.Embeddings == nil || !*c.Embeddings { + return false + } + } + if (u & FLAG_IMAGE) == FLAG_IMAGE { + imageBackends := []string{"diffusers", "tinydream", "stablediffusion"} + if !slices.Contains(imageBackends, c.Backend) { + return false + } + + if c.Backend == "diffusers" && c.Diffusers.PipelineType == "" { + return false + } + + } + if (u & FLAG_RERANK) == FLAG_RERANK { + if c.Backend != "rerankers" { + return false + } + } + if (u & FLAG_TRANSCRIPT) == FLAG_TRANSCRIPT { + if c.Backend != "whisper" { + return false + } + } + if (u & FLAG_TTS) == FLAG_TTS { + ttsBackends := []string{"piper", "transformers-musicgen", "parler-tts"} + if !slices.Contains(ttsBackends, c.Backend) { + return false + } + } + + if (u & FLAG_SOUND_GENERATION) == FLAG_SOUND_GENERATION { + if c.Backend != "transformers-musicgen" { + return false + } + } + + return true +} diff --git a/core/config/backend_config_filter.go b/core/config/backend_config_filter.go new file mode 100644 index 00000000..f1eb2488 --- /dev/null +++ b/core/config/backend_config_filter.go @@ -0,0 +1,35 @@ +package config + +import "regexp" + +type BackendConfigFilterFn func(string, *BackendConfig) bool + +func NoFilterFn(_ string, _ *BackendConfig) bool { return true } + +func BuildNameFilterFn(filter string) (BackendConfigFilterFn, error) { + if filter == "" { + return NoFilterFn, nil + } + rxp, err := regexp.Compile(filter) + if err != nil { + return nil, err + } + return func(name string, config *BackendConfig) bool { + if config != nil { + return rxp.MatchString(config.Name) + } + return rxp.MatchString(name) + }, nil +} + +func BuildUsecaseFilterFn(usecases BackendConfigUsecases) BackendConfigFilterFn { + if usecases == FLAG_ANY { + return NoFilterFn + } + return func(name string, config *BackendConfig) bool { + if config == nil { + return false // TODO: Potentially make this a param, for now, no known usecase to include + } + return config.HasUsecases(usecases) + } +} diff --git a/core/config/backend_config_loader.go b/core/config/backend_config_loader.go index 45fe259e..7fe49bab 100644 --- a/core/config/backend_config_loader.go +++ b/core/config/backend_config_loader.go @@ -201,6 +201,26 @@ func (bcl *BackendConfigLoader) GetAllBackendConfigs() []BackendConfig { return res } +func (bcl *BackendConfigLoader) GetBackendConfigsByFilter(filter BackendConfigFilterFn) []BackendConfig { + bcl.Lock() + defer bcl.Unlock() + var res []BackendConfig + + if filter == nil { + filter = NoFilterFn + } + + for n, v := range bcl.configs { + if filter(n, &v) { + res = append(res, v) + } + } + + // TODO: I don't think this one needs to Sort on name... but we'll see what breaks. + + return res +} + func (bcl *BackendConfigLoader) RemoveBackendConfig(m string) { bcl.Lock() defer bcl.Unlock() diff --git a/core/config/backend_config_test.go b/core/config/backend_config_test.go index da245933..04eacb7e 100644 --- a/core/config/backend_config_test.go +++ b/core/config/backend_config_test.go @@ -19,12 +19,17 @@ var _ = Describe("Test cases for config related functions", func() { `backend: "../foo-bar" name: "foo" parameters: - model: "foo-bar"`) + model: "foo-bar" +known_usecases: +- chat +- COMPLETION +`) Expect(err).ToNot(HaveOccurred()) config, err := readBackendConfigFromFile(tmp.Name()) Expect(err).To(BeNil()) Expect(config).ToNot(BeNil()) Expect(config.Validate()).To(BeFalse()) + Expect(config.KnownUsecases).ToNot(BeNil()) }) It("Test Validate", func() { tmp, err := os.CreateTemp("", "config.yaml") @@ -61,4 +66,99 @@ parameters: Expect(config.Validate()).To(BeTrue()) }) }) + It("Properly handles backend usecase matching", func() { + + a := BackendConfig{ + Name: "a", + } + Expect(a.HasUsecases(FLAG_ANY)).To(BeTrue()) // FLAG_ANY just means the config _exists_ essentially. + + b := BackendConfig{ + Name: "b", + Backend: "stablediffusion", + } + Expect(b.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(b.HasUsecases(FLAG_IMAGE)).To(BeTrue()) + Expect(b.HasUsecases(FLAG_CHAT)).To(BeFalse()) + + c := BackendConfig{ + Name: "c", + Backend: "llama-cpp", + TemplateConfig: TemplateConfig{ + Chat: "chat", + }, + } + Expect(c.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(c.HasUsecases(FLAG_IMAGE)).To(BeFalse()) + Expect(c.HasUsecases(FLAG_COMPLETION)).To(BeFalse()) + Expect(c.HasUsecases(FLAG_CHAT)).To(BeTrue()) + + d := BackendConfig{ + Name: "d", + Backend: "llama-cpp", + TemplateConfig: TemplateConfig{ + Chat: "chat", + Completion: "completion", + }, + } + Expect(d.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(d.HasUsecases(FLAG_IMAGE)).To(BeFalse()) + Expect(d.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(d.HasUsecases(FLAG_CHAT)).To(BeTrue()) + + trueValue := true + e := BackendConfig{ + Name: "e", + Backend: "llama-cpp", + TemplateConfig: TemplateConfig{ + Completion: "completion", + }, + Embeddings: &trueValue, + } + + Expect(e.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(e.HasUsecases(FLAG_IMAGE)).To(BeFalse()) + Expect(e.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(e.HasUsecases(FLAG_CHAT)).To(BeFalse()) + Expect(e.HasUsecases(FLAG_EMBEDDINGS)).To(BeTrue()) + + f := BackendConfig{ + Name: "f", + Backend: "piper", + } + Expect(f.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(f.HasUsecases(FLAG_TTS)).To(BeTrue()) + Expect(f.HasUsecases(FLAG_CHAT)).To(BeFalse()) + + g := BackendConfig{ + Name: "g", + Backend: "whisper", + } + Expect(g.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(g.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) + Expect(g.HasUsecases(FLAG_TTS)).To(BeFalse()) + + h := BackendConfig{ + Name: "h", + Backend: "transformers-musicgen", + } + Expect(h.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(h.HasUsecases(FLAG_TRANSCRIPT)).To(BeFalse()) + Expect(h.HasUsecases(FLAG_TTS)).To(BeTrue()) + Expect(h.HasUsecases(FLAG_SOUND_GENERATION)).To(BeTrue()) + + knownUsecases := FLAG_CHAT | FLAG_COMPLETION + i := BackendConfig{ + Name: "i", + Backend: "whisper", + // Earlier test checks parsing, this just needs to set final values + KnownUsecases: &knownUsecases, + } + Expect(i.HasUsecases(FLAG_ANY)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_TRANSCRIPT)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_TTS)).To(BeFalse()) + Expect(i.HasUsecases(FLAG_COMPLETION)).To(BeTrue()) + Expect(i.HasUsecases(FLAG_CHAT)).To(BeTrue()) + + }) }) diff --git a/core/http/ctx/fiber.go b/core/http/ctx/fiber.go index 94059847..28a35ac4 100644 --- a/core/http/ctx/fiber.go +++ b/core/http/ctx/fiber.go @@ -21,12 +21,12 @@ func ModelFromContext(ctx *fiber.Ctx, cl *config.BackendConfigLoader, loader *mo } // Set model from bearer token, if available - bearer := strings.TrimLeft(ctx.Get("authorization"), "Bearer ") + bearer := strings.TrimLeft(ctx.Get("authorization"), "Bear ") // Reduced duplicate characters of Bearer bearerExists := bearer != "" && loader.ExistsInModelPath(bearer) // If no model was specified, take the first available if modelInput == "" && !bearerExists && firstModel { - models, _ := services.ListModels(cl, loader, "", true) + models, _ := services.ListModels(cl, loader, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if len(models) > 0 { modelInput = models[0] log.Debug().Msgf("No model specified, using: %s", modelInput) diff --git a/core/http/endpoints/localai/welcome.go b/core/http/endpoints/localai/welcome.go index 396c4084..0518ceac 100644 --- a/core/http/endpoints/localai/welcome.go +++ b/core/http/endpoints/localai/welcome.go @@ -13,7 +13,7 @@ import ( func WelcomeEndpoint(appConfig *config.ApplicationConfig, cl *config.BackendConfigLoader, ml *model.ModelLoader, modelStatus func() (map[string]string, map[string]string)) func(*fiber.Ctx) error { return func(c *fiber.Ctx) error { - models, _ := services.ListModels(cl, ml, "", true) + models, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) backendConfigs := cl.GetAllBackendConfigs() galleryConfigs := map[string]*gallery.Config{} @@ -32,18 +32,10 @@ func WelcomeEndpoint(appConfig *config.ApplicationConfig, // Get model statuses to display in the UI the operation in progress processingModels, taskTypes := modelStatus() - modelsWithoutConfig := []string{} - - for _, m := range models { - if _, ok := modelsWithBackendConfig[m]; !ok { - modelsWithoutConfig = append(modelsWithoutConfig, m) - } - } - summary := fiber.Map{ "Title": "LocalAI API - " + internal.PrintableVersion(), "Version": internal.PrintableVersion(), - "Models": modelsWithoutConfig, + "Models": models, "ModelsConfig": backendConfigs, "GalleryConfig": galleryConfigs, "IsP2PEnabled": p2p.IsP2PEnabled(), diff --git a/core/http/endpoints/openai/assistant.go b/core/http/endpoints/openai/assistant.go index ff218730..3240e8ee 100644 --- a/core/http/endpoints/openai/assistant.go +++ b/core/http/endpoints/openai/assistant.go @@ -225,7 +225,7 @@ func filterAssistantsAfterID(assistants []Assistant, id string) []Assistant { func modelExists(cl *config.BackendConfigLoader, ml *model.ModelLoader, modelName string) (found bool) { found = false - models, err := services.ListModels(cl, ml, "", true) + models, err := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if err != nil { return } diff --git a/core/http/endpoints/openai/list.go b/core/http/endpoints/openai/list.go index d446b100..80dcb3e4 100644 --- a/core/http/endpoints/openai/list.go +++ b/core/http/endpoints/openai/list.go @@ -18,32 +18,32 @@ func ListModelsEndpoint(bcl *config.BackendConfigLoader, ml *model.ModelLoader) filter := c.Query("filter") // By default, exclude any loose files that are already referenced by a configuration file. - excludeConfigured := c.QueryBool("excludeConfigured", true) + var policy services.LooseFilePolicy + if c.QueryBool("excludeConfigured", true) { + policy = services.SKIP_IF_CONFIGURED + } else { + policy = services.ALWAYS_INCLUDE // This replicates current behavior. TODO: give more options to the user? + } - dataModels, err := modelList(bcl, ml, filter, excludeConfigured) + filterFn, err := config.BuildNameFilterFn(filter) if err != nil { return err } + + modelNames, err := services.ListModels(bcl, ml, filterFn, policy) + if err != nil { + return err + } + + // Map from a slice of names to a slice of OpenAIModel response objects + dataModels := []schema.OpenAIModel{} + for _, m := range modelNames { + dataModels = append(dataModels, schema.OpenAIModel{ID: m, Object: "model"}) + } + return c.JSON(schema.ModelsDataResponse{ Object: "list", Data: dataModels, }) } } - -func modelList(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter string, excludeConfigured bool) ([]schema.OpenAIModel, error) { - - models, err := services.ListModels(bcl, ml, filter, excludeConfigured) - if err != nil { - return nil, err - } - - dataModels := []schema.OpenAIModel{} - - // Then iterate through the loose files: - for _, m := range models { - dataModels = append(dataModels, schema.OpenAIModel{ID: m, Object: "model"}) - } - - return dataModels, nil -} diff --git a/core/http/routes/ui.go b/core/http/routes/ui.go index 7b2c6ae7..cfe9368c 100644 --- a/core/http/routes/ui.go +++ b/core/http/routes/ui.go @@ -303,7 +303,7 @@ func RegisterUIRoutes(app *fiber.App, // Show the Chat page app.Get("/chat/:model", func(c *fiber.Ctx) error { - backendConfigs, _ := services.ListModels(cl, ml, "", true) + backendConfigs, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) summary := fiber.Map{ "Title": "LocalAI - Chat with " + c.Params("model"), @@ -318,7 +318,7 @@ func RegisterUIRoutes(app *fiber.App, }) app.Get("/talk/", func(c *fiber.Ctx) error { - backendConfigs, _ := services.ListModels(cl, ml, "", true) + backendConfigs, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if len(backendConfigs) == 0 { // If no model is available redirect to the index which suggests how to install models @@ -339,7 +339,7 @@ func RegisterUIRoutes(app *fiber.App, app.Get("/chat/", func(c *fiber.Ctx) error { - backendConfigs, _ := services.ListModels(cl, ml, "", true) + backendConfigs, _ := services.ListModels(cl, ml, config.NoFilterFn, services.SKIP_IF_CONFIGURED) if len(backendConfigs) == 0 { // If no model is available redirect to the index which suggests how to install models diff --git a/core/services/list_models.go b/core/services/list_models.go index 4b578e25..c310ac15 100644 --- a/core/services/list_models.go +++ b/core/services/list_models.go @@ -1,55 +1,47 @@ package services import ( - "regexp" - "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/model" ) -func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter string, excludeConfigured bool) ([]string, error) { +type LooseFilePolicy int - models, err := ml.ListFilesInModelPath() - if err != nil { - return nil, err - } +const ( + SKIP_IF_CONFIGURED LooseFilePolicy = iota + SKIP_ALWAYS + ALWAYS_INCLUDE + LOOSE_ONLY +) - var mm map[string]interface{} = map[string]interface{}{} +func ListModels(bcl *config.BackendConfigLoader, ml *model.ModelLoader, filter config.BackendConfigFilterFn, looseFilePolicy LooseFilePolicy) ([]string, error) { + + var skipMap map[string]interface{} = map[string]interface{}{} dataModels := []string{} - var filterFn func(name string) bool - - // If filter is not specified, do not filter the list by model name - if filter == "" { - filterFn = func(_ string) bool { return true } - } else { - // If filter _IS_ specified, we compile it to a regex which is used to create the filterFn - rxp, err := regexp.Compile(filter) - if err != nil { - return nil, err - } - filterFn = func(name string) bool { - return rxp.MatchString(name) - } - } - - // Start with the known configurations - for _, c := range bcl.GetAllBackendConfigs() { - if excludeConfigured { - mm[c.Model] = nil - } - - if filterFn(c.Name) { + // Start with known configurations + if looseFilePolicy != LOOSE_ONLY { + for _, c := range bcl.GetBackendConfigsByFilter(filter) { + if looseFilePolicy == SKIP_IF_CONFIGURED { + skipMap[c.Model] = nil + } dataModels = append(dataModels, c.Name) } } - // Then iterate through the loose files: - for _, m := range models { - // And only adds them if they shouldn't be skipped. - if _, exists := mm[m]; !exists && filterFn(m) { - dataModels = append(dataModels, m) + // Then iterate through the loose files if requested. + if looseFilePolicy != SKIP_ALWAYS { + + models, err := ml.ListFilesInModelPath() + if err != nil { + return nil, err + } + for _, m := range models { + // And only adds them if they shouldn't be skipped. + if _, exists := skipMap[m]; !exists && filter(m, nil) { + dataModels = append(dataModels, m) + } } }