diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4276a7b9..084d016d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -110,7 +110,7 @@ jobs: # Pre-build stable diffusion before we install a newer version of abseil (not compatible with stablediffusion-ncn) PATH="$PATH:/root/go/bin" GO_TAGS="stablediffusion tts" GRPC_BACKENDS=backend-assets/grpc/stablediffusion make build env: - CUDA_VERSION: 12-5 + CUDA_VERSION: 12-4 - name: Cache grpc id: cache-grpc uses: actions/cache@v4 diff --git a/core/cli/federated.go b/core/cli/federated.go index b99ef4f8..b1de1840 100644 --- a/core/cli/federated.go +++ b/core/cli/federated.go @@ -34,7 +34,9 @@ func (f *FederatedCLI) Run(ctx *cliContext.Context) error { return fmt.Errorf("creating a new node: %w", err) } - if err := p2p.ServiceDiscoverer(context.Background(), n, f.Peer2PeerToken, p2p.FederatedID, nil); err != nil { + if err := p2p.ServiceDiscoverer(context.Background(), n, f.Peer2PeerToken, p2p.FederatedID, func(servicesID string, tunnel p2p.NodeData) { + log.Debug().Msgf("Discovered node: %s", tunnel.ID) + }); err != nil { return err } @@ -98,6 +100,10 @@ func Proxy(ctx context.Context, node *node.Node, listenAddr, service string) err } } + if len(tunnelAddresses) == 0 { + log.Error().Msg("No available nodes yet") + return + } // open a TCP stream to one of the tunnels // chosen randomly // TODO: optimize this and track usage diff --git a/core/cli/run.go b/core/cli/run.go index d7b45f77..b3d91632 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -119,7 +119,7 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { } log.Info().Msg("Starting P2P server discovery...") - if err := p2p.ServiceDiscoverer(context.Background(), node, token, "", func() { + if err := p2p.ServiceDiscoverer(context.Background(), node, token, "", func(serviceID string, node p2p.NodeData) { var tunnelAddresses []string for _, v := range p2p.GetAvailableNodes("") { if v.IsOnline() { diff --git a/core/http/endpoints/openai/chat.go b/core/http/endpoints/openai/chat.go index 763e3f69..f63a9913 100644 --- a/core/http/endpoints/openai/chat.go +++ b/core/http/endpoints/openai/chat.go @@ -225,18 +225,10 @@ func ChatEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, startup } // Update input grammar - // Handle if we should return "name" instead of "functions" - if config.FunctionsConfig.FunctionName { - jsStruct := funcs.ToJSONNameStructure() - config.Grammar = jsStruct.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) - } else { - jsStruct := funcs.ToJSONFunctionStructure() - config.Grammar = jsStruct.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) - } + jsStruct := funcs.ToJSONStructure(config.FunctionsConfig.FunctionNameKey, config.FunctionsConfig.FunctionNameKey) + config.Grammar = jsStruct.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) case input.JSONFunctionGrammarObject != nil: config.Grammar = input.JSONFunctionGrammarObject.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) - case input.JSONFunctionGrammarObjectName != nil: - config.Grammar = input.JSONFunctionGrammarObjectName.Grammar(config.FunctionsConfig.GrammarConfig.Options()...) default: // Force picking one of the functions by the request if config.FunctionToCall() != "" { diff --git a/core/p2p/p2p.go b/core/p2p/p2p.go index e0e46170..9b71f7de 100644 --- a/core/p2p/p2p.go +++ b/core/p2p/p2p.go @@ -144,7 +144,7 @@ func copyStream(closer chan struct{}, dst io.Writer, src io.Reader) { // This is the main of the server (which keeps the env variable updated) // This starts a goroutine that keeps LLAMACPP_GRPC_SERVERS updated with the discovered services -func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID string, discoveryFunc func()) error { +func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID string, discoveryFunc func(serviceID string, node NodeData)) error { if servicesID == "" { servicesID = defaultServicesID } @@ -166,7 +166,7 @@ func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID stri case tunnel := <-tunnels: AddNode(servicesID, tunnel) if discoveryFunc != nil { - discoveryFunc() + discoveryFunc(servicesID, tunnel) } } } diff --git a/core/p2p/p2p_disabled.go b/core/p2p/p2p_disabled.go index 340a4fb4..b1d1d04a 100644 --- a/core/p2p/p2p_disabled.go +++ b/core/p2p/p2p_disabled.go @@ -14,7 +14,7 @@ func GenerateToken() string { return "not implemented" } -func ServiceDiscoverer(ctx context.Context, node *node.Node, token, servicesID string, fn func()) error { +func ServiceDiscoverer(ctx context.Context, node *node.Node, token, servicesID string, fn func(string, NodeData)) error { return fmt.Errorf("not implemented") } diff --git a/core/schema/openai.go b/core/schema/openai.go index 354b1f05..3b39eaf3 100644 --- a/core/schema/openai.go +++ b/core/schema/openai.go @@ -179,8 +179,7 @@ type OpenAIRequest struct { // A grammar to constrain the LLM output Grammar string `json:"grammar" yaml:"grammar"` - JSONFunctionGrammarObject *functions.JSONFunctionStructureFunction `json:"grammar_json_functions" yaml:"grammar_json_functions"` - JSONFunctionGrammarObjectName *functions.JSONFunctionStructureName `json:"grammar_json_name" yaml:"grammar_json_name"` + JSONFunctionGrammarObject *functions.JSONFunctionStructure `json:"grammar_json_functions" yaml:"grammar_json_functions"` Backend string `json:"backend" yaml:"backend"` diff --git a/docs/content/docs/advanced/advanced-usage.md b/docs/content/docs/advanced/advanced-usage.md index a5f47865..35d3a2e4 100644 --- a/docs/content/docs/advanced/advanced-usage.md +++ b/docs/content/docs/advanced/advanced-usage.md @@ -152,7 +152,8 @@ function: replace_function_results: [] # Placeholder to replace function call results with arbitrary strings or patterns. replace_llm_results: [] # Replace language model results with arbitrary strings or patterns. capture_llm_results: [] # Capture language model results as text result, among JSON, in function calls. For instance, if a model returns a block for "thinking" and a block for "response", this will allow you to capture the thinking block. - return_name_in_function_response: false # Some models might prefer to use "name" rather then "function" when returning JSON data. This will allow to use "name" as a key in the JSON response. + function_name_key: "name" + function_arguments_key: "arguments" # Feature gating flags to enable experimental or optional features. feature_flags: {} diff --git a/pkg/functions/functions.go b/pkg/functions/functions.go index f13ffe01..49e9fc93 100644 --- a/pkg/functions/functions.go +++ b/pkg/functions/functions.go @@ -6,6 +6,11 @@ import ( "github.com/rs/zerolog/log" ) +const ( + defaultFunctionNameKey = "name" + defaultFunctionArgumentsKey = "arguments" +) + type Function struct { Name string `json:"name"` Description string `json:"description"` @@ -19,50 +24,18 @@ type Tool struct { } type Tools []Tool -// ToJSONFunctionStructure converts a list of functions to a JSON structure that can be parsed to a grammar -// This allows the LLM to return a response of the type: { "function": "function_name", "arguments": { "arg1": "value1", "arg2": "value2" } } -func (f Functions) ToJSONFunctionStructure() JSONFunctionStructureFunction { - js := JSONFunctionStructureFunction{} - for _, function := range f { - // t := function.Parameters["type"] - //tt := t.(string) - - properties := function.Parameters["properties"] - defs := function.Parameters["$defs"] - dat, _ := json.Marshal(properties) - dat2, _ := json.Marshal(defs) - prop := map[string]interface{}{} - defsD := map[string]interface{}{} - - err := json.Unmarshal(dat, &prop) - if err != nil { - log.Error().Err(err).Msg("error unmarshalling dat") - } - err = json.Unmarshal(dat2, &defsD) - if err != nil { - log.Error().Err(err).Msg("error unmarshalling dat2") - } - if js.Defs == nil { - js.Defs = defsD - } - js.OneOf = append(js.OneOf, ItemFunction{ - Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{Const: function.Name}, - Arguments: Argument{ - Type: "object", - Properties: prop, - }, - }, - }) - } - return js -} - // ToJSONNameStructure converts a list of functions to a JSON structure that can be parsed to a grammar // This allows the LLM to return a response of the type: { "name": "function_name", "arguments": { "arg1": "value1", "arg2": "value2" } } -func (f Functions) ToJSONNameStructure() JSONFunctionStructureName { - js := JSONFunctionStructureName{} +func (f Functions) ToJSONStructure(name, args string) JSONFunctionStructure { + nameKey := defaultFunctionNameKey + argsKey := defaultFunctionArgumentsKey + if name != "" { + nameKey = name + } + if args != "" { + argsKey = args + } + js := JSONFunctionStructure{} for _, function := range f { // t := function.Parameters["type"] //tt := t.(string) @@ -85,15 +58,16 @@ func (f Functions) ToJSONNameStructure() JSONFunctionStructureName { if js.Defs == nil { js.Defs = defsD } - js.OneOf = append(js.OneOf, ItemName{ - Type: "object", - Properties: NameProperties{ - Function: FunctionName{Const: function.Name}, - Arguments: Argument{ - Type: "object", - Properties: prop, - }, - }, + + property := map[string]interface{}{} + property[nameKey] = FunctionName{Const: function.Name} + property[argsKey] = Argument{ + Type: "object", + Properties: prop, + } + js.OneOf = append(js.OneOf, Item{ + Type: "object", + Properties: property, }) } return js diff --git a/pkg/functions/functions_test.go b/pkg/functions/functions_test.go index e82a5f71..2eb0946a 100644 --- a/pkg/functions/functions_test.go +++ b/pkg/functions/functions_test.go @@ -35,21 +35,35 @@ var _ = Describe("LocalAI grammar functions", func() { }, } - js := functions.ToJSONFunctionStructure() + js := functions.ToJSONStructure("function", "arguments") Expect(len(js.OneOf)).To(Equal(2)) - Expect(js.OneOf[0].Properties.Function.Const).To(Equal("create_event")) - Expect(js.OneOf[0].Properties.Arguments.Properties["event_name"].(map[string]interface{})["type"]).To(Equal("string")) - Expect(js.OneOf[0].Properties.Arguments.Properties["event_date"].(map[string]interface{})["type"]).To(Equal("string")) - Expect(js.OneOf[1].Properties.Function.Const).To(Equal("search")) - Expect(js.OneOf[1].Properties.Arguments.Properties["query"].(map[string]interface{})["type"]).To(Equal("string")) + fnName := js.OneOf[0].Properties["function"].(FunctionName) + fnArgs := js.OneOf[0].Properties["arguments"].(Argument) + Expect(fnName.Const).To(Equal("create_event")) + Expect(fnArgs.Properties["event_name"].(map[string]interface{})["type"]).To(Equal("string")) + Expect(fnArgs.Properties["event_date"].(map[string]interface{})["type"]).To(Equal("string")) - jsN := functions.ToJSONNameStructure() + fnName = js.OneOf[1].Properties["function"].(FunctionName) + fnArgs = js.OneOf[1].Properties["arguments"].(Argument) + Expect(fnName.Const).To(Equal("search")) + Expect(fnArgs.Properties["query"].(map[string]interface{})["type"]).To(Equal("string")) + + // Test with custom keys + jsN := functions.ToJSONStructure("name", "arguments") Expect(len(jsN.OneOf)).To(Equal(2)) - Expect(jsN.OneOf[0].Properties.Function.Const).To(Equal("create_event")) - Expect(jsN.OneOf[0].Properties.Arguments.Properties["event_name"].(map[string]interface{})["type"]).To(Equal("string")) - Expect(jsN.OneOf[0].Properties.Arguments.Properties["event_date"].(map[string]interface{})["type"]).To(Equal("string")) - Expect(jsN.OneOf[1].Properties.Function.Const).To(Equal("search")) - Expect(jsN.OneOf[1].Properties.Arguments.Properties["query"].(map[string]interface{})["type"]).To(Equal("string")) + + fnName = jsN.OneOf[0].Properties["name"].(FunctionName) + fnArgs = jsN.OneOf[0].Properties["arguments"].(Argument) + + Expect(fnName.Const).To(Equal("create_event")) + Expect(fnArgs.Properties["event_name"].(map[string]interface{})["type"]).To(Equal("string")) + Expect(fnArgs.Properties["event_date"].(map[string]interface{})["type"]).To(Equal("string")) + + fnName = jsN.OneOf[1].Properties["name"].(FunctionName) + fnArgs = jsN.OneOf[1].Properties["arguments"].(Argument) + + Expect(fnName.Const).To(Equal("search")) + Expect(fnArgs.Properties["query"].(map[string]interface{})["type"]).To(Equal("string")) }) }) Context("Select()", func() { diff --git a/pkg/functions/grammar_json_schema.go b/pkg/functions/grammar_json_schema.go index 2c42e291..7356d01d 100644 --- a/pkg/functions/grammar_json_schema.go +++ b/pkg/functions/grammar_json_schema.go @@ -331,6 +331,7 @@ func (sc *JSONSchemaConverter) resolveReference(ref string, rootSchema map[strin return def } + func (sc *JSONSchemaConverter) Grammar(schema map[string]interface{}, options ...func(*GrammarOption)) string { sc.addRule("freestring", PRIMITIVE_RULES["freestring"]) sc.visit(schema, "", schema) @@ -352,52 +353,23 @@ type FunctionName struct { Const string `json:"const"` } -type FunctionProperties struct { - Function FunctionName `json:"function"` - Arguments Argument `json:"arguments"` -} - -type NameProperties struct { - Function FunctionName `json:"name"` - Arguments Argument `json:"arguments"` -} - type Argument struct { Type string `json:"type"` Properties map[string]interface{} `json:"properties"` } -type ItemName struct { - Type string `json:"type"` - Properties NameProperties `json:"properties"` +type Item struct { + Type string `json:"type"` + Properties map[string]interface{} `json:"properties"` } -type ItemFunction struct { - Type string `json:"type"` - Properties FunctionProperties `json:"properties"` -} - -type JSONFunctionStructureName struct { - OneOf []ItemName `json:"oneOf,omitempty"` - AnyOf []ItemName `json:"anyOf,omitempty"` +type JSONFunctionStructure struct { + OneOf []Item `json:"oneOf,omitempty"` + AnyOf []Item `json:"anyOf,omitempty"` Defs map[string]interface{} `json:"$defs,omitempty"` } -func (j JSONFunctionStructureName) Grammar(options ...func(*GrammarOption)) string { - grammarOpts := &GrammarOption{} - grammarOpts.Apply(options...) - - dat, _ := json.Marshal(j) - return NewJSONSchemaConverter(grammarOpts.PropOrder).GrammarFromBytes(dat, options...) -} - -type JSONFunctionStructureFunction struct { - OneOf []ItemFunction `json:"oneOf,omitempty"` - AnyOf []ItemFunction `json:"anyOf,omitempty"` - Defs map[string]interface{} `json:"$defs,omitempty"` -} - -func (j JSONFunctionStructureFunction) Grammar(options ...func(*GrammarOption)) string { +func (j JSONFunctionStructure) Grammar(options ...func(*GrammarOption)) string { grammarOpts := &GrammarOption{} grammarOpts.Apply(options...) diff --git a/pkg/functions/grammar_json_schema_test.go b/pkg/functions/grammar_json_schema_test.go index 5f72dbd5..bf52bd8d 100644 --- a/pkg/functions/grammar_json_schema_test.go +++ b/pkg/functions/grammar_json_schema_test.go @@ -9,69 +9,65 @@ import ( . "github.com/onsi/gomega" ) -var testFunctions = []ItemFunction{ +func createFunction(field1 string, field2 string, name string, properties map[string]interface{}) map[string]interface{} { + property := map[string]interface{}{} + property[field1] = FunctionName{Const: name} + property[field2] = Argument{ + Type: "object", + Properties: properties, + } + return property +} + +var testFunctions = []Item{ { Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{ - Const: "create_event", + Properties: createFunction( + "function", + "arguments", + "create_event", + map[string]interface{}{ + "title": map[string]string{"type": "string"}, + "date": map[string]string{"type": "string"}, + "time": map[string]string{"type": "string"}, }, - Arguments: Argument{ // this is OpenAI's parameter - Type: "object", - Properties: map[string]interface{}{ - "title": map[string]string{"type": "string"}, - "date": map[string]string{"type": "string"}, - "time": map[string]string{"type": "string"}, - }, - }, - }, + ), }, { Type: "object", - Properties: FunctionProperties{ - Function: FunctionName{ - Const: "search", - }, - Arguments: Argument{ - Type: "object", - Properties: map[string]interface{}{ - "query": map[string]string{"type": "string"}, - }, - }, - }, + Properties: createFunction( + "function", + "arguments", + "search", + map[string]interface{}{ + "query": map[string]string{"type": "string"}, + }), }, } -var testFunctionsName = []ItemName{ +var testFunctionsName = []Item{ { Type: "object", - Properties: NameProperties{ - Function: FunctionName{ - Const: "create_event", + Properties: createFunction( + "name", + "arguments", + "create_event", + map[string]interface{}{ + "title": map[string]string{"type": "string"}, + "date": map[string]string{"type": "string"}, + "time": map[string]string{"type": "string"}, }, - Arguments: Argument{ // this is OpenAI's parameter - Type: "object", - Properties: map[string]interface{}{ - "title": map[string]string{"type": "string"}, - "date": map[string]string{"type": "string"}, - "time": map[string]string{"type": "string"}, - }, - }, - }, + ), }, { Type: "object", - Properties: NameProperties{ - Function: FunctionName{ - Const: "search", - }, - Arguments: Argument{ - Type: "object", - Properties: map[string]interface{}{ - "query": map[string]string{"type": "string"}, - }, - }, - }, + Properties: createFunction( + "name", + "arguments", + "search", + map[string]interface{}{ + "query": map[string]string{"type": "string"}, + }), }, } @@ -270,7 +266,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates a valid grammar from JSON Objects", func() { - structuredGrammar := JSONFunctionStructureFunction{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctions} grammar := structuredGrammar.Grammar() @@ -284,7 +280,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates a valid grammar from JSON Objects for multiple function return", func() { - structuredGrammar := JSONFunctionStructureFunction{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctions} grammar := structuredGrammar.Grammar(functions.EnableMaybeArray) @@ -302,7 +298,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates a valid grammar from JSON Objects for multiple function return", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar(functions.EnableMaybeArray) @@ -320,7 +316,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates a valid grammar from JSON Objects for multiple function return with a suffix and array", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar( @@ -340,7 +336,7 @@ var _ = Describe("JSON schema grammar tests", func() { Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) }) It("generates a valid grammar from JSON Objects with a suffix", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar(functions.SetPrefix("suffix")) @@ -357,7 +353,7 @@ var _ = Describe("JSON schema grammar tests", func() { Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) }) It("generates a valid grammar from JSON Objects with a suffix and could return string", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar(functions.SetPrefix("suffix"), functions.EnableMaybeString) @@ -374,7 +370,7 @@ var _ = Describe("JSON schema grammar tests", func() { Expect(len(results)).To(Equal(len(strings.Split(grammar, "\n"))), grammar) }) It("generates a valid grammar from JSON Objects with a suffix that could return text or an array of tools", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar(functions.SetPrefix("suffix"), functions.EnableMaybeString, functions.EnableMaybeArray) @@ -393,7 +389,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates a valid grammar from JSON Objects without a suffix that could return text or an array of tools or just string", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar(functions.EnableMaybeString, functions.EnableMaybeArray) @@ -411,7 +407,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates a valid grammar from JSON Objects without a suffix that could return text or an array of tools or just string. Disables mixedstring", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} grammar := structuredGrammar.Grammar(functions.EnableMaybeString, functions.EnableMaybeArray, functions.NoMixedFreeString) @@ -429,7 +425,7 @@ var _ = Describe("JSON schema grammar tests", func() { }) It("generates parallel tools without newlines in JSON", func() { - structuredGrammar := JSONFunctionStructureName{ + structuredGrammar := JSONFunctionStructure{ OneOf: testFunctionsName} content := `arr ::= "[" ( diff --git a/pkg/functions/parse.go b/pkg/functions/parse.go index a9eef658..8e848a60 100644 --- a/pkg/functions/parse.go +++ b/pkg/functions/parse.go @@ -2,6 +2,8 @@ package functions import ( "encoding/json" + "errors" + "io" "regexp" "strings" @@ -76,7 +78,8 @@ type FunctionsConfig struct { // FunctionName enable the LLM to return { "name": "function_name", "arguments": { "arg1": "value1", "arg2": "value2" } } // instead of { "function": "function_name", "arguments": { "arg1": "value1", "arg2": "value2" } }. // This might be useful for certain models trained with the function name as the first token. - FunctionName bool `yaml:"return_name_in_function_response"` + FunctionNameKey string `yaml:"function_name_key"` + FunctionArgumentsKey string `yaml:"function_arguments_key"` } type ReplaceResult struct { @@ -145,6 +148,47 @@ func ParseTextContent(llmresult string, functionConfig FunctionsConfig) string { return "" } +// ParseJSON is a function that parses a JSON string that might contain multiple JSON objects +// and syntax errors in between by shifting the offset +// This for e.g. allow to parse +// { "foo": "bar" } invalid { "baz": "qux" } +// into +// [ { "foo": "bar" }, { "baz": "qux" } ] +// Credits to Michael Yang (https://github.com/mxyng) for the original implementation +// This is a slighly reworked version, improved for readability and error handling +func ParseJSON(s string) ([]map[string]any, error) { + var objs []map[string]any + offset := 0 + + for offset < len(s) { + var obj map[string]any + decoder := json.NewDecoder(strings.NewReader(s[offset:])) + + err := decoder.Decode(&obj) + switch { + case errors.Is(err, io.EOF): + return objs, nil + case err == nil: + offset += int(decoder.InputOffset()) + objs = append(objs, obj) + default: // handle the error type + var syntaxErr *json.SyntaxError + var unmarshalTypeErr *json.UnmarshalTypeError + + switch { + case errors.As(err, &syntaxErr): + offset += int(syntaxErr.Offset) + case errors.As(err, &unmarshalTypeErr): + offset += int(unmarshalTypeErr.Offset) + default: + return objs, err + } + } + } + + return objs, nil +} + func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncCallResults { log.Debug().Msgf("LLM result: %s", llmresult) @@ -157,9 +201,13 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC } log.Debug().Msgf("LLM result(function cleanup): %s", llmresult) - functionNameKey := "function" - if functionConfig.FunctionName { - functionNameKey = "name" + functionNameKey := defaultFunctionNameKey + functionArgumentsKey := defaultFunctionArgumentsKey + if functionConfig.FunctionNameKey != "" { + functionNameKey = functionConfig.FunctionNameKey + } + if functionConfig.FunctionArgumentsKey != "" { + functionArgumentsKey = functionConfig.FunctionArgumentsKey } results := []FuncCallResults{} @@ -170,19 +218,13 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC result = make([]FuncCallResults, 0) for _, s := range results { - var ss []map[string]interface{} + var ss []map[string]any s = utils.EscapeNewLines(s) - err := json.Unmarshal([]byte(s), &ss) + ss, err := ParseJSON(s) + //err := json.Unmarshal([]byte(s), &ss) if err != nil { - // If the LLM result is a single object, try unmarshaling it into a single map - var singleObj map[string]interface{} - err = json.Unmarshal([]byte(s), &singleObj) - if err != nil { - log.Debug().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result in a single object or an array of JSON objects") - } else { - ss = []map[string]interface{}{singleObj} - } + log.Debug().Err(err).Str("escapedLLMResult", s).Msg("unable to unmarshal llm result in a single object or an array of JSON objects") } log.Debug().Msgf("Function return: %s %+v", s, ss) @@ -195,7 +237,7 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC //return result, fmt.Errorf("unable to find function name in result") } // Similarly, while here arguments is a map[string]interface{}, OpenAI actually want a stringified object - args, ok := s["arguments"] // arguments needs to be a string, but we return an object from the grammar result (TODO: fix) + args, ok := s[functionArgumentsKey] // arguments needs to be a string, but we return an object from the grammar result (TODO: fix) if !ok { continue //return result, fmt.Errorf("unable to find arguments in result") @@ -253,7 +295,7 @@ func ParseFunctionCall(llmresult string, functionConfig FunctionsConfig) []FuncC if functionName == "" { return results } - results = append(results, FuncCallResults{Name: result[functionNameKey], Arguments: result["arguments"]}) + results = append(results, FuncCallResults{Name: result[functionNameKey], Arguments: result[functionArgumentsKey]}) } } } else { diff --git a/pkg/functions/parse_test.go b/pkg/functions/parse_test.go index 0e38f9d9..f0783524 100644 --- a/pkg/functions/parse_test.go +++ b/pkg/functions/parse_test.go @@ -16,7 +16,7 @@ var _ = Describe("LocalAI function parse tests", func() { Context("when using grammars and single result expected", func() { It("should parse the function name and arguments correctly", func() { - input := `{"function": "add", "arguments": {"x": 5, "y": 3}}` + input := `{"name": "add", "arguments": {"x": 5, "y": 3}}` results := ParseFunctionCall(input, functionConfig) Expect(results).To(HaveLen(1)) @@ -28,13 +28,22 @@ var _ = Describe("LocalAI function parse tests", func() { Context("when not using grammars and regex is needed", func() { It("should extract function name and arguments from the regex", func() { input := `add({"x":5,"y":3})` - functionConfig.ResponseRegex = []string{`(?P\w+)\s*\((?P.*)\)`} + functionConfig.ResponseRegex = []string{`(?P\w+)\s*\((?P.*)\)`} results := ParseFunctionCall(input, functionConfig) Expect(results).To(HaveLen(1)) Expect(results[0].Name).To(Equal("add")) Expect(results[0].Arguments).To(Equal(`{"x":5,"y":3}`)) }) + It("should extract function name and arguments from the regex", func() { + input := `add({"x":5,"y":3})` + functionConfig.ResponseRegex = []string{`(?P\w+)\s*\((?P.*)\)`} + functionConfig.FunctionNameKey = "function" + results := ParseFunctionCall(input, functionConfig) + Expect(results).To(HaveLen(1)) + Expect(results[0].Name).To(Equal("add")) + Expect(results[0].Arguments).To(Equal(`{"x":5,"y":3}`)) + }) }) Context("when having invalid input", func() { @@ -53,7 +62,7 @@ var _ = Describe("LocalAI function parse tests", func() { Context("when parallel calls are enabled", func() { It("should handle multiple function calls", func() { - input := `[{"function": "add", "arguments": {"x": 5, "y": 3}}, {"function": "subtract", "arguments": {"x": 10, "y": 7}}]` + input := `[{"name": "add", "arguments": {"x": 5, "y": 3}}, {"name": "subtract", "arguments": {"x": 10, "y": 7}}]` results := ParseFunctionCall(input, functionConfig) Expect(results).To(HaveLen(2)) @@ -66,8 +75,8 @@ var _ = Describe("LocalAI function parse tests", func() { Context("without grammars and without regex", func() { It("should parse the function name and arguments correctly with the name key", func() { - input := `{"name": "add", "arguments": {"x": 5, "y": 3}}` - functionConfig.FunctionName = true + input := `{"function": "add", "arguments": {"x": 5, "y": 3}}` + functionConfig.FunctionNameKey = "function" results := ParseFunctionCall(input, functionConfig) Expect(results).To(HaveLen(1)) @@ -76,7 +85,7 @@ var _ = Describe("LocalAI function parse tests", func() { }) It("should parse the function name and arguments correctly with the function key", func() { - input := `{"function": "add", "arguments": {"x": 5, "y": 3}}` + input := `{"name": "add", "arguments": {"x": 5, "y": 3}}` results := ParseFunctionCall(input, functionConfig) Expect(results).To(HaveLen(1)) @@ -87,7 +96,7 @@ var _ = Describe("LocalAI function parse tests", func() { It("should parse the result by matching the JSONRegexMatch", func() { input := ` -{"function": "add", "arguments": {"x": 5, "y": 3}} +{"name": "add", "arguments": {"x": 5, "y": 3}} ` functionConfig.JSONRegexMatch = []string{`(?s)(.*?)`} @@ -100,7 +109,7 @@ var _ = Describe("LocalAI function parse tests", func() { It("should parse the result by matching the JSONRegexMatch", func() { input := ` -{"function": "add", "arguments": {"x": 5, "y": 3}} +{"name": "add", "arguments": {"x": 5, "y": 3}} ` functionConfig.JSONRegexMatch = []string{`(?s)(.*?)`} @@ -110,13 +119,21 @@ var _ = Describe("LocalAI function parse tests", func() { Expect(results[0].Name).To(Equal("add")) Expect(results[0].Arguments).To(Equal(`{"x":5,"y":3}`)) }) + + It("should parse the result even with invalid JSON", func() { + input := `{"name": "add", "arguments": {"x": 5, "y": 3}} invalid {"name": "add", "arguments": {"x": 5, "y": 3}}` + results := ParseFunctionCall(input, functionConfig) + Expect(results).To(HaveLen(2)) + Expect(results[0].Name).To(Equal("add")) + Expect(results[0].Arguments).To(Equal(`{"x":5,"y":3}`)) + }) }) Context("when using ReplaceResults to clean up input", func() { It("should replace text before and after JSON blob", func() { input := ` Some text before the JSON -{"function": "add", "arguments": {"x": 5, "y": 3}} +{"name": "add", "arguments": {"x": 5, "y": 3}} Some text after the JSON ` @@ -134,7 +151,7 @@ Some text after the JSON It("should replace text before and after array JSON blob", func() { input := ` Some text before the JSON -[{"function": "add", "arguments": {"x": 5, "y": 3}}, {"function": "subtract", "arguments": {"x": 10, "y": 7}}] +[{"name": "add", "arguments": {"x": 5, "y": 3}}, {"name": "subtract", "arguments": {"x": 10, "y": 7}}] Some text after the JSON ` functionConfig.ReplaceFunctionResults = []ReplaceResult{ @@ -153,7 +170,7 @@ Some text after the JSON It("should convert single-quoted key-value pairs to double-quoted and escape double quotes within values", func() { input := ` Some text before the JSON -{'function': '"add"', 'arguments': {'x': 5, 'z': '"v"', 'y': 'v"value"'}} +{'name': '"add"', 'arguments': {'x': 5, 'z': '"v"', 'y': 'v"value"'}} Some text after the JSON ` functionConfig.JSONRegexMatch = []string{`(?s)(.*?)`} @@ -186,7 +203,7 @@ Some text after the JSON It("should convert single-quoted key-value pairs to double-quoted and escape double quotes within values", func() { input := ` Some text before the JSON -{'function': '"add"', 'arguments': {'x': 5, 'z': '"v"', 'y': 'v"value"'}} +{'name': '"add"', 'arguments': {'x': 5, 'z': '"v"', 'y': 'v"value"'}} Some text after the JSON ` functionConfig.JSONRegexMatch = []string{`(?s)(.*?)`} @@ -219,8 +236,8 @@ Some text after the JSON It("should detect multiple functions call where the JSONRegexMatch is repeated", func() { input := ` Some text before the JSON -{"function": "add", "arguments": {"x": 5, "y": 3}} -{"function": "subtract", "arguments": {"x": 10, "y": 7}} +{"name": "add", "arguments": {"x": 5, "y": 3}} +{"name": "subtract", "arguments": {"x": 10, "y": 7}} Some text after the JSON ` functionConfig.JSONRegexMatch = []string{`(?s)(.*?)`} @@ -240,7 +257,7 @@ Some text after the JSON roses are red - {"function": "subtract", "arguments": {"x": 10, "y": 7}} + {"name": "subtract", "arguments": {"x": 10, "y": 7}} Some text after the JSON ` functionConfig.CaptureLLMResult = []string{`(?s)(.*?)`} @@ -251,7 +268,7 @@ roses are red It("Defaults to empty if doesn't catch any", func() { input := ` Some text before the JSON - {"function": "subtract", "arguments": {"x": 10, "y": 7}} + {"name": "subtract", "arguments": {"x": 10, "y": 7}} Some text after the JSON ` functionConfig.CaptureLLMResult = []string{`(?s)(.*?)`} @@ -259,4 +276,74 @@ roses are red Expect(results).To(Equal("")) }) }) + Context("ParseJSON - when given valid JSON strings", func() { + It("should parse multiple JSON objects", func() { + input := `{"key1": "value1"} {"key2": "value2"}` + expected := []map[string]any{ + {"key1": "value1"}, + {"key2": "value2"}, + } + result, err := ParseJSON(input) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(expected)) + }) + + It("should parse a single JSON object with various types", func() { + input := `{"key1": "value1", "key2": 2}` + expected := []map[string]any{ + {"key1": "value1", "key2": float64(2)}, + } + result, err := ParseJSON(input) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(expected)) + }) + It("should handle JSON without syntax errors gracefully", func() { + input := `{"key1": "value1"}` + expected := []map[string]any{ + {"key1": "value1"}, + } + result, err := ParseJSON(input) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(expected)) + }) + It("should handle JSON without syntax errors gracefully", func() { + input := `[{"key1": "value1"}]` + expected := []map[string]any{ + {"key1": "value1"}, + } + result, err := ParseJSON(input) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(expected)) + }) + }) + + Context("ParseJSON - when given invalid JSON strings", func() { + It("should return an error for completely invalid JSON", func() { + input := `invalid json` + result, err := ParseJSON(input) + Expect(err).To(HaveOccurred()) + Expect(result).To(BeNil()) + }) + + It("should skip invalid JSON parts and parse valid parts", func() { + input := `{"key1": "value1"} invalid {"key2": "value2"}` + expected := []map[string]any{ + {"key1": "value1"}, + {"key2": "value2"}, + } + result, err := ParseJSON(input) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(expected)) + }) + + PIt("should handle JSON with syntax errors gracefully", func() { + input := `{"key1": "value1", "key2": }` + expected := []map[string]any{ + {"key1": "value1"}, + } + result, err := ParseJSON(input) + Expect(err).NotTo(HaveOccurred()) + Expect(result).To(Equal(expected)) + }) + }) }) diff --git a/pkg/model/loader.go b/pkg/model/loader.go index c7159f7e..b2570c71 100644 --- a/pkg/model/loader.go +++ b/pkg/model/loader.go @@ -98,6 +98,9 @@ var knownModelsNameSuffixToSkip []string = []string{ ".yaml", ".yml", ".json", + ".txt", + ".md", + ".MD", ".DS_Store", ".", ".partial", diff --git a/swagger/docs.go b/swagger/docs.go index e9cb40e3..d5ffbc09 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -51,6 +51,64 @@ const docTemplate = `{ } } }, + "/backend/monitor": { + "get": { + "summary": "Backend monitor endpoint", + "parameters": [ + { + "description": "Backend statistics request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.BackendMonitorRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/proto.StatusResponse" + } + } + } + } + }, + "/backend/shutdown": { + "post": { + "summary": "Backend monitor endpoint", + "parameters": [ + { + "description": "Backend statistics request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.BackendMonitorRequest" + } + } + ], + "responses": {} + } + }, + "/metrics": { + "get": { + "summary": "Prometheus metrics endpoint", + "parameters": [ + { + "description": "Gallery details", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.Gallery" + } + } + ], + "responses": {} + } + }, "/models/apply": { "post": { "summary": "Install models to LocalAI.", @@ -179,6 +237,35 @@ const docTemplate = `{ } } }, + "/models/jobs": { + "get": { + "summary": "Returns all the jobs status progress", + "responses": { + "200": { + "description": "Response", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/gallery.GalleryOpStatus" + } + } + } + } + } + }, + "/models/jobs/{uuid}": { + "get": { + "summary": "Returns the job status", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/gallery.GalleryOpStatus" + } + } + } + } + }, "/tts": { "post": { "consumes": [ @@ -210,6 +297,46 @@ const docTemplate = `{ } }, "/v1/assistants": { + "get": { + "summary": "List available assistents", + "parameters": [ + { + "type": "integer", + "description": "Limit the number of assistants returned", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Order of assistants returned", + "name": "order", + "in": "query" + }, + { + "type": "string", + "description": "Return assistants created after the given ID", + "name": "after", + "in": "query" + }, + { + "type": "string", + "description": "Return assistants created before the given ID", + "name": "before", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/openai.Assistant" + } + } + } + } + }, "post": { "summary": "Create an assistant with a model and instructions.", "parameters": [ @@ -233,6 +360,30 @@ const docTemplate = `{ } } }, + "/v1/assistants/{assistant_id}": { + "get": { + "summary": "Get assistent data", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/openai.Assistant" + } + } + } + }, + "delete": { + "summary": "Delete assistents", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.DeleteAssistantResponse" + } + } + } + } + }, "/v1/audio/speech": { "post": { "consumes": [ @@ -346,6 +497,30 @@ const docTemplate = `{ } } }, + "/v1/edits": { + "post": { + "summary": "OpenAI edit endpoint", + "parameters": [ + { + "description": "query params", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.OpenAIRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.OpenAIResponse" + } + } + } + } + }, "/v1/embeddings": { "post": { "summary": "Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms.", @@ -370,6 +545,19 @@ const docTemplate = `{ } } }, + "/v1/files": { + "get": { + "summary": "List files.", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.ListFiles" + } + } + } + } + }, "/v1/files/{file_id}": { "get": { "summary": "Returns information about a specific file.", @@ -377,7 +565,7 @@ const docTemplate = `{ "200": { "description": "Response", "schema": { - "$ref": "#/definitions/openai.File" + "$ref": "#/definitions/schema.File" } } } @@ -719,6 +907,37 @@ const docTemplate = `{ } } }, + "gallery.GalleryOpStatus": { + "type": "object", + "properties": { + "deletion": { + "description": "Deletion is true if the operation is a deletion", + "type": "boolean" + }, + "downloaded_size": { + "type": "string" + }, + "error": {}, + "file_name": { + "type": "string" + }, + "file_size": { + "type": "string" + }, + "gallery_model_name": { + "type": "string" + }, + "message": { + "type": "string" + }, + "processed": { + "type": "boolean" + }, + "progress": { + "type": "number" + } + } + }, "localai.GalleryModel": { "type": "object", "properties": { @@ -889,35 +1108,6 @@ const docTemplate = `{ } } }, - "openai.File": { - "type": "object", - "properties": { - "bytes": { - "description": "Size of the file in bytes", - "type": "integer" - }, - "created_at": { - "description": "The time at which the file was created", - "type": "string" - }, - "filename": { - "description": "The name of the file", - "type": "string" - }, - "id": { - "description": "Unique identifier for the file", - "type": "string" - }, - "object": { - "description": "Type of the object (e.g., \"file\")", - "type": "string" - }, - "purpose": { - "description": "The purpose of the file (e.g., \"fine-tune\", \"classifications\", etc.)", - "type": "string" - } - } - }, "openai.Tool": { "type": "object", "properties": { @@ -956,6 +1146,54 @@ const docTemplate = `{ } } }, + "proto.MemoryUsageData": { + "type": "object", + "properties": { + "breakdown": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "total": { + "type": "integer" + } + } + }, + "proto.StatusResponse": { + "type": "object", + "properties": { + "memory": { + "$ref": "#/definitions/proto.MemoryUsageData" + }, + "state": { + "$ref": "#/definitions/proto.StatusResponse_State" + } + } + }, + "proto.StatusResponse_State": { + "type": "integer", + "enum": [ + 0, + 1, + 2, + -1 + ], + "x-enum-varnames": [ + "StatusResponse_UNINITIALIZED", + "StatusResponse_BUSY", + "StatusResponse_READY", + "StatusResponse_ERROR" + ] + }, + "schema.BackendMonitorRequest": { + "type": "object", + "properties": { + "model": { + "type": "string" + } + } + }, "schema.Choice": { "type": "object", "properties": { @@ -976,6 +1214,49 @@ const docTemplate = `{ } } }, + "schema.DeleteAssistantResponse": { + "type": "object", + "properties": { + "deleted": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "object": { + "type": "string" + } + } + }, + "schema.File": { + "type": "object", + "properties": { + "bytes": { + "description": "Size of the file in bytes", + "type": "integer" + }, + "created_at": { + "description": "The time at which the file was created", + "type": "string" + }, + "filename": { + "description": "The name of the file", + "type": "string" + }, + "id": { + "description": "Unique identifier for the file", + "type": "string" + }, + "object": { + "description": "Type of the object (e.g., \"file\")", + "type": "string" + }, + "purpose": { + "description": "The purpose of the file (e.g., \"fine-tune\", \"classifications\", etc.)", + "type": "string" + } + } + }, "schema.FunctionCall": { "type": "object", "properties": { @@ -1092,6 +1373,20 @@ const docTemplate = `{ } } }, + "schema.ListFiles": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.File" + } + }, + "object": { + "type": "string" + } + } + }, "schema.Message": { "type": "object", "properties": { diff --git a/swagger/swagger.json b/swagger/swagger.json index 7b83b6c5..71feb8bb 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -44,6 +44,64 @@ } } }, + "/backend/monitor": { + "get": { + "summary": "Backend monitor endpoint", + "parameters": [ + { + "description": "Backend statistics request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.BackendMonitorRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/proto.StatusResponse" + } + } + } + } + }, + "/backend/shutdown": { + "post": { + "summary": "Backend monitor endpoint", + "parameters": [ + { + "description": "Backend statistics request", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.BackendMonitorRequest" + } + } + ], + "responses": {} + } + }, + "/metrics": { + "get": { + "summary": "Prometheus metrics endpoint", + "parameters": [ + { + "description": "Gallery details", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/config.Gallery" + } + } + ], + "responses": {} + } + }, "/models/apply": { "post": { "summary": "Install models to LocalAI.", @@ -172,6 +230,35 @@ } } }, + "/models/jobs": { + "get": { + "summary": "Returns all the jobs status progress", + "responses": { + "200": { + "description": "Response", + "schema": { + "type": "object", + "additionalProperties": { + "$ref": "#/definitions/gallery.GalleryOpStatus" + } + } + } + } + } + }, + "/models/jobs/{uuid}": { + "get": { + "summary": "Returns the job status", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/gallery.GalleryOpStatus" + } + } + } + } + }, "/tts": { "post": { "consumes": [ @@ -203,6 +290,46 @@ } }, "/v1/assistants": { + "get": { + "summary": "List available assistents", + "parameters": [ + { + "type": "integer", + "description": "Limit the number of assistants returned", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Order of assistants returned", + "name": "order", + "in": "query" + }, + { + "type": "string", + "description": "Return assistants created after the given ID", + "name": "after", + "in": "query" + }, + { + "type": "string", + "description": "Return assistants created before the given ID", + "name": "before", + "in": "query" + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "type": "array", + "items": { + "$ref": "#/definitions/openai.Assistant" + } + } + } + } + }, "post": { "summary": "Create an assistant with a model and instructions.", "parameters": [ @@ -226,6 +353,30 @@ } } }, + "/v1/assistants/{assistant_id}": { + "get": { + "summary": "Get assistent data", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/openai.Assistant" + } + } + } + }, + "delete": { + "summary": "Delete assistents", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.DeleteAssistantResponse" + } + } + } + } + }, "/v1/audio/speech": { "post": { "consumes": [ @@ -339,6 +490,30 @@ } } }, + "/v1/edits": { + "post": { + "summary": "OpenAI edit endpoint", + "parameters": [ + { + "description": "query params", + "name": "request", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/schema.OpenAIRequest" + } + } + ], + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.OpenAIResponse" + } + } + } + } + }, "/v1/embeddings": { "post": { "summary": "Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms.", @@ -363,6 +538,19 @@ } } }, + "/v1/files": { + "get": { + "summary": "List files.", + "responses": { + "200": { + "description": "Response", + "schema": { + "$ref": "#/definitions/schema.ListFiles" + } + } + } + } + }, "/v1/files/{file_id}": { "get": { "summary": "Returns information about a specific file.", @@ -370,7 +558,7 @@ "200": { "description": "Response", "schema": { - "$ref": "#/definitions/openai.File" + "$ref": "#/definitions/schema.File" } } } @@ -712,6 +900,37 @@ } } }, + "gallery.GalleryOpStatus": { + "type": "object", + "properties": { + "deletion": { + "description": "Deletion is true if the operation is a deletion", + "type": "boolean" + }, + "downloaded_size": { + "type": "string" + }, + "error": {}, + "file_name": { + "type": "string" + }, + "file_size": { + "type": "string" + }, + "gallery_model_name": { + "type": "string" + }, + "message": { + "type": "string" + }, + "processed": { + "type": "boolean" + }, + "progress": { + "type": "number" + } + } + }, "localai.GalleryModel": { "type": "object", "properties": { @@ -882,35 +1101,6 @@ } } }, - "openai.File": { - "type": "object", - "properties": { - "bytes": { - "description": "Size of the file in bytes", - "type": "integer" - }, - "created_at": { - "description": "The time at which the file was created", - "type": "string" - }, - "filename": { - "description": "The name of the file", - "type": "string" - }, - "id": { - "description": "Unique identifier for the file", - "type": "string" - }, - "object": { - "description": "Type of the object (e.g., \"file\")", - "type": "string" - }, - "purpose": { - "description": "The purpose of the file (e.g., \"fine-tune\", \"classifications\", etc.)", - "type": "string" - } - } - }, "openai.Tool": { "type": "object", "properties": { @@ -949,6 +1139,54 @@ } } }, + "proto.MemoryUsageData": { + "type": "object", + "properties": { + "breakdown": { + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "total": { + "type": "integer" + } + } + }, + "proto.StatusResponse": { + "type": "object", + "properties": { + "memory": { + "$ref": "#/definitions/proto.MemoryUsageData" + }, + "state": { + "$ref": "#/definitions/proto.StatusResponse_State" + } + } + }, + "proto.StatusResponse_State": { + "type": "integer", + "enum": [ + 0, + 1, + 2, + -1 + ], + "x-enum-varnames": [ + "StatusResponse_UNINITIALIZED", + "StatusResponse_BUSY", + "StatusResponse_READY", + "StatusResponse_ERROR" + ] + }, + "schema.BackendMonitorRequest": { + "type": "object", + "properties": { + "model": { + "type": "string" + } + } + }, "schema.Choice": { "type": "object", "properties": { @@ -969,6 +1207,49 @@ } } }, + "schema.DeleteAssistantResponse": { + "type": "object", + "properties": { + "deleted": { + "type": "boolean" + }, + "id": { + "type": "string" + }, + "object": { + "type": "string" + } + } + }, + "schema.File": { + "type": "object", + "properties": { + "bytes": { + "description": "Size of the file in bytes", + "type": "integer" + }, + "created_at": { + "description": "The time at which the file was created", + "type": "string" + }, + "filename": { + "description": "The name of the file", + "type": "string" + }, + "id": { + "description": "Unique identifier for the file", + "type": "string" + }, + "object": { + "description": "Type of the object (e.g., \"file\")", + "type": "string" + }, + "purpose": { + "description": "The purpose of the file (e.g., \"fine-tune\", \"classifications\", etc.)", + "type": "string" + } + } + }, "schema.FunctionCall": { "type": "object", "properties": { @@ -1085,6 +1366,20 @@ } } }, + "schema.ListFiles": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/definitions/schema.File" + } + }, + "object": { + "type": "string" + } + } + }, "schema.Message": { "type": "object", "properties": { diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index 759753fe..e40ef119 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -145,6 +145,27 @@ definitions: type: string type: array type: object + gallery.GalleryOpStatus: + properties: + deletion: + description: Deletion is true if the operation is a deletion + type: boolean + downloaded_size: + type: string + error: {} + file_name: + type: string + file_size: + type: string + gallery_model_name: + type: string + message: + type: string + processed: + type: boolean + progress: + type: number + type: object localai.GalleryModel: properties: config_file: @@ -263,28 +284,6 @@ definitions: object: type: string type: object - openai.File: - properties: - bytes: - description: Size of the file in bytes - type: integer - created_at: - description: The time at which the file was created - type: string - filename: - description: The name of the file - type: string - id: - description: Unique identifier for the file - type: string - object: - description: Type of the object (e.g., "file") - type: string - purpose: - description: The purpose of the file (e.g., "fine-tune", "classifications", - etc.) - type: string - type: object openai.Tool: properties: type: @@ -311,6 +310,39 @@ definitions: tunnelAddress: type: string type: object + proto.MemoryUsageData: + properties: + breakdown: + additionalProperties: + type: integer + type: object + total: + type: integer + type: object + proto.StatusResponse: + properties: + memory: + $ref: '#/definitions/proto.MemoryUsageData' + state: + $ref: '#/definitions/proto.StatusResponse_State' + type: object + proto.StatusResponse_State: + enum: + - 0 + - 1 + - 2 + - -1 + type: integer + x-enum-varnames: + - StatusResponse_UNINITIALIZED + - StatusResponse_BUSY + - StatusResponse_READY + - StatusResponse_ERROR + schema.BackendMonitorRequest: + properties: + model: + type: string + type: object schema.Choice: properties: delta: @@ -324,6 +356,37 @@ definitions: text: type: string type: object + schema.DeleteAssistantResponse: + properties: + deleted: + type: boolean + id: + type: string + object: + type: string + type: object + schema.File: + properties: + bytes: + description: Size of the file in bytes + type: integer + created_at: + description: The time at which the file was created + type: string + filename: + description: The name of the file + type: string + id: + description: Unique identifier for the file + type: string + object: + description: Type of the object (e.g., "file") + type: string + purpose: + description: The purpose of the file (e.g., "fine-tune", "classifications", + etc.) + type: string + type: object schema.FunctionCall: properties: arguments: @@ -399,6 +462,15 @@ definitions: total_tokens: type: integer type: object + schema.ListFiles: + properties: + data: + items: + $ref: '#/definitions/schema.File' + type: array + object: + type: string + type: object schema.Message: properties: content: @@ -656,6 +728,43 @@ paths: schema: type: string summary: Show the P2P token + /backend/monitor: + get: + parameters: + - description: Backend statistics request + in: body + name: request + required: true + schema: + $ref: '#/definitions/schema.BackendMonitorRequest' + responses: + "200": + description: Response + schema: + $ref: '#/definitions/proto.StatusResponse' + summary: Backend monitor endpoint + /backend/shutdown: + post: + parameters: + - description: Backend statistics request + in: body + name: request + required: true + schema: + $ref: '#/definitions/schema.BackendMonitorRequest' + responses: {} + summary: Backend monitor endpoint + /metrics: + get: + parameters: + - description: Gallery details + in: body + name: request + required: true + schema: + $ref: '#/definitions/config.Gallery' + responses: {} + summary: Prometheus metrics endpoint /models/apply: post: parameters: @@ -737,6 +846,24 @@ paths: $ref: '#/definitions/config.Gallery' type: array summary: Adds a gallery in LocalAI + /models/jobs: + get: + responses: + "200": + description: Response + schema: + additionalProperties: + $ref: '#/definitions/gallery.GalleryOpStatus' + type: object + summary: Returns all the jobs status progress + /models/jobs/{uuid}: + get: + responses: + "200": + description: Response + schema: + $ref: '#/definitions/gallery.GalleryOpStatus' + summary: Returns the job status /tts: post: consumes: @@ -757,6 +884,32 @@ paths: type: string summary: Generates audio from the input text. /v1/assistants: + get: + parameters: + - description: Limit the number of assistants returned + in: query + name: limit + type: integer + - description: Order of assistants returned + in: query + name: order + type: string + - description: Return assistants created after the given ID + in: query + name: after + type: string + - description: Return assistants created before the given ID + in: query + name: before + type: string + responses: + "200": + description: Response + schema: + items: + $ref: '#/definitions/openai.Assistant' + type: array + summary: List available assistents post: parameters: - description: query params @@ -771,6 +924,21 @@ paths: schema: $ref: '#/definitions/openai.Assistant' summary: Create an assistant with a model and instructions. + /v1/assistants/{assistant_id}: + delete: + responses: + "200": + description: Response + schema: + $ref: '#/definitions/schema.DeleteAssistantResponse' + summary: Delete assistents + get: + responses: + "200": + description: Response + schema: + $ref: '#/definitions/openai.Assistant' + summary: Get assistent data /v1/audio/speech: post: consumes: @@ -843,6 +1011,21 @@ paths: schema: $ref: '#/definitions/schema.OpenAIResponse' summary: Generate completions for a given prompt and model. + /v1/edits: + post: + parameters: + - description: query params + in: body + name: request + required: true + schema: + $ref: '#/definitions/schema.OpenAIRequest' + responses: + "200": + description: Response + schema: + $ref: '#/definitions/schema.OpenAIResponse' + summary: OpenAI edit endpoint /v1/embeddings: post: parameters: @@ -859,6 +1042,14 @@ paths: $ref: '#/definitions/schema.OpenAIResponse' summary: Get a vector representation of a given input that can be easily consumed by machine learning models and algorithms. + /v1/files: + get: + responses: + "200": + description: Response + schema: + $ref: '#/definitions/schema.ListFiles' + summary: List files. /v1/files/{file_id}: delete: responses: @@ -872,7 +1063,7 @@ paths: "200": description: Response schema: - $ref: '#/definitions/openai.File' + $ref: '#/definitions/schema.File' summary: Returns information about a specific file. /v1/files/{file_id}/content: get: