From 44a70457322c734eed43a7f8dc90ef4fe0ee7bb1 Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 11 Oct 2023 12:18:12 -0400 Subject: [PATCH] Feats: bruno example, gallery improvements for new scraper (#1161) This PR bundles together two unrelated features: 1. Model Gallery improvements - specifically, the ability to follow ".ref" gallery links (which I made up for this specific application) to an actual gallery yaml file (in order to have stable URLs) and the ability to load self-contained configurations, rather than always using a base.yaml + overrides. This is groundwork for my python-based huggingface scraper. 2. A while ago I introduced some Insomnia request templates for people to use. Unfortunately, Insomnia has decided to tank their product... So I've personally switched to using [bruno](https://github.com/usebruno/bruno/). Corresponding equivalent files that I use for my testing have been added. Just open the folder from bruno and everything will work. No import process required. --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- .../backend monitor/backend monitor.bru | 11 ++++ .../backend monitor/backend-shutdown.bru | 21 ++++++++ .../bruno/LocalAI Test Requests/bruno.json | 5 ++ .../environments/localhost.bru | 6 +++ .../LocalAI Test Requests/get models list.bru | 11 ++++ .../llm text/-completions.bru | 24 +++++++++ .../LocalAI Test Requests/llm text/-edits.bru | 23 +++++++++ .../llm text/-embeddings.bru | 22 ++++++++ .../chat completion -simple- 1 message-.bru | 24 +++++++++ .../llm text/chat/chat-completions -long-.bru | 29 +++++++++++ .../chat/chat-completions -stream-.bru | 25 +++++++++ .../model gallery/add model gallery.bru | 22 ++++++++ .../model gallery/delete model gallery.bru | 21 ++++++++ .../list MODELS in galleries.bru | 11 ++++ .../model gallery/list model GALLERIES.bru | 11 ++++ .../model gallery apply -gist-.bru | 21 ++++++++ .../model gallery/model gallery apply.bru | 22 ++++++++ .../bruno/LocalAI Test Requests/tts/-tts.bru | 22 ++++++++ pkg/gallery/gallery.go | 51 +++++++++++++++++-- pkg/gallery/models.go | 1 + pkg/gallery/request.go | 9 ++-- 21 files changed, 383 insertions(+), 9 deletions(-) create mode 100644 examples/bruno/LocalAI Test Requests/backend monitor/backend monitor.bru create mode 100644 examples/bruno/LocalAI Test Requests/backend monitor/backend-shutdown.bru create mode 100644 examples/bruno/LocalAI Test Requests/bruno.json create mode 100644 examples/bruno/LocalAI Test Requests/environments/localhost.bru create mode 100644 examples/bruno/LocalAI Test Requests/get models list.bru create mode 100644 examples/bruno/LocalAI Test Requests/llm text/-completions.bru create mode 100644 examples/bruno/LocalAI Test Requests/llm text/-edits.bru create mode 100644 examples/bruno/LocalAI Test Requests/llm text/-embeddings.bru create mode 100644 examples/bruno/LocalAI Test Requests/llm text/chat/chat completion -simple- 1 message-.bru create mode 100644 examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -long-.bru create mode 100644 examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -stream-.bru create mode 100644 examples/bruno/LocalAI Test Requests/model gallery/add model gallery.bru create mode 100644 examples/bruno/LocalAI Test Requests/model gallery/delete model gallery.bru create mode 100644 examples/bruno/LocalAI Test Requests/model gallery/list MODELS in galleries.bru create mode 100644 examples/bruno/LocalAI Test Requests/model gallery/list model GALLERIES.bru create mode 100644 examples/bruno/LocalAI Test Requests/model gallery/model gallery apply -gist-.bru create mode 100644 examples/bruno/LocalAI Test Requests/model gallery/model gallery apply.bru create mode 100644 examples/bruno/LocalAI Test Requests/tts/-tts.bru diff --git a/examples/bruno/LocalAI Test Requests/backend monitor/backend monitor.bru b/examples/bruno/LocalAI Test Requests/backend monitor/backend monitor.bru new file mode 100644 index 00000000..e3f72134 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/backend monitor/backend monitor.bru @@ -0,0 +1,11 @@ +meta { + name: backend monitor + type: http + seq: 4 +} + +get { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/backend/monitor + body: none + auth: none +} diff --git a/examples/bruno/LocalAI Test Requests/backend monitor/backend-shutdown.bru b/examples/bruno/LocalAI Test Requests/backend monitor/backend-shutdown.bru new file mode 100644 index 00000000..f75f259a --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/backend monitor/backend-shutdown.bru @@ -0,0 +1,21 @@ +meta { + name: backend-shutdown + type: http + seq: 3 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/backend/shutdown + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}" + } +} diff --git a/examples/bruno/LocalAI Test Requests/bruno.json b/examples/bruno/LocalAI Test Requests/bruno.json new file mode 100644 index 00000000..9491e3a5 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/bruno.json @@ -0,0 +1,5 @@ +{ + "version": "1", + "name": "LocalAI Test Requests", + "type": "collection" +} \ No newline at end of file diff --git a/examples/bruno/LocalAI Test Requests/environments/localhost.bru b/examples/bruno/LocalAI Test Requests/environments/localhost.bru new file mode 100644 index 00000000..fb97edb2 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/environments/localhost.bru @@ -0,0 +1,6 @@ +vars { + HOST: localhost + PORT: 8080 + DEFAULT_MODEL: gpt-3.5-turbo + PROTOCOL: http:// +} diff --git a/examples/bruno/LocalAI Test Requests/get models list.bru b/examples/bruno/LocalAI Test Requests/get models list.bru new file mode 100644 index 00000000..4bf1628f --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/get models list.bru @@ -0,0 +1,11 @@ +meta { + name: get models list + type: http + seq: 2 +} + +get { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models + body: none + auth: none +} diff --git a/examples/bruno/LocalAI Test Requests/llm text/-completions.bru b/examples/bruno/LocalAI Test Requests/llm text/-completions.bru new file mode 100644 index 00000000..6e16a244 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/llm text/-completions.bru @@ -0,0 +1,24 @@ +meta { + name: -completions + type: http + seq: 4 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/completions + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "prompt": "function downloadFile(string url, string outputPath) {", + "max_tokens": 256, + "temperature": 0.5 + } +} diff --git a/examples/bruno/LocalAI Test Requests/llm text/-edits.bru b/examples/bruno/LocalAI Test Requests/llm text/-edits.bru new file mode 100644 index 00000000..838afa27 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/llm text/-edits.bru @@ -0,0 +1,23 @@ +meta { + name: -edits + type: http + seq: 5 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/edits + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "input": "What day of the wek is it?", + "instruction": "Fix the spelling mistakes" + } +} diff --git a/examples/bruno/LocalAI Test Requests/llm text/-embeddings.bru b/examples/bruno/LocalAI Test Requests/llm text/-embeddings.bru new file mode 100644 index 00000000..a3045df2 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/llm text/-embeddings.bru @@ -0,0 +1,22 @@ +meta { + name: -embeddings + type: http + seq: 6 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/embeddings + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "input": "A STRANGE GAME.\nTHE ONLY WINNING MOVE IS NOT TO PLAY.\n\nHOW ABOUT A NICE GAME OF CHESS?" + } +} diff --git a/examples/bruno/LocalAI Test Requests/llm text/chat/chat completion -simple- 1 message-.bru b/examples/bruno/LocalAI Test Requests/llm text/chat/chat completion -simple- 1 message-.bru new file mode 100644 index 00000000..cf51bb7c --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/llm text/chat/chat completion -simple- 1 message-.bru @@ -0,0 +1,24 @@ +meta { + name: chat completion -simple- 1 message- + type: http + seq: 4 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/chat/completions + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "messages": [{"role": "user", "content": "How could one use friction to cook an egg?"}], + "max_tokens": 256, + "temperature": 0.2 + } +} diff --git a/examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -long-.bru b/examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -long-.bru new file mode 100644 index 00000000..5d9a49a0 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -long-.bru @@ -0,0 +1,29 @@ +meta { + name: chat-completions -long- + type: http + seq: 5 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/chat/completions + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "messages": [{"role": "system", "content": "You are a helpful, intelligent pirate assistant. Always answer as helpfully as possible, while being a pirate and using pirate language. If a question does not make any sense, or is not factually coherent, explain why instead of answering something not correct. If you don't know the answer to a question, please don't share false information."}, + {"role": "user", "content": "How could one use electricity to cook an egg?"}, + {"role": "assistant", + "content": "Shiver me timbers! Using electricity to cook an egg? Well, matey, I reckon that be a right curious idea! *adjusts spectacles* Now, I ain't sure if this be possible, but I'll do me best to help ye out.\n\nFirst things first, ye gotta understand that electricity be a powerful force, and it can't just cook an egg on its own. Ye see, electricity be like a mighty wind that can make things happen, but it needs somethin' to work with. So, if ye want to use electricity to cook an egg, ye gotta find a way to harness that power and make it do the cookin'.\n\nNow, I know what ye might be thinkin': \"How do I harness the power of electricity to cook an egg?\" Well, matey, there be a few ways to do it. One way be to use a special device called an \"electric frying pan.\" This be a pan that has a built-in heating element that gets hot when ye plug it into a wall socket. When the element gets hot, ye can crack an egg into the pan and watch as it cook" + }, + {"role": "user", "content": "I don't have one of those, just a raw wire and plenty of power! How do we get it done?"}], + "max_tokens": 1024, + "temperature": 0.5 + } +} diff --git a/examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -stream-.bru b/examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -stream-.bru new file mode 100644 index 00000000..cf0539fa --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/llm text/chat/chat-completions -stream-.bru @@ -0,0 +1,25 @@ +meta { + name: chat-completions -stream- + type: http + seq: 6 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/chat/completions + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "messages": [{"role": "user", "content": "Explain how I can set sail on the ocean using only power generated by seagulls?"}], + "max_tokens": 256, + "temperature": 0.9, + "stream": true + } +} diff --git a/examples/bruno/LocalAI Test Requests/model gallery/add model gallery.bru b/examples/bruno/LocalAI Test Requests/model gallery/add model gallery.bru new file mode 100644 index 00000000..1463160f --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/model gallery/add model gallery.bru @@ -0,0 +1,22 @@ +meta { + name: add model gallery + type: http + seq: 10 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "url": "file:///home/dave/projects/model-gallery/huggingface/TheBloke__CodeLlama-7B-Instruct-GGML.yaml", + "name": "test" + } +} diff --git a/examples/bruno/LocalAI Test Requests/model gallery/delete model gallery.bru b/examples/bruno/LocalAI Test Requests/model gallery/delete model gallery.bru new file mode 100644 index 00000000..3e211aa6 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/model gallery/delete model gallery.bru @@ -0,0 +1,21 @@ +meta { + name: delete model gallery + type: http + seq: 11 +} + +delete { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "name": "test" + } +} diff --git a/examples/bruno/LocalAI Test Requests/model gallery/list MODELS in galleries.bru b/examples/bruno/LocalAI Test Requests/model gallery/list MODELS in galleries.bru new file mode 100644 index 00000000..1d866f8a --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/model gallery/list MODELS in galleries.bru @@ -0,0 +1,11 @@ +meta { + name: list MODELS in galleries + type: http + seq: 7 +} + +get { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/available + body: none + auth: none +} diff --git a/examples/bruno/LocalAI Test Requests/model gallery/list model GALLERIES.bru b/examples/bruno/LocalAI Test Requests/model gallery/list model GALLERIES.bru new file mode 100644 index 00000000..f7664672 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/model gallery/list model GALLERIES.bru @@ -0,0 +1,11 @@ +meta { + name: list model GALLERIES + type: http + seq: 8 +} + +get { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/galleries + body: none + auth: none +} diff --git a/examples/bruno/LocalAI Test Requests/model gallery/model gallery apply -gist-.bru b/examples/bruno/LocalAI Test Requests/model gallery/model gallery apply -gist-.bru new file mode 100644 index 00000000..d94c75a2 --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/model gallery/model gallery apply -gist-.bru @@ -0,0 +1,21 @@ +meta { + name: model gallery apply -gist- + type: http + seq: 12 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/apply + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "id": "TheBloke__CodeLlama-7B-Instruct-GGML__codellama-7b-instruct.ggmlv3.Q2_K.bin" + } +} diff --git a/examples/bruno/LocalAI Test Requests/model gallery/model gallery apply.bru b/examples/bruno/LocalAI Test Requests/model gallery/model gallery apply.bru new file mode 100644 index 00000000..aa308e1e --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/model gallery/model gallery apply.bru @@ -0,0 +1,22 @@ +meta { + name: model gallery apply + type: http + seq: 9 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/models/apply + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "id": "dave@TheBloke__CodeLlama-7B-Instruct-GGML__codellama-7b-instruct.ggmlv3.Q3_K_S.bin", + "name": "codellama7b" + } +} diff --git a/examples/bruno/LocalAI Test Requests/tts/-tts.bru b/examples/bruno/LocalAI Test Requests/tts/-tts.bru new file mode 100644 index 00000000..643f2ace --- /dev/null +++ b/examples/bruno/LocalAI Test Requests/tts/-tts.bru @@ -0,0 +1,22 @@ +meta { + name: -tts + type: http + seq: 2 +} + +post { + url: {{PROTOCOL}}{{HOST}}:{{PORT}}/tts + body: json + auth: none +} + +headers { + Content-Type: application/json +} + +body:json { + { + "model": "{{DEFAULT_MODEL}}", + "input": "A STRANGE GAME.\nTHE ONLY WINNING MOVE IS NOT TO PLAY.\n\nHOW ABOUT A NICE GAME OF CHESS?" + } +} diff --git a/pkg/gallery/gallery.go b/pkg/gallery/gallery.go index eacd9aa1..1fd96172 100644 --- a/pkg/gallery/gallery.go +++ b/pkg/gallery/gallery.go @@ -21,9 +21,31 @@ func InstallModelFromGallery(galleries []Gallery, name string, basePath string, applyModel := func(model *GalleryModel) error { name = strings.ReplaceAll(name, string(os.PathSeparator), "__") - config, err := GetGalleryConfigFromURL(model.URL) - if err != nil { - return err + var config Config + + if len(model.URL) > 0 { + var err error + config, err = GetGalleryConfigFromURL(model.URL) + if err != nil { + return err + } + } else if len(model.ConfigFile) > 0 { + // TODO: is this worse than using the override method with a blank cfg yaml? + reYamlConfig, err := yaml.Marshal(model.ConfigFile) + if err != nil { + return err + } + config = Config{ + ConfigFile: string(reYamlConfig), + Description: model.Description, + License: model.License, + URLs: model.URLs, + Name: model.Name, + Files: make([]File, 0), // Real values get added below, must be blank + // Prompt Template Skipped for now - I expect in this mode that they will be delivered as files. + } + } else { + return fmt.Errorf("invalid gallery model %+v", model) } installName := model.Name @@ -115,13 +137,36 @@ func AvailableGalleryModels(galleries []Gallery, basePath string) ([]*GalleryMod return models, nil } +func findGalleryURLFromReferenceURL(url string) (string, error) { + var refFile string + err := utils.GetURI(url, func(url string, d []byte) error { + refFile = string(d) + if len(refFile) == 0 { + return fmt.Errorf("invalid reference file at url %s: %s", url, d) + } + cutPoint := strings.LastIndex(url, "/") + refFile = url[:cutPoint+1] + refFile + return nil + }) + return refFile, err +} + func getGalleryModels(gallery Gallery, basePath string) ([]*GalleryModel, error) { var models []*GalleryModel = []*GalleryModel{} + if strings.HasSuffix(gallery.URL, ".ref") { + var err error + gallery.URL, err = findGalleryURLFromReferenceURL(gallery.URL) + if err != nil { + return models, err + } + } + err := utils.GetURI(gallery.URL, func(url string, d []byte) error { return yaml.Unmarshal(d, &models) }) if err != nil { + return models, err } diff --git a/pkg/gallery/models.go b/pkg/gallery/models.go index 4295a995..db68f525 100644 --- a/pkg/gallery/models.go +++ b/pkg/gallery/models.go @@ -71,6 +71,7 @@ func GetGalleryConfigFromURL(url string) (Config, error) { return yaml.Unmarshal(d, &config) }) if err != nil { + log.Error().Msgf("GetGalleryConfigFromURL error for url %s\n%s", url, err.Error()) return config, err } return config, nil diff --git a/pkg/gallery/request.go b/pkg/gallery/request.go index 030ee16b..d14f3f5c 100644 --- a/pkg/gallery/request.go +++ b/pkg/gallery/request.go @@ -11,8 +11,9 @@ type GalleryModel struct { URLs []string `json:"urls,omitempty" yaml:"urls,omitempty"` Icon string `json:"icon,omitempty" yaml:"icon,omitempty"` Tags []string `json:"tags,omitempty" yaml:"tags,omitempty"` - - // Overrides are used to override the configuration of the model + // config_file is read in the situation where URL is blank - and therefore this is a base config. + ConfigFile map[string]interface{} `json:"config_file,omitempty" yaml:"config_file,omitempty"` + // Overrides are used to override the configuration of the model located at URL Overrides map[string]interface{} `json:"overrides,omitempty" yaml:"overrides,omitempty"` // AdditionalFiles are used to add additional files to the model AdditionalFiles []File `json:"files,omitempty" yaml:"files,omitempty"` @@ -21,7 +22,3 @@ type GalleryModel struct { // Installed is used to indicate if the model is installed or not Installed bool `json:"installed,omitempty" yaml:"installed,omitempty"` } - -const ( - githubURI = "github:" -)