ExternalVendorCode/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2025-06-13 04:28:10 +00:00
Code Issues Actions 14 Packages Projects Releases Wiki Activity

feat: cancel stream generation if client disappears (#792)

Browse Source
This commit is contained in:
Aman Gupta Karmani
2023-07-24 14:10:54 -07:00
committed by GitHub
parent 72e3e236de
commit 12fe0932c4
12 changed files with 37 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
api/openai/inference.go
View File

@ -7,7 +7,8 @@ import (
model "github.com/go-skynet/LocalAI/pkg/model"
)
func ComputeChoices(predInput string, n int, config *config.Config, o *options.Option, loader *model.ModelLoader, cb func(string, *[]Choice), tokenCallback func(string) bool) ([]Choice, error) {
func ComputeChoices(req *OpenAIRequest, predInput string, config *config.Config, o *options.Option, loader *model.ModelLoader, cb func(string, *[]Choice), tokenCallback func(string) bool) ([]Choice, error) {
n := req.N
result := []Choice{}
if n == 0 {
@ -15,7 +16,7 @@ func ComputeChoices(predInput string, n int, config *config.Config, o *options.O
}
// get the model function to call for the result
predFunc, err := backend.ModelInference(predInput, loader, *config, o, tokenCallback)
predFunc, err := backend.ModelInference(req.Context, predInput, loader, *config, o, tokenCallback)
if err != nil {
return result, err
}