ExternalVendorCode/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2024-12-18 20:27:57 +00:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

fix: pkg/downloader should respect basePath for file:// urls (#2481)

Browse Source 
* pass basePath down to pkg/downloader

Signed-off-by: Dave Lee <dave@gray101.com>

* enforce

Signed-off-by: Dave Lee <dave@gray101.com>

---------

Signed-off-by: Dave Lee <dave@gray101.com>
This commit is contained in:
Dave 2024-06-04 10:32:47 -04:00 committed by GitHub
parent bdd6769b2d
commit 2fc6fe806b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
core/http/app_test.go
View File

@ -73,7 +73,8 @@ func getModelStatus(url string) (response map[string]interface{}) {
}
func getModels(url string) (response []gallery.GalleryModel) {
downloader.GetURI(url, func(url string, i []byte) error {
// TODO: No tests currently seem to exercise file:// urls. Fix?
downloader.GetURI(url, "", func(url string, i []byte) error {
// Unmarshal YAML data into a struct
return json.Unmarshal(i, &response)
})

2
core/services/gallery.go
View File

@ -32,7 +32,7 @@ func NewGalleryService(modelPath string) *GalleryService {
func prepareModel(modelPath string, req gallery.GalleryModel, cl *config.BackendConfigLoader, downloadStatus func(string, string, string, float64)) error {
config, err := gallery.GetGalleryConfigFromURL(req.URL)
config, err := gallery.GetGalleryConfigFromURL(req.URL, modelPath)
if err != nil {
return err
}

4
embedded/embedded.go
View File

@ -36,10 +36,10 @@ func init() {
}
}
func GetRemoteLibraryShorteners(url string) (map[string]string, error) {
func GetRemoteLibraryShorteners(url string, basePath string) (map[string]string, error) {
remoteLibrary := map[string]string{}
err := downloader.GetURI(url, func(_ string, i []byte) error {
err := downloader.GetURI(url, basePath, func(_ string, i []byte) error {
return yaml.Unmarshal(i, &remoteLibrary)
})
if err != nil {

7
pkg/downloader/uri.go
View File

@ -23,7 +23,7 @@ const (
GithubURI2 = "github://"
)
func GetURI(url string, f func(url string, i []byte) error) error {
func GetURI(url string, basePath string, f func(url string, i []byte) error) error {
url = ConvertURL(url)
if strings.HasPrefix(url, "file://") {
@ -33,6 +33,11 @@ func GetURI(url string, f func(url string, i []byte) error) error {
if err != nil {
return err
}
// Check if the local file is rooted in basePath
err = utils.VerifyPath(resolvedFile, basePath)
if err != nil {
return err
}
// Read the response body
body, err := os.ReadFile(resolvedFile)
if err != nil {

6
pkg/downloader/uri_test.go
View File

@ -10,7 +10,7 @@ var _ = Describe("Gallery API tests", func() {
Context("URI", func() {
It("parses github with a branch", func() {
Expect(
GetURI("github:go-skynet/model-gallery/gpt4all-j.yaml", func(url string, i []byte) error {
GetURI("github:go-skynet/model-gallery/gpt4all-j.yaml", "", func(url string, i []byte) error {
Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml"))
return nil
}),
@ -18,7 +18,7 @@ var _ = Describe("Gallery API tests", func() {
})
It("parses github without a branch", func() {
Expect(
GetURI("github:go-skynet/model-gallery/gpt4all-j.yaml@main", func(url string, i []byte) error {
GetURI("github:go-skynet/model-gallery/gpt4all-j.yaml@main", "", func(url string, i []byte) error {
Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml"))
return nil
}),
@ -26,7 +26,7 @@ var _ = Describe("Gallery API tests", func() {
})
It("parses github with urls", func() {
Expect(
GetURI("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml", func(url string, i []byte) error {
GetURI("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml", "", func(url string, i []byte) error {
Expect(url).To(Equal("https://raw.githubusercontent.com/go-skynet/model-gallery/main/gpt4all-j.yaml"))
return nil
}),

10
pkg/gallery/gallery.go
View File

@ -27,7 +27,7 @@ func InstallModelFromGallery(galleries []Gallery, name string, basePath string,
if len(model.URL) > 0 {
var err error
config, err = GetGalleryConfigFromURL(model.URL)
config, err = GetGalleryConfigFromURL(model.URL, basePath)
if err != nil {
return err
}
@ -142,9 +142,9 @@ func AvailableGalleryModels(galleries []Gallery, basePath string) ([]*GalleryMod
return models, nil
}
func findGalleryURLFromReferenceURL(url string) (string, error) {
func findGalleryURLFromReferenceURL(url string, basePath string) (string, error) {
var refFile string
err := downloader.GetURI(url, func(url string, d []byte) error {
err := downloader.GetURI(url, basePath, func(url string, d []byte) error {
refFile = string(d)
if len(refFile) == 0 {
return fmt.Errorf("invalid reference file at url %s: %s", url, d)
@ -161,13 +161,13 @@ func getGalleryModels(gallery Gallery, basePath string) ([]*GalleryModel, error)
if strings.HasSuffix(gallery.URL, ".ref") {
var err error
gallery.URL, err = findGalleryURLFromReferenceURL(gallery.URL)
gallery.URL, err = findGalleryURLFromReferenceURL(gallery.URL, basePath)
if err != nil {
return models, err
}
}
err := downloader.GetURI(gallery.URL, func(url string, d []byte) error {
err := downloader.GetURI(gallery.URL, basePath, func(url string, d []byte) error {
return yaml.Unmarshal(d, &models)
})
if err != nil {

4
pkg/gallery/models.go
View File

@ -63,9 +63,9 @@ type PromptTemplate struct {
Content string `yaml:"content"`
}
func GetGalleryConfigFromURL(url string) (Config, error) {
func GetGalleryConfigFromURL(url string, basePath string) (Config, error) {
var config Config
err := downloader.GetURI(url, func(url string, d []byte) error {
err := downloader.GetURI(url, basePath, func(url string, d []byte) error {
return yaml.Unmarshal(d, &config)
})
if err != nil {

2
pkg/gallery/request_test.go
View File

@ -10,7 +10,7 @@ var _ = Describe("Gallery API tests", func() {
Context("requests", func() {
It("parses github with a branch", func() {
req := GalleryModel{URL: "github:go-skynet/model-gallery/gpt4all-j.yaml@main"}
e, err := GetGalleryConfigFromURL(req.URL)
e, err := GetGalleryConfigFromURL(req.URL, "")
Expect(err).ToNot(HaveOccurred())
Expect(e.Name).To(Equal("gpt4all-j"))
})

2
pkg/startup/model_preload.go
View File

@ -20,7 +20,7 @@ func PreloadModelsConfigurations(modelLibraryURL string, modelPath string, model
// As a best effort, try to resolve the model from the remote library
// if it's not resolved we try with the other method below
if modelLibraryURL != "" {
lib, err := embedded.GetRemoteLibraryShorteners(modelLibraryURL)
lib, err := embedded.GetRemoteLibraryShorteners(modelLibraryURL, modelPath)
if err == nil {
if lib[url] != "" {
log.Debug().Msgf("[startup] model configuration is defined remotely: %s (%s)", url, lib[url])