diff --git a/Makefile b/Makefile index faa82d6b..944cad37 100644 --- a/Makefile +++ b/Makefile @@ -22,10 +22,6 @@ PIPER_VERSION?=e10ca041a885d4a8f3871d52924b47792d5e5aa0 STABLEDIFFUSION_REPO?=https://github.com/mudler/go-stable-diffusion STABLEDIFFUSION_VERSION?=4a3cd6aeae6f66ee57eae9a0075f8c58c3a6a38f -# tinydream version -TINYDREAM_REPO?=https://github.com/M0Rf30/go-tiny-dream -TINYDREAM_VERSION?=c04fa463ace9d9a6464313aa5f9cd0f953b6c057 - # bark.cpp BARKCPP_REPO?=https://github.com/PABannier/bark.cpp.git BARKCPP_VERSION?=v1.0.0 @@ -188,11 +184,6 @@ ifeq ($(findstring stablediffusion,$(GO_TAGS)),stablediffusion) OPTIONAL_GRPC+=backend-assets/grpc/stablediffusion endif -ifeq ($(findstring tinydream,$(GO_TAGS)),tinydream) -# OPTIONAL_TARGETS+=go-tiny-dream/libtinydream.a - OPTIONAL_GRPC+=backend-assets/grpc/tinydream -endif - ifeq ($(findstring tts,$(GO_TAGS)),tts) # OPTIONAL_TARGETS+=go-piper/libpiper_binding.a # OPTIONAL_TARGETS+=backend-assets/espeak-ng-data @@ -327,19 +318,6 @@ else mv backend-assets/lib/libonnxruntime.so.$(ONNX_VERSION) backend-assets/lib/libonnxruntime.so.1 endif -## tiny-dream -sources/go-tiny-dream: - mkdir -p sources/go-tiny-dream - cd sources/go-tiny-dream && \ - git init && \ - git remote add origin $(TINYDREAM_REPO) && \ - git fetch origin && \ - git checkout $(TINYDREAM_VERSION) && \ - git submodule update --init --recursive --depth 1 --single-branch - -sources/go-tiny-dream/libtinydream.a: sources/go-tiny-dream - $(MAKE) -C sources/go-tiny-dream libtinydream.a - ## whisper sources/whisper.cpp: mkdir -p sources/whisper.cpp @@ -353,12 +331,11 @@ sources/whisper.cpp: sources/whisper.cpp/libwhisper.a: sources/whisper.cpp cd sources/whisper.cpp && $(MAKE) libwhisper.a libggml.a -get-sources: sources/go-llama.cpp sources/go-piper sources/stablediffusion-ggml.cpp sources/bark.cpp sources/whisper.cpp sources/go-stable-diffusion sources/go-tiny-dream backend/cpp/llama/llama.cpp +get-sources: sources/go-llama.cpp sources/go-piper sources/stablediffusion-ggml.cpp sources/bark.cpp sources/whisper.cpp sources/go-stable-diffusion backend/cpp/llama/llama.cpp replace: $(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp=$(CURDIR)/sources/whisper.cpp $(GOCMD) mod edit -replace github.com/ggerganov/whisper.cpp/bindings/go=$(CURDIR)/sources/whisper.cpp/bindings/go - $(GOCMD) mod edit -replace github.com/M0Rf30/go-tiny-dream=$(CURDIR)/sources/go-tiny-dream $(GOCMD) mod edit -replace github.com/mudler/go-piper=$(CURDIR)/sources/go-piper $(GOCMD) mod edit -replace github.com/mudler/go-stable-diffusion=$(CURDIR)/sources/go-stable-diffusion $(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(CURDIR)/sources/go-llama.cpp @@ -366,7 +343,6 @@ replace: dropreplace: $(GOCMD) mod edit -dropreplace github.com/ggerganov/whisper.cpp $(GOCMD) mod edit -dropreplace github.com/ggerganov/whisper.cpp/bindings/go - $(GOCMD) mod edit -dropreplace github.com/M0Rf30/go-tiny-dream $(GOCMD) mod edit -dropreplace github.com/mudler/go-piper $(GOCMD) mod edit -dropreplace github.com/mudler/go-stable-diffusion $(GOCMD) mod edit -dropreplace github.com/go-skynet/go-llama.cpp @@ -381,7 +357,6 @@ rebuild: ## Rebuilds the project $(MAKE) -C sources/whisper.cpp clean $(MAKE) -C sources/go-stable-diffusion clean $(MAKE) -C sources/go-piper clean - $(MAKE) -C sources/go-tiny-dream clean $(MAKE) build prepare: prepare-sources $(OPTIONAL_TARGETS) @@ -855,13 +830,6 @@ ifneq ($(UPX),) $(UPX) backend-assets/grpc/silero-vad endif -backend-assets/grpc/tinydream: sources/go-tiny-dream sources/go-tiny-dream/libtinydream.a backend-assets/grpc - CGO_LDFLAGS="$(CGO_LDFLAGS)" LIBRARY_PATH=$(CURDIR)/go-tiny-dream \ - $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/tinydream ./backend/go/image/tinydream -ifneq ($(UPX),) - $(UPX) backend-assets/grpc/tinydream -endif - backend-assets/grpc/whisper: sources/whisper.cpp sources/whisper.cpp/libwhisper.a backend-assets/grpc CGO_LDFLAGS="$(CGO_LDFLAGS) $(CGO_LDFLAGS_WHISPER)" C_INCLUDE_PATH="$(CURDIR)/sources/whisper.cpp/include:$(CURDIR)/sources/whisper.cpp/ggml/include" LIBRARY_PATH=$(CURDIR)/sources/whisper.cpp \ $(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./backend/go/transcribe/whisper diff --git a/backend/go/image/tinydream/main.go b/backend/go/image/tinydream/main.go deleted file mode 100644 index ae259fa7..00000000 --- a/backend/go/image/tinydream/main.go +++ /dev/null @@ -1,21 +0,0 @@ -package main - -// Note: this is started internally by LocalAI and a server is allocated for each model - -import ( - "flag" - - grpc "github.com/mudler/LocalAI/pkg/grpc" -) - -var ( - addr = flag.String("addr", "localhost:50051", "the address to connect to") -) - -func main() { - flag.Parse() - - if err := grpc.StartServer(*addr, &Image{}); err != nil { - panic(err) - } -} diff --git a/backend/go/image/tinydream/tinydream.go b/backend/go/image/tinydream/tinydream.go deleted file mode 100644 index ad364c47..00000000 --- a/backend/go/image/tinydream/tinydream.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -// This is a wrapper to statisfy the GRPC service interface -// It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc) -import ( - "github.com/mudler/LocalAI/pkg/grpc/base" - pb "github.com/mudler/LocalAI/pkg/grpc/proto" - "github.com/mudler/LocalAI/pkg/tinydream" -) - -type Image struct { - base.SingleThread - tinydream *tinydream.TinyDream -} - -func (image *Image) Load(opts *pb.ModelOptions) error { - var err error - // Note: the Model here is a path to a directory containing the model files - image.tinydream, err = tinydream.New(opts.ModelFile) - return err -} - -func (image *Image) GenerateImage(opts *pb.GenerateImageRequest) error { - return image.tinydream.GenerateImage( - int(opts.Height), - int(opts.Width), - int(opts.Step), - int(opts.Seed), - opts.PositivePrompt, - opts.NegativePrompt, - opts.Dst) -} diff --git a/core/config/backend_config.go b/core/config/backend_config.go index bb2fa643..a488f2a0 100644 --- a/core/config/backend_config.go +++ b/core/config/backend_config.go @@ -515,7 +515,7 @@ func (c *BackendConfig) GuessUsecases(u BackendConfigUsecases) bool { } } if (u & FLAG_IMAGE) == FLAG_IMAGE { - imageBackends := []string{"diffusers", "tinydream", "stablediffusion"} + imageBackends := []string{"diffusers", "stablediffusion"} if !slices.Contains(imageBackends, c.Backend) { return false } diff --git a/core/http/endpoints/openai/image.go b/core/http/endpoints/openai/image.go index 3fdb64d4..baaecd4e 100644 --- a/core/http/endpoints/openai/image.go +++ b/core/http/endpoints/openai/image.go @@ -130,8 +130,6 @@ func ImageEndpoint(cl *config.BackendConfigLoader, ml *model.ModelLoader, appCon switch config.Backend { case "stablediffusion": config.Backend = model.StableDiffusionBackend - case "tinydream": - config.Backend = model.TinyDreamBackend case "": config.Backend = model.StableDiffusionBackend } diff --git a/docs/content/docs/getting-started/build.md b/docs/content/docs/getting-started/build.md index f21a5b48..9fff1989 100644 --- a/docs/content/docs/getting-started/build.md +++ b/docs/content/docs/getting-started/build.md @@ -88,7 +88,7 @@ Here is the list of the variables available that can be used to customize the bu | Variable | Default | Description | | ---------------------| ------- | ----------- | | `BUILD_TYPE` | None | Build type. Available: `cublas`, `openblas`, `clblas`, `metal`,`hipblas`, `sycl_f16`, `sycl_f32` | -| `GO_TAGS` | `tts stablediffusion` | Go tags. Available: `stablediffusion`, `tts`, `tinydream` | +| `GO_TAGS` | `tts stablediffusion` | Go tags. Available: `stablediffusion`, `tts` | | `CLBLAST_DIR` | | Specify a CLBlast directory | | `CUDA_LIBPATH` | | Specify a CUDA library path | | `BUILD_API_ONLY` | false | Set to true to build only the API (no backends will be built) | @@ -202,7 +202,7 @@ make build **Requirements**: OpenCV, Gomp -Image generation requires `GO_TAGS=stablediffusion` or `GO_TAGS=tinydream` to be set during build: +Image generation requires `GO_TAGS=stablediffusion` to be set during build: ``` make GO_TAGS=stablediffusion build diff --git a/docs/content/docs/getting-started/container-images.md b/docs/content/docs/getting-started/container-images.md index 967fc28b..64f6dbc9 100644 --- a/docs/content/docs/getting-started/container-images.md +++ b/docs/content/docs/getting-started/container-images.md @@ -16,7 +16,7 @@ For GPU Acceleration support for Nvidia video graphic cards, use the Nvidia/CUDA **Available Images Types**: -- Images ending with `-core` are smaller images without predownload python dependencies. Use these images if you plan to use `llama.cpp`, `stablediffusion-ncn`, `tinydream` or `rwkv` backends - if you are not sure which one to use, do **not** use these images. +- Images ending with `-core` are smaller images without predownload python dependencies. Use these images if you plan to use `llama.cpp`, `stablediffusion-ncn` or `rwkv` backends - if you are not sure which one to use, do **not** use these images. - Images containing the `aio` tag are all-in-one images with all the features enabled, and come with an opinionated set of configuration. - FFMpeg is **not** included in the default images due to [its licensing](https://www.ffmpeg.org/legal.html). If you need FFMpeg, use the images ending with `-ffmpeg`. Note that `ffmpeg` is needed in case of using `audio-to-text` LocalAI's features. - If using old and outdated CPUs and no GPUs you might need to set `REBUILD` to `true` as environment variable along with options to disable the flags which your CPU does not support, however note that inference will perform poorly and slow. See also [flagset compatibility]({{%relref "docs/getting-started/build#cpu-flagset-compatibility" %}}). diff --git a/docs/content/docs/reference/compatibility-table.md b/docs/content/docs/reference/compatibility-table.md index 7056f4a5..d2f4d8ac 100644 --- a/docs/content/docs/reference/compatibility-table.md +++ b/docs/content/docs/reference/compatibility-table.md @@ -32,7 +32,7 @@ LocalAI will attempt to automatically load models which are not explicitly confi | `mamba` | Mamba models architecture | yes | GPT | no | no | CPU/CUDA | | `exllama2` | GPTQ | yes | GPT only | no | no | N/A | | `transformers-musicgen` | | no | Audio generation | no | no | N/A | -| [tinydream](https://github.com/symisc/tiny-dream#tiny-dreaman-embedded-header-only-stable-diffusion-inference-c-librarypixlabiotiny-dream) | stablediffusion | no | Image | no | no | N/A | +| stablediffusion | no | Image | no | no | N/A | | `coqui` | Coqui | no | Audio generation and Voice cloning | no | no | CPU/CUDA | | `openvoice` | Open voice | no | Audio generation and Voice cloning | no | no | CPU/CUDA | | `parler-tts` | Open voice | no | Audio generation and Voice cloning | no | no | CPU/CUDA | diff --git a/gallery/index.yaml b/gallery/index.yaml index 349cd419..35fac331 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -11187,15 +11187,6 @@ description: | Stable Diffusion in NCNN with c++, supported txt2img and img2img name: stablediffusion-cpp -## Tiny Dream -- url: github:mudler/LocalAI/gallery/tinydream.yaml@master - name: tinydream - license: "BSD-3" - urls: - - https://github.com/symisc/tiny-dream - - https://github.com/symisc/tiny-dream/blob/main/LICENSE - description: | - An embedded, Header Only, Stable Diffusion C++ implementation - &piper ## Piper TTS url: github:mudler/LocalAI/gallery/piper.yaml@master diff --git a/gallery/tinydream.yaml b/gallery/tinydream.yaml deleted file mode 100644 index e4a79ad7..00000000 --- a/gallery/tinydream.yaml +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: "tinydream" - -config_file: | - name: tinydream - backend: tinydream - parameters: - model: tinydream_assets - -files: - - filename: "tinydream_assets/AutoencoderKL-fp16.bin" - sha256: "f02e71f80e70252734724bbfaed5c4ddd3a8ed7e61bb2175ff5f53099f0e35dd" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/AutoencoderKL-fp16.bin" - - filename: "tinydream_assets/AutoencoderKL-fp16.param" - sha256: "0254a056dce61b0c27dc9ec1b78b53bcf55315c540f55f051eb841aa992701ba" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/AutoencoderKL-fp16.param" - - filename: "tinydream_assets/FrozenCLIPEmbedder-fp16.bin" - sha256: "1c9a12f4e1dd1b295a388045f7f28a2352a4d70c3dc96a542189a3dd7051fdd6" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/FrozenCLIPEmbedder-fp16.bin" - - filename: "tinydream_assets/FrozenCLIPEmbedder-fp16.param" - sha256: "471afbe678dd1fd3fe764ef9c6eccaccb0a7d7e601f27b462aa926b20eb368c9" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/FrozenCLIPEmbedder-fp16.param" - - filename: "tinydream_assets/RealESRGAN_x4plus_anime.bin" - sha256: "fe01c269cfd10cdef8e018ab66ebe750cf79c7af4d1f9c16c737e1295229bacc" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/RealESRGAN_x4plus_anime.bin" - - filename: "tinydream_assets/RealESRGAN_x4plus_anime.param" - sha256: "2b8fb6e0ae4d2d85704ca08c119a2f5ea40add4f2ecd512eb7f4cd44b6127ed4" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/RealESRGAN_x4plus_anime.param" - - filename: "tinydream_assets/UNetModel-fp16.bin" - sha256: "d618918d011bfc1f644c0f2a33bf84931bd53b28a98492b0a8ed6f3a818852c3" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/UNetModel-fp16.bin" - - filename: "tinydream_assets/UNetModel-fp16.param" - sha256: "696f6975de49f4325b53ce32aff81861a6d6c07cd9ce3f0aae2cc405350af38d" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/UNetModel-fp16.param" - - filename: "tinydream_assets/vocab.txt" - sha256: "e30e57b6f1e47616982ef898d8922be24e535b4fa3d0110477b3a6f02ebbae7d" - uri: "https://github.com/M0Rf30/tiny-dream-bins/releases/download/1.0/vocab.txt" diff --git a/go.mod b/go.mod index 8aecf14d..adfa7357 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ toolchain go1.23.1 require ( dario.cat/mergo v1.0.1 - github.com/M0Rf30/go-tiny-dream v0.0.0-20240425104733-c04fa463ace9 github.com/Masterminds/sprig/v3 v3.3.0 github.com/alecthomas/kong v0.9.0 github.com/census-instrumentation/opencensus-proto v0.4.1 diff --git a/go.sum b/go.sum index a1a487b2..4a744ed8 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= -github.com/M0Rf30/go-tiny-dream v0.0.0-20240425104733-c04fa463ace9 h1:ASsbvw7wQPldWpwKdmYRszJ2A8Cj3oJDr4zO0DiXvN4= -github.com/M0Rf30/go-tiny-dream v0.0.0-20240425104733-c04fa463ace9/go.mod h1:UOf2Mb/deUri5agct5OJ4SLWjhI+kZKbsUVUeRb24I0= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= github.com/Masterminds/semver/v3 v3.3.0 h1:B8LGeaivUe71a5qox1ICM/JLl0NqZSW5CHyL+hmvYS0= diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index eb3e4fdf..756deea7 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -56,7 +56,6 @@ const ( WhisperBackend = "whisper" StableDiffusionBackend = "stablediffusion" - TinyDreamBackend = "tinydream" PiperBackend = "piper" LCHuggingFaceBackend = "huggingface" diff --git a/pkg/tinydream/generate.go b/pkg/tinydream/generate.go deleted file mode 100644 index cfcd23cc..00000000 --- a/pkg/tinydream/generate.go +++ /dev/null @@ -1,36 +0,0 @@ -//go:build tinydream -// +build tinydream - -package tinydream - -import ( - "fmt" - "path/filepath" - - tinyDream "github.com/M0Rf30/go-tiny-dream" -) - -func GenerateImage(height, width, step, seed int, positive_prompt, negative_prompt, dst, asset_dir string) error { - fmt.Println(dst) - if height > 512 || width > 512 { - return tinyDream.GenerateImage( - 1, - step, - seed, - positive_prompt, - negative_prompt, - filepath.Dir(dst), - asset_dir, - ) - } - - return tinyDream.GenerateImage( - 0, - step, - seed, - positive_prompt, - negative_prompt, - filepath.Dir(dst), - asset_dir, - ) -} diff --git a/pkg/tinydream/generate_unsupported.go b/pkg/tinydream/generate_unsupported.go deleted file mode 100644 index 4ffd421a..00000000 --- a/pkg/tinydream/generate_unsupported.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build !tinydream -// +build !tinydream - -package tinydream - -import "fmt" - -func GenerateImage(height, width, step, seed int, positive_prompt, negative_prompt, dst, asset_dir string) error { - return fmt.Errorf("This version of LocalAI was built without the tinytts tag") -} diff --git a/pkg/tinydream/tinydream.go b/pkg/tinydream/tinydream.go deleted file mode 100644 index a316e641..00000000 --- a/pkg/tinydream/tinydream.go +++ /dev/null @@ -1,20 +0,0 @@ -package tinydream - -import "os" - -type TinyDream struct { - assetDir string -} - -func New(assetDir string) (*TinyDream, error) { - if _, err := os.Stat(assetDir); err != nil { - return nil, err - } - return &TinyDream{ - assetDir: assetDir, - }, nil -} - -func (td *TinyDream) GenerateImage(height, width, step, seed int, positive_prompt, negative_prompt, dst string) error { - return GenerateImage(height, width, step, seed, positive_prompt, negative_prompt, dst, td.assetDir) -}