diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 2ece8a57..3f86c1c1 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -78,7 +78,7 @@ jobs:
           sudo apt-get install -y libopencv-dev && sudo ln -s /usr/include/opencv4/opencv2 /usr/include/opencv2
           
           sudo rm -rfv /usr/bin/conda || true
-          PATH=$PATH:/opt/conda/bin make -C extra/grpc/huggingface
+          PATH=$PATH:/opt/conda/bin make -C backend/python/huggingface
 
           # Pre-build piper before we start tests in order to have shared libraries in place
           make go-piper && \
diff --git a/Dockerfile b/Dockerfile
index 2091e2e3..be6d747b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -12,7 +12,7 @@ ARG TARGETARCH
 ARG TARGETVARIANT
 
 ENV BUILD_TYPE=${BUILD_TYPE}
-ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/extra/grpc/huggingface/run.sh,autogptq:/build/extra/grpc/autogptq/run.sh,bark:/build/extra/grpc/bark/run.sh,diffusers:/build/extra/grpc/diffusers/run.sh,exllama:/build/extra/grpc/exllama/run.sh,vall-e-x:/build/extra/grpc/vall-e-x/run.sh,vllm:/build/extra/grpc/vllm/run.sh"
+ENV EXTERNAL_GRPC_BACKENDS="huggingface-embeddings:/build/backend/python/huggingface/run.sh,autogptq:/build/backend/python/autogptq/run.sh,bark:/build/backend/python/bark/run.sh,diffusers:/build/backend/python/diffusers/run.sh,exllama:/build/backend/python/exllama/run.sh,vall-e-x:/build/backend/python/vall-e-x/run.sh,vllm:/build/backend/python/vllm/run.sh"
 ENV GALLERIES='[{"name":"model-gallery", "url":"github:go-skynet/model-gallery/index.yaml"}, {"url": "github:go-skynet/model-gallery/huggingface.yaml","name":"huggingface"}]'
 ARG GO_TAGS="stablediffusion tts"
 
@@ -64,20 +64,10 @@ RUN curl https://repo.anaconda.com/pkgs/misc/gpgkeys/anaconda.asc | gpg --dearmo
     apt-get update && \
     apt-get install -y conda
 
-COPY extra/requirements.txt /build/extra/requirements.txt
 ENV PATH="/root/.cargo/bin:${PATH}"
 RUN pip install --upgrade pip
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
-#RUN if [ "${TARGETARCH}" = "amd64" ]; then \
-#        pip install git+https://github.com/suno-ai/bark.git diffusers invisible_watermark transformers accelerate safetensors;\
-#    fi
-#RUN if [ "${BUILD_TYPE}" = "cublas" ] && [ "${TARGETARCH}" = "amd64" ]; then \
-#        pip install torch vllm && pip install auto-gptq https://github.com/jllllll/exllama/releases/download/0.0.10/exllama-0.0.10+cu${CUDA_MAJOR_VERSION}${CUDA_MINOR_VERSION}-cp39-cp39-linux_x86_64.whl;\
- #   fi
-#RUN pip install -r /build/extra/requirements.txt && rm -rf /build/extra/requirements.txt
 
-# Vall-e-X
-RUN git clone https://github.com/Plachtaa/VALL-E-X.git /usr/lib/vall-e-x && cd /usr/lib/vall-e-x && pip install -r requirements.txt
 
 # \
 #    ; fi
@@ -162,25 +152,25 @@ COPY --from=builder /build/backend-assets/grpc/stablediffusion ./backend-assets/
 
 ## Duplicated from Makefile to avoid having a big layer that's hard to push
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/autogptq \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/autogptq \
     ; fi
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/bark \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/bark \
     ; fi
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/diffusers \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/diffusers \
     ; fi
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/vllm \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/vllm \
     ; fi
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/huggingface \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/huggingface \
     ; fi
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/vall-e-x \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/vall-e-x \
     ; fi
 RUN if [ "${IMAGE_TYPE}" = "extras" ]; then \
-	PATH=$PATH:/opt/conda/bin make -C extra/grpc/exllama \
+	PATH=$PATH:/opt/conda/bin make -C backend/python/exllama \
     ; fi
 
 # Copy VALLE-X as it's not a real "lib"
@@ -190,7 +180,7 @@ RUN if [ -d /usr/lib/vall-e-x ]; then \
 
 # we also copy exllama libs over to resolve exllama import error
 RUN if [ -d /usr/local/lib/python3.9/dist-packages/exllama ]; then \
-        cp -rfv /usr/local/lib/python3.9/dist-packages/exllama extra/grpc/exllama/;\
+        cp -rfv /usr/local/lib/python3.9/dist-packages/exllama backend/python/exllama/;\
     fi
 
 # Define the health check command
diff --git a/Makefile b/Makefile
index 1b03de09..b36e2d0b 100644
--- a/Makefile
+++ b/Makefile
@@ -306,7 +306,7 @@ test: prepare test-models/testmodel grpcs
 	@echo 'Running tests'
 	export GO_TAGS="tts stablediffusion"
 	$(MAKE) prepare-test
-	HUGGINGFACE_GRPC=$(abspath ./)/extra/grpc/huggingface/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \
+	HUGGINGFACE_GRPC=$(abspath ./)/backend/python/huggingface/run.sh TEST_DIR=$(abspath ./)/test-dir/ FIXTURES=$(abspath ./)/tests/fixtures CONFIG_FILE=$(abspath ./)/test-models/config.yaml MODELS_PATH=$(abspath ./)/test-models \
 	$(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter="!gpt4all && !llama && !llama-gguf"  --flake-attempts 5 --fail-fast -v -r ./api ./pkg
 	$(MAKE) test-gpt4all
 	$(MAKE) test-llama
@@ -374,27 +374,27 @@ protogen: protogen-go protogen-python
 
 protogen-go:
 	protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative \
-    pkg/grpc/proto/backend.proto
+    backend/backend.proto
 
 protogen-python:
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/huggingface/ --grpc_python_out=extra/grpc/huggingface/ pkg/grpc/proto/backend.proto
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/autogptq/ --grpc_python_out=extra/grpc/autogptq/ pkg/grpc/proto/backend.proto
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/exllama/ --grpc_python_out=extra/grpc/exllama/ pkg/grpc/proto/backend.proto
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/bark/ --grpc_python_out=extra/grpc/bark/ pkg/grpc/proto/backend.proto
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/diffusers/ --grpc_python_out=extra/grpc/diffusers/ pkg/grpc/proto/backend.proto
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/vall-e-x/ --grpc_python_out=extra/grpc/vall-e-x/ pkg/grpc/proto/backend.proto
-	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=extra/grpc/vllm/ --grpc_python_out=extra/grpc/vllm/ pkg/grpc/proto/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/huggingface/ --grpc_python_out=backend/python/huggingface/ backend/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/autogptq/ --grpc_python_out=backend/python/autogptq/ backend/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/exllama/ --grpc_python_out=backend/python/exllama/ backend/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/bark/ --grpc_python_out=backend/python/bark/ backend/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/diffusers/ --grpc_python_out=backend/python/diffusers/ backend/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/vall-e-x/ --grpc_python_out=backend/python/vall-e-x/ backend/backend.proto
+	python3 -m grpc_tools.protoc -Ipkg/grpc/proto/ --python_out=backend/python/vllm/ --grpc_python_out=backend/python/vllm/ backend/backend.proto
 
 ## GRPC
 # Note: it is duplicated in the Dockerfile
 prepare-extra-conda-environments:
-	$(MAKE) -C extra/grpc/autogptq
-	$(MAKE) -C extra/grpc/bark
-	$(MAKE) -C extra/grpc/diffusers
-	$(MAKE) -C extra/grpc/vllm
-	$(MAKE) -C extra/grpc/huggingface
-	$(MAKE) -C extra/grpc/vall-e-x
-	$(MAKE) -C extra/grpc/exllama
+	$(MAKE) -C backend/python/autogptq
+	$(MAKE) -C backend/python/bark
+	$(MAKE) -C backend/python/diffusers
+	$(MAKE) -C backend/python/vllm
+	$(MAKE) -C backend/python/huggingface
+	$(MAKE) -C backend/python/vall-e-x
+	$(MAKE) -C backend/python/exllama
 
 
 backend-assets/grpc:
@@ -403,7 +403,7 @@ backend-assets/grpc:
 backend-assets/grpc/llama: backend-assets/grpc go-llama/libbinding.a
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama LIBRARY_PATH=$(shell pwd)/go-llama \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama ./cmd/grpc/llama/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama ./backend/go/llm/llama/
 # TODO: every binary should have its own folder instead, so can have different metal implementations
 ifeq ($(BUILD_TYPE),metal)
 	cp go-llama/build/bin/ggml-metal.metal backend-assets/grpc/
@@ -443,68 +443,68 @@ endif
 backend-assets/grpc/llama-stable: backend-assets/grpc go-llama-stable/libbinding.a
 	$(GOCMD) mod edit -replace github.com/go-skynet/go-llama.cpp=$(shell pwd)/go-llama-stable
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-llama-stable LIBRARY_PATH=$(shell pwd)/go-llama \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-stable ./cmd/grpc/llama-stable/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/llama-stable ./backend/go/llm/llama-stable/
 
 backend-assets/grpc/gpt4all: backend-assets/grpc backend-assets/gpt4all gpt4all/gpt4all-bindings/golang/libgpt4all.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/gpt4all/gpt4all-bindings/golang/ LIBRARY_PATH=$(shell pwd)/gpt4all/gpt4all-bindings/golang/ \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt4all ./cmd/grpc/gpt4all/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt4all ./backend/go/llm/gpt4all/
 
 backend-assets/grpc/dolly: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/dolly ./cmd/grpc/dolly/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/dolly ./backend/go/llm/dolly/
 
 backend-assets/grpc/gpt2: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt2 ./cmd/grpc/gpt2/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gpt2 ./backend/go/llm/gpt2/
 
 backend-assets/grpc/gptj: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptj ./cmd/grpc/gptj/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptj ./backend/go/llm/gptj/
 
 backend-assets/grpc/gptneox: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptneox ./cmd/grpc/gptneox/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/gptneox ./backend/go/llm/gptneox/
 
 backend-assets/grpc/mpt: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/mpt ./cmd/grpc/mpt/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/mpt ./backend/go/llm/mpt/
 
 backend-assets/grpc/replit: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/replit ./cmd/grpc/replit/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/replit ./backend/go/llm/replit/
 
 backend-assets/grpc/falcon-ggml: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/falcon-ggml ./cmd/grpc/falcon-ggml/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/falcon-ggml ./backend/go/llm/falcon-ggml/
 
 backend-assets/grpc/starcoder: backend-assets/grpc go-ggml-transformers/libtransformers.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-ggml-transformers LIBRARY_PATH=$(shell pwd)/go-ggml-transformers \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/starcoder ./cmd/grpc/starcoder/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/starcoder ./backend/go/llm/starcoder/
 
 backend-assets/grpc/rwkv: backend-assets/grpc go-rwkv/librwkv.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-rwkv LIBRARY_PATH=$(shell pwd)/go-rwkv \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./cmd/grpc/rwkv/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/rwkv ./backend/go/llm/rwkv
 
 backend-assets/grpc/bert-embeddings: backend-assets/grpc go-bert/libgobert.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-bert LIBRARY_PATH=$(shell pwd)/go-bert \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bert-embeddings ./cmd/grpc/bert-embeddings/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/bert-embeddings ./backend/go/llm/bert/
 
 backend-assets/grpc/langchain-huggingface: backend-assets/grpc
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./cmd/grpc/langchain-huggingface/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/langchain-huggingface ./backend/go/llm/langchain/
 
 backend-assets/grpc/stablediffusion: backend-assets/grpc
 	if [ ! -f backend-assets/grpc/stablediffusion ]; then \
 		$(MAKE) go-stable-diffusion/libstablediffusion.a; \
 		CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/go-stable-diffusion/ LIBRARY_PATH=$(shell pwd)/go-stable-diffusion/ \
-		$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./cmd/grpc/stablediffusion/; \
+		$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/stablediffusion ./backend/go/image/; \
 	fi
 
 backend-assets/grpc/piper: backend-assets/grpc backend-assets/espeak-ng-data go-piper/libpiper_binding.a
 	CGO_CXXFLAGS="$(PIPER_CGO_CXXFLAGS)" CGO_LDFLAGS="$(PIPER_CGO_LDFLAGS)" LIBRARY_PATH=$(shell pwd)/go-piper \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./cmd/grpc/piper/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/piper ./backend/go/tts/
 
 backend-assets/grpc/whisper: backend-assets/grpc whisper.cpp/libwhisper.a
 	CGO_LDFLAGS="$(CGO_LDFLAGS)" C_INCLUDE_PATH=$(shell pwd)/whisper.cpp LIBRARY_PATH=$(shell pwd)/whisper.cpp \
-	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./cmd/grpc/whisper/
+	$(GOCMD) build -ldflags "$(LD_FLAGS)" -tags "$(GO_TAGS)" -o backend-assets/grpc/whisper ./backend/go/transcribe/
 
 grpcs: prepare $(GRPC_BACKENDS)
diff --git a/pkg/grpc/proto/backend.proto b/backend/backend.proto
similarity index 100%
rename from pkg/grpc/proto/backend.proto
rename to backend/backend.proto
diff --git a/backend/cpp/llama/CMakeLists.txt b/backend/cpp/llama/CMakeLists.txt
index 79938566..7bf17871 100644
--- a/backend/cpp/llama/CMakeLists.txt
+++ b/backend/cpp/llama/CMakeLists.txt
@@ -36,7 +36,7 @@ include_directories(${Protobuf_INCLUDE_DIRS})
 message(STATUS "Using protobuf version ${Protobuf_VERSION} | Protobuf_INCLUDE_DIRS: ${Protobuf_INCLUDE_DIRS} | CMAKE_CURRENT_BINARY_DIR: ${CMAKE_CURRENT_BINARY_DIR}")
 
 # Proto file
-get_filename_component(hw_proto "../../../../../../pkg/grpc/proto/backend.proto" ABSOLUTE)
+get_filename_component(hw_proto "../../../../../../backend/backend.proto" ABSOLUTE)
 get_filename_component(hw_proto_path "${hw_proto}" PATH)
 
 # Generated sources
diff --git a/cmd/grpc/bert-embeddings/main.go b/backend/go/image/main.go
similarity index 71%
rename from cmd/grpc/bert-embeddings/main.go
rename to backend/go/image/main.go
index 90fae8b2..425b2340 100644
--- a/cmd/grpc/bert-embeddings/main.go
+++ b/backend/go/image/main.go
@@ -5,7 +5,6 @@ package main
 import (
 	"flag"
 
-	bert "github.com/go-skynet/LocalAI/pkg/backend/llm/bert"
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
 
@@ -16,7 +15,7 @@ var (
 func main() {
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &bert.Embeddings{}); err != nil {
+	if err := grpc.StartServer(*addr, &StableDiffusion{}); err != nil {
 		panic(err)
 	}
 }
diff --git a/pkg/backend/image/stablediffusion.go b/backend/go/image/stablediffusion.go
similarity index 98%
rename from pkg/backend/image/stablediffusion.go
rename to backend/go/image/stablediffusion.go
index 50d299da..f04ee1de 100644
--- a/pkg/backend/image/stablediffusion.go
+++ b/backend/go/image/stablediffusion.go
@@ -1,4 +1,4 @@
-package image
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/pkg/backend/llm/bert/bert.go b/backend/go/llm/bert/bert.go
similarity index 98%
rename from pkg/backend/llm/bert/bert.go
rename to backend/go/llm/bert/bert.go
index 423ff79c..4330762d 100644
--- a/pkg/backend/llm/bert/bert.go
+++ b/backend/go/llm/bert/bert.go
@@ -1,4 +1,4 @@
-package bert
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/cmd/grpc/rwkv/main.go b/backend/go/llm/bert/main.go
similarity index 71%
rename from cmd/grpc/rwkv/main.go
rename to backend/go/llm/bert/main.go
index 74724dc2..2e582ec5 100644
--- a/cmd/grpc/rwkv/main.go
+++ b/backend/go/llm/bert/main.go
@@ -5,8 +5,6 @@ package main
 import (
 	"flag"
 
-	rwkv "github.com/go-skynet/LocalAI/pkg/backend/llm/rwkv"
-
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
 
@@ -17,7 +15,7 @@ var (
 func main() {
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &rwkv.LLM{}); err != nil {
+	if err := grpc.StartServer(*addr, &Embeddings{}); err != nil {
 		panic(err)
 	}
 }
diff --git a/cmd/grpc/dolly/main.go b/backend/go/llm/dolly/main.go
similarity index 83%
rename from cmd/grpc/dolly/main.go
rename to backend/go/llm/dolly/main.go
index cc040cb6..d00a30f0 100644
--- a/cmd/grpc/dolly/main.go
+++ b/backend/go/llm/dolly/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/cmd/grpc/falcon-ggml/main.go b/backend/go/llm/falcon-ggml/main.go
similarity index 83%
rename from cmd/grpc/falcon-ggml/main.go
rename to backend/go/llm/falcon-ggml/main.go
index 5a99a915..426ae823 100644
--- a/cmd/grpc/falcon-ggml/main.go
+++ b/backend/go/llm/falcon-ggml/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/cmd/grpc/gpt2/main.go b/backend/go/llm/gpt2/main.go
similarity index 83%
rename from cmd/grpc/gpt2/main.go
rename to backend/go/llm/gpt2/main.go
index 42f6ba47..972e39d3 100644
--- a/cmd/grpc/gpt2/main.go
+++ b/backend/go/llm/gpt2/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/pkg/backend/llm/gpt4all/gpt4all.go b/backend/go/llm/gpt4all/gpt4all.go
similarity index 99%
rename from pkg/backend/llm/gpt4all/gpt4all.go
rename to backend/go/llm/gpt4all/gpt4all.go
index 86d4baa3..11685b3d 100644
--- a/pkg/backend/llm/gpt4all/gpt4all.go
+++ b/backend/go/llm/gpt4all/gpt4all.go
@@ -1,4 +1,4 @@
-package gpt4all
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/cmd/grpc/gpt4all/main.go b/backend/go/llm/gpt4all/main.go
similarity index 69%
rename from cmd/grpc/gpt4all/main.go
rename to backend/go/llm/gpt4all/main.go
index e659fe77..79829efd 100644
--- a/cmd/grpc/gpt4all/main.go
+++ b/backend/go/llm/gpt4all/main.go
@@ -5,8 +5,6 @@ package main
 import (
 	"flag"
 
-	gpt4all "github.com/go-skynet/LocalAI/pkg/backend/llm/gpt4all"
-
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
 
@@ -17,7 +15,7 @@ var (
 func main() {
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &gpt4all.LLM{}); err != nil {
+	if err := grpc.StartServer(*addr, &LLM{}); err != nil {
 		panic(err)
 	}
 }
diff --git a/cmd/grpc/gptj/main.go b/backend/go/llm/gptj/main.go
similarity index 83%
rename from cmd/grpc/gptj/main.go
rename to backend/go/llm/gptj/main.go
index 3530a727..10f16a29 100644
--- a/cmd/grpc/gptj/main.go
+++ b/backend/go/llm/gptj/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/cmd/grpc/gptneox/main.go b/backend/go/llm/gptneox/main.go
similarity index 83%
rename from cmd/grpc/gptneox/main.go
rename to backend/go/llm/gptneox/main.go
index d4e8be47..450cd5da 100644
--- a/cmd/grpc/gptneox/main.go
+++ b/backend/go/llm/gptneox/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/pkg/backend/llm/langchain/langchain.go b/backend/go/llm/langchain/langchain.go
similarity index 98%
rename from pkg/backend/llm/langchain/langchain.go
rename to backend/go/llm/langchain/langchain.go
index 5d5f94bd..f1dee360 100644
--- a/pkg/backend/llm/langchain/langchain.go
+++ b/backend/go/llm/langchain/langchain.go
@@ -1,4 +1,4 @@
-package langchain
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/backend/go/llm/langchain/main.go b/backend/go/llm/langchain/main.go
new file mode 100644
index 00000000..79829efd
--- /dev/null
+++ b/backend/go/llm/langchain/main.go
@@ -0,0 +1,21 @@
+package main
+
+// Note: this is started internally by LocalAI and a server is allocated for each model
+
+import (
+	"flag"
+
+	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &LLM{}); err != nil {
+		panic(err)
+	}
+}
diff --git a/pkg/backend/llm/llama-stable/llama.go b/backend/go/llm/llama-stable/llama.go
similarity index 99%
rename from pkg/backend/llm/llama-stable/llama.go
rename to backend/go/llm/llama-stable/llama.go
index b9e25754..321d476a 100644
--- a/pkg/backend/llm/llama-stable/llama.go
+++ b/backend/go/llm/llama-stable/llama.go
@@ -1,4 +1,4 @@
-package llama
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/cmd/grpc/llama-stable/main.go b/backend/go/llm/llama-stable/main.go
similarity index 61%
rename from cmd/grpc/llama-stable/main.go
rename to backend/go/llm/llama-stable/main.go
index afe98f7d..a2e61270 100644
--- a/cmd/grpc/llama-stable/main.go
+++ b/backend/go/llm/llama-stable/main.go
@@ -3,8 +3,6 @@ package main
 import (
 	"flag"
 
-	llama "github.com/go-skynet/LocalAI/pkg/backend/llm/llama-stable"
-
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
 
@@ -15,7 +13,7 @@ var (
 func main() {
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &llama.LLM{}); err != nil {
+	if err := grpc.StartServer(*addr, &LLM{}); err != nil {
 		panic(err)
 	}
 }
diff --git a/pkg/backend/llm/llama/llama.go b/backend/go/llm/llama/llama.go
similarity index 99%
rename from pkg/backend/llm/llama/llama.go
rename to backend/go/llm/llama/llama.go
index d461d9f2..34eb03c2 100644
--- a/pkg/backend/llm/llama/llama.go
+++ b/backend/go/llm/llama/llama.go
@@ -1,4 +1,4 @@
-package llama
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/cmd/grpc/piper/main.go b/backend/go/llm/llama/main.go
similarity index 72%
rename from cmd/grpc/piper/main.go
rename to backend/go/llm/llama/main.go
index d77189ae..42868e96 100644
--- a/cmd/grpc/piper/main.go
+++ b/backend/go/llm/llama/main.go
@@ -1,12 +1,12 @@
 package main
 
+// GRPC Falcon server
+
 // Note: this is started internally by LocalAI and a server is allocated for each model
 
 import (
 	"flag"
 
-	tts "github.com/go-skynet/LocalAI/pkg/backend/tts"
-
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
 
@@ -17,7 +17,7 @@ var (
 func main() {
 	flag.Parse()
 
-	if err := grpc.StartServer(*addr, &tts.Piper{}); err != nil {
+	if err := grpc.StartServer(*addr, &LLM{}); err != nil {
 		panic(err)
 	}
 }
diff --git a/cmd/grpc/mpt/main.go b/backend/go/llm/mpt/main.go
similarity index 83%
rename from cmd/grpc/mpt/main.go
rename to backend/go/llm/mpt/main.go
index 6f46d187..3455078f 100644
--- a/cmd/grpc/mpt/main.go
+++ b/backend/go/llm/mpt/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/cmd/grpc/replit/main.go b/backend/go/llm/replit/main.go
similarity index 83%
rename from cmd/grpc/replit/main.go
rename to backend/go/llm/replit/main.go
index 0f54b6ab..df605149 100644
--- a/cmd/grpc/replit/main.go
+++ b/backend/go/llm/replit/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/backend/go/llm/rwkv/main.go b/backend/go/llm/rwkv/main.go
new file mode 100644
index 00000000..79829efd
--- /dev/null
+++ b/backend/go/llm/rwkv/main.go
@@ -0,0 +1,21 @@
+package main
+
+// Note: this is started internally by LocalAI and a server is allocated for each model
+
+import (
+	"flag"
+
+	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &LLM{}); err != nil {
+		panic(err)
+	}
+}
diff --git a/pkg/backend/llm/rwkv/rwkv.go b/backend/go/llm/rwkv/rwkv.go
similarity index 99%
rename from pkg/backend/llm/rwkv/rwkv.go
rename to backend/go/llm/rwkv/rwkv.go
index 501e5699..d315fcc2 100644
--- a/pkg/backend/llm/rwkv/rwkv.go
+++ b/backend/go/llm/rwkv/rwkv.go
@@ -1,4 +1,4 @@
-package rwkv
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/cmd/grpc/starcoder/main.go b/backend/go/llm/starcoder/main.go
similarity index 83%
rename from cmd/grpc/starcoder/main.go
rename to backend/go/llm/starcoder/main.go
index c08cef20..38534d6c 100644
--- a/cmd/grpc/starcoder/main.go
+++ b/backend/go/llm/starcoder/main.go
@@ -5,7 +5,7 @@ package main
 import (
 	"flag"
 
-	transformers "github.com/go-skynet/LocalAI/pkg/backend/llm/transformers"
+	transformers "github.com/go-skynet/LocalAI/backend/go/llm/transformers"
 
 	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
 )
diff --git a/pkg/backend/llm/transformers/dolly.go b/backend/go/llm/transformers/dolly.go
similarity index 100%
rename from pkg/backend/llm/transformers/dolly.go
rename to backend/go/llm/transformers/dolly.go
diff --git a/pkg/backend/llm/transformers/falcon.go b/backend/go/llm/transformers/falcon.go
similarity index 100%
rename from pkg/backend/llm/transformers/falcon.go
rename to backend/go/llm/transformers/falcon.go
diff --git a/pkg/backend/llm/transformers/gpt2.go b/backend/go/llm/transformers/gpt2.go
similarity index 100%
rename from pkg/backend/llm/transformers/gpt2.go
rename to backend/go/llm/transformers/gpt2.go
diff --git a/pkg/backend/llm/transformers/gptj.go b/backend/go/llm/transformers/gptj.go
similarity index 100%
rename from pkg/backend/llm/transformers/gptj.go
rename to backend/go/llm/transformers/gptj.go
diff --git a/pkg/backend/llm/transformers/gptneox.go b/backend/go/llm/transformers/gptneox.go
similarity index 100%
rename from pkg/backend/llm/transformers/gptneox.go
rename to backend/go/llm/transformers/gptneox.go
diff --git a/pkg/backend/llm/transformers/mpt.go b/backend/go/llm/transformers/mpt.go
similarity index 100%
rename from pkg/backend/llm/transformers/mpt.go
rename to backend/go/llm/transformers/mpt.go
diff --git a/pkg/backend/llm/transformers/predict.go b/backend/go/llm/transformers/predict.go
similarity index 100%
rename from pkg/backend/llm/transformers/predict.go
rename to backend/go/llm/transformers/predict.go
diff --git a/pkg/backend/llm/transformers/replit.go b/backend/go/llm/transformers/replit.go
similarity index 100%
rename from pkg/backend/llm/transformers/replit.go
rename to backend/go/llm/transformers/replit.go
diff --git a/pkg/backend/llm/transformers/starcoder.go b/backend/go/llm/transformers/starcoder.go
similarity index 100%
rename from pkg/backend/llm/transformers/starcoder.go
rename to backend/go/llm/transformers/starcoder.go
diff --git a/backend/go/transcribe/main.go b/backend/go/transcribe/main.go
new file mode 100644
index 00000000..ad77da1a
--- /dev/null
+++ b/backend/go/transcribe/main.go
@@ -0,0 +1,21 @@
+package main
+
+// Note: this is started internally by LocalAI and a server is allocated for each model
+
+import (
+	"flag"
+
+	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &Whisper{}); err != nil {
+		panic(err)
+	}
+}
diff --git a/pkg/backend/transcribe/transcript.go b/backend/go/transcribe/transcript.go
similarity index 99%
rename from pkg/backend/transcribe/transcript.go
rename to backend/go/transcribe/transcript.go
index e428b324..ebd43eca 100644
--- a/pkg/backend/transcribe/transcript.go
+++ b/backend/go/transcribe/transcript.go
@@ -1,4 +1,4 @@
-package transcribe
+package main
 
 import (
 	"fmt"
diff --git a/pkg/backend/transcribe/whisper.go b/backend/go/transcribe/whisper.go
similarity index 97%
rename from pkg/backend/transcribe/whisper.go
rename to backend/go/transcribe/whisper.go
index 493d8229..a033afb0 100644
--- a/pkg/backend/transcribe/whisper.go
+++ b/backend/go/transcribe/whisper.go
@@ -1,4 +1,4 @@
-package transcribe
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/backend/go/tts/main.go b/backend/go/tts/main.go
new file mode 100644
index 00000000..faa6d5a5
--- /dev/null
+++ b/backend/go/tts/main.go
@@ -0,0 +1,21 @@
+package main
+
+// Note: this is started internally by LocalAI and a server is allocated for each model
+
+import (
+	"flag"
+
+	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
+)
+
+var (
+	addr = flag.String("addr", "localhost:50051", "the address to connect to")
+)
+
+func main() {
+	flag.Parse()
+
+	if err := grpc.StartServer(*addr, &Piper{}); err != nil {
+		panic(err)
+	}
+}
diff --git a/pkg/backend/tts/piper.go b/backend/go/tts/piper.go
similarity index 98%
rename from pkg/backend/tts/piper.go
rename to backend/go/tts/piper.go
index 41f10049..fb025d82 100644
--- a/pkg/backend/tts/piper.go
+++ b/backend/go/tts/piper.go
@@ -1,4 +1,4 @@
-package tts
+package main
 
 // This is a wrapper to statisfy the GRPC service interface
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
diff --git a/extra/grpc/README.md b/backend/python/README.md
similarity index 100%
rename from extra/grpc/README.md
rename to backend/python/README.md
diff --git a/extra/grpc/autogptq/Makefile b/backend/python/autogptq/Makefile
similarity index 100%
rename from extra/grpc/autogptq/Makefile
rename to backend/python/autogptq/Makefile
diff --git a/extra/grpc/autogptq/README.md b/backend/python/autogptq/README.md
similarity index 100%
rename from extra/grpc/autogptq/README.md
rename to backend/python/autogptq/README.md
diff --git a/extra/grpc/autogptq/autogptq.py b/backend/python/autogptq/autogptq.py
similarity index 100%
rename from extra/grpc/autogptq/autogptq.py
rename to backend/python/autogptq/autogptq.py
diff --git a/extra/grpc/autogptq/autogptq.yml b/backend/python/autogptq/autogptq.yml
similarity index 100%
rename from extra/grpc/autogptq/autogptq.yml
rename to backend/python/autogptq/autogptq.yml
diff --git a/extra/grpc/autogptq/backend_pb2.py b/backend/python/autogptq/backend_pb2.py
similarity index 100%
rename from extra/grpc/autogptq/backend_pb2.py
rename to backend/python/autogptq/backend_pb2.py
diff --git a/extra/grpc/autogptq/backend_pb2_grpc.py b/backend/python/autogptq/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/autogptq/backend_pb2_grpc.py
rename to backend/python/autogptq/backend_pb2_grpc.py
diff --git a/extra/grpc/autogptq/run.sh b/backend/python/autogptq/run.sh
similarity index 100%
rename from extra/grpc/autogptq/run.sh
rename to backend/python/autogptq/run.sh
diff --git a/extra/grpc/bark/Makefile b/backend/python/bark/Makefile
similarity index 100%
rename from extra/grpc/bark/Makefile
rename to backend/python/bark/Makefile
diff --git a/extra/grpc/bark/README.md b/backend/python/bark/README.md
similarity index 100%
rename from extra/grpc/bark/README.md
rename to backend/python/bark/README.md
diff --git a/extra/grpc/bark/backend_pb2.py b/backend/python/bark/backend_pb2.py
similarity index 100%
rename from extra/grpc/bark/backend_pb2.py
rename to backend/python/bark/backend_pb2.py
diff --git a/extra/grpc/bark/backend_pb2_grpc.py b/backend/python/bark/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/bark/backend_pb2_grpc.py
rename to backend/python/bark/backend_pb2_grpc.py
diff --git a/extra/grpc/bark/run.sh b/backend/python/bark/run.sh
similarity index 100%
rename from extra/grpc/bark/run.sh
rename to backend/python/bark/run.sh
diff --git a/extra/grpc/bark/test_ttsbark.py b/backend/python/bark/test_ttsbark.py
similarity index 100%
rename from extra/grpc/bark/test_ttsbark.py
rename to backend/python/bark/test_ttsbark.py
diff --git a/extra/grpc/bark/ttsbark.py b/backend/python/bark/ttsbark.py
similarity index 100%
rename from extra/grpc/bark/ttsbark.py
rename to backend/python/bark/ttsbark.py
diff --git a/extra/grpc/bark/ttsbark.yml b/backend/python/bark/ttsbark.yml
similarity index 100%
rename from extra/grpc/bark/ttsbark.yml
rename to backend/python/bark/ttsbark.yml
diff --git a/extra/grpc/diffusers/Makefile b/backend/python/diffusers/Makefile
similarity index 100%
rename from extra/grpc/diffusers/Makefile
rename to backend/python/diffusers/Makefile
diff --git a/extra/grpc/diffusers/README.md b/backend/python/diffusers/README.md
similarity index 100%
rename from extra/grpc/diffusers/README.md
rename to backend/python/diffusers/README.md
diff --git a/extra/grpc/diffusers/backend_diffusers.py b/backend/python/diffusers/backend_diffusers.py
similarity index 100%
rename from extra/grpc/diffusers/backend_diffusers.py
rename to backend/python/diffusers/backend_diffusers.py
diff --git a/extra/grpc/diffusers/backend_pb2.py b/backend/python/diffusers/backend_pb2.py
similarity index 100%
rename from extra/grpc/diffusers/backend_pb2.py
rename to backend/python/diffusers/backend_pb2.py
diff --git a/extra/grpc/diffusers/backend_pb2_grpc.py b/backend/python/diffusers/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/diffusers/backend_pb2_grpc.py
rename to backend/python/diffusers/backend_pb2_grpc.py
diff --git a/extra/grpc/diffusers/diffusers.yml b/backend/python/diffusers/diffusers.yml
similarity index 100%
rename from extra/grpc/diffusers/diffusers.yml
rename to backend/python/diffusers/diffusers.yml
diff --git a/extra/grpc/diffusers/run.sh b/backend/python/diffusers/run.sh
similarity index 100%
rename from extra/grpc/diffusers/run.sh
rename to backend/python/diffusers/run.sh
diff --git a/extra/grpc/exllama/Makefile b/backend/python/exllama/Makefile
similarity index 100%
rename from extra/grpc/exllama/Makefile
rename to backend/python/exllama/Makefile
diff --git a/extra/grpc/exllama/README.md b/backend/python/exllama/README.md
similarity index 100%
rename from extra/grpc/exllama/README.md
rename to backend/python/exllama/README.md
diff --git a/extra/grpc/exllama/backend_pb2.py b/backend/python/exllama/backend_pb2.py
similarity index 100%
rename from extra/grpc/exllama/backend_pb2.py
rename to backend/python/exllama/backend_pb2.py
diff --git a/extra/grpc/exllama/backend_pb2_grpc.py b/backend/python/exllama/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/exllama/backend_pb2_grpc.py
rename to backend/python/exllama/backend_pb2_grpc.py
diff --git a/extra/grpc/exllama/exllama.py b/backend/python/exllama/exllama.py
similarity index 100%
rename from extra/grpc/exllama/exllama.py
rename to backend/python/exllama/exllama.py
diff --git a/extra/grpc/exllama/exllama.yml b/backend/python/exllama/exllama.yml
similarity index 100%
rename from extra/grpc/exllama/exllama.yml
rename to backend/python/exllama/exllama.yml
diff --git a/extra/grpc/exllama/run.sh b/backend/python/exllama/run.sh
similarity index 100%
rename from extra/grpc/exllama/run.sh
rename to backend/python/exllama/run.sh
diff --git a/extra/grpc/huggingface/Makefile b/backend/python/huggingface/Makefile
similarity index 100%
rename from extra/grpc/huggingface/Makefile
rename to backend/python/huggingface/Makefile
diff --git a/extra/grpc/huggingface/README.md b/backend/python/huggingface/README.md
similarity index 100%
rename from extra/grpc/huggingface/README.md
rename to backend/python/huggingface/README.md
diff --git a/extra/grpc/huggingface/backend_pb2.py b/backend/python/huggingface/backend_pb2.py
similarity index 100%
rename from extra/grpc/huggingface/backend_pb2.py
rename to backend/python/huggingface/backend_pb2.py
diff --git a/extra/grpc/huggingface/backend_pb2_grpc.py b/backend/python/huggingface/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/huggingface/backend_pb2_grpc.py
rename to backend/python/huggingface/backend_pb2_grpc.py
diff --git a/extra/grpc/huggingface/huggingface.py b/backend/python/huggingface/huggingface.py
similarity index 100%
rename from extra/grpc/huggingface/huggingface.py
rename to backend/python/huggingface/huggingface.py
diff --git a/extra/grpc/huggingface/huggingface.yml b/backend/python/huggingface/huggingface.yml
similarity index 100%
rename from extra/grpc/huggingface/huggingface.yml
rename to backend/python/huggingface/huggingface.yml
diff --git a/extra/grpc/huggingface/run.sh b/backend/python/huggingface/run.sh
similarity index 100%
rename from extra/grpc/huggingface/run.sh
rename to backend/python/huggingface/run.sh
diff --git a/extra/grpc/huggingface/test.sh b/backend/python/huggingface/test.sh
similarity index 100%
rename from extra/grpc/huggingface/test.sh
rename to backend/python/huggingface/test.sh
diff --git a/extra/grpc/huggingface/test_huggingface.py b/backend/python/huggingface/test_huggingface.py
similarity index 100%
rename from extra/grpc/huggingface/test_huggingface.py
rename to backend/python/huggingface/test_huggingface.py
diff --git a/extra/grpc/vall-e-x/Makefile b/backend/python/vall-e-x/Makefile
similarity index 100%
rename from extra/grpc/vall-e-x/Makefile
rename to backend/python/vall-e-x/Makefile
diff --git a/extra/grpc/vall-e-x/README.md b/backend/python/vall-e-x/README.md
similarity index 100%
rename from extra/grpc/vall-e-x/README.md
rename to backend/python/vall-e-x/README.md
diff --git a/extra/grpc/vall-e-x/backend_pb2.py b/backend/python/vall-e-x/backend_pb2.py
similarity index 100%
rename from extra/grpc/vall-e-x/backend_pb2.py
rename to backend/python/vall-e-x/backend_pb2.py
diff --git a/extra/grpc/vall-e-x/backend_pb2_grpc.py b/backend/python/vall-e-x/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/vall-e-x/backend_pb2_grpc.py
rename to backend/python/vall-e-x/backend_pb2_grpc.py
diff --git a/extra/grpc/vall-e-x/run.sh b/backend/python/vall-e-x/run.sh
similarity index 100%
rename from extra/grpc/vall-e-x/run.sh
rename to backend/python/vall-e-x/run.sh
diff --git a/extra/grpc/vall-e-x/ttsvalle.py b/backend/python/vall-e-x/ttsvalle.py
similarity index 100%
rename from extra/grpc/vall-e-x/ttsvalle.py
rename to backend/python/vall-e-x/ttsvalle.py
diff --git a/extra/grpc/vall-e-x/ttsvalle.yml b/backend/python/vall-e-x/ttsvalle.yml
similarity index 100%
rename from extra/grpc/vall-e-x/ttsvalle.yml
rename to backend/python/vall-e-x/ttsvalle.yml
diff --git a/extra/grpc/vllm/Makefile b/backend/python/vllm/Makefile
similarity index 100%
rename from extra/grpc/vllm/Makefile
rename to backend/python/vllm/Makefile
diff --git a/extra/grpc/vllm/README.md b/backend/python/vllm/README.md
similarity index 100%
rename from extra/grpc/vllm/README.md
rename to backend/python/vllm/README.md
diff --git a/extra/grpc/vllm/backend_pb2.py b/backend/python/vllm/backend_pb2.py
similarity index 100%
rename from extra/grpc/vllm/backend_pb2.py
rename to backend/python/vllm/backend_pb2.py
diff --git a/extra/grpc/vllm/backend_pb2_grpc.py b/backend/python/vllm/backend_pb2_grpc.py
similarity index 100%
rename from extra/grpc/vllm/backend_pb2_grpc.py
rename to backend/python/vllm/backend_pb2_grpc.py
diff --git a/extra/grpc/vllm/backend_vllm.py b/backend/python/vllm/backend_vllm.py
similarity index 100%
rename from extra/grpc/vllm/backend_vllm.py
rename to backend/python/vllm/backend_vllm.py
diff --git a/extra/grpc/vllm/run.sh b/backend/python/vllm/run.sh
similarity index 100%
rename from extra/grpc/vllm/run.sh
rename to backend/python/vllm/run.sh
diff --git a/extra/grpc/vllm/test_backend_vllm.py b/backend/python/vllm/test_backend_vllm.py
similarity index 100%
rename from extra/grpc/vllm/test_backend_vllm.py
rename to backend/python/vllm/test_backend_vllm.py
diff --git a/extra/grpc/vllm/vllm.yml b/backend/python/vllm/vllm.yml
similarity index 100%
rename from extra/grpc/vllm/vllm.yml
rename to backend/python/vllm/vllm.yml
diff --git a/cmd/grpc/langchain-huggingface/main.go b/cmd/grpc/langchain-huggingface/main.go
deleted file mode 100644
index 938908d2..00000000
--- a/cmd/grpc/langchain-huggingface/main.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package main
-
-// Note: this is started internally by LocalAI and a server is allocated for each model
-
-import (
-	"flag"
-
-	langchain "github.com/go-skynet/LocalAI/pkg/backend/llm/langchain"
-
-	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
-)
-
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
-)
-
-func main() {
-	flag.Parse()
-
-	if err := grpc.StartServer(*addr, &langchain.LLM{}); err != nil {
-		panic(err)
-	}
-}
diff --git a/cmd/grpc/llama/main.go b/cmd/grpc/llama/main.go
deleted file mode 100644
index 442f84ef..00000000
--- a/cmd/grpc/llama/main.go
+++ /dev/null
@@ -1,25 +0,0 @@
-package main
-
-// GRPC Falcon server
-
-// Note: this is started internally by LocalAI and a server is allocated for each model
-
-import (
-	"flag"
-
-	llama "github.com/go-skynet/LocalAI/pkg/backend/llm/llama"
-
-	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
-)
-
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
-)
-
-func main() {
-	flag.Parse()
-
-	if err := grpc.StartServer(*addr, &llama.LLM{}); err != nil {
-		panic(err)
-	}
-}
diff --git a/cmd/grpc/stablediffusion/main.go b/cmd/grpc/stablediffusion/main.go
deleted file mode 100644
index d12fe3c5..00000000
--- a/cmd/grpc/stablediffusion/main.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package main
-
-// Note: this is started internally by LocalAI and a server is allocated for each model
-
-import (
-	"flag"
-
-	image "github.com/go-skynet/LocalAI/pkg/backend/image"
-
-	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
-)
-
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
-)
-
-func main() {
-	flag.Parse()
-
-	if err := grpc.StartServer(*addr, &image.StableDiffusion{}); err != nil {
-		panic(err)
-	}
-}
diff --git a/cmd/grpc/whisper/main.go b/cmd/grpc/whisper/main.go
deleted file mode 100644
index 4896c4f9..00000000
--- a/cmd/grpc/whisper/main.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package main
-
-// Note: this is started internally by LocalAI and a server is allocated for each model
-
-import (
-	"flag"
-
-	transcribe "github.com/go-skynet/LocalAI/pkg/backend/transcribe"
-
-	grpc "github.com/go-skynet/LocalAI/pkg/grpc"
-)
-
-var (
-	addr = flag.String("addr", "localhost:50051", "the address to connect to")
-)
-
-func main() {
-	flag.Parse()
-
-	if err := grpc.StartServer(*addr, &transcribe.Whisper{}); err != nil {
-		panic(err)
-	}
-}
diff --git a/extra/requirements.txt b/extra/requirements.txt
deleted file mode 100644
index fb3cc012..00000000
--- a/extra/requirements.txt
+++ /dev/null
@@ -1,7 +0,0 @@
-sentence_transformers
-grpcio
-google
-protobuf
-six
-omegaconf
-compel
\ No newline at end of file
diff --git a/pkg/grpc/proto/backend.pb.go b/pkg/grpc/proto/backend.pb.go
index a9ac6057..259f8e0a 100644
--- a/pkg/grpc/proto/backend.pb.go
+++ b/pkg/grpc/proto/backend.pb.go
@@ -2,7 +2,7 @@
 // versions:
 // 	protoc-gen-go v1.26.0
 // 	protoc        v4.23.4
-// source: pkg/grpc/proto/backend.proto
+// source: backend/backend.proto
 
 package proto
 
diff --git a/pkg/grpc/proto/backend_grpc.pb.go b/pkg/grpc/proto/backend_grpc.pb.go
index 1c3cd8f5..5c97691d 100644
--- a/pkg/grpc/proto/backend_grpc.pb.go
+++ b/pkg/grpc/proto/backend_grpc.pb.go
@@ -2,7 +2,7 @@
 // versions:
 // - protoc-gen-go-grpc v1.2.0
 // - protoc             v4.23.4
-// source: pkg/grpc/proto/backend.proto
+// source: backend/backend.proto
 
 package proto
 
@@ -453,5 +453,5 @@ var Backend_ServiceDesc = grpc.ServiceDesc{
 			ServerStreams: true,
 		},
 	},
-	Metadata: "pkg/grpc/proto/backend.proto",
+	Metadata: "backend/backend.proto",
 }