MQTT Startup Refactoring Part 1: core/ packages part 1 (#1728)

This PR specifically introduces a `core` folder and moves the following packages over, without any other changes:

- `api/backend`
- `api/config`
- `api/options`
- `api/schema`

Once this is merged and we confirm there's no regressions, I can migrate over the remaining changes piece by piece to split up application startup, backend services, http, and mqtt as was the goal of the earlier PRs!

api/localai/backend_monitor.go

@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"strings"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
 	"github.com/go-skynet/LocalAI/pkg/grpc/proto"
 
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/gofiber/fiber/v2"
 	"github.com/rs/zerolog/log"
 

api/localai/gallery.go

@@ -11,7 +11,7 @@ import (
 	json "github.com/json-iterator/go"
 	"gopkg.in/yaml.v3"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
 	"github.com/go-skynet/LocalAI/pkg/gallery"
 	"github.com/go-skynet/LocalAI/pkg/utils"
 

api/localai/localai.go

@@ -1,12 +1,12 @@
 package localai
 
 import (
-	"github.com/go-skynet/LocalAI/api/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
 	fiberContext "github.com/go-skynet/LocalAI/api/ctx"
+	"github.com/go-skynet/LocalAI/core/backend"
+	config "github.com/go-skynet/LocalAI/core/config"
 	"github.com/rs/zerolog/log"
 
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/gofiber/fiber/v2"
 )
 

api/openai/chat.go

@@ -8,10 +8,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/go-skynet/LocalAI/pkg/grammar"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/go-skynet/LocalAI/pkg/utils"

api/openai/completion.go

@@ -8,10 +8,10 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/go-skynet/LocalAI/pkg/grammar"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/gofiber/fiber/v2"

api/openai/edit.go

@@ -5,10 +5,10 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/gofiber/fiber/v2"
 	"github.com/google/uuid"

api/openai/embeddings.go

@@ -5,12 +5,12 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/google/uuid"
 
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/gofiber/fiber/v2"
 	"github.com/rs/zerolog/log"
 )

api/openai/files.go

@@ -4,14 +4,15 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	config "github.com/go-skynet/LocalAI/api/config"
-	"github.com/go-skynet/LocalAI/api/options"
-	"github.com/go-skynet/LocalAI/pkg/utils"
-	"github.com/gofiber/fiber/v2"
-	"github.com/rs/zerolog/log"
 	"os"
 	"path/filepath"
 	"time"
+
+	config "github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/core/options"
+	"github.com/go-skynet/LocalAI/pkg/utils"
+	"github.com/gofiber/fiber/v2"
+	"github.com/rs/zerolog/log"
 )
 
 var uploadedFiles []File

api/openai/files_test.go

@@ -3,11 +3,6 @@ package openai
 import (
 	"encoding/json"
 	"fmt"
-	config "github.com/go-skynet/LocalAI/api/config"
-	"github.com/go-skynet/LocalAI/api/options"
-	utils2 "github.com/go-skynet/LocalAI/pkg/utils"
-	"github.com/gofiber/fiber/v2"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"mime/multipart"
 	"net/http"
@@ -16,6 +11,12 @@ import (
 	"path/filepath"
 	"strings"
 
+	config "github.com/go-skynet/LocalAI/core/config"
+	"github.com/go-skynet/LocalAI/core/options"
+	utils2 "github.com/go-skynet/LocalAI/pkg/utils"
+	"github.com/gofiber/fiber/v2"
+	"github.com/stretchr/testify/assert"
+
 	"testing"
 )
 

api/openai/image.go

@@ -13,12 +13,12 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/google/uuid"
 
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/gofiber/fiber/v2"
 	"github.com/rs/zerolog/log"

api/openai/inference.go

@@ -1,10 +1,10 @@
 package openai
 
 import (
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 )
 

api/openai/list.go

@@ -3,8 +3,8 @@ package openai
 import (
 	"regexp"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/gofiber/fiber/v2"
 )

api/openai/request.go

@@ -9,10 +9,10 @@ import (
 	"net/http"
 	"strings"
 
-	config "github.com/go-skynet/LocalAI/api/config"
 	fiberContext "github.com/go-skynet/LocalAI/api/ctx"
-	options "github.com/go-skynet/LocalAI/api/options"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/schema"
+	options "github.com/go-skynet/LocalAI/core/options"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/go-skynet/LocalAI/pkg/grammar"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/gofiber/fiber/v2"

api/openai/transcription.go

@@ -8,9 +8,9 @@ import (
 	"path"
 	"path/filepath"
 
-	"github.com/go-skynet/LocalAI/api/backend"
+	"github.com/go-skynet/LocalAI/core/backend"
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 
 	"github.com/gofiber/fiber/v2"
 	"github.com/rs/zerolog/log"

backend/go/transcribe/transcript.go

@@ -8,7 +8,7 @@ import (
 
 	"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
 	"github.com/go-audio/wav"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 )
 
 func sh(c string) (string, error) {

backend/go/transcribe/whisper.go

@@ -4,7 +4,7 @@ package main
 // It is meant to be used by the main executable that is the server for the specific backend type (falcon, gpt3, etc)
 import (
 	"github.com/ggerganov/whisper.cpp/bindings/go/pkg/whisper"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/go-skynet/LocalAI/pkg/grpc/base"
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 )

core/backend/embeddings.go

@@ -3,8 +3,8 @@ package backend
 import (
 	"fmt"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/pkg/grpc"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 )

core/backend/image.go

@@ -1,8 +1,8 @@
 package backend
 
 import (
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 )

core/backend/llm.go

@@ -8,8 +8,8 @@ import (
 	"sync"
 	"unicode/utf8"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/pkg/gallery"
 	"github.com/go-skynet/LocalAI/pkg/grpc"
 	model "github.com/go-skynet/LocalAI/pkg/model"

core/backend/options.go

@@ -7,8 +7,8 @@ import (
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 )
 
 func modelOpts(c config.Config, o *options.Option, opts []model.Option) []model.Option {

core/backend/transcript.go

@@ -4,10 +4,10 @@ import (
 	"context"
 	"fmt"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 )

core/backend/tts.go

@@ -6,9 +6,8 @@ import (
 	"os"
 	"path/filepath"
 
-	api_config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
-	config "github.com/go-skynet/LocalAI/api/config"
+	"github.com/go-skynet/LocalAI/core/options"
-	"github.com/go-skynet/LocalAI/api/options"
 	"github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	"github.com/go-skynet/LocalAI/pkg/utils"
@@ -38,7 +37,7 @@ func ModelTTS(backend, text, modelFile string, loader *model.ModelLoader, o *opt
 
 	grpcOpts := gRPCModelOpts(c)
 
-	opts := modelOpts(api_config.Config{}, o, []model.Option{
+	opts := modelOpts(config.Config{}, o, []model.Option{
 		model.WithBackendString(bb),
 		model.WithModel(modelFile),
 		model.WithContext(o.Context),

core/config/config.go

@@ -1,4 +1,4 @@
-package api_config
+package config
 
 import (
 	"errors"

core/config/config_test.go

@@ -1,10 +1,10 @@
-package api_config_test
+package config_test
 
 import (
 	"os"
 
-	. "github.com/go-skynet/LocalAI/api/config"
+	. "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/pkg/model"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"

core/config/prediction.go

@@ -1,4 +1,4 @@
-package api_config
+package config
 
 type PredictionOptions struct {
 

core/http/api.go

@@ -1,4 +1,4 @@
-package api
+package http
 
 import (
 	"encoding/json"
@@ -7,11 +7,11 @@ import (
 	"os"
 	"strings"
 
-	config "github.com/go-skynet/LocalAI/api/config"
 	"github.com/go-skynet/LocalAI/api/localai"
 	"github.com/go-skynet/LocalAI/api/openai"
-	"github.com/go-skynet/LocalAI/api/options"
+	config "github.com/go-skynet/LocalAI/core/config"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/options"
+	"github.com/go-skynet/LocalAI/core/schema"
 	"github.com/go-skynet/LocalAI/internal"
 	"github.com/go-skynet/LocalAI/metrics"
 	"github.com/go-skynet/LocalAI/pkg/assets"

core/http/api_test.go

@@ -1,4 +1,4 @@
-package api_test
+package http_test
 
 import (
 	"bytes"
@@ -13,8 +13,8 @@ import (
 	"path/filepath"
 	"runtime"
 
-	. "github.com/go-skynet/LocalAI/api"
+	. "github.com/go-skynet/LocalAI/core/http"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/metrics"
 	"github.com/go-skynet/LocalAI/pkg/downloader"
 	"github.com/go-skynet/LocalAI/pkg/gallery"

core/http/apt_suite_test.go

@@ -1,4 +1,4 @@
-package api_test
+package http_test
 
 import (
 	"testing"

core/schema/openai.go

@@ -3,7 +3,7 @@ package schema
 import (
 	"context"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
 
 	"github.com/go-skynet/LocalAI/pkg/grammar"
 )

main.go

@@ -12,10 +12,10 @@ import (
 	"syscall"
 	"time"
 
-	api "github.com/go-skynet/LocalAI/api"
+	"github.com/go-skynet/LocalAI/core/backend"
-	"github.com/go-skynet/LocalAI/api/backend"
+	config "github.com/go-skynet/LocalAI/core/config"
-	config "github.com/go-skynet/LocalAI/api/config"
+	api "github.com/go-skynet/LocalAI/core/http"
-	"github.com/go-skynet/LocalAI/api/options"
+	"github.com/go-skynet/LocalAI/core/options"
 	"github.com/go-skynet/LocalAI/internal"
 	"github.com/go-skynet/LocalAI/metrics"
 	"github.com/go-skynet/LocalAI/pkg/gallery"

pkg/grpc/backend.go

@@ -2,7 +2,8 @@ package grpc
 
 import (
 	"context"
-	"github.com/go-skynet/LocalAI/api/schema"
+
+	"github.com/go-skynet/LocalAI/core/schema"
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	"google.golang.org/grpc"
 )

pkg/grpc/base/base.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	gopsutil "github.com/shirou/gopsutil/v3/process"
 )

pkg/grpc/client.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"

pkg/grpc/embed.go

@@ -2,11 +2,12 @@ package grpc
 
 import (
 	"context"
-	"github.com/go-skynet/LocalAI/api/schema"
+	"time"
+
+	"github.com/go-skynet/LocalAI/core/schema"
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
-	"time"
 )
 
 var _ Backend = new(embedBackend)

pkg/grpc/interface.go

@@ -1,7 +1,7 @@
 package grpc
 
 import (
-	"github.com/go-skynet/LocalAI/api/schema"
+	"github.com/go-skynet/LocalAI/core/schema"
 	pb "github.com/go-skynet/LocalAI/pkg/grpc/proto"
 )
 

tests/integration/reflect_test.go

@@ -3,7 +3,7 @@ package integration_test
 import (
 	"reflect"
 
-	config "github.com/go-skynet/LocalAI/api/config"
+	config "github.com/go-skynet/LocalAI/core/config"
 	model "github.com/go-skynet/LocalAI/pkg/model"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"