Use go-llama.cpp

Earthfile

@@ -14,10 +14,11 @@ go-deps:
 build:
     FROM +go-deps
     WORKDIR /build
-    RUN git clone https://github.com/go-skynet/llama
+    RUN git clone --recurse-submodules https://github.com/go-skynet/go-llama.cpp
-    RUN cd llama && make libllama.a
+    RUN cd go-llama.cpp && make libbinding.a
     COPY . .
-    RUN C_INCLUDE_PATH=/build/llama LIBRARY_PATH=/build/llama go build -o llama-cli ./
+    RUN go mod edit -replace github.com/go-skynet/go-llama.cpp=/build/go-llama.cpp
+    RUN C_INCLUDE_PATH=$GOPATH/src/github.com/go-skynet/go-llama.cpp LIBRARY_PATH=$GOPATH/src/github.com/go-skynet/go-llama.cpp go build -o llama-cli ./
     SAVE ARTIFACT llama-cli AS LOCAL llama-cli
 
 image:

api.go

@@ -6,7 +6,7 @@ import (
 	"strconv"
 	"sync"
 
-	llama "github.com/go-skynet/llama/go"
+	llama "github.com/go-skynet/go-llama.cpp"
 	"github.com/gofiber/fiber/v2"
 	"github.com/gofiber/fiber/v2/middleware/filesystem"
 )

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
 	github.com/containerd/console v1.0.3 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
+	github.com/go-skynet/go-llama.cpp v0.0.0-20230405204601-5429d2339021 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect

go.sum

@@ -19,6 +19,10 @@ github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARu
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/go-skynet/go-llama.cpp v0.0.0-20230404185816-24b85a924f09 h1:WPUWvw7DOv3WUuhtNfv+xJVE2CCTGa1op1PKGcNk2Bk=
+github.com/go-skynet/go-llama.cpp v0.0.0-20230404185816-24b85a924f09/go.mod h1:yD5HHNAHPReBlvWGWUr9OcMeE5BJH3xOUDtKCwjxdEQ=
+github.com/go-skynet/go-llama.cpp v0.0.0-20230405204601-5429d2339021 h1:SsUkTjdCCAJjULfspizf99Sfw8Fx9OAHF30kp3i6cxc=
+github.com/go-skynet/go-llama.cpp v0.0.0-20230405204601-5429d2339021/go.mod h1:yD5HHNAHPReBlvWGWUr9OcMeE5BJH3xOUDtKCwjxdEQ=
 github.com/go-skynet/llama v0.0.0-20230321172246-7be5326e18cc h1:NcmO8mA7iRZIX0Qy2SjcsSaV14+g87MiTey1neUJaFQ=
 github.com/go-skynet/llama v0.0.0-20230321172246-7be5326e18cc/go.mod h1:ZtYsAIud4cvP9VTTI9uhdgR1uCwaO/gGKnZZ95h9i7w=
 github.com/go-skynet/llama v0.0.0-20230325223742-a3563a2690ba h1:u6OhAqlWFHsTjfWKePdK2kP4/mTyXX5vsmKwrK5QX6o=

interactive.go

@@ -11,7 +11,7 @@ import (
 	"github.com/charmbracelet/bubbles/viewport"
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/charmbracelet/lipgloss"
-	llama "github.com/go-skynet/llama/go"
+	llama "github.com/go-skynet/go-llama.cpp"
 )
 
 func startInteractive(l *llama.LLama, opts ...llama.PredictOption) error {

main.go

@@ -8,7 +8,7 @@ import (
 	"runtime"
 	"text/template"
 
-	llama "github.com/go-skynet/llama/go"
+	llama "github.com/go-skynet/go-llama.cpp"
 	"github.com/urfave/cli/v2"
 )
 
@@ -33,12 +33,6 @@ var nonEmptyInput string = `Below is an instruction that describes a task, paire
 
 func llamaFromOptions(ctx *cli.Context) (*llama.LLama, error) {
 	opts := []llama.ModelOption{llama.SetContext(ctx.Int("context-size"))}
-	if ctx.Bool("alpaca") {
-		opts = append(opts, llama.EnableAlpaca)
-	}
-	if ctx.Bool("gpt4all") {
-		opts = append(opts, llama.EnableGPT4All)
-	}
 	return llama.New(ctx.String("model"), opts...)
 }
 
@@ -92,16 +86,6 @@ var modelFlags = []cli.Flag{
 		EnvVars: []string{"TOP_K"},
 		Value:   20,
 	},
-	&cli.BoolFlag{
-		Name:    "alpaca",
-		EnvVars: []string{"ALPACA"},
-		Value:   true,
-	},
-	&cli.BoolFlag{
-		Name:    "gpt4all",
-		EnvVars: []string{"GPT4ALL"},
-		Value:   false,
-	},
 }
 
 func main() {
@@ -170,16 +154,6 @@ echo "An Alpaca (Vicugna pacos) is a domesticated species of South American came
 						EnvVars: []string{"ADDRESS"},
 						Value:   ":8080",
 					},
-					&cli.BoolFlag{
-						Name:    "alpaca",
-						EnvVars: []string{"ALPACA"},
-						Value:   true,
-					},
-					&cli.BoolFlag{
-						Name:    "gpt4all",
-						EnvVars: []string{"GPT4ALL"},
-						Value:   false,
-					},
 					&cli.IntFlag{
 						Name:    "context-size",
 						EnvVars: []string{"CONTEXT_SIZE"},