feat(p2p): Federation and AI swarms (#2723)

* Wip p2p enhancements

* get online state

* Pass-by token to show in the dashboard

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Style

* Minor fixups

* parametrize SearchID

* Refactoring

* Allow to expose/bind more services

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Add federation

* Display federated mode in the WebUI

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* Small fixups

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* make federated nodes visible from the WebUI

* Fix version display

* improve web page

* live page update

* visual enhancements

* enhancements

* visual enhancements

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

.github/release.yml

@@ -13,6 +13,9 @@ changelog:
       labels:
         - bug
         - regression
+    - title: "🖧 P2P area"
+      labels:
+         - area/p2p
     - title: Exciting New Features 🎉
       labels:
         - Semver-Minor

Makefile

@@ -53,8 +53,8 @@ RANDOM := $(shell bash -c 'echo $$RANDOM')
 VERSION?=$(shell git describe --always --tags || echo "dev" )
 # go tool nm ./local-ai | grep Commit
 LD_FLAGS?=
-override LD_FLAGS += -X "github.com/go-skynet/LocalAI/internal.Version=$(VERSION)"
+override LD_FLAGS += -X "github.com/mudler/LocalAI/internal.Version=$(VERSION)"
-override LD_FLAGS += -X "github.com/go-skynet/LocalAI/internal.Commit=$(shell git rev-parse HEAD)"
+override LD_FLAGS += -X "github.com/mudler/LocalAI/internal.Commit=$(shell git rev-parse HEAD)"
 
 OPTIONAL_TARGETS?=
 
@@ -147,7 +147,7 @@ endif
 
 # glibc-static or glibc-devel-static required
 ifeq ($(STATIC),true)
-	LD_FLAGS=-linkmode external -extldflags -static
+	LD_FLAGS+=-linkmode external -extldflags -static
 endif
 
 ifeq ($(findstring stablediffusion,$(GO_TAGS)),stablediffusion)

core/cli/cli.go

@@ -9,6 +9,7 @@ var CLI struct {
 	cliContext.Context `embed:""`
 
 	Run        RunCMD        `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"`
+	Federated  FederatedCLI  `cmd:"" help:"Run LocalAI in federated mode"`
 	Models     ModelsCMD     `cmd:"" help:"Manage LocalAI models and definitions"`
 	TTS        TTSCMD        `cmd:"" help:"Convert text to speech"`
 	Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"`

core/cli/federated.go

@@ -0,0 +1,130 @@
+package cli
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"time"
+
+	"math/rand/v2"
+
+	cliContext "github.com/mudler/LocalAI/core/cli/context"
+	"github.com/mudler/LocalAI/core/p2p"
+	"github.com/mudler/edgevpn/pkg/node"
+	"github.com/mudler/edgevpn/pkg/protocol"
+	"github.com/mudler/edgevpn/pkg/types"
+	"github.com/rs/zerolog/log"
+)
+
+type FederatedCLI struct {
+	Address        string `env:"LOCALAI_ADDRESS,ADDRESS" default:":8080" help:"Bind address for the API server" group:"api"`
+	Peer2PeerToken string `env:"LOCALAI_P2P_TOKEN,P2P_TOKEN,TOKEN" name:"p2ptoken" help:"Token for P2P mode (optional)" group:"p2p"`
+}
+
+func (f *FederatedCLI) Run(ctx *cliContext.Context) error {
+
+	n, err := p2p.NewNode(f.Peer2PeerToken)
+	if err != nil {
+		return fmt.Errorf("creating a new node: %w", err)
+	}
+	err = n.Start(context.Background())
+	if err != nil {
+		return fmt.Errorf("creating a new node: %w", err)
+	}
+
+	if err := p2p.ServiceDiscoverer(context.Background(), n, f.Peer2PeerToken, p2p.FederatedID, nil); err != nil {
+		return err
+	}
+
+	return Proxy(context.Background(), n, f.Address, p2p.FederatedID)
+}
+
+func Proxy(ctx context.Context, node *node.Node, listenAddr, service string) error {
+
+	log.Info().Msgf("Allocating service '%s' on: %s", service, listenAddr)
+	// Open local port for listening
+	l, err := net.Listen("tcp", listenAddr)
+	if err != nil {
+		log.Error().Err(err).Msg("Error listening")
+		return err
+	}
+	//	ll.Info("Binding local port on", srcaddr)
+
+	ledger, _ := node.Ledger()
+
+	// Announce ourselves so nodes accepts our connection
+	ledger.Announce(
+		ctx,
+		10*time.Second,
+		func() {
+			// Retrieve current ID for ip in the blockchain
+			//_, found := ledger.GetKey(protocol.UsersLedgerKey, node.Host().ID().String())
+			// If mismatch, update the blockchain
+			//if !found {
+			updatedMap := map[string]interface{}{}
+			updatedMap[node.Host().ID().String()] = &types.User{
+				PeerID:    node.Host().ID().String(),
+				Timestamp: time.Now().String(),
+			}
+			ledger.Add(protocol.UsersLedgerKey, updatedMap)
+			//	}
+		},
+	)
+
+	defer l.Close()
+	for {
+		select {
+		case <-ctx.Done():
+			return errors.New("context canceled")
+		default:
+			log.Debug().Msg("New for connection")
+			// Listen for an incoming connection.
+			conn, err := l.Accept()
+			if err != nil {
+				fmt.Println("Error accepting: ", err.Error())
+				continue
+			}
+
+			// Handle connections in a new goroutine, forwarding to the p2p service
+			go func() {
+				var tunnelAddresses []string
+				for _, v := range p2p.GetAvailableNodes(p2p.FederatedID) {
+					if v.IsOnline() {
+						tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
+					} else {
+						log.Info().Msgf("Node %s is offline", v.ID)
+					}
+				}
+
+				// open a TCP stream to one of the tunnels
+				// chosen randomly
+				// TODO: optimize this and track usage
+				tunnelAddr := tunnelAddresses[rand.IntN(len(tunnelAddresses))]
+
+				tunnelConn, err := net.Dial("tcp", tunnelAddr)
+				if err != nil {
+					log.Error().Err(err).Msg("Error connecting to tunnel")
+					return
+				}
+
+				log.Info().Msgf("Redirecting %s to %s", conn.LocalAddr().String(), tunnelConn.RemoteAddr().String())
+				closer := make(chan struct{}, 2)
+				go copyStream(closer, tunnelConn, conn)
+				go copyStream(closer, conn, tunnelConn)
+				<-closer
+
+				tunnelConn.Close()
+				conn.Close()
+				//	ll.Infof("(service %s) Done handling %s", serviceID, l.Addr().String())
+			}()
+		}
+	}
+
+}
+
+func copyStream(closer chan struct{}, dst io.Writer, src io.Reader) {
+	defer func() { closer <- struct{}{} }() // connection is closed, send signal to stop proxy
+	io.Copy(dst, src)
+}

core/cli/run.go

@@ -3,6 +3,8 @@ package cli
 import (
 	"context"
 	"fmt"
+	"net"
+	"os"
 	"strings"
 	"time"
 
@@ -50,7 +52,7 @@ type RunCMD struct {
 	DisableWebUI         bool     `env:"LOCALAI_DISABLE_WEBUI,DISABLE_WEBUI" default:"false" help:"Disable webui" group:"api"`
 	OpaqueErrors         bool     `env:"LOCALAI_OPAQUE_ERRORS" default:"false" help:"If true, all error responses are replaced with blank 500 errors. This is intended only for hardening against information leaks and is normally not recommended." group:"api"`
 	Peer2Peer            bool     `env:"LOCALAI_P2P,P2P" name:"p2p" default:"false" help:"Enable P2P mode" group:"p2p"`
-	Peer2PeerToken       string   `env:"LOCALAI_P2P_TOKEN,P2P_TOKEN" name:"p2ptoken" help:"Token for P2P mode (optional)" group:"p2p"`
+	Peer2PeerToken       string   `env:"LOCALAI_P2P_TOKEN,P2P_TOKEN,TOKEN" name:"p2ptoken" help:"Token for P2P mode (optional)" group:"p2p"`
 	ParallelRequests     bool     `env:"LOCALAI_PARALLEL_REQUESTS,PARALLEL_REQUESTS" help:"Enable backends to handle multiple requests in parallel if they support it (e.g.: llama.cpp or vllm)" group:"backends"`
 	SingleActiveBackend  bool     `env:"LOCALAI_SINGLE_ACTIVE_BACKEND,SINGLE_ACTIVE_BACKEND" help:"Allow only one backend to be run at a time" group:"backends"`
 	PreloadBackendOnly   bool     `env:"LOCALAI_PRELOAD_BACKEND_ONLY,PRELOAD_BACKEND_ONLY" default:"false" help:"Do not launch the API services, only the preloaded models / backends are started (useful for multi-node setups)" group:"backends"`
@@ -59,6 +61,7 @@ type RunCMD struct {
 	WatchdogIdleTimeout  string   `env:"LOCALAI_WATCHDOG_IDLE_TIMEOUT,WATCHDOG_IDLE_TIMEOUT" default:"15m" help:"Threshold beyond which an idle backend should be stopped" group:"backends"`
 	EnableWatchdogBusy   bool     `env:"LOCALAI_WATCHDOG_BUSY,WATCHDOG_BUSY" default:"false" help:"Enable watchdog for stopping backends that are busy longer than the watchdog-busy-timeout" group:"backends"`
 	WatchdogBusyTimeout  string   `env:"LOCALAI_WATCHDOG_BUSY_TIMEOUT,WATCHDOG_BUSY_TIMEOUT" default:"5m" help:"Threshold beyond which a busy backend should be stopped" group:"backends"`
+	Federated            bool     `env:"LOCALAI_FEDERATED,FEDERATED" help:"Enable federated instance" group:"federated"`
 }
 
 func (r *RunCMD) Run(ctx *cliContext.Context) error {
@@ -91,9 +94,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		config.WithOpaqueErrors(r.OpaqueErrors),
 	}
 
+	token := ""
 	if r.Peer2Peer || r.Peer2PeerToken != "" {
 		log.Info().Msg("P2P mode enabled")
-		token := r.Peer2PeerToken
+		token = r.Peer2PeerToken
 		if token == "" {
 			// IF no token is provided, and p2p is enabled,
 			// we generate one and wait for the user to pick up the token (this is for interactive)
@@ -104,14 +108,46 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 
 			log.Info().Msg("To use the token, you can run the following command in another node or terminal:")
 			fmt.Printf("export TOKEN=\"%s\"\nlocal-ai worker p2p-llama-cpp-rpc\n", token)
-
-			// Ask for user confirmation
-			log.Info().Msg("Press a button to proceed")
-			var input string
-			fmt.Scanln(&input)
 		}
+		opts = append(opts, config.WithP2PToken(token))
+
+		node, err := p2p.NewNode(token)
+		if err != nil {
+			return err
+		}
+
 		log.Info().Msg("Starting P2P server discovery...")
-		if err := p2p.LLamaCPPRPCServerDiscoverer(context.Background(), token); err != nil {
+		if err := p2p.ServiceDiscoverer(context.Background(), node, token, "", func() {
+			var tunnelAddresses []string
+			for _, v := range p2p.GetAvailableNodes("") {
+				if v.IsOnline() {
+					tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
+				} else {
+					log.Info().Msgf("Node %s is offline", v.ID)
+				}
+			}
+			tunnelEnvVar := strings.Join(tunnelAddresses, ",")
+
+			os.Setenv("LLAMACPP_GRPC_SERVERS", tunnelEnvVar)
+			log.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", tunnelEnvVar)
+		}); err != nil {
+			return err
+		}
+	}
+
+	if r.Federated {
+		_, port, err := net.SplitHostPort(r.Address)
+		if err != nil {
+			return err
+		}
+		if err := p2p.ExposeService(context.Background(), "localhost", port, token, p2p.FederatedID); err != nil {
+			return err
+		}
+		node, err := p2p.NewNode(token)
+		if err != nil {
+			return err
+		}
+		if err := p2p.ServiceDiscoverer(context.Background(), node, token, p2p.FederatedID, nil); err != nil {
 			return err
 		}
 	}

core/cli/worker/worker_p2p.go

@@ -20,7 +20,7 @@ import (
 
 type P2P struct {
 	WorkerFlags       `embed:""`
-	Token             string   `env:"LOCALAI_TOKEN,TOKEN" help:"JSON list of galleries"`
+	Token             string   `env:"LOCALAI_TOKEN,LOCALAI_P2P_TOKEN,TOKEN" help:"P2P token to use"`
 	NoRunner          bool     `env:"LOCALAI_NO_RUNNER,NO_RUNNER" help:"Do not start the llama-cpp-rpc-server"`
 	RunnerAddress     string   `env:"LOCALAI_RUNNER_ADDRESS,RUNNER_ADDRESS" help:"Address of the llama-cpp-rpc-server"`
 	RunnerPort        string   `env:"LOCALAI_RUNNER_PORT,RUNNER_PORT" help:"Port of the llama-cpp-rpc-server"`
@@ -59,7 +59,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
 			p = r.RunnerPort
 		}
 
-		err = p2p.BindLLamaCPPWorker(context.Background(), address, p, r.Token)
+		err = p2p.ExposeService(context.Background(), address, p, r.Token, "")
 		if err != nil {
 			return err
 		}
@@ -99,7 +99,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
 		}
 	}()
 
-	err = p2p.BindLLamaCPPWorker(context.Background(), address, fmt.Sprint(port), r.Token)
+	err = p2p.ExposeService(context.Background(), address, fmt.Sprint(port), r.Token, "")
 	if err != nil {
 		return err
 	}

core/config/application_config.go

@@ -32,6 +32,7 @@ type ApplicationConfig struct {
 	CORSAllowOrigins                    string
 	ApiKeys                             []string
 	OpaqueErrors                        bool
+	P2PToken                            string
 
 	ModelLibraryURL string
 
@@ -95,6 +96,12 @@ func WithCsrf(b bool) AppOption {
 	}
 }
 
+func WithP2PToken(s string) AppOption {
+	return func(o *ApplicationConfig) {
+		o.P2PToken = s
+	}
+}
+
 func WithModelLibraryURL(url string) AppOption {
 	return func(o *ApplicationConfig) {
 		o.ModelLibraryURL = url

core/http/elements/gallery.go

@@ -7,6 +7,7 @@ import (
 	"github.com/chasefleming/elem-go"
 	"github.com/chasefleming/elem-go/attrs"
 	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/core/services"
 	"github.com/mudler/LocalAI/pkg/xsync"
 )
@@ -15,6 +16,14 @@ const (
 	noImage = "https://upload.wikimedia.org/wikipedia/commons/6/65/No-Image-Placeholder.svg"
 )
 
+func renderElements(n []elem.Node) string {
+	render := ""
+	for _, r := range n {
+		render += r.Render()
+	}
+	return render
+}
+
 func DoneProgress(galleryID, text string, showDelete bool) string {
 	var modelName = galleryID
 	// Split by @ and grab the name
@@ -72,6 +81,135 @@ func ProgressBar(progress string) string {
 	).Render()
 }
 
+func P2PNodeStats(nodes []p2p.NodeData) string {
+	/*
+	   <div class="bg-gray-800 p-6 rounded-lg shadow-lg text-left">
+	                       <p class="text-xl font-semibold text-gray-200">Total Workers Detected: {{ len .Nodes }}</p>
+	                       {{ $online := 0 }}
+	                       {{ range .Nodes }}
+	                           {{ if .IsOnline }}
+	                               {{ $online = add $online 1 }}
+	                           {{ end }}
+	                       {{ end }}
+	                       <p class="text-xl font-semibold text-gray-200">Total Online Workers: {{$online}}</p>
+	                   </div>
+	*/
+
+	online := 0
+	for _, n := range nodes {
+		if n.IsOnline() {
+			online++
+		}
+	}
+
+	class := "text-green-500"
+	if online == 0 {
+		class = "text-red-500"
+	}
+	/*
+	   <i class="fas fa-circle animate-pulse text-green-500 ml-2 mr-1"></i>
+	*/
+	circle := elem.I(attrs.Props{
+		"class": "fas fa-circle animate-pulse " + class + " ml-2 mr-1",
+	})
+	nodesElements := []elem.Node{
+		elem.Span(
+			attrs.Props{
+				"class": class,
+			},
+			circle,
+			elem.Text(fmt.Sprintf("%d", online)),
+		),
+		elem.Span(
+			attrs.Props{
+				"class": "text-gray-200",
+			},
+			elem.Text(fmt.Sprintf("/%d", len(nodes))),
+		),
+	}
+
+	return renderElements(nodesElements)
+}
+
+func P2PNodeBoxes(nodes []p2p.NodeData) string {
+	/*
+			<div class="bg-gray-800 p-4 rounded-lg shadow-lg text-left">
+			<div class="flex items-center mb-2">
+				<i class="fas fa-desktop text-gray-400 mr-2"></i>
+				<span class="text-gray-200 font-semibold">{{.ID}}</span>
+			</div>
+			<p class="text-sm text-gray-400 mt-2 flex items-center">
+				Status:
+				<i class="fas fa-circle {{ if .IsOnline }}text-green-500{{ else }}text-red-500{{ end }} ml-2 mr-1"></i>
+				<span class="{{ if .IsOnline }}text-green-400{{ else }}text-red-400{{ end }}">
+					{{ if .IsOnline }}Online{{ else }}Offline{{ end }}
+				</span>
+			</p>
+		</div>
+	*/
+
+	nodesElements := []elem.Node{}
+
+	for _, n := range nodes {
+
+		nodesElements = append(nodesElements,
+			elem.Div(
+				attrs.Props{
+					"class": "bg-gray-700 p-6 rounded-lg shadow-lg text-left",
+				},
+				elem.P(
+					attrs.Props{
+						"class": "text-sm text-gray-400 mt-2 flex",
+					},
+					elem.I(
+						attrs.Props{
+							"class": "fas fa-desktop text-gray-400 mr-2",
+						},
+					),
+					elem.Text("Name: "),
+					elem.Span(
+						attrs.Props{
+							"class": "text-gray-200 font-semibold ml-2 mr-1",
+						},
+						elem.Text(n.ID),
+					),
+					elem.Text("Status: "),
+					elem.If(
+						n.IsOnline(),
+						elem.I(
+							attrs.Props{
+								"class": "fas fa-circle animate-pulse text-green-500 ml-2 mr-1",
+							},
+						),
+						elem.I(
+							attrs.Props{
+								"class": "fas fa-circle animate-pulse text-red-500 ml-2 mr-1",
+							},
+						),
+					),
+					elem.If(
+						n.IsOnline(),
+						elem.Span(
+							attrs.Props{
+								"class": "text-green-400",
+							},
+
+							elem.Text("Online"),
+						),
+						elem.Span(
+							attrs.Props{
+								"class": "text-red-400",
+							},
+							elem.Text("Offline"),
+						),
+					),
+				),
+			))
+	}
+
+	return renderElements(nodesElements)
+}
+
 func StartProgressBar(uid, progress, text string) string {
 	if progress == "" {
 		progress = "0"

core/http/endpoints/localai/welcome.go

@@ -4,6 +4,7 @@ import (
 	"github.com/gofiber/fiber/v2"
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/gallery"
+	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/internal"
 	"github.com/mudler/LocalAI/pkg/model"
 )
@@ -33,6 +34,7 @@ func WelcomeEndpoint(appConfig *config.ApplicationConfig,
 			"Models":            models,
 			"ModelsConfig":      backendConfigs,
 			"GalleryConfig":     galleryConfigs,
+			"IsP2PEnabled":      p2p.IsP2PEnabled(),
 			"ApplicationConfig": appConfig,
 			"ProcessingModels":  processingModels,
 			"TaskTypes":         taskTypes,

core/http/routes/localai.go

@@ -5,6 +5,7 @@ import (
 	"github.com/gofiber/swagger"
 	"github.com/mudler/LocalAI/core/config"
 	"github.com/mudler/LocalAI/core/http/endpoints/localai"
+	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/core/services"
 	"github.com/mudler/LocalAI/internal"
 	"github.com/mudler/LocalAI/pkg/model"
@@ -56,6 +57,20 @@ func RegisterLocalAIRoutes(app *fiber.App,
 	app.Get("/backend/monitor", auth, localai.BackendMonitorEndpoint(backendMonitorService))
 	app.Post("/backend/shutdown", auth, localai.BackendShutdownEndpoint(backendMonitorService))
 
+	// p2p
+	if p2p.IsP2PEnabled() {
+		app.Get("/api/p2p", auth, func(c *fiber.Ctx) error {
+			// Render index
+			return c.JSON(map[string]interface{}{
+				"Nodes":          p2p.GetAvailableNodes(""),
+				"FederatedNodes": p2p.GetAvailableNodes(p2p.FederatedID),
+			})
+		})
+		app.Get("/api/p2p/token", auth, func(c *fiber.Ctx) error {
+			return c.Send([]byte(appConfig.P2PToken))
+		})
+	}
+
 	app.Get("/version", auth, func(c *fiber.Ctx) error {
 		return c.JSON(struct {
 			Version string `json:"version"`

core/http/routes/ui.go

@@ -10,6 +10,7 @@ import (
 	"github.com/mudler/LocalAI/core/gallery"
 	"github.com/mudler/LocalAI/core/http/elements"
 	"github.com/mudler/LocalAI/core/http/endpoints/localai"
+	"github.com/mudler/LocalAI/core/p2p"
 	"github.com/mudler/LocalAI/core/services"
 	"github.com/mudler/LocalAI/internal"
 	"github.com/mudler/LocalAI/pkg/model"
@@ -53,6 +54,37 @@ func RegisterUIRoutes(app *fiber.App,
 
 	app.Get("/", auth, localai.WelcomeEndpoint(appConfig, cl, ml, modelStatus))
 
+	if p2p.IsP2PEnabled() {
+		app.Get("/p2p", auth, func(c *fiber.Ctx) error {
+			summary := fiber.Map{
+				"Title":   "LocalAI - P2P dashboard",
+				"Version": internal.PrintableVersion(),
+				//"Nodes":          p2p.GetAvailableNodes(""),
+				//"FederatedNodes": p2p.GetAvailableNodes(p2p.FederatedID),
+				"IsP2PEnabled": p2p.IsP2PEnabled(),
+				"P2PToken":     appConfig.P2PToken,
+			}
+
+			// Render index
+			return c.Render("views/p2p", summary)
+		})
+
+		/* show nodes live! */
+		app.Get("/p2p/ui/workers", auth, func(c *fiber.Ctx) error {
+			return c.SendString(elements.P2PNodeBoxes(p2p.GetAvailableNodes("")))
+		})
+		app.Get("/p2p/ui/workers-federation", auth, func(c *fiber.Ctx) error {
+			return c.SendString(elements.P2PNodeBoxes(p2p.GetAvailableNodes(p2p.FederatedID)))
+		})
+
+		app.Get("/p2p/ui/workers-stats", auth, func(c *fiber.Ctx) error {
+			return c.SendString(elements.P2PNodeStats(p2p.GetAvailableNodes("")))
+		})
+		app.Get("/p2p/ui/workers-federation-stats", auth, func(c *fiber.Ctx) error {
+			return c.SendString(elements.P2PNodeStats(p2p.GetAvailableNodes(p2p.FederatedID)))
+		})
+	}
+
 	// Show the Models page (all models)
 	app.Get("/browse", auth, func(c *fiber.Ctx) error {
 		term := c.Query("term")
@@ -87,7 +119,9 @@ func RegisterUIRoutes(app *fiber.App,
 			"AllTags":          tags,
 			"ProcessingModels": processingModelsData,
 			"AvailableModels":  len(models),
-			"TaskTypes":        taskTypes,
+			"IsP2PEnabled":     p2p.IsP2PEnabled(),
+
+			"TaskTypes": taskTypes,
 			//	"ApplicationConfig": appConfig,
 		}
 
@@ -243,6 +277,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"ModelsConfig": backendConfigs,
 			"Model":        c.Params("model"),
 			"Version":      internal.PrintableVersion(),
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 		}
 
 		// Render index
@@ -261,6 +296,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"Title":        "LocalAI - Talk",
 			"ModelsConfig": backendConfigs,
 			"Model":        backendConfigs[0].ID,
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 			"Version":      internal.PrintableVersion(),
 		}
 
@@ -282,6 +318,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"ModelsConfig": backendConfigs,
 			"Model":        backendConfigs[0].ID,
 			"Version":      internal.PrintableVersion(),
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 		}
 
 		// Render index
@@ -296,6 +333,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"ModelsConfig": backendConfigs,
 			"Model":        c.Params("model"),
 			"Version":      internal.PrintableVersion(),
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 		}
 
 		// Render index
@@ -316,6 +354,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"ModelsConfig": backendConfigs,
 			"Model":        backendConfigs[0].Name,
 			"Version":      internal.PrintableVersion(),
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 		}
 
 		// Render index
@@ -330,6 +369,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"ModelsConfig": backendConfigs,
 			"Model":        c.Params("model"),
 			"Version":      internal.PrintableVersion(),
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 		}
 
 		// Render index
@@ -349,6 +389,7 @@ func RegisterUIRoutes(app *fiber.App,
 			"Title":        "LocalAI - Generate audio with " + backendConfigs[0].Name,
 			"ModelsConfig": backendConfigs,
 			"Model":        backendConfigs[0].Name,
+			"IsP2PEnabled": p2p.IsP2PEnabled(),
 			"Version":      internal.PrintableVersion(),
 		}
 

core/http/static/general.css

@@ -81,10 +81,10 @@ ul {
 li {
     font-size: 0.875rem; /* Small text size */
     color: #4a5568; /* Dark gray text */
-    background-color: #f7fafc; /* Very light gray background */
+   /*  background-color: #f7fafc; Very light gray background */
     border-radius: 0.375rem; /* Rounded corners */
     padding: 0.5rem; /* Padding inside each list item */
-    box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.1), 0 1px 2px 0 rgba(0, 0, 0, 0.06); /* Subtle shadow */
+    /*box-shadow: 0 1px 3px 0 rgba(0, 0, 0, 0.1), 0 1px 2px 0 rgba(0, 0, 0, 0.06);  Subtle shadow */
     margin-bottom: 0.5rem; /* Vertical space between list items */
 }
 

core/http/views/chat.html

@@ -37,7 +37,7 @@ SOFTWARE.
   <body class="bg-gray-900 text-gray-200" x-data="{ key: $store.chat.key }">
     <div class="flex flex-col min-h-screen">
 
-    {{template "views/partials/navbar"}}
+    {{template "views/partials/navbar" .}}
     <div class="chat-container mt-2 mr-2 ml-2 mb-2 bg-gray-800 shadow-lg rounded-lg" >
      <!-- Chat Header -->
     <div class="border-b border-gray-700 p-4"  x-data="{ component: 'menu' }">

core/http/views/p2p.html

@@ -0,0 +1,150 @@
+<!DOCTYPE html>
+<html lang="en">
+{{template "views/partials/head" .}}
+
+<body class="bg-gray-900 text-gray-200">
+<div class="flex flex-col min-h-screen">
+   
+    {{template "views/partials/navbar" .}}
+    <div class="container mx-auto px-4 flex-grow">
+        <div class="workers mt-12 text-center">
+      
+            <h2 class="text-3xl font-semibold text-gray-100 mb-8">
+                <i class="fa-solid fa-circle-nodes"></i> Distributed inference with P2P   
+                <a href="https://localai.io/features/distribute/" target="_blank">
+                    <i class="fas fa-circle-info pr-2"></i>
+                </a> 
+            </h2> 
+            <h5 class="mb-4 text-justify">LocalAI uses P2P technologies to enable distribution of work between peers. It is possible to share an instance with Federation and/or split the weights of a model across peers (only available with llama.cpp models). You can now share computational resources between your devices or your friends!</h5>
+            
+            <!-- Federation Box -->
+            <div class="bg-gray-800 p-6 rounded-lg shadow-lg mb-12 text-left">
+
+                <p class="text-xl font-semibold text-gray-200"> <i class="text-gray-200 fa-solid fa-circle-nodes"></i> Federated Nodes: <span hx-get="/p2p/ui/workers-federation-stats" hx-trigger="every 1s"></span> </p>
+                <p class="mb-4">You can start LocalAI in federated mode to share your instance, or start the federated server to balance requests between nodes of the federation.</p>
+
+                <div class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4 mb-12">
+                    <div hx-get="/p2p/ui/workers-federation" hx-trigger="every 1s"></div>
+                </div>
+
+                <hr class="border-gray-700 mb-12">
+
+                <h3 class="text-2xl font-semibold text-gray-100 mb-6"><i class="fa-solid fa-book"></i> Start a federated instance</h3>
+
+                
+                <!-- Tabs navigation -->
+                <ul class="mb-5 flex list-none flex-row flex-wrap ps-0" role="tablist" data-twe-nav-ref>
+                    <li role="presentation" class="flex-auto text-center">
+                        <a href="#tabs-federated-cli" class="tablink my-2 block border-0 bg-gray-800 px-7 pb-3.5 pt-4 text-xs font-medium uppercase leading-tight text-white hover:bg-gray-700 focus:bg-gray-700 data-[twe-nav-active]:border-yellow-500 data-[twe-nav-active]:text-yellow-500 data-[twe-nav-active]:bg-gray-700 active" data-twe-toggle="pill" data-twe-target="#tabs-federated-cli" data-twe-nav-active role="tab" aria-controls="tabs-federated-cli" aria-selected="true"><i class="fa-solid fa-terminal"></i> CLI</a>
+                    </li>
+                    <li role="presentation" class="flex-auto text-center">
+                        <a href="#tabs-federated-docker" class="tablink my-2 block border-0 bg-gray-800 px-7 pb-3.5 pt-4 text-xs font-medium uppercase leading-tight text-white hover:bg-gray-700 focus:bg-gray-700 data-[twe-nav-active]:border-yellow-500 data-[twe-nav-active]:text-yellow-500 data-[twe-nav-active]:bg-gray-700" data-twe-toggle="pill" data-twe-target="#tabs-federated-docker" role="tab" aria-controls="tabs-federated-docker" aria-selected="false"><i class="fa-solid fa-box-open"></i> Container images</a>
+                    </li>
+                </ul>
+
+                <!-- Tabs content -->
+                <div class="mb-6">
+                    
+                    <div class="tabcontent hidden opacity-100 transition-opacity duration-150 ease-linear data-[twe-tab-active]:block p-4" id="tabs-federated-cli" role="tabpanel" aria-labelledby="tabs-federated-cli" data-twe-tab-active>
+
+
+                        <p class="mb-2">To start a new instance to share:</p>
+                        <code class="block bg-gray-700 text-yellow-300 p-4 rounded-lg break-words">
+                            # Start a new instance to share with --federated and a TOKEN<br>
+                            export TOKEN="<span class="token">{{.P2PToken}}</span>"<br>
+                            local-ai run --federated --p2p
+                        </code>
+
+                        <p class="mt-2">Note: If you don't have a token do not specify it and use the generated one that you can find in this page.</p>
+
+                        <p class="mb-2">To start a new federated load balancer:</p>
+                        <code class="block bg-gray-700 text-yellow-300 p-4 rounded-lg break-words">
+                            export TOKEN="<span class="token">{{.P2PToken}}</span>"<br>
+                            local-ai federated
+                        </code>
+
+                        <p class="mt-2">Note: Token is needed when starting the federated server.</p>
+
+                        <p class="mt-2">For all the options available, please refer to the <a href="https://localai.io/features/distribute/#starting-workers" target="_blank" class="text-yellow-300 hover:text-yellow-400">documentation</a>.</p>
+                    </div>
+                    <div class="tabcontent hidden opacity-0 transition-opacity duration-150 ease-linear data-[twe-tab-active]:block p-4" id="tabs-federated-docker" role="tabpanel" aria-labelledby="tabs-federated-docker">
+                        <p class="mb-2">To start a new federated instance:</p>
+                        <code class="block bg-gray-700 text-yellow-300 p-4 rounded-lg break-words">
+                            docker run -ti --net host -e TOKEN="<span class="token">{{.P2PToken}}</span>" --name local-ai -p 8080:8080 localai/localai:latest-cpu run --federated --p2p
+                        </code>
+
+                        <p class="mb-2">To start a new federated server (port to 9090):</p>
+                        <code class="block bg-gray-700 text-yellow-300 p-4 rounded-lg break-words">
+                            docker run -ti --net host -e TOKEN="<span class="token">{{.P2PToken}}</span>" --name local-ai -p 9090:8080 localai/localai:latest-cpu federated
+                        </code>
+
+                        <p class="mt-2">For all the options available and see what image to use, please refer to the <a href="https://localai.io/basics/container/" target="_blank" class="text-yellow-300 hover:text-yellow-400">Container images documentation</a> and <a href="https://localai.io/advanced/#cli-parameters" target="_blank" class="text-yellow-300 hover:text-yellow-400">CLI parameters documentation</a>.</p>
+                    </div>
+                </div>
+            </div>
+
+            <!-- Llama.cpp Box -->
+
+            <div class="bg-gray-800 p-6 rounded-lg shadow-lg mb-12 text-left">
+
+                <p class="text-xl font-semibold text-gray-200"> <i class="text-gray-200 fa-solid fa-circle-nodes"></i> Workers (llama.cpp): <span hx-get="/p2p/ui/workers-stats" hx-trigger="every 1s"></span> </p>
+                <p class="mb-4">You can start llama.cpp workers to distribute weights between the workers and offload part of the computation. To start a new worker, you can use the CLI or Docker.</p>
+
+                <div class="grid grid-cols-1 sm:grid-cols-2 md:grid-cols-3 gap-4 mb-12">
+                    <div hx-get="/p2p/ui/workers" hx-trigger="every 1s"></div>
+                </div>
+                <hr class="border-gray-700 mb-12">
+
+                <h3 class="text-2xl font-semibold text-gray-100 mb-6"><i class="fa-solid fa-book"></i> Start a new llama.cpp P2P worker</h3>
+                
+                <!-- Tabs navigation -->
+                <ul class="mb-5 flex list-none flex-row flex-wrap ps-0" role="tablist" data-twe-nav-ref>
+                    <li role="presentation" class="flex-auto text-center">
+                        <a href="#tabs-cli" class="tablink my-2 block border-0 bg-gray-800 px-7 pb-3.5 pt-4 text-xs font-medium uppercase leading-tight text-white hover:bg-gray-700 focus:bg-gray-700 data-[twe-nav-active]:border-yellow-500 data-[twe-nav-active]:text-yellow-500 data-[twe-nav-active]:bg-gray-700 active" data-twe-toggle="pill" data-twe-target="#tabs-cli" data-twe-nav-active role="tab" aria-controls="tabs-cli" aria-selected="true"><i class="fa-solid fa-terminal"></i> CLI</a>
+                    </li>
+                    <li role="presentation" class="flex-auto text-center">
+                        <a href="#tabs-docker" class="tablink my-2 block border-0 bg-gray-800 px-7 pb-3.5 pt-4 text-xs font-medium uppercase leading-tight text-white hover:bg-gray-700 focus:bg-gray-700 data-[twe-nav-active]:border-yellow-500 data-[twe-nav-active]:text-yellow-500 data-[twe-nav-active]:bg-gray-700" data-twe-toggle="pill" data-twe-target="#tabs-docker" role="tab" aria-controls="tabs-docker" aria-selected="false"><i class="fa-solid fa-box-open"></i> Container images</a>
+                    </li>
+                </ul>
+
+                <!-- Tabs content -->
+                <div class="mb-6">
+                    <div class="tabcontent hidden opacity-100 transition-opacity duration-150 ease-linear data-[twe-tab-active]:block p-4" id="tabs-cli" role="tabpanel" aria-labelledby="tabs-cli" data-twe-tab-active>
+                        <p class="mb-2">To start a new worker, run the following command:</p>
+                        <code class="block bg-gray-700 text-yellow-300 p-4 rounded-lg break-words">
+                            export TOKEN="<span class="token">{{.P2PToken}}</span>"<br>
+                            local-ai worker p2p-llama-cpp-rpc
+                        </code>
+
+                        <p class="mt-2">For all the options available, please refer to the <a href="https://localai.io/features/distribute/#starting-workers" target="_blank" class="text-yellow-300 hover:text-yellow-400">documentation</a>.</p>
+                    </div>
+                    <div class="tabcontent hidden opacity-0 transition-opacity duration-150 ease-linear data-[twe-tab-active]:block p-4" id="tabs-docker" role="tabpanel" aria-labelledby="tabs-docker">
+                        <p class="mb-2">To start a new worker with docker, run the following command:</p>
+                        <code class="block bg-gray-700 text-yellow-300 p-4 rounded-lg break-words">
+                            docker run -ti --net host -e TOKEN="<span class="token">{{.P2PToken}}</span>" --name local-ai -p 8080:8080 localai/localai:latest-cpu worker p2p-llama-cpp-rpc
+                        </code>
+
+                        <p class="mt-2">For all the options available and see what image to use, please refer to the <a href="https://localai.io/basics/container/" target="_blank" class="text-yellow-300 hover:text-yellow-400">Container images documentation</a> and <a href="https://localai.io/advanced/#cli-parameters" target="_blank" class="text-yellow-300 hover:text-yellow-400">CLI parameters documentation</a>.</p>
+                    </div>
+                </div>
+            </div>
+            <!-- Llama.cpp Box END -->       
+        </div>
+    </div>
+
+    {{template "views/partials/footer" .}}
+</div>
+
+<style>
+    .token {
+        word-break: break-all;
+    }
+    .workers .grid div {
+        display: flex;
+        flex-direction: column;
+        justify-content: space-between;
+    }
+</style>
+
+</body>
+</html>

core/http/views/partials/footer.html

@@ -1,4 +1,5 @@
 <footer class="text-center py-8">
     LocalAI Version {{.Version}}<br>
     <a href='https://localai.io' class="text-blue-400 hover:text-blue-600" target="_blank">LocalAI</a> © 2023-2024 <a href='https://mudler.pm' class="text-blue-400 hover:text-blue-600" target="_blank">Ettore Di Giacinto</a>
-</footer>
+</footer>
+<script src="/static/assets/tw-elements.js"></script>

core/http/views/partials/navbar.html

@@ -21,6 +21,9 @@
                 <a href="/text2image/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-image pr-2"></i> Generate images</a>
                 <a href="/tts/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fa-solid fa-music pr-2"></i> TTS </a>
                 <a href="/talk/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fa-solid fa-phone pr-2"></i> Talk </a>
+                {{ if .IsP2PEnabled }}
+                <a href="/p2p/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fa-solid fa-circle-nodes"></i> Swarm </a>
+                {{ end }}
                 <a href="/swagger/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-code pr-2"></i> API</a>
             </div>
         </div>
@@ -34,6 +37,9 @@
                 <a href="/text2image/" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1"><i class="fas fa-image pr-2"></i> Generate images</a>
                 <a href="/tts/" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1"><i class="fa-solid fa-music pr-2"></i> TTS </a>
                 <a href="/talk/" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1"><i class="fa-solid fa-phone pr-2"></i> Talk </a>
+                {{ if .IsP2PEnabled }}
+                <a href="/p2p/" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1"><i class="fa-solid fa-circle-nodes"></i> Swarm </a>
+                {{ end }}
                 <a href="/swagger/" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1"><i class="fas fa-code pr-2"></i> API</a>
             </div>
         </div>

core/http/views/talk.html

@@ -10,7 +10,7 @@
   <body class="bg-gray-900 text-gray-200" x-data="{ key: $store.chat.key }">
     <div class="flex flex-col min-h-screen">
 
-    {{template "views/partials/navbar"}}
+    {{template "views/partials/navbar" .}}
     <div class="chat-container mt-2 mr-2 ml-2 mb-2 bg-gray-800 shadow-lg rounded-lg " >
      <!-- Chat Header -->
       <div class="border-b border-gray-700 p-4"  x-data="{ component: 'menu' }">

core/p2p/node.go

@@ -0,0 +1,50 @@
+package p2p
+
+import (
+	"sync"
+	"time"
+)
+
+const defaultServicesID = "services_localai"
+const FederatedID = "federated"
+
+type NodeData struct {
+	Name          string
+	ID            string
+	TunnelAddress string
+	LastSeen      time.Time
+}
+
+func (d NodeData) IsOnline() bool {
+	now := time.Now()
+	// if the node was seen in the last 40 seconds, it's online
+	return now.Sub(d.LastSeen) < 40*time.Second
+}
+
+var mu sync.Mutex
+var nodes = map[string]map[string]NodeData{}
+
+func GetAvailableNodes(serviceID string) []NodeData {
+	if serviceID == "" {
+		serviceID = defaultServicesID
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	var availableNodes = []NodeData{}
+	for _, v := range nodes[serviceID] {
+		availableNodes = append(availableNodes, v)
+	}
+	return availableNodes
+}
+
+func AddNode(serviceID string, node NodeData) {
+	if serviceID == "" {
+		serviceID = defaultServicesID
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	if nodes[serviceID] == nil {
+		nodes[serviceID] = map[string]NodeData{}
+	}
+	nodes[serviceID][node.ID] = node
+}

core/p2p/p2p.go

@@ -10,19 +10,18 @@ import (
 	"io"
 	"net"
 	"os"
-	"strings"
+	"sync"
 	"time"
 
+	"github.com/ipfs/go-log"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/mudler/LocalAI/pkg/utils"
+	"github.com/mudler/edgevpn/pkg/config"
 	"github.com/mudler/edgevpn/pkg/node"
 	"github.com/mudler/edgevpn/pkg/protocol"
+	"github.com/mudler/edgevpn/pkg/services"
 	"github.com/mudler/edgevpn/pkg/types"
 	"github.com/phayes/freeport"
-
-	"github.com/ipfs/go-log"
-	"github.com/mudler/edgevpn/pkg/config"
-	"github.com/mudler/edgevpn/pkg/services"
 	zlog "github.com/rs/zerolog/log"
 
 	"github.com/mudler/edgevpn/pkg/logger"
@@ -34,6 +33,15 @@ func GenerateToken() string {
 	return newData.Base64()
 }
 
+func IsP2PEnabled() bool {
+	return true
+}
+
+func nodeID(s string) string {
+	hostname, _ := os.Hostname()
+	return fmt.Sprintf("%s-%s", hostname, s)
+}
+
 func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, service string) error {
 
 	zlog.Info().Msgf("Allocating service '%s' on: %s", service, listenAddr)
@@ -53,16 +61,16 @@ func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, serv
 		10*time.Second,
 		func() {
 			// Retrieve current ID for ip in the blockchain
-			_, found := ledger.GetKey(protocol.UsersLedgerKey, node.Host().ID().String())
+			//_, found := ledger.GetKey(protocol.UsersLedgerKey, node.Host().ID().String())
 			// If mismatch, update the blockchain
-			if !found {
+			//if !found {
-				updatedMap := map[string]interface{}{}
+			updatedMap := map[string]interface{}{}
-				updatedMap[node.Host().ID().String()] = &types.User{
+			updatedMap[node.Host().ID().String()] = &types.User{
-					PeerID:    node.Host().ID().String(),
+				PeerID:    node.Host().ID().String(),
-					Timestamp: time.Now().String(),
+				Timestamp: time.Now().String(),
-				}
-				ledger.Add(protocol.UsersLedgerKey, updatedMap)
 			}
+			ledger.Add(protocol.UsersLedgerKey, updatedMap)
+			//	}
 		},
 	)
 
@@ -80,7 +88,6 @@ func allocateLocalService(ctx context.Context, node *node.Node, listenAddr, serv
 				continue
 			}
 
-			//	ll.Info("New connection from", l.Addr().String())
 			// Handle connections in a new goroutine, forwarding to the p2p service
 			go func() {
 				// Retrieve current ID for ip in the blockchain
@@ -137,24 +144,30 @@ func copyStream(closer chan struct{}, dst io.Writer, src io.Reader) {
 
 // This is the main of the server (which keeps the env variable updated)
 // This starts a goroutine that keeps LLAMACPP_GRPC_SERVERS updated with the discovered services
-func LLamaCPPRPCServerDiscoverer(ctx context.Context, token string) error {
+func ServiceDiscoverer(ctx context.Context, n *node.Node, token, servicesID string, discoveryFunc func()) error {
-	tunnels, err := discoveryTunnels(ctx, token)
+	if servicesID == "" {
+		servicesID = defaultServicesID
+	}
+	tunnels, err := discoveryTunnels(ctx, n, token, servicesID)
 	if err != nil {
 		return err
 	}
-
+	// TODO: discoveryTunnels should return all the nodes that are available?
+	// In this way we updated availableNodes here instead of appending
+	// e.g. we have a LastSeen field in NodeData that is updated in discoveryTunnels
+	// each time the node is seen
+	// In this case the below function should be idempotent and just keep track of the nodes
 	go func() {
-		totalTunnels := []string{}
 		for {
 			select {
 			case <-ctx.Done():
 				zlog.Error().Msg("Discoverer stopped")
 				return
 			case tunnel := <-tunnels:
-
+				AddNode(servicesID, tunnel)
-				totalTunnels = append(totalTunnels, tunnel)
+				if discoveryFunc != nil {
-				os.Setenv("LLAMACPP_GRPC_SERVERS", strings.Join(totalTunnels, ","))
+					discoveryFunc()
-				zlog.Debug().Msgf("setting LLAMACPP_GRPC_SERVERS to %s", strings.Join(totalTunnels, ","))
+				}
 			}
 		}
 	}()
@@ -162,19 +175,10 @@ func LLamaCPPRPCServerDiscoverer(ctx context.Context, token string) error {
 	return nil
 }
 
-func discoveryTunnels(ctx context.Context, token string) (chan string, error) {
+func discoveryTunnels(ctx context.Context, n *node.Node, token, servicesID string) (chan NodeData, error) {
-	tunnels := make(chan string)
+	tunnels := make(chan NodeData)
 
-	nodeOpts, err := newNodeOpts(token)
+	err := n.Start(ctx)
-	if err != nil {
-		return nil, err
-	}
-
-	n, err := node.New(nodeOpts...)
-	if err != nil {
-		return nil, fmt.Errorf("creating a new node: %w", err)
-	}
-	err = n.Start(ctx)
 	if err != nil {
 		return nil, fmt.Errorf("creating a new node: %w", err)
 	}
@@ -184,8 +188,14 @@ func discoveryTunnels(ctx context.Context, token string) (chan string, error) {
 	}
 
 	// get new services, allocate and return to the channel
+
+	// TODO:
+	// a function ensureServices that:
+	// - starts a service if not started, if the worker is Online
+	// - checks that workers are Online, if not cancel the context of allocateLocalService
+	// - discoveryTunnels should return all the nodes and addresses associated with it
+	// - the caller should take now care of the fact that we are always returning fresh informations
 	go func() {
-		emitted := map[string]bool{}
 		for {
 			select {
 			case <-ctx.Done():
@@ -195,20 +205,20 @@ func discoveryTunnels(ctx context.Context, token string) (chan string, error) {
 				time.Sleep(5 * time.Second)
 				zlog.Debug().Msg("Searching for workers")
 
-				data := ledger.LastBlock().Storage["services_localai"]
+				data := ledger.LastBlock().Storage[servicesID]
-				for k := range data {
+				for k, v := range data {
 					zlog.Info().Msgf("Found worker %s", k)
-					if _, found := emitted[k]; !found {
+					nd := &NodeData{}
-						emitted[k] = true
+					if err := v.Unmarshal(nd); err != nil {
-						//discoveredPeers <- k
+						zlog.Error().Msg("cannot unmarshal node data")
-						port, err := freeport.GetFreePort()
+						continue
-						if err != nil {
-							fmt.Print(err)
-						}
-						tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
-						go allocateLocalService(ctx, n, tunnelAddress, k)
-						tunnels <- tunnelAddress
 					}
+					ensureService(ctx, n, nd, k)
+					muservice.Lock()
+					if _, ok := service[nd.Name]; ok {
+						tunnels <- service[nd.Name].NodeData
+					}
+					muservice.Unlock()
 				}
 			}
 		}
@@ -217,8 +227,60 @@ func discoveryTunnels(ctx context.Context, token string) (chan string, error) {
 	return tunnels, err
 }
 
+type nodeServiceData struct {
+	NodeData   NodeData
+	CancelFunc context.CancelFunc
+}
+
+var service = map[string]nodeServiceData{}
+var muservice sync.Mutex
+
+func ensureService(ctx context.Context, n *node.Node, nd *NodeData, sserv string) {
+	muservice.Lock()
+	defer muservice.Unlock()
+	if ndService, found := service[nd.Name]; !found {
+		if !nd.IsOnline() {
+			// if node is offline and not present, do nothing
+			return
+		}
+		newCtxm, cancel := context.WithCancel(ctx)
+		// Start the service
+		port, err := freeport.GetFreePort()
+		if err != nil {
+			fmt.Print(err)
+		}
+		tunnelAddress := fmt.Sprintf("127.0.0.1:%d", port)
+		nd.TunnelAddress = tunnelAddress
+		service[nd.Name] = nodeServiceData{
+			NodeData:   *nd,
+			CancelFunc: cancel,
+		}
+		go allocateLocalService(newCtxm, n, tunnelAddress, sserv)
+		zlog.Debug().Msgf("Starting service %s on %s", sserv, tunnelAddress)
+	} else {
+		// Check if the service is still alive
+		// if not cancel the context
+		if !nd.IsOnline() && !ndService.NodeData.IsOnline() {
+			ndService.CancelFunc()
+			delete(service, nd.Name)
+			zlog.Info().Msgf("Node %s is offline, deleting", nd.ID)
+		} else if nd.IsOnline() {
+			// update last seen inside service
+			nd.TunnelAddress = ndService.NodeData.TunnelAddress
+			service[nd.Name] = nodeServiceData{
+				NodeData:   *nd,
+				CancelFunc: ndService.CancelFunc,
+			}
+			zlog.Debug().Msgf("Node %s is still online", nd.ID)
+		}
+	}
+}
+
 // This is the P2P worker main
-func BindLLamaCPPWorker(ctx context.Context, host, port, token string) error {
+func ExposeService(ctx context.Context, host, port, token, servicesID string) error {
+	if servicesID == "" {
+		servicesID = defaultServicesID
+	}
 	llger := logger.New(log.LevelFatal)
 
 	nodeOpts, err := newNodeOpts(token)
@@ -248,22 +310,40 @@ func BindLLamaCPPWorker(ctx context.Context, host, port, token string) error {
 
 	ledger.Announce(
 		ctx,
-		10*time.Second,
+		20*time.Second,
 		func() {
 			// Retrieve current ID for ip in the blockchain
-			_, found := ledger.GetKey("services_localai", name)
+			//_, found := ledger.GetKey("services_localai", name)
 			// If mismatch, update the blockchain
-			if !found {
+			//if !found {
-				updatedMap := map[string]interface{}{}
+			updatedMap := map[string]interface{}{}
-				updatedMap[name] = "p2p"
+			updatedMap[name] = &NodeData{
-				ledger.Add("services_localai", updatedMap)
+				Name:     name,
+				LastSeen: time.Now(),
+				ID:       nodeID(name),
 			}
+			ledger.Add(servicesID, updatedMap)
+			//	}
 		},
 	)
 
 	return err
 }
 
+func NewNode(token string) (*node.Node, error) {
+	nodeOpts, err := newNodeOpts(token)
+	if err != nil {
+		return nil, err
+	}
+
+	n, err := node.New(nodeOpts...)
+	if err != nil {
+		return nil, fmt.Errorf("creating a new node: %w", err)
+	}
+
+	return n, nil
+}
+
 func newNodeOpts(token string) ([]node.Option, error) {
 	llger := logger.New(log.LevelFatal)
 	defaultInterval := 10 * time.Second

core/p2p/p2p_disabled.go

@@ -6,16 +6,26 @@ package p2p
 import (
 	"context"
 	"fmt"
+
+	"github.com/mudler/edgevpn/pkg/node"
 )
 
 func GenerateToken() string {
 	return "not implemented"
 }
 
-func LLamaCPPRPCServerDiscoverer(ctx context.Context, token string) error {
+func ServiceDiscoverer(ctx context.Context, node *node.Node, token, servicesID string, fn func()) error {
 	return fmt.Errorf("not implemented")
 }
 
-func BindLLamaCPPWorker(ctx context.Context, host, port, token string) error {
+func ExposeService(ctx context.Context, host, port, token, servicesID string) error {
 	return fmt.Errorf("not implemented")
 }
+
+func IsP2PEnabled() bool {
+	return false
+}
+
+func NewNode(token string) (*node.Node, error) {
+	return nil, fmt.Errorf("not implemented")
+}

docs/static/install.sh

@@ -76,6 +76,8 @@ DOCKER_INSTALL=${DOCKER_INSTALL:-$docker_found}
 USE_AIO=${USE_AIO:-false}
 API_KEY=${API_KEY:-}
 CORE_IMAGES=${CORE_IMAGES:-false}
+P2P_TOKEN=${P2P_TOKEN:-}
+WORKER=${WORKER:-false}
 # nprocs -1
 if available nproc; then
     procs=$(nproc)
@@ -132,7 +134,14 @@ configure_systemd() {
 
     info "Adding current user to local-ai group..."
     $SUDO usermod -a -G local-ai $(whoami)
-
+    STARTCOMMAND="run"
+    if [ "$WORKER" = true ]; then
+        if [ -n "$P2P_TOKEN" ]; then
+            STARTCOMMAND="worker p2p-llama-cpp-rpc"
+        else 
+            STARTCOMMAND="worker llama-cpp-rpc"
+        fi
+    fi
     info "Creating local-ai systemd service..."
     cat <<EOF | $SUDO tee /etc/systemd/system/local-ai.service >/dev/null
 [Unit]
@@ -140,7 +149,7 @@ Description=LocalAI Service
 After=network-online.target
 
 [Service]
-ExecStart=$BINDIR/local-ai run
+ExecStart=$BINDIR/local-ai $STARTCOMMAND
 User=local-ai
 Group=local-ai
 Restart=always
@@ -159,6 +168,11 @@ EOF
     $SUDO echo "THREADS=$THREADS" | $SUDO tee -a /etc/localai.env >/dev/null
     $SUDO echo "MODELS_PATH=$MODELS_PATH" | $SUDO tee -a /etc/localai.env >/dev/null
 
+    if [ -n "$P2P_TOKEN" ]; then
+        $SUDO echo "LOCALAI_P2P_TOKEN=$P2P_TOKEN" | $SUDO tee -a /etc/localai.env >/dev/null
+        $SUDO echo "LOCALAI_P2P=true" | $SUDO tee -a /etc/localai.env >/dev/null
+    fi
+
     SYSTEMCTL_RUNNING="$(systemctl is-system-running || true)"
     case $SYSTEMCTL_RUNNING in
         running|degraded)
@@ -407,6 +421,19 @@ install_docker() {
         # exit 0
     fi
 
+    STARTCOMMAND="run"
+    if [ "$WORKER" = true ]; then
+        if [ -n "$P2P_TOKEN" ]; then
+            STARTCOMMAND="worker p2p-llama-cpp-rpc"
+        else 
+            STARTCOMMAND="worker llama-cpp-rpc"
+        fi
+    fi
+    envs=""
+    if [ -n "$P2P_TOKEN" ]; then
+        envs="-e LOCALAI_P2P_TOKEN=$P2P_TOKEN -e LOCALAI_P2P=true"
+    fi
+
     IMAGE_TAG=
     if [ "$HAS_CUDA" ]; then
         IMAGE_TAG=${VERSION}-cublas-cuda12-ffmpeg
@@ -430,7 +457,8 @@ install_docker() {
             --restart=always \
             -e API_KEY=$API_KEY \
             -e THREADS=$THREADS \
-            -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG
+            $envs \
+            -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND
     elif [ "$HAS_AMD" ]; then
         IMAGE_TAG=${VERSION}-hipblas-ffmpeg
         # CORE
@@ -448,7 +476,8 @@ install_docker() {
             --restart=always \
             -e API_KEY=$API_KEY \
             -e THREADS=$THREADS \
-            -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG
+            $envs \
+            -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND
     elif [ "$HAS_INTEL" ]; then
         IMAGE_TAG=${VERSION}-sycl-f32-ffmpeg
         # CORE
@@ -465,7 +494,8 @@ install_docker() {
             --restart=always \
             -e API_KEY=$API_KEY \
             -e THREADS=$THREADS \
-            -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG
+            $envs \
+            -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND
     else
         IMAGE_TAG=${VERSION}-ffmpeg
         # CORE
@@ -481,7 +511,8 @@ install_docker() {
                 -e MODELS_PATH=/models \
                 -e API_KEY=$API_KEY \
                 -e THREADS=$THREADS \
-                -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG
+                $envs \
+                -d -p $PORT:8080 --name local-ai localai/localai:$IMAGE_TAG $STARTCOMMAND
     fi
 
     install_success

embedded/webui_static.yaml

@@ -16,6 +16,9 @@
 - filename: "tw-elements.css"
   url: "https://cdn.jsdelivr.net/npm/tw-elements/css/tw-elements.min.css"
   sha: "72746af5326d6eb3647f504efa81b5e0f50ed486f37cc8262a4169781ad310d3"
+- filename: "tw-elements.js"
+  url: "https://cdn.jsdelivr.net/npm/tw-elements/js/tw-elements.umd.min.js"
+  sha: "2985706362e92360b65c8697cc32490bb9c0a5df9cd9b7251a97c1c5a661a40a"
 - filename: "tailwindcss.js"
   url: "https://cdn.tailwindcss.com/3.3.0"
   sha: "dbff048aa4581e6eae7f1cb2c641f72655ea833b3bb82923c4a59822e11ca594"