feat(p2p): add network explorer and community pools (#3125)

* WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Wire up a simple explorer DB Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * wip Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * WIP Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * refactor: group services id so can be identified easily in the ledger table Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(discovery): discovery service now gather worker informations correctly Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display network token Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display form to add new networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): stop from overwriting networks Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): display only networks with active workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(explorer): list only clusters in a network if it has online workers Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * remove invalid and inactive networks if networks have no workers delete them from the database, similarly, if invalid. Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci: add workflow to deploy new explorer versions automatically Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * build-api: build with p2p tag Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to specify a connection timeout Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * logging Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Better p2p defaults Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Set loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Fix dht enable Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Default to info for loglevel Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Add navbar Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Slightly improve rendering Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * Allow to copy the token easily Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * ci fixups Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
2024-12-18 20:27:57 +00:00 · 2024-08-09 20:12:01 +02:00 · 2024-08-09 20:12:01 +02:00 · 9e3e892ac7
commit 9e3e892ac7
parent 5fcafc3d1e
19 changed files with 1082 additions and 17 deletions
--- a/.github/workflows/deploy-explorer.yaml
+++ b/.github/workflows/deploy-explorer.yaml
@ -0,0 +1,64 @@
+name: Explorer deployment
+
+on:
+  push:
+    branches:
+      - master
+    tags:
+      - 'v*'
+
+concurrency:
+  group: ci-deploy-${{ github.head_ref || github.ref }}-${{ github.repository }}
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Clone
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: actions/setup-go@v5
+        with:
+          go-version: '1.21.x'
+          cache: false
+      - name: Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y wget curl build-essential ffmpeg protobuf-compiler ccache upx-ucl gawk cmake libgmock-dev
+          go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@1958fcbe2ca8bd93af633f11e97d44e567e945af
+          go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.34.2
+          make protogen-go
+      - name: Build api
+        run: |
+          make build-api
+      - name: rm
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+            host: ${{ secrets.EXPLORER_SSH_HOST }}
+            username: ${{ secrets.EXPLORER_SSH_USERNAME }}
+            key: ${{ secrets.EXPLORER_SSH_KEY }}
+            port: ${{ secrets.EXPLORER_SSH_PORT }}
+            script: |
+                sudo rm -rf local-ai/ || true
+      - name: copy file via ssh
+        uses: appleboy/scp-action@v0.1.7
+        with:
+            host: ${{ secrets.EXPLORER_SSH_HOST }}
+            username: ${{ secrets.EXPLORER_SSH_USERNAME }}
+            key: ${{ secrets.EXPLORER_SSH_KEY }}
+            port: ${{ secrets.EXPLORER_SSH_PORT }}
+            source: "local-ai"
+            overwrite: true
+            rm: true
+            target: ./local-ai
+      - name: restarting
+        uses: appleboy/ssh-action@v1.0.3
+        with:
+            host: ${{ secrets.EXPLORER_SSH_HOST }}
+            username: ${{ secrets.EXPLORER_SSH_USERNAME }}
+            key: ${{ secrets.EXPLORER_SSH_KEY }}
+            port: ${{ secrets.EXPLORER_SSH_PORT }}
+            script: |
+                sudo cp -rfv local-ai/local-ai /usr/bin/local-ai
+                sudo systemctl restart local-ai
--- a/2
+++ b/2
@ -376,7 +376,7 @@ build-minimal:
 	BUILD_GRPC_FOR_BACKEND_LLAMA=true GRPC_BACKENDS="backend-assets/grpc/llama-cpp-avx2" GO_TAGS=p2p $(MAKE) build

 build-api:
-	BUILD_GRPC_FOR_BACKEND_LLAMA=true BUILD_API_ONLY=true GO_TAGS=none $(MAKE) build
+	BUILD_GRPC_FOR_BACKEND_LLAMA=true BUILD_API_ONLY=true GO_TAGS=p2p $(MAKE) build

 backend-assets/lib:
 	mkdir -p backend-assets/lib
--- a/core/cli/cli.go
+++ b/core/cli/cli.go
@ -15,4 +15,5 @@ var CLI struct {
 	Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"`
 	Worker     worker.Worker `cmd:"" help:"Run workers to distribute workload (llama.cpp-only)"`
 	Util       UtilCMD       `cmd:"" help:"Utility commands"`
+	Explorer   ExplorerCMD   `cmd:"" help:"Run p2p explorer"`
 }
--- a/core/cli/explorer.go
+++ b/core/cli/explorer.go
@ -0,0 +1,35 @@
+package cli
+
+import (
+	"context"
+	"time"
+
+	cliContext "github.com/mudler/LocalAI/core/cli/context"
+	"github.com/mudler/LocalAI/core/explorer"
+	"github.com/mudler/LocalAI/core/http"
+)
+
+type ExplorerCMD struct {
+	Address           string `env:"LOCALAI_ADDRESS,ADDRESS" default:":8080" help:"Bind address for the API server" group:"api"`
+	PoolDatabase      string `env:"LOCALAI_POOL_DATABASE,POOL_DATABASE" default:"explorer.json" help:"Path to the pool database" group:"api"`
+	ConnectionTimeout string `env:"LOCALAI_CONNECTION_TIMEOUT,CONNECTION_TIMEOUT" default:"2m" help:"Connection timeout for the explorer" group:"api"`
+}
+
+func (e *ExplorerCMD) Run(ctx *cliContext.Context) error {
+
+	db, err := explorer.NewDatabase(e.PoolDatabase)
+	if err != nil {
+		return err
+	}
+
+	dur, err := time.ParseDuration(e.ConnectionTimeout)
+	if err != nil {
+		return err
+	}
+	ds := explorer.NewDiscoveryServer(db, dur)
+
+	go ds.Start(context.Background())
+	appHTTP := http.Explorer(db, ds)
+
+	return appHTTP.Listen(e.Address)
+}
--- a/core/cli/run.go
+++ b/core/cli/run.go
@ -121,9 +121,9 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error {
 		}

 		log.Info().Msg("Starting P2P server discovery...")
-		if err := p2p.ServiceDiscoverer(context.Background(), node, token, p2p.NetworkID(r.Peer2PeerNetworkID, ""), func(serviceID string, node p2p.NodeData) {
+		if err := p2p.ServiceDiscoverer(context.Background(), node, token, p2p.NetworkID(r.Peer2PeerNetworkID, p2p.WorkerID), func(serviceID string, node p2p.NodeData) {
 			var tunnelAddresses []string
-			for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(r.Peer2PeerNetworkID, "")) {
+			for _, v := range p2p.GetAvailableNodes(p2p.NetworkID(r.Peer2PeerNetworkID, p2p.WorkerID)) {
 				if v.IsOnline() {
 					tunnelAddresses = append(tunnelAddresses, v.TunnelAddress)
 				} else {
--- a/core/cli/worker/worker_p2p.go
+++ b/core/cli/worker/worker_p2p.go
@ -60,7 +60,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
 			p = r.RunnerPort
 		}

-		err = p2p.ExposeService(context.Background(), address, p, r.Token, p2p.NetworkID(r.Peer2PeerNetworkID, ""))
+		err = p2p.ExposeService(context.Background(), address, p, r.Token, p2p.NetworkID(r.Peer2PeerNetworkID, p2p.WorkerID))
 		if err != nil {
 			return err
 		}
@ -100,7 +100,7 @@ func (r *P2P) Run(ctx *cliContext.Context) error {
 		}
 	}()

-	err = p2p.ExposeService(context.Background(), address, fmt.Sprint(port), r.Token, p2p.NetworkID(r.Peer2PeerNetworkID, ""))
+	err = p2p.ExposeService(context.Background(), address, fmt.Sprint(port), r.Token, p2p.NetworkID(r.Peer2PeerNetworkID, p2p.WorkerID))
 	if err != nil {
 		return err
 	}
--- a/core/explorer/database.go
+++ b/core/explorer/database.go
@ -0,0 +1,106 @@
+package explorer
+
+// A simple JSON database for storing and retrieving p2p network tokens and a name and description.
+
+import (
+	"encoding/json"
+	"os"
+	"sort"
+	"sync"
+)
+
+// Database is a simple JSON database for storing and retrieving p2p network tokens and a name and description.
+type Database struct {
+	sync.RWMutex
+	path string
+	data map[string]TokenData
+}
+
+// TokenData is a p2p network token with a name and description.
+type TokenData struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+// NewDatabase creates a new Database with the given path.
+func NewDatabase(path string) (*Database, error) {
+	db := &Database{
+		data: make(map[string]TokenData),
+		path: path,
+	}
+	return db, db.load()
+}
+
+// Get retrieves a Token from the Database by its token.
+func (db *Database) Get(token string) (TokenData, bool) {
+	db.RLock()
+	defer db.RUnlock()
+	t, ok := db.data[token]
+	return t, ok
+}
+
+// Set stores a Token in the Database by its token.
+func (db *Database) Set(token string, t TokenData) error {
+	db.Lock()
+	db.data[token] = t
+	db.Unlock()
+
+	return db.Save()
+}
+
+// Delete removes a Token from the Database by its token.
+func (db *Database) Delete(token string) error {
+	db.Lock()
+	delete(db.data, token)
+	db.Unlock()
+	return db.Save()
+}
+
+func (db *Database) TokenList() []string {
+	db.RLock()
+	defer db.RUnlock()
+	tokens := []string{}
+	for k := range db.data {
+		tokens = append(tokens, k)
+	}
+
+	sort.Slice(tokens, func(i, j int) bool {
+		// sort by token
+		return tokens[i] < tokens[j]
+	})
+
+	return tokens
+}
+
+// load reads the Database from disk.
+func (db *Database) load() error {
+	db.Lock()
+	defer db.Unlock()
+
+	if _, err := os.Stat(db.path); os.IsNotExist(err) {
+		return nil
+	}
+
+	// Read the file from disk
+	// Unmarshal the JSON into db.data
+	f, err := os.ReadFile(db.path)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(f, &db.data)
+}
+
+// Save writes the Database to disk.
+func (db *Database) Save() error {
+	db.RLock()
+	defer db.RUnlock()
+
+	// Marshal db.data into JSON
+	// Write the JSON to the file
+	f, err := os.Create(db.path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return json.NewEncoder(f).Encode(db.data)
+}
--- a/core/explorer/database_test.go
+++ b/core/explorer/database_test.go
@ -0,0 +1,92 @@
+package explorer_test
+
+import (
+	"os"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/explorer"
+)
+
+var _ = Describe("Database", func() {
+	var (
+		dbPath string
+		db     *explorer.Database
+		err    error
+	)
+
+	BeforeEach(func() {
+		// Create a temporary file path for the database
+		dbPath = "test_db.json"
+		db, err = explorer.NewDatabase(dbPath)
+		Expect(err).To(BeNil())
+	})
+
+	AfterEach(func() {
+		// Clean up the temporary database file
+		os.Remove(dbPath)
+	})
+
+	Context("when managing tokens", func() {
+		It("should add and retrieve a token", func() {
+			token := "token123"
+			t := explorer.TokenData{Name: "TokenName", Description: "A test token"}
+
+			err = db.Set(token, t)
+			Expect(err).To(BeNil())
+
+			retrievedToken, exists := db.Get(token)
+			Expect(exists).To(BeTrue())
+			Expect(retrievedToken).To(Equal(t))
+		})
+
+		It("should delete a token", func() {
+			token := "token123"
+			t := explorer.TokenData{Name: "TokenName", Description: "A test token"}
+
+			err = db.Set(token, t)
+			Expect(err).To(BeNil())
+
+			err = db.Delete(token)
+			Expect(err).To(BeNil())
+
+			_, exists := db.Get(token)
+			Expect(exists).To(BeFalse())
+		})
+
+		It("should persist data to disk", func() {
+			token := "token123"
+			t := explorer.TokenData{Name: "TokenName", Description: "A test token"}
+
+			err = db.Set(token, t)
+			Expect(err).To(BeNil())
+
+			// Recreate the database object to simulate reloading from disk
+			db, err = explorer.NewDatabase(dbPath)
+			Expect(err).To(BeNil())
+
+			retrievedToken, exists := db.Get(token)
+			Expect(exists).To(BeTrue())
+			Expect(retrievedToken).To(Equal(t))
+
+			// Check the token list
+			tokenList := db.TokenList()
+			Expect(tokenList).To(ContainElement(token))
+		})
+	})
+
+	Context("when loading an empty or non-existent file", func() {
+		It("should start with an empty database", func() {
+			dbPath = "empty_db.json"
+			db, err = explorer.NewDatabase(dbPath)
+			Expect(err).To(BeNil())
+
+			_, exists := db.Get("nonexistent")
+			Expect(exists).To(BeFalse())
+
+			// Clean up
+			os.Remove(dbPath)
+		})
+	})
+})
--- a/core/explorer/discovery.go
+++ b/core/explorer/discovery.go
@ -0,0 +1,203 @@
+package explorer
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/rs/zerolog/log"
+
+	"github.com/mudler/LocalAI/core/p2p"
+	"github.com/mudler/edgevpn/pkg/blockchain"
+)
+
+type DiscoveryServer struct {
+	sync.Mutex
+	database       *Database
+	networkState   *NetworkState
+	connectionTime time.Duration
+}
+
+type NetworkState struct {
+	Networks map[string]Network
+}
+
+func (s *DiscoveryServer) NetworkState() *NetworkState {
+	s.Lock()
+	defer s.Unlock()
+	return s.networkState
+}
+
+// NewDiscoveryServer creates a new DiscoveryServer with the given Database.
+// it keeps the db state in sync with the network state
+func NewDiscoveryServer(db *Database, dur time.Duration) *DiscoveryServer {
+	if dur == 0 {
+		dur = 50 * time.Second
+	}
+	return &DiscoveryServer{
+		database:       db,
+		connectionTime: dur,
+		networkState: &NetworkState{
+			Networks: map[string]Network{},
+		},
+	}
+}
+
+type Network struct {
+	Clusters []ClusterData
+}
+
+func (s *DiscoveryServer) runBackground() {
+	if len(s.database.TokenList()) == 0 {
+		time.Sleep(5 * time.Second) // avoid busy loop
+		return
+	}
+
+	for _, token := range s.database.TokenList() {
+		c, cancel := context.WithTimeout(context.Background(), s.connectionTime)
+		defer cancel()
+
+		// Connect to the network
+		// Get the number of nodes
+		// save it in the current state (mutex)
+		// do not do in parallel
+		n, err := p2p.NewNode(token)
+		if err != nil {
+			log.Err(err).Msg("Failed to create node")
+			s.database.Delete(token)
+			continue
+		}
+
+		err = n.Start(c)
+		if err != nil {
+			log.Err(err).Msg("Failed to start node")
+			s.database.Delete(token)
+			continue
+		}
+
+		ledger, err := n.Ledger()
+		if err != nil {
+			log.Err(err).Msg("Failed to start ledger")
+			s.database.Delete(token)
+			continue
+		}
+
+		networkData := make(chan ClusterData)
+
+		// get the network data - it takes the whole timeout
+		// as we might not be connected to the network yet,
+		// and few attempts would have to be made before bailing out
+		go s.retrieveNetworkData(c, ledger, networkData)
+
+		hasWorkers := false
+		ledgerK := []ClusterData{}
+		for key := range networkData {
+			ledgerK = append(ledgerK, key)
+			if len(key.Workers) > 0 {
+				hasWorkers = true
+			}
+		}
+
+		log.Debug().Any("network", token).Msgf("Network has %d clusters", len(ledgerK))
+		if len(ledgerK) != 0 {
+			for _, k := range ledgerK {
+				log.Debug().Any("network", token).Msgf("Clusterdata %+v", k)
+			}
+		}
+
+		if hasWorkers {
+			s.Lock()
+			s.networkState.Networks[token] = Network{
+				Clusters: ledgerK,
+			}
+			s.Unlock()
+		} else {
+			log.Info().Any("network", token).Msg("No workers found in the network. Removing it from the database")
+			s.database.Delete(token)
+		}
+	}
+}
+
+type ClusterData struct {
+	Workers   []string
+	Type      string
+	NetworkID string
+}
+
+func (s *DiscoveryServer) retrieveNetworkData(c context.Context, ledger *blockchain.Ledger, networkData chan ClusterData) {
+	clusters := map[string]ClusterData{}
+
+	defer func() {
+		for _, n := range clusters {
+			networkData <- n
+		}
+		close(networkData)
+	}()
+
+	for {
+		select {
+		case <-c.Done():
+			return
+		default:
+			time.Sleep(5 * time.Second)
+
+			data := ledger.LastBlock().Storage
+		LEDGER:
+			for d := range data {
+				toScanForWorkers := false
+				cd := ClusterData{}
+				isWorkerCluster := d == p2p.WorkerID || (strings.Contains(d, "_") && strings.Contains(d, p2p.WorkerID))
+				isFederatedCluster := d == p2p.FederatedID || (strings.Contains(d, "_") && strings.Contains(d, p2p.FederatedID))
+				switch {
+				case isWorkerCluster:
+					toScanForWorkers = true
+					cd.Type = "worker"
+				case isFederatedCluster:
+					toScanForWorkers = true
+					cd.Type = "federated"
+				}
+
+				if strings.Contains(d, "_") {
+					cd.NetworkID = strings.Split(d, "_")[0]
+				}
+
+				if !toScanForWorkers {
+					continue LEDGER
+				}
+
+				atLeastOneWorker := false
+			DATA:
+				for _, v := range data[d] {
+					nd := &p2p.NodeData{}
+					if err := v.Unmarshal(nd); err != nil {
+						continue DATA
+					}
+
+					if nd.IsOnline() {
+						atLeastOneWorker = true
+						(&cd).Workers = append(cd.Workers, nd.ID)
+					}
+				}
+
+				if atLeastOneWorker {
+					clusters[d] = cd
+				}
+			}
+		}
+	}
+}
+
+// Start the discovery server. This is meant to be run in to a goroutine.
+func (s *DiscoveryServer) Start(ctx context.Context) error {
+	for {
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("context cancelled")
+		default:
+			// Collect data
+			s.runBackground()
+		}
+	}
+}
--- a/core/explorer/explorer_suite_test.go
+++ b/core/explorer/explorer_suite_test.go
@ -0,0 +1,13 @@
+package explorer_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestExplorer(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Explorer test suite")
+}
--- a/core/http/endpoints/explorer/dashboard.go
+++ b/core/http/endpoints/explorer/dashboard.go
@ -0,0 +1,105 @@
+package explorer
+
+import (
+	"encoding/base64"
+	"sort"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/mudler/LocalAI/core/explorer"
+	"github.com/mudler/LocalAI/internal"
+)
+
+func Dashboard() func(*fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+
+		summary := fiber.Map{
+			"Title":   "LocalAI API - " + internal.PrintableVersion(),
+			"Version": internal.PrintableVersion(),
+		}
+
+		if string(c.Context().Request.Header.ContentType()) == "application/json" || len(c.Accepts("html")) == 0 {
+			// The client expects a JSON response
+			return c.Status(fiber.StatusOK).JSON(summary)
+		} else {
+			// Render index
+			return c.Render("views/explorer", summary)
+		}
+	}
+}
+
+type AddNetworkRequest struct {
+	Token       string `json:"token"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+}
+
+type Network struct {
+	explorer.Network
+	explorer.TokenData
+	Token string `json:"token"`
+}
+
+func ShowNetworks(db *explorer.Database, ds *explorer.DiscoveryServer) func(*fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		networkState := ds.NetworkState()
+		results := []Network{}
+		for token, network := range networkState.Networks {
+			networkData, exists := db.Get(token) // get the token data
+			hasWorkers := false
+			for _, cluster := range network.Clusters {
+				if len(cluster.Workers) > 0 {
+					hasWorkers = true
+					break
+				}
+			}
+			if exists && hasWorkers {
+				results = append(results, Network{Network: network, TokenData: networkData, Token: token})
+			}
+		}
+
+		// order by number of clusters
+		sort.Slice(results, func(i, j int) bool {
+			return len(results[i].Clusters) > len(results[j].Clusters)
+		})
+
+		return c.JSON(results)
+	}
+}
+
+func AddNetwork(db *explorer.Database) func(*fiber.Ctx) error {
+	return func(c *fiber.Ctx) error {
+		request := new(AddNetworkRequest)
+		if err := c.BodyParser(request); err != nil {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Cannot parse JSON"})
+		}
+
+		if request.Token == "" {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Token is required"})
+		}
+
+		if request.Name == "" {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Name is required"})
+		}
+
+		if request.Description == "" {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Description is required"})
+		}
+
+		// TODO: check if token is valid, otherwise reject
+		// try to decode the token from base64
+		_, err := base64.StdEncoding.DecodeString(request.Token)
+		if err != nil {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Invalid token"})
+		}
+
+		if _, exists := db.Get(request.Token); exists {
+			return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{"error": "Token already exists"})
+		}
+		err = db.Set(request.Token, explorer.TokenData{Name: request.Name, Description: request.Description})
+		if err != nil {
+			return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{"error": "Cannot add token"})
+		}
+
+		return c.Status(fiber.StatusOK).JSON(fiber.Map{"message": "Token added"})
+	}
+}
--- a/core/http/endpoints/localai/p2p.go
+++ b/core/http/endpoints/localai/p2p.go
@ -15,7 +15,7 @@ func ShowP2PNodes(appConfig *config.ApplicationConfig) func(*fiber.Ctx) error {
 	// Render index
 	return func(c *fiber.Ctx) error {
 		return c.JSON(schema.P2PNodesResponse{
-			Nodes:          p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, "")),
+			Nodes:          p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.WorkerID)),
 			FederatedNodes: p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.FederatedID)),
 		})
 	}
--- a/core/http/explorer.go
+++ b/core/http/explorer.go
@ -0,0 +1,46 @@
+package http
+
+import (
+	"net/http"
+
+	"github.com/gofiber/fiber/v2"
+	"github.com/gofiber/fiber/v2/middleware/favicon"
+	"github.com/gofiber/fiber/v2/middleware/filesystem"
+	"github.com/mudler/LocalAI/core/explorer"
+	"github.com/mudler/LocalAI/core/http/routes"
+)
+
+func Explorer(db *explorer.Database, discoveryServer *explorer.DiscoveryServer) *fiber.App {
+
+	fiberCfg := fiber.Config{
+		Views: renderEngine(),
+		// We disable the Fiber startup message as it does not conform to structured logging.
+		// We register a startup log line with connection information in the OnListen hook to keep things user friendly though
+		DisableStartupMessage: false,
+		// Override default error handler
+	}
+
+	app := fiber.New(fiberCfg)
+
+	routes.RegisterExplorerRoutes(app, db, discoveryServer)
+
+	httpFS := http.FS(embedDirStatic)
+
+	app.Use(favicon.New(favicon.Config{
+		URL:        "/favicon.ico",
+		FileSystem: httpFS,
+		File:       "static/favicon.ico",
+	}))
+
+	app.Use("/static", filesystem.New(filesystem.Config{
+		Root:       httpFS,
+		PathPrefix: "static",
+		Browse:     true,
+	}))
+
+	// Define a custom 404 handler
+	// Note: keep this at the bottom!
+	app.Use(notFoundHandler)
+
+	return app
+}
--- a/core/http/routes/explorer.go
+++ b/core/http/routes/explorer.go
@ -0,0 +1,13 @@
+package routes
+
+import (
+	"github.com/gofiber/fiber/v2"
+	coreExplorer "github.com/mudler/LocalAI/core/explorer"
+	"github.com/mudler/LocalAI/core/http/endpoints/explorer"
+)
+
+func RegisterExplorerRoutes(app *fiber.App, db *coreExplorer.Database, ds *coreExplorer.DiscoveryServer) {
+	app.Get("/", explorer.Dashboard())
+	app.Post("/network/add", explorer.AddNetwork(db))
+	app.Get("/networks", explorer.ShowNetworks(db, ds))
+}
--- a/core/http/routes/ui.go
+++ b/core/http/routes/ui.go
@ -105,14 +105,14 @@ func RegisterUIRoutes(app *fiber.App,

 		/* show nodes live! */
 		app.Get("/p2p/ui/workers", auth, func(c *fiber.Ctx) error {
-			return c.SendString(elements.P2PNodeBoxes(p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, ""))))
+			return c.SendString(elements.P2PNodeBoxes(p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.WorkerID))))
 		})
 		app.Get("/p2p/ui/workers-federation", auth, func(c *fiber.Ctx) error {
 			return c.SendString(elements.P2PNodeBoxes(p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.FederatedID))))
 		})

 		app.Get("/p2p/ui/workers-stats", auth, func(c *fiber.Ctx) error {
-			return c.SendString(elements.P2PNodeStats(p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, ""))))
+			return c.SendString(elements.P2PNodeStats(p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.WorkerID))))
 		})
 		app.Get("/p2p/ui/workers-federation-stats", auth, func(c *fiber.Ctx) error {
 			return c.SendString(elements.P2PNodeStats(p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.FederatedID))))
--- a/core/http/views/explorer.html
+++ b/core/http/views/explorer.html
@ -0,0 +1,342 @@
+<!DOCTYPE html>
+<html lang="en">
+
+{{template "views/partials/head" .}}
+
+<style>
+    body {
+        background-color: #1a202c;
+        color: #e2e8f0;
+        font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
+    }
+    .container {
+        max-width: 800px;
+        margin: 0 auto;
+        padding: 20px;
+        position: relative;
+    }
+    .network-card {
+        background-color: #2d3748;
+        padding: 20px;
+        border-radius: 8px;
+        margin-bottom: 20px;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+        transition: transform 0.3s ease, box-shadow 0.3s ease;
+    }
+    .network-card:hover {
+        transform: translateY(-5px);
+        box-shadow: 0 6px 10px rgba(0, 0, 0, 0.15);
+    }
+    .network-title {
+        font-size: 24px;
+        font-weight: bold;
+        margin-bottom: 10px;
+        color: #63b3ed;
+    }
+    .network-token {
+        font-size: 14px;
+        font-style: italic;
+        color: #cbd5e0;
+        margin-bottom: 10px;
+        word-break: break-word; /* Breaks words to prevent overflow */
+        overflow-wrap: break-word; /* Ensures long strings break */
+        white-space: pre-wrap; /* Preserves whitespace for breaking */
+    }
+    .cluster {
+        margin-top: 10px;
+        background-color: #4a5568;
+        padding: 10px;
+        border-radius: 6px;
+        transition: background-color 0.3s ease;
+    }
+    .cluster:hover {
+        background-color: #5a6b78;
+    }
+    .cluster-title {
+        font-size: 18px;
+        font-weight: bold;
+        color: #e2e8f0;
+    }
+    .form-container {
+        background-color: #2d3748;
+        padding: 20px;
+        border-radius: 8px;
+        margin-bottom: 20px;
+        box-shadow: 0 4px 6px rgba(0, 0, 0, 0.1);
+    }
+    .form-control {
+        margin-bottom: 15px;
+    }
+    label {
+        display: block;
+        margin-bottom: 5px;
+        font-weight: bold;
+    }
+    input[type="text"],
+    textarea {
+        width: 100%;
+        padding: 10px;
+        border-radius: 4px;
+        border: 1px solid #4a5568;
+        background-color: #3a4250;
+        color: #e2e8f0;
+        transition: border-color 0.3s ease, background-color 0.3s ease;
+    }
+    input[type="text"]:focus,
+    textarea:focus {
+        border-color: #63b3ed;
+        background-color: #4a5568;
+    }
+    button {
+        background-color: #3182ce;
+        color: #e2e8f0;
+        padding: 10px 20px;
+        border: none;
+        border-radius: 4px;
+        cursor: pointer;
+        transition: background-color 0.3s ease;
+    }
+    .error {
+        color: #e53e3e;
+        margin-top: 5px;
+    }
+    .success {
+        color: #38a169;
+        margin-top: 5px;
+    }
+    /* Spinner Styles */
+    .spinner {
+        display: inline-block;
+        width: 50px;
+        height: 50px;
+        border: 5px solid rgba(255, 255, 255, 0.2);
+        border-radius: 50%;
+        border-top-color: #3182ce;
+        animation: spin 1s linear infinite;
+        margin: 0 auto;
+    }
+
+    @keyframes spin {
+        to { transform: rotate(360deg); }
+    }
+
+    /* Center the loading text and spinner */
+    .loading-container {
+        text-align: center;
+        padding: 50px;
+    }
+    .warning-box {
+            border-radius: 5px;
+    }
+    .warning-box i {
+        margin-right: 10px;
+    }
+    .token-box {
+            background-color: #4a5568;
+            padding: 10px;
+            border-radius: 4px;
+            margin-top: 10px;
+            position: relative;
+            cursor: pointer;
+        }
+        .token-box:hover {
+            background-color: #5a6b7e;
+        }
+        .token-text {
+            overflow-wrap: break-word;
+            font-family: monospace;
+        }
+        .copy-icon {
+            position: absolute;
+            top: 10px;
+            right: 10px;
+            color: #e2e8f0;
+        }
+</style>
+
+<body class="bg-gray-900 text-gray-200">
+    <div class="flex flex-col min-h-screen" x-data="networkClusters()" x-init="init()">
+        {{template "views/partials/navbar_explorer" .}}
+
+        <header class="text-center py-12">
+            <h1 class="text-5xl font-bold text-gray-100">Network Clusters Explorer</h1>
+            <p class="mt-4 text-lg">View the clusters and workers available in each network.</p>
+        </header>
+
+        <div class="container mx-auto px-4 flex-grow">
+        <!-- Warning Box -->
+        <div class="warning-box bg-yellow-100 text-gray-800 mb-20 pt-5 pb-5 pr-5 pl-5 text-lg">
+            <i class="fa-solid fa-triangle-exclamation"></i>
+            The explorer is a global, community-driven tool to share network tokens and view available clusters in the globe.
+            Anyone can use the tokens to offload computation and use the clusters available or share resources.
+            This is provided without any warranty. Use it at your own risk. We are not responsible for any potential harm or misuse. Sharing tokens globally allows anyone from the internet to use your instances. 
+            Although the community will address bugs, this is experimental software and may be insecure to deploy on your hardware unless you take all necessary precautions.
+        </div>
+            <div class="flow-root">
+            <!-- Toggle button for showing/hiding the form -->
+            <button class="bg-red-600 hover:bg-blue-600 float-right mb-2 flex items-center px-4 py-2 rounded" @click="toggleForm()">
+                <!-- Conditional icon display -->
+                <i :class="showForm ? 'fa-solid fa-times' : 'fa-solid fa-plus'" class="mr-2"></i>
+                <span x-text="showForm ? 'Close' : 'Add New Network'"></span>
+            </button>
+        </div>
+            <!-- Form for adding a new network -->
+            <div class="form-container" x-show="showForm" @click.outside="showForm = false">
+                <h2 class="text-3xl font-bold mb-4"><i class="fa-solid fa-plus"></i> Add New Network</h2>
+                <div class="form-control">
+                    <label for="name">Network Name</label>
+                    <input type="text" id="name" x-model="newNetwork.name" placeholder="Enter network name" />
+                </div>
+                <div class="form-control">
+                    <label for="description">Description</label>
+                    <textarea id="description" x-model="newNetwork.description" placeholder="Enter description"></textarea>
+                </div>
+                <div class="form-control">
+                    <label for="token">Token</label>
+                    <textarea id="token" x-model="newNetwork.token" placeholder="Enter token"></textarea>
+                </div>
+                <button @click="addNetwork"><i class="fa-solid fa-plus"></i> Add Network</button>
+                <template x-if="errorMessage">
+                    <p class="error" x-text="errorMessage"></p>
+                </template>
+                <template x-if="successMessage">
+                    <p class="success" x-text="successMessage"></p>
+                </template>
+            </div>
+
+            <!-- Loading Spinner -->
+            <template x-if="networks.length === 0 && !loadingComplete">
+                <div class="loading-container">
+                    <div class="spinner"></div>
+                    <p class="text-center mt-4">Loading networks...</p>
+                </div>
+            </template>
+
+            <template x-if="networks.length === 0 && loadingComplete">
+                <div class="loading-container">
+                    <p class="text-center mt-4">No networks available with online workers</p>
+                </div>
+            </template>
+
+            <!-- Display Networks -->
+            <template x-for="network in networks" :key="network.name">
+                <div class="network-card">
+                    <div class="network-title" x-text="network.name"></div>
+                    <div class="token-box" @click="copyToken(network.token)">
+                        <i class="fa-solid fa-copy copy-icon"></i>
+                        Token (click to copy): <br>
+                        <span class="token-text" x-text="network.token"></span>
+                    </div>
+
+                    <div class="cluster">
+                        <p class="text-lg">Description</p>
+                        <p x-text="network.description"></p>
+                    </div>
+
+                    <template x-for="cluster in network.Clusters" :key="cluster.NetworkID + cluster.Type">
+                        <div class="cluster">
+                            <div class="cluster-title" x-text="'Cluster Type: ' + cluster.Type"></div>
+                            <p x-show="cluster.NetworkID" x-text="'Network ID: ' + (cluster.NetworkID || 'N/A')"></p>
+                            <p x-text="'Number of Workers: ' + cluster.Workers.length"></p>
+                        </div>
+                    </template>
+                </div>
+            </template>
+        </div>
+        <script>
+            function networkClusters() {
+                return {
+                    networks: [],
+                    newNetwork: {
+                        name: '',
+                        description: '',
+                        token: ''
+                    },
+                    errorMessage: '',
+                    successMessage: '',
+                    showForm: false, // Form visibility state
+                    loadingComplete: false, // To track if loading is complete
+                    toggleForm() {
+                        this.showForm = !this.showForm;
+                        console.log('Toggling form:', this.showForm);
+                    },
+                    fetchNetworks() {
+                        console.log('Fetching networks...');
+                        fetch('/networks')
+                            .then(response => response.json())
+                            .then(data => {
+                                console.log('Data fetched successfully:', data);
+                                this.networks = data;
+                                this.loadingComplete = true; // Set loading complete
+                            })
+                            .catch(error => {
+                                console.error('Error fetching networks:', error);
+                                this.loadingComplete = true; // Ensure spinner is hidden if error occurs
+                            });
+                    },
+
+                    addNetwork() {
+                        this.errorMessage = '';
+                        this.successMessage = '';
+                        console.log('Adding new network:', this.newNetwork);
+
+                        // Validate input
+                        if (!this.newNetwork.name || !this.newNetwork.description || !this.newNetwork.token) {
+                            this.errorMessage = 'All fields are required.';
+                            return;
+                        }
+
+                        fetch('/network/add', {
+                            method: 'POST',
+                            headers: {
+                                'Content-Type': 'application/json'
+                            },
+                            body: JSON.stringify(this.newNetwork)
+                        })
+                            .then(response => {
+                                if (!response.ok) {
+                                    return response.json().then(err => { throw err; });
+                                }
+                                return response.json();
+                            })
+                            .then(data => {
+                                console.log('Network added successfully:', data);
+                                this.successMessage = 'Network added successfully!';
+                                this.fetchNetworks(); // Refresh the networks list
+                                this.newNetwork = { name: '', description: '', token: '' }; // Clear form
+                            })
+                            .catch(error => {
+                                console.error('Error adding network:', error);
+                                this.errorMessage = 'Failed to add network. Please try again.'
+                                if (error.error) {
+                                    this.errorMessage += " Error : " + error.error;
+                                }
+                            });
+                    },
+                    copyToken(token) {
+                        navigator.clipboard.writeText(token)
+                        .then(() => {
+                            console.log('Token copied to clipboard:', token);
+                            alert('Token copied to clipboard!');
+                        })
+                        .catch(err => {
+                            console.error('Failed to copy token:', err);
+                        });
+                    },
+                    init() {
+                        console.log('Initializing Alpine component...');
+                        this.fetchNetworks();
+                        setInterval(() => {
+                            this.fetchNetworks();
+                        }, 5000); // Refresh every 5 seconds
+                    }
+                }
+            }
+        </script>
+
+        {{template "views/partials/footer" .}}
+    </div>
+
+</body>
+
+</html>
--- a/core/http/views/partials/navbar_explorer.html
+++ b/core/http/views/partials/navbar_explorer.html
@ -0,0 +1,39 @@
+<nav class="bg-gray-800 shadow-lg">
+    <div class="container mx-auto px-4 py-4">
+        <div class="flex items-center justify-between">
+            <div class="flex items-center">
+                <!-- Logo Image: Replace 'logo_url_here' with your actual logo URL -->
+                <a href="/" class="text-white text-xl font-bold"><img src="https://github.com/go-skynet/LocalAI/assets/2420543/0966aa2a-166e-4f99-a3e5-6c915fc997dd" alt="LocalAI Logo" class="h-10 mr-3 border-2 border-gray-300 shadow rounded"></a>
+                <a href="/" class="text-white text-xl font-bold">LocalAI</a>
+            </div>
+            <!-- Menu button for small screens -->
+            <div class="lg:hidden">
+                <button id="menu-toggle" class="text-gray-400 hover:text-white focus:outline-none">
+                    <i class="fas fa-bars fa-lg"></i>
+                </button>
+            </div>
+            <!-- Navigation links -->
+            <div class="hidden lg:flex lg:items-center lg:justify-end lg:flex-1 lg:w-0">
+                <a href="/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-home pr-2"></i>Home</a>
+                <a href="https://localai.io" class="text-gray-400 hover:text-white px-3 py-2 rounded" target="_blank" ><i class="fas fa-book-reader pr-2"></i> Documentation</a>
+                <a href="https://models.localai.io/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-brain pr-2"></i> Models</a>
+            </div>
+        </div>
+        <!-- Collapsible menu for small screens -->
+        <div class="hidden lg:hidden" id="mobile-menu">
+            <div class="pt-4 pb-3 border-t border-gray-700">
+                <a href="/" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1"><i class="fas fa-home pr-2"></i>Home</a>
+                <a href="https://localai.io" class="block text-gray-400 hover:text-white px-3 py-2 rounded mt-1" target="_blank" ><i class="fas fa-book-reader pr-2"></i> Documentation</a>
+                <a href="https://models.localai.io/" class="text-gray-400 hover:text-white px-3 py-2 rounded"><i class="fas fa-brain pr-2"></i> Models</a>
+            </div>
+        </div>
+    </div>
+</nav>
+
+<script>
+    // JavaScript to toggle the mobile menu
+    document.getElementById('menu-toggle').addEventListener('click', function () {
+        var mobileMenu = document.getElementById('mobile-menu');
+        mobileMenu.classList.toggle('hidden');
+    });
+</script>
--- a/core/p2p/node.go
+++ b/core/p2p/node.go
@ -5,7 +5,10 @@ import (
 	"time"
 )

-const defaultServicesID = "services_localai"
+const (
+	defaultServicesID = "services"
+	WorkerID          = "worker"
+)

 type NodeData struct {
 	Name          string
--- a/core/p2p/p2p.go
+++ b/core/p2p/p2p.go
@ -345,13 +345,16 @@ func newNodeOpts(token string) ([]node.Option, error) {

 	// TODO: move this up, expose more config options when creating a node
 	noDHT := os.Getenv("LOCALAI_P2P_DISABLE_DHT") == "true"
-	noLimits := os.Getenv("LOCALAI_P2P_DISABLE_LIMITS") == "true"
+	noLimits := os.Getenv("LOCALAI_P2P_ENABLE_LIMITS") == "true"

-	loglevel := "info"
+	loglevel := os.Getenv("LOCALAI_P2P_LOGLEVEL")
+	if loglevel == "" {
+		loglevel = "info"
+	}

 	c := config.Config{
 		Limit: config.ResourceLimit{
-			Enable:   !noLimits,
+			Enable:   noLimits,
 			MaxConns: 100,
 		},
 		NetworkToken:   token,
@ -366,19 +369,19 @@ func newNodeOpts(token string) ([]node.Option, error) {
 			Service:           true,
 			Map:               true,
 			RateLimit:         true,
-			RateLimitGlobal:   10,
-			RateLimitPeer:     10,
+			RateLimitGlobal:   100,
+			RateLimitPeer:     100,
 			RateLimitInterval: defaultInterval,
 		},
 		Discovery: config.Discovery{
-			DHT:      noDHT,
+			DHT:      !noDHT,
 			MDNS:     true,
-			Interval: 30 * time.Second,
+			Interval: 10 * time.Second,
 		},
 		Connection: config.Connection{
 			HolePunch:      true,
 			AutoRelay:      true,
-			MaxConnections: 100,
+			MaxConnections: 1000,
 		},
 	}