markdown rendering + websocket

README.md

@@ -106,6 +106,16 @@ You can also refuse to download the model during the install procedure and downl
 - [Vicuna 7B rev 1](https://huggingface.co/eachadea/legacy-ggml-vicuna-7b-4bit/resolve/main/ggml-vicuna-7b-4bit-rev1.bin) or visit [repository](https://huggingface.co/eachadea/legacy-ggml-vicuna-7b-4bit)  
 - [Vicuna 13B rev 1](https://huggingface.co/eachadea/ggml-vicuna-13b-4bit/resolve/main/ggml-vicuna-13b-4bit-rev1.bin) or visit [repository](https://huggingface.co/eachadea/ggml-vicuna-13b-4bit)
 
+-[ggml-gpt4all-j-v1.3-groovy](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin)
+-[ggml-gpt4all-j-v1.2-jazzy](https://gpt4all.io/models/ggml-gpt4all-j-v1.2-jazzy.bin)
+-[ggml-gpt4all-l13b-snoozy](https://gpt4all.io/models/ggml-gpt4all-l13b-snoozy.bin)
+
+-[ggml-gpt4all-j-v1.1-breezy](https://gpt4all.io/models/ggml-gpt4all-j-v1.1-breezy.bin)
+-[ggml-gpt4all-j](https://gpt4all.io/models/ggml-gpt4all-j.bin)
+-[ggml-vicuna-7b-1.1-q4_2](https://gpt4all.io/models/ggml-vicuna-7b-1.1-q4_2.bin)
+-[ggml-vicuna-13b-1.1-q4_2](https://gpt4all.io/models/ggml-vicuna-13b-1.1-q4_2.bin)
+
+
 We also support GPT-j models out of the box
 
 ## GPT-j models

app.py

@@ -14,7 +14,7 @@ __github__ = "https://github.com/nomic-ai/gpt4all-ui"
 __copyright__ = "Copyright 2023, "
 __license__ = "Apache 2.0"
 
-
+import os
 import logging
 import argparse
 import json
@@ -37,9 +37,13 @@ from pathlib import Path
 import gc
 app = Flask("GPT4All-WebUI", static_url_path="/static", static_folder="static")
 socketio = SocketIO(app)
+app.config['SECRET_KEY'] = 'secret!'
 # Set the logging level to WARNING or higher
 logging.getLogger('socketio').setLevel(logging.WARNING)
 logging.getLogger('engineio').setLevel(logging.WARNING)
+# Suppress Flask's default console output
+log = logging.getLogger('werkzeug')
+log.setLevel(logging.ERROR)
 
 import time
 from pyGpt4All.config import load_config, save_config
@@ -91,10 +95,8 @@ class Gpt4AllWebUI(GPT4AllAPI):
         self.add_endpoint(
             "/new_discussion", "new_discussion", self.new_discussion, methods=["GET"]
         )
-        self.add_endpoint("/generate", "generate", self.generate, methods=["POST"])
         self.add_endpoint("/stop_gen", "stop_gen", self.stop_gen, methods=["GET"])
 
-        self.add_endpoint("/run_to", "run_to", self.run_to, methods=["POST"])
         self.add_endpoint("/rename", "rename", self.rename, methods=["POST"])
         self.add_endpoint(
             "/load_discussion", "load_discussion", self.load_discussion, methods=["POST"]
@@ -158,27 +160,15 @@ class Gpt4AllWebUI(GPT4AllAPI):
         
         # Socket IO stuff    
         @socketio.on('connect')
-        def test_connect():
+        def connect():
             print('Client connected')
 
         @socketio.on('disconnect')
-        def test_disconnect():
+        def disconnect():
             print('Client disconnected')
         
-        @socketio.on('stream-text')
+        @socketio.on('generate_msg')
-        def handle_stream_text(data):
+        def generate_msg(data):
-            text = data['prompt']
-            words = text.split()
-            for word in words:
-                emit('stream-word', {'word': word})
-                time.sleep(1)  # sleep for 1 second to simulate processing time
-            emit('stream-end')    
-        
-        
-        @socketio.on('connected')
-        def handle_connection(data):
-            if "data" in data and data["data"]=='Connected!':
-                return
             if self.current_discussion is None:
                 if self.db.does_last_discussion_have_messages():
                     self.current_discussion = self.db.create_discussion()
@@ -194,12 +184,16 @@ class Gpt4AllWebUI(GPT4AllAPI):
             tpe = threading.Thread(target=self.parse_to_prompt_stream, args=(message, message_id))
             tpe.start()
 
-            # self.parse_to_prompt_stream(message, message_id)
+        @socketio.on('generate_msg_from')
+        def handle_connection(data):
+            message_id = int(data['id'])
+            message = data["prompt"]
+            self.current_message_id = message_id
+            tpe = threading.Thread(target=self.parse_to_prompt_stream, args=(message, message_id))
+            tpe.start()
+
 
-            #self.socketio.emit('message', {'data': 'WebSocket connected!'})
 
-            #for i in range(10):
-            #    socketio.emit('message', {'data': 'Message ' + str(i)})  
 
     def list_backends(self):
         backends_dir = Path('./backends')  # replace with the actual path to the models folder
@@ -297,6 +291,7 @@ class Gpt4AllWebUI(GPT4AllAPI):
 
         # send the message to the bot
         print(f"Received message : {message}")
+        if self.current_discussion:
             # First we need to send the new message ID to the client
             response_id = self.current_discussion.add_message(
                 self.personality["name"], "", parent = message_id
@@ -310,7 +305,7 @@ class Gpt4AllWebUI(GPT4AllAPI):
                         "id": message_id,
                         "response_id": response_id,
                     }
-            )  
+            );
 
 
             # prepare query and reception
@@ -319,57 +314,22 @@ class Gpt4AllWebUI(GPT4AllAPI):
             self.generating = True
             # app.config['executor'] = ThreadPoolExecutor(max_workers=1)
             # app.config['executor'].submit(self.generate_message)
+            print("## Generate message ##")
             self.generate_message()
             print("## Done ##")
             self.current_discussion.update_message(response_id, self.bot_says)
             self.full_message_list.append(self.bot_says)
-        bot_says = markdown.markdown(self.bot_says)
-
-        socketio.emit('final', {'data': bot_says})
             self.cancel_gen = False
             return bot_says
-    
-    
-  
-
-    def generate(self):
-
-        if self.current_discussion is None:
-            if self.db.does_last_discussion_have_messages():
-                self.current_discussion = self.db.create_discussion()
         else:
-                self.current_discussion = self.db.load_last_discussion()
+            print("## Done ##")
+            return ""
     
-        message = request.json["message"]
-        message_id = self.current_discussion.add_message(
-            "user", message, parent=self.current_message_id
-        )
-        message = f"{request.json['message']}"
-        self.current_message_id = message_id
-
-        # Segmented (the user receives the output as it comes)
-        # We will first send a json entry that contains the message id and so on, then the text as it goes
-        return Response(
-            stream_with_context(
-                self.parse_to_prompt_stream(message, message_id)
-            ), content_type='text/plain; charset=utf-8'
-        )
      
     def stop_gen(self):
         self.cancel_gen = True
         return jsonify({"status": "ok"}) 
            
-    def run_to(self):
-        data = request.get_json()
-        message_id = int(data["id"])
-        # Segmented (the user receives the output as it comes)
-        # We will first send a json entry that contains the message id and so on, then the text as it goes
-        return Response(
-            stream_with_context(
-                self.parse_to_prompt_stream("",message_id)
-            )
-        )
-
     def rename(self):
         data = request.get_json()
         title = data["title"]
@@ -419,8 +379,11 @@ class Gpt4AllWebUI(GPT4AllAPI):
 
     def delete_message(self):
         discussion_id = request.args.get("id")
+        if self.current_discussion is None:
+            return jsonify({"status": False,"message":"No discussion is selected"})
+        else:
             new_rank = self.current_discussion.delete_message(discussion_id)
-        return jsonify({"new_rank": new_rank})
+            return jsonify({"status":True,"new_rank": new_rank})
 
 
     def new_discussion(self):

backends/llama_cpp/__init__.py

@@ -35,6 +35,8 @@ class LLAMACPP(GPTBackend):
                 seed=self.config['seed'],
                 )
 
+    def stop_generation(self):
+        self.model._grab_text_callback()
 
     def generate(self, 
                  prompt:str,                  

pyGpt4All/api.py

@@ -9,11 +9,11 @@
 ######
 import gc
 import sys
-from queue import Queue
 from datetime import datetime
 from pyGpt4All.db import DiscussionsDB
 from pathlib import Path
 import importlib
+import markdown
 
 __author__ = "parisneo"
 __github__ = "https://github.com/nomic-ai/gpt4all-ui"
@@ -27,9 +27,6 @@ class GPT4AllAPI():
         self.config_file_path = config_file_path
         self.cancel_gen = False
 
-        # This is the queue used to stream text to the ui as the bot spits out its response
-        self.text_queue = Queue(0)
-
         # Keeping track of current discussion and message
         self.current_discussion = None
         self.current_message_id = 0
@@ -106,6 +103,7 @@ class GPT4AllAPI():
             0
         )
         self.current_message_id = message_id
+        if self.personality["welcome_message"]!="":
             if self.personality["welcome_message"]!="":
                 message_id = self.current_discussion.add_message(
                     self.personality["name"], self.personality["welcome_message"], 
@@ -193,16 +191,16 @@ class GPT4AllAPI():
         return string
 
 
-
     def new_text_callback(self, text: str):
         if self.cancel_gen:
             return False
         print(text, end="")
         sys.stdout.flush()
+        
         if self.chatbot_bindings.inline:
             self.bot_says += text
             if not self.personality["user_message_prefix"].lower() in self.bot_says.lower():
-                self.socketio.emit('message', {'data': text});
+                self.socketio.emit('message', {'data': markdown.markdown(self.bot_says)});
                 if self.cancel_gen:
                     print("Generation canceled")
                     return False
@@ -211,27 +209,33 @@ class GPT4AllAPI():
             else:
                 self.bot_says = self.remove_text_from_string(self.bot_says, self.personality["user_message_prefix"].lower())
                 print("The model is halucinating")
+                self.socketio.emit('final', {'data': self.bot_says})
                 return False
         else:
             self.full_text += text
             if self.is_bot_text_started:
                 self.bot_says += text
                 if not self.personality["user_message_prefix"].lower() in self.bot_says.lower():
-                    self.text_queue.put(text)
+                    self.socketio.emit('message', {'data': markdown.markdown(self.bot_says)});
+                    #self.socketio.emit('message', {'data': text});
                     if self.cancel_gen:
                         print("Generation canceled")
+                        self.socketio.emit('final', {'data': self.bot_says})
                         return False
                     else:
                         return True
                 else:
                     self.bot_says = self.remove_text_from_string(self.bot_says, self.personality["user_message_prefix"].lower())
                     print("The model is halucinating")
+                    self.socketio.emit('final', {'data': self.bot_says})
                     self.cancel_gen=True
                     return False
                 
             #if self.current_message in self.full_text:
             if len(self.discussion_messages) < len(self.full_text):
                 self.is_bot_text_started = True
+            else:
+                self.socketio.emit('waiter', {'wait': (len(self.discussion_messages)-len(self.full_text))/len(self.discussion_messages)});
         
     def generate_message(self):
         self.generating=True

static/js/chat.js

@@ -15,8 +15,6 @@ function addMessage(sender, message, id, rank = 0, can_edit = false) {
     const chatForm = document.getElementById('chat-form');
     const userInput = document.getElementById('user-input');
     
-    console.log(id)
-    
     const messageElement = document.createElement('div');
     messageElement.classList.add('bg-secondary', 'drop-shadow-sm', 'p-4', 'mx-6', 'my-4', 'flex', 'flex-col', 'space-x-2', 'rounded-lg', 'shadow-lg', 'bg-gray-300', 'text-black', 'dark:text-gray-200', 'dark:bg-gray-800', 'hover:bg-gray-400', 'dark:hover:bg-gray-700', 'transition-colors', 'duration-300');
 
@@ -88,85 +86,14 @@ function addMessage(sender, message, id, rank = 0, can_edit = false) {
             waitAnimation.style.display = "block";
             stopGeneration.style.display = "block";
 
-            elements = addMessage("", "", 0, 0, can_edit = true);
+            globals.bot_msg = addMessage("", "", 0, 0, can_edit = true);
+            globals.user_msg = undefined
             // scroll to bottom of chat window
             chatWindow.scrollTop = chatWindow.scrollHeight;
-            fetch("/run_to", {
+            send_message('generate_msg_from',{prompt: message, id: messageElement.id})
-                method: 'POST',
+            entry_counter = 0;
-                headers: {
-                    'Content-Type': 'application/json'
-                },
-                body: JSON.stringify({
-                    id: messageElement.id,
-                    message: message
-                })
-            })
-                .then(function (response) {
-                    const stream = new ReadableStream({
-                        start(controller) {
-                            const reader = response.body.getReader();
-                            function push() {
-                                reader.read().then(function (result) {
-                                    if (result.done) {
-                                        console.log(result)
-                                        sendbtn.style.display = "block";
-                                        waitAnimation.style.display = "none";
-                                        stopGeneration.style.display = "none";
-                                        controller.close();
-                                        return;
-                                    }
-                                    controller.enqueue(result.value);
-                                    push();
-                                })
-                            }
-                            push();
-                        }
-                    });
-                    const textDecoder = new TextDecoder();
-                    const readableStreamDefaultReader = stream.getReader();
-                    let entry_counter = 0
-                    function readStream() {
-                        readableStreamDefaultReader.read().then(function (result) {
-                            if (result.done) {
-                                console.log(result)
-                                return;
-                            }
         
-                            text = textDecoder.decode(result.value);
 
-                            // The server will first send a json containing information about the message just sent
-                            if (entry_counter == 0) {
-                                // We parse it and
-                                infos = JSON.parse(text)
-                                elements.setID(infos.response_id)
-                                elements.setSender(infos.bot)
-                                entry_counter++;
-                            }
-                            else {
-                                entry_counter++;
-                                prefix = "FINAL:";
-                                if(text.startsWith(prefix)){
-                                    console.log("Final text found")
-                                    text = text.substring(prefix.length);
-                                    elements.hiddenElement.innerHTML         = text
-                                    elements.messageTextElement.innerHTML    = text
-                                }
-                                else{
-                                    // For the other enrtries, these are just the text of the chatbot
-                                    txt = hiddenElement_.innerHTML;
-                                    txt += text
-                                    elements.hiddenElement.innerHTML = txt
-                                    elements.messageTextElement.innerHTML = txt
-                                    // scroll to bottom of chat window
-                                    chatWindow.scrollTop = chatWindow.scrollHeight;
-                                }
-                            }
-
-                            readStream();
-                        });
-                    }
-                    readStream();
-                });
         });
 
         const editButton = document.createElement('button');

static/js/main.js

@@ -1,10 +1,72 @@
-function update_main(){
+var globals={
-  const chatWindow = document.getElementById('chat-window');
+  chatWindow:undefined,
-  const chatForm = document.getElementById('chat-form');
+  chatForm:undefined,
-  const userInput = document.getElementById('user-input');
+  userInput:undefined,
-  const stopGeneration = document.getElementById("stop-generation")
+  stopGeneration:undefined,
+  sendbtn:undefined,
+  waitAnimation:undefined
+}
 
-  stopGeneration.addEventListener('click', (event) =>{
+function send_message(service_name, parameters){
+  var socket = io.connect('http://' + document.domain + ':' + location.port);
+  globals.socket = socket
+  socket.on('connect', function() {
+      globals.sendbtn.style.display="block";
+      globals.waitAnimation.style.display="none";
+      globals.stopGeneration.style.display = "none";
+      entry_counter = 0;
+
+  });
+  socket.on('disconnect', function() {
+    console.log("disconnected")
+    entry_counter = 0;
+  });
+
+
+  socket.on('infos', function(msg) {
+    if(globals.user_msg){
+      globals.user_msg.setSender(msg.user);
+      globals.user_msg.setMessage(msg.message);
+      globals.user_msg.setID(msg.id);
+    }
+    globals.bot_msg.setSender(msg.bot);
+    globals.bot_msg.setID(msg.response_id);
+  });
+
+  socket.on('waiter', function(msg) {
+    globals.bot_msg.messageTextElement.innerHTML    = `Remaining words ${Math.floor(msg.wait * 100)}%`;    
+  });
+  
+  socket.on('message', function(msg) {
+        text = msg.data;
+        // For the other enrtries, these are just the text of the chatbot
+        globals.bot_msg.messageTextElement.innerHTML    = text;
+        // scroll to bottom of chat window
+        globals.chatWindow.scrollTop = globals.chatWindow.scrollHeight;
+  });
+
+  socket.on('final',function(msg){
+    text = msg.data;
+    globals.bot_msg.hiddenElement.innerHTML         = text
+    globals.bot_msg.messageTextElement.innerHTML    = text
+    globals.sendbtn.style.display="block";
+    globals.waitAnimation.style.display="none";
+    globals.stopGeneration.style.display = "none";
+  });  
+  setTimeout(()=>{
+    globals.socket.emit(service_name, parameters);
+  },1000);
+}
+
+function update_main(){
+  globals.chatWindow = document.getElementById('chat-window');
+  globals.chatForm = document.getElementById('chat-form');
+  globals.userInput = document.getElementById('user-input');
+  globals.stopGeneration = document.getElementById("stop-generation")
+  globals.sendbtn = document.querySelector("#submit-input")
+  globals.waitAnimation = document.querySelector("#wait-animation")
+  
+  globals.stopGeneration.addEventListener('click', (event) =>{
     event.preventDefault();
     console.log("Stop clicked");
     fetch('/stop_gen')
@@ -20,73 +82,29 @@ function update_main(){
     console.log("Submitting")
   
     // get user input and clear input field
-    message = userInput.value;
+    message = globals.userInput.value;
-    userInput.value = '';
+    globals.userInput.value = '';    
-    
+    globals.sendbtn.style.display="none";
-    // add user message to chat window
+    globals.waitAnimation.style.display="block";
-    const sendbtn = document.querySelector("#submit-input")
+    globals.stopGeneration.style.display = "block";
-    const waitAnimation = document.querySelector("#wait-animation")
-    const stopGeneration = document.querySelector("#stop-generation")
-    
-    sendbtn.style.display="none";
-    waitAnimation.style.display="block";
-    stopGeneration.style.display = "block";
     console.log("Sending message to bot")
 
-    user_msg = addMessage('',message, 0, 0, can_edit=true);
+    globals.user_msg = addMessage('',message, 0, 0, can_edit=true);
-    bot_msg = addMessage('', '', 0, 0, can_edit=true);
+    globals.bot_msg = addMessage('', '', 0, 0, can_edit=true);
     // scroll to bottom of chat window
-    chatWindow.scrollTop = chatWindow.scrollHeight;
+    globals.chatWindow.scrollTop = globals.chatWindow.scrollHeight;
-
-    var socket = io.connect('http://' + document.domain + ':' + location.port);
-    entry_counter = 0;
-
-    socket.on('connect', function() {
-        socket.emit('connected', {prompt: message});
-        entry_counter = 0;
-    });
-    socket.on('disconnect', function() {
 
     entry_counter = 0;
-    });
+    send_message('generate_msg',{prompt: message})
-    socket.on()
 
-    socket.on('infos', function(msg) {
-      user_msg.setSender(msg.user);
-      user_msg.setMessage(msg.message);
-      user_msg.setID(msg.id);
-      bot_msg.setSender(msg.bot);
-      bot_msg.setID(msg.response_id);
-    });
 
-    socket.on('message', function(msg) {
-          text = msg.data;
-          console.log(text)
-          // For the other enrtries, these are just the text of the chatbot
-          txt = bot_msg.hiddenElement.innerHTML;
-          txt += text
-          bot_msg.hiddenElement.innerHTML         = txt;
-          bot_msg.messageTextElement.innerHTML    = txt;
-          // scroll to bottom of chat window
-          chatWindow.scrollTop = chatWindow.scrollHeight;
-    });
-
-    socket.on('final',function(msg){
-      text = msg.data;
-      bot_msg.hiddenElement.innerHTML         = text
-      bot_msg.messageTextElement.innerHTML    = text
-      sendbtn.style.display="block";
-      waitAnimation.style.display="none";
-      stopGeneration.style.display = "none";
-      socket.disconnect();
-    });
     //socket.emit('stream-text', {text: text});
   }
-  chatForm.addEventListener('submit', event => {
+  globals.chatForm.addEventListener('submit', event => {
       event.preventDefault();
       submit_form();
   });  
-  userInput.addEventListener("keyup", function(event) {
+  globals.userInput.addEventListener("keyup", function(event) {
     // Check if Enter key was pressed while holding Shift
     // Also check if Shift + Ctrl keys were pressed while typing
     // These combinations override the submit action
@@ -99,9 +117,9 @@ function update_main(){
     // Restore original functionality for the remaining cases
     else if (!shiftPressed && ctrlPressed) {
       setTimeout(() => {
-        userInput.focus();
+        globals.userInput.focus();
         contentEditable.value += event.data;
-        lastValue.innerHTML = userInput.value;
+        lastValue.innerHTML = globals.userInput.value;
       }, 0);
     }
   });