markdown rendering + websocket

This commit is contained in:
ParisNeo 2023-04-28 01:39:57 +02:00
parent 62e6a1c68c
commit 92be2d0d5b
6 changed files with 170 additions and 246 deletions

View File

@ -106,6 +106,16 @@ You can also refuse to download the model during the install procedure and downl
- [Vicuna 7B rev 1](https://huggingface.co/eachadea/legacy-ggml-vicuna-7b-4bit/resolve/main/ggml-vicuna-7b-4bit-rev1.bin) or visit [repository](https://huggingface.co/eachadea/legacy-ggml-vicuna-7b-4bit)
- [Vicuna 13B rev 1](https://huggingface.co/eachadea/ggml-vicuna-13b-4bit/resolve/main/ggml-vicuna-13b-4bit-rev1.bin) or visit [repository](https://huggingface.co/eachadea/ggml-vicuna-13b-4bit)
-[ggml-gpt4all-j-v1.3-groovy](https://gpt4all.io/models/ggml-gpt4all-j-v1.3-groovy.bin)
-[ggml-gpt4all-j-v1.2-jazzy](https://gpt4all.io/models/ggml-gpt4all-j-v1.2-jazzy.bin)
-[ggml-gpt4all-l13b-snoozy](https://gpt4all.io/models/ggml-gpt4all-l13b-snoozy.bin)
-[ggml-gpt4all-j-v1.1-breezy](https://gpt4all.io/models/ggml-gpt4all-j-v1.1-breezy.bin)
-[ggml-gpt4all-j](https://gpt4all.io/models/ggml-gpt4all-j.bin)
-[ggml-vicuna-7b-1.1-q4_2](https://gpt4all.io/models/ggml-vicuna-7b-1.1-q4_2.bin)
-[ggml-vicuna-13b-1.1-q4_2](https://gpt4all.io/models/ggml-vicuna-13b-1.1-q4_2.bin)
We also support GPT-j models out of the box
## GPT-j models

147
app.py
View File

@ -14,7 +14,7 @@ __github__ = "https://github.com/nomic-ai/gpt4all-ui"
__copyright__ = "Copyright 2023, "
__license__ = "Apache 2.0"
import os
import logging
import argparse
import json
@ -37,9 +37,13 @@ from pathlib import Path
import gc
app = Flask("GPT4All-WebUI", static_url_path="/static", static_folder="static")
socketio = SocketIO(app)
app.config['SECRET_KEY'] = 'secret!'
# Set the logging level to WARNING or higher
logging.getLogger('socketio').setLevel(logging.WARNING)
logging.getLogger('engineio').setLevel(logging.WARNING)
# Suppress Flask's default console output
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
import time
from pyGpt4All.config import load_config, save_config
@ -91,10 +95,8 @@ class Gpt4AllWebUI(GPT4AllAPI):
self.add_endpoint(
"/new_discussion", "new_discussion", self.new_discussion, methods=["GET"]
)
self.add_endpoint("/generate", "generate", self.generate, methods=["POST"])
self.add_endpoint("/stop_gen", "stop_gen", self.stop_gen, methods=["GET"])
self.add_endpoint("/run_to", "run_to", self.run_to, methods=["POST"])
self.add_endpoint("/rename", "rename", self.rename, methods=["POST"])
self.add_endpoint(
"/load_discussion", "load_discussion", self.load_discussion, methods=["POST"]
@ -158,27 +160,15 @@ class Gpt4AllWebUI(GPT4AllAPI):
# Socket IO stuff
@socketio.on('connect')
def test_connect():
def connect():
print('Client connected')
@socketio.on('disconnect')
def test_disconnect():
def disconnect():
print('Client disconnected')
@socketio.on('stream-text')
def handle_stream_text(data):
text = data['prompt']
words = text.split()
for word in words:
emit('stream-word', {'word': word})
time.sleep(1) # sleep for 1 second to simulate processing time
emit('stream-end')
@socketio.on('connected')
def handle_connection(data):
if "data" in data and data["data"]=='Connected!':
return
@socketio.on('generate_msg')
def generate_msg(data):
if self.current_discussion is None:
if self.db.does_last_discussion_have_messages():
self.current_discussion = self.db.create_discussion()
@ -194,12 +184,16 @@ class Gpt4AllWebUI(GPT4AllAPI):
tpe = threading.Thread(target=self.parse_to_prompt_stream, args=(message, message_id))
tpe.start()
# self.parse_to_prompt_stream(message, message_id)
#self.socketio.emit('message', {'data': 'WebSocket connected!'})
@socketio.on('generate_msg_from')
def handle_connection(data):
message_id = int(data['id'])
message = data["prompt"]
self.current_message_id = message_id
tpe = threading.Thread(target=self.parse_to_prompt_stream, args=(message, message_id))
tpe.start()
#for i in range(10):
# socketio.emit('message', {'data': 'Message ' + str(i)})
def list_backends(self):
backends_dir = Path('./backends') # replace with the actual path to the models folder
@ -297,79 +291,45 @@ class Gpt4AllWebUI(GPT4AllAPI):
# send the message to the bot
print(f"Received message : {message}")
# First we need to send the new message ID to the client
response_id = self.current_discussion.add_message(
self.personality["name"], "", parent = message_id
) # first the content is empty, but we'll fill it at the end
socketio.emit('infos',
{
"type": "input_message_infos",
"bot": self.personality["name"],
"user": self.personality["user_name"],
"message":markdown.markdown(message),
"id": message_id,
"response_id": response_id,
}
)
if self.current_discussion:
# First we need to send the new message ID to the client
response_id = self.current_discussion.add_message(
self.personality["name"], "", parent = message_id
) # first the content is empty, but we'll fill it at the end
socketio.emit('infos',
{
"type": "input_message_infos",
"bot": self.personality["name"],
"user": self.personality["user_name"],
"message":markdown.markdown(message),
"id": message_id,
"response_id": response_id,
}
);
# prepare query and reception
self.discussion_messages = self.prepare_query(message_id)
self.prepare_reception()
self.generating = True
# app.config['executor'] = ThreadPoolExecutor(max_workers=1)
# app.config['executor'].submit(self.generate_message)
self.generate_message()
print("## Done ##")
self.current_discussion.update_message(response_id, self.bot_says)
self.full_message_list.append(self.bot_says)
bot_says = markdown.markdown(self.bot_says)
socketio.emit('final', {'data': bot_says})
self.cancel_gen = False
return bot_says
def generate(self):
if self.current_discussion is None:
if self.db.does_last_discussion_have_messages():
self.current_discussion = self.db.create_discussion()
else:
self.current_discussion = self.db.load_last_discussion()
message = request.json["message"]
message_id = self.current_discussion.add_message(
"user", message, parent=self.current_message_id
)
message = f"{request.json['message']}"
self.current_message_id = message_id
# Segmented (the user receives the output as it comes)
# We will first send a json entry that contains the message id and so on, then the text as it goes
return Response(
stream_with_context(
self.parse_to_prompt_stream(message, message_id)
), content_type='text/plain; charset=utf-8'
)
# prepare query and reception
self.discussion_messages = self.prepare_query(message_id)
self.prepare_reception()
self.generating = True
# app.config['executor'] = ThreadPoolExecutor(max_workers=1)
# app.config['executor'].submit(self.generate_message)
print("## Generate message ##")
self.generate_message()
print("## Done ##")
self.current_discussion.update_message(response_id, self.bot_says)
self.full_message_list.append(self.bot_says)
self.cancel_gen = False
return bot_says
else:
print("## Done ##")
return ""
def stop_gen(self):
self.cancel_gen = True
return jsonify({"status": "ok"})
def run_to(self):
data = request.get_json()
message_id = int(data["id"])
# Segmented (the user receives the output as it comes)
# We will first send a json entry that contains the message id and so on, then the text as it goes
return Response(
stream_with_context(
self.parse_to_prompt_stream("",message_id)
)
)
def rename(self):
data = request.get_json()
title = data["title"]
@ -419,8 +379,11 @@ class Gpt4AllWebUI(GPT4AllAPI):
def delete_message(self):
discussion_id = request.args.get("id")
new_rank = self.current_discussion.delete_message(discussion_id)
return jsonify({"new_rank": new_rank})
if self.current_discussion is None:
return jsonify({"status": False,"message":"No discussion is selected"})
else:
new_rank = self.current_discussion.delete_message(discussion_id)
return jsonify({"status":True,"new_rank": new_rank})
def new_discussion(self):

View File

@ -35,6 +35,8 @@ class LLAMACPP(GPTBackend):
seed=self.config['seed'],
)
def stop_generation(self):
self.model._grab_text_callback()
def generate(self,
prompt:str,

View File

@ -9,11 +9,11 @@
######
import gc
import sys
from queue import Queue
from datetime import datetime
from pyGpt4All.db import DiscussionsDB
from pathlib import Path
import importlib
import markdown
__author__ = "parisneo"
__github__ = "https://github.com/nomic-ai/gpt4all-ui"
@ -27,9 +27,6 @@ class GPT4AllAPI():
self.config_file_path = config_file_path
self.cancel_gen = False
# This is the queue used to stream text to the ui as the bot spits out its response
self.text_queue = Queue(0)
# Keeping track of current discussion and message
self.current_discussion = None
self.current_message_id = 0
@ -107,12 +104,13 @@ class GPT4AllAPI():
)
self.current_message_id = message_id
if self.personality["welcome_message"]!="":
message_id = self.current_discussion.add_message(
self.personality["name"], self.personality["welcome_message"],
DiscussionsDB.MSG_TYPE_NORMAL,
0,
self.current_message_id
)
if self.personality["welcome_message"]!="":
message_id = self.current_discussion.add_message(
self.personality["name"], self.personality["welcome_message"],
DiscussionsDB.MSG_TYPE_NORMAL,
0,
self.current_message_id
)
self.current_message_id = message_id
return message_id
@ -193,16 +191,16 @@ class GPT4AllAPI():
return string
def new_text_callback(self, text: str):
if self.cancel_gen:
return False
print(text, end="")
sys.stdout.flush()
if self.chatbot_bindings.inline:
self.bot_says += text
if not self.personality["user_message_prefix"].lower() in self.bot_says.lower():
self.socketio.emit('message', {'data': text});
self.socketio.emit('message', {'data': markdown.markdown(self.bot_says)});
if self.cancel_gen:
print("Generation canceled")
return False
@ -211,27 +209,33 @@ class GPT4AllAPI():
else:
self.bot_says = self.remove_text_from_string(self.bot_says, self.personality["user_message_prefix"].lower())
print("The model is halucinating")
self.socketio.emit('final', {'data': self.bot_says})
return False
else:
self.full_text += text
if self.is_bot_text_started:
self.bot_says += text
if not self.personality["user_message_prefix"].lower() in self.bot_says.lower():
self.text_queue.put(text)
self.socketio.emit('message', {'data': markdown.markdown(self.bot_says)});
#self.socketio.emit('message', {'data': text});
if self.cancel_gen:
print("Generation canceled")
self.socketio.emit('final', {'data': self.bot_says})
return False
else:
return True
else:
self.bot_says = self.remove_text_from_string(self.bot_says, self.personality["user_message_prefix"].lower())
print("The model is halucinating")
self.socketio.emit('final', {'data': self.bot_says})
self.cancel_gen=True
return False
#if self.current_message in self.full_text:
if len(self.discussion_messages) < len(self.full_text):
self.is_bot_text_started = True
else:
self.socketio.emit('waiter', {'wait': (len(self.discussion_messages)-len(self.full_text))/len(self.discussion_messages)});
def generate_message(self):
self.generating=True

View File

@ -14,8 +14,6 @@ function addMessage(sender, message, id, rank = 0, can_edit = false) {
const chatWindow = document.getElementById('chat-window');
const chatForm = document.getElementById('chat-form');
const userInput = document.getElementById('user-input');
console.log(id)
const messageElement = document.createElement('div');
messageElement.classList.add('bg-secondary', 'drop-shadow-sm', 'p-4', 'mx-6', 'my-4', 'flex', 'flex-col', 'space-x-2', 'rounded-lg', 'shadow-lg', 'bg-gray-300', 'text-black', 'dark:text-gray-200', 'dark:bg-gray-800', 'hover:bg-gray-400', 'dark:hover:bg-gray-700', 'transition-colors', 'duration-300');
@ -88,85 +86,14 @@ function addMessage(sender, message, id, rank = 0, can_edit = false) {
waitAnimation.style.display = "block";
stopGeneration.style.display = "block";
elements = addMessage("", "", 0, 0, can_edit = true);
globals.bot_msg = addMessage("", "", 0, 0, can_edit = true);
globals.user_msg = undefined
// scroll to bottom of chat window
chatWindow.scrollTop = chatWindow.scrollHeight;
fetch("/run_to", {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
id: messageElement.id,
message: message
})
})
.then(function (response) {
const stream = new ReadableStream({
start(controller) {
const reader = response.body.getReader();
function push() {
reader.read().then(function (result) {
if (result.done) {
console.log(result)
sendbtn.style.display = "block";
waitAnimation.style.display = "none";
stopGeneration.style.display = "none";
controller.close();
return;
}
controller.enqueue(result.value);
push();
})
}
push();
}
});
const textDecoder = new TextDecoder();
const readableStreamDefaultReader = stream.getReader();
let entry_counter = 0
function readStream() {
readableStreamDefaultReader.read().then(function (result) {
if (result.done) {
console.log(result)
return;
}
send_message('generate_msg_from',{prompt: message, id: messageElement.id})
entry_counter = 0;
text = textDecoder.decode(result.value);
// The server will first send a json containing information about the message just sent
if (entry_counter == 0) {
// We parse it and
infos = JSON.parse(text)
elements.setID(infos.response_id)
elements.setSender(infos.bot)
entry_counter++;
}
else {
entry_counter++;
prefix = "FINAL:";
if(text.startsWith(prefix)){
console.log("Final text found")
text = text.substring(prefix.length);
elements.hiddenElement.innerHTML = text
elements.messageTextElement.innerHTML = text
}
else{
// For the other enrtries, these are just the text of the chatbot
txt = hiddenElement_.innerHTML;
txt += text
elements.hiddenElement.innerHTML = txt
elements.messageTextElement.innerHTML = txt
// scroll to bottom of chat window
chatWindow.scrollTop = chatWindow.scrollHeight;
}
}
readStream();
});
}
readStream();
});
});
const editButton = document.createElement('button');

View File

@ -1,10 +1,72 @@
var globals={
chatWindow:undefined,
chatForm:undefined,
userInput:undefined,
stopGeneration:undefined,
sendbtn:undefined,
waitAnimation:undefined
}
function send_message(service_name, parameters){
var socket = io.connect('http://' + document.domain + ':' + location.port);
globals.socket = socket
socket.on('connect', function() {
globals.sendbtn.style.display="block";
globals.waitAnimation.style.display="none";
globals.stopGeneration.style.display = "none";
entry_counter = 0;
});
socket.on('disconnect', function() {
console.log("disconnected")
entry_counter = 0;
});
socket.on('infos', function(msg) {
if(globals.user_msg){
globals.user_msg.setSender(msg.user);
globals.user_msg.setMessage(msg.message);
globals.user_msg.setID(msg.id);
}
globals.bot_msg.setSender(msg.bot);
globals.bot_msg.setID(msg.response_id);
});
socket.on('waiter', function(msg) {
globals.bot_msg.messageTextElement.innerHTML = `Remaining words ${Math.floor(msg.wait * 100)}%`;
});
socket.on('message', function(msg) {
text = msg.data;
// For the other enrtries, these are just the text of the chatbot
globals.bot_msg.messageTextElement.innerHTML = text;
// scroll to bottom of chat window
globals.chatWindow.scrollTop = globals.chatWindow.scrollHeight;
});
socket.on('final',function(msg){
text = msg.data;
globals.bot_msg.hiddenElement.innerHTML = text
globals.bot_msg.messageTextElement.innerHTML = text
globals.sendbtn.style.display="block";
globals.waitAnimation.style.display="none";
globals.stopGeneration.style.display = "none";
});
setTimeout(()=>{
globals.socket.emit(service_name, parameters);
},1000);
}
function update_main(){
const chatWindow = document.getElementById('chat-window');
const chatForm = document.getElementById('chat-form');
const userInput = document.getElementById('user-input');
const stopGeneration = document.getElementById("stop-generation")
stopGeneration.addEventListener('click', (event) =>{
globals.chatWindow = document.getElementById('chat-window');
globals.chatForm = document.getElementById('chat-form');
globals.userInput = document.getElementById('user-input');
globals.stopGeneration = document.getElementById("stop-generation")
globals.sendbtn = document.querySelector("#submit-input")
globals.waitAnimation = document.querySelector("#wait-animation")
globals.stopGeneration.addEventListener('click', (event) =>{
event.preventDefault();
console.log("Stop clicked");
fetch('/stop_gen')
@ -20,73 +82,29 @@ function update_main(){
console.log("Submitting")
// get user input and clear input field
message = userInput.value;
userInput.value = '';
// add user message to chat window
const sendbtn = document.querySelector("#submit-input")
const waitAnimation = document.querySelector("#wait-animation")
const stopGeneration = document.querySelector("#stop-generation")
sendbtn.style.display="none";
waitAnimation.style.display="block";
stopGeneration.style.display = "block";
message = globals.userInput.value;
globals.userInput.value = '';
globals.sendbtn.style.display="none";
globals.waitAnimation.style.display="block";
globals.stopGeneration.style.display = "block";
console.log("Sending message to bot")
user_msg = addMessage('',message, 0, 0, can_edit=true);
bot_msg = addMessage('', '', 0, 0, can_edit=true);
globals.user_msg = addMessage('',message, 0, 0, can_edit=true);
globals.bot_msg = addMessage('', '', 0, 0, can_edit=true);
// scroll to bottom of chat window
chatWindow.scrollTop = chatWindow.scrollHeight;
globals.chatWindow.scrollTop = globals.chatWindow.scrollHeight;
var socket = io.connect('http://' + document.domain + ':' + location.port);
entry_counter = 0;
send_message('generate_msg',{prompt: message})
socket.on('connect', function() {
socket.emit('connected', {prompt: message});
entry_counter = 0;
});
socket.on('disconnect', function() {
entry_counter = 0;
});
socket.on()
socket.on('infos', function(msg) {
user_msg.setSender(msg.user);
user_msg.setMessage(msg.message);
user_msg.setID(msg.id);
bot_msg.setSender(msg.bot);
bot_msg.setID(msg.response_id);
});
socket.on('message', function(msg) {
text = msg.data;
console.log(text)
// For the other enrtries, these are just the text of the chatbot
txt = bot_msg.hiddenElement.innerHTML;
txt += text
bot_msg.hiddenElement.innerHTML = txt;
bot_msg.messageTextElement.innerHTML = txt;
// scroll to bottom of chat window
chatWindow.scrollTop = chatWindow.scrollHeight;
});
socket.on('final',function(msg){
text = msg.data;
bot_msg.hiddenElement.innerHTML = text
bot_msg.messageTextElement.innerHTML = text
sendbtn.style.display="block";
waitAnimation.style.display="none";
stopGeneration.style.display = "none";
socket.disconnect();
});
//socket.emit('stream-text', {text: text});
}
chatForm.addEventListener('submit', event => {
globals.chatForm.addEventListener('submit', event => {
event.preventDefault();
submit_form();
});
userInput.addEventListener("keyup", function(event) {
globals.userInput.addEventListener("keyup", function(event) {
// Check if Enter key was pressed while holding Shift
// Also check if Shift + Ctrl keys were pressed while typing
// These combinations override the submit action
@ -99,9 +117,9 @@ function update_main(){
// Restore original functionality for the remaining cases
else if (!shiftPressed && ctrlPressed) {
setTimeout(() => {
userInput.focus();
globals.userInput.focus();
contentEditable.value += event.data;
lastValue.innerHTML = userInput.value;
lastValue.innerHTML = globals.userInput.value;
}, 0);
}
});