ExternalVendorCode/lollms-webui
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms-webui.git synced 2024-12-19 04:17:52 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Beta1 is out

Browse Source
This commit is contained in:
Saifeddine ALOUI 2023-10-09 01:18:21 +02:00
parent 747dcb7759
commit ea66b5f30d
11 changed files with 164 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
api/__init__.py
View File

@ -19,8 +19,8 @@ from lollms.binding import LOLLMSConfig, BindingBuilder, LLMBinding, ModelBuilde
from lollms.paths import LollmsPaths
from lollms.helpers import ASCIIColors, trace_exception
from lollms.app import LollmsApplication
from lollms.utilities import File64BitsManager, PromptReshaper, TextVectorizer
from lollms.utilities import File64BitsManager, PromptReshaper
from safe_store import TextVectorizer, VectorizationMethod
import threading
from tqdm import tqdm
import traceback
@ -136,7 +136,7 @@ class LoLLMsAPPI(LollmsApplication):
self.db = DiscussionsDB(self.db_path)
else:
# Create database object
self.db = DiscussionsDB(self.lollms_paths.personal_path/"databases"/self.db_path)
self.db = DiscussionsDB(self.lollms_paths.personal_databases_path/self.db_path)
# If the database is empty, populate it with tables
ASCIIColors.info("Checking discussions database... ",end="")
@ -1162,14 +1162,11 @@ class LoLLMsAPPI(LollmsApplication):
if self.config.use_discussions_history:
if self.discussion_store is None:
self.discussion_store = TextVectorizer(self.config.data_vectorization_method, # supported "model_embedding" or "ftidf_vectorizer"
self.discussion_store = TextVectorizer(self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
database_path=self.lollms_paths.personal_databases_path/"discussionsdb.json",
save_db=self.config.data_vectorization_save_db,
visualize_data_at_startup=False,
visualize_data_at_add_file=False,
visualize_data_at_generate=False,
data_visualization_method="PCA",
data_visualization_method=VectorizationMethod.PCA,
database_dict=None)
corpus = self.db.export_all_as_markdown()
self.discussion_store.add_document("discussions", corpus, self.config.data_vectorization_chunk_size, self.config.data_vectorization_overlap_size )
@ -1182,8 +1179,7 @@ class LoLLMsAPPI(LollmsApplication):
if len(self.personality.files)>0 and self.personality.vectorizer:
emb = self.personality.vectorizer.embed_query(message.content)
docs, sorted_similarities = self.personality.vectorizer.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)
docs, sorted_similarities = self.personality.vectorizer.recover_text(message.content, top_k=self.config.data_vectorization_nb_chunks)
for doc, infos in zip(docs, sorted_similarities):
str_docs+=f"document chunk:\nchunk path: {infos[0]}\nchunk content:{doc}"

24
api/db.py
View File

@ -47,7 +47,7 @@ class DiscussionsDB:
personality TEXT,
sender TEXT NOT NULL,
content TEXT NOT NULL,
type INT NOT NULL,
message_type INT NOT NULL,
sender_type INT DEFAULT 0,
rank INT NOT NULL DEFAULT 0,
parent_message_id INT,
@ -231,7 +231,7 @@ class DiscussionsDB:
discussion_id = row[0]
discussion_title = row[1]
discussion = {"id": discussion_id, "title":discussion_title, "messages": []}
rows = self.select(f"SELECT sender, content, type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at FROM message WHERE discussion_id=?",(discussion_id,))
rows = self.select(f"SELECT sender, content, message_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at FROM message WHERE discussion_id=?",(discussion_id,))
for message_row in rows:
sender = message_row[1]
content = message_row[2]
@ -245,7 +245,7 @@ class DiscussionsDB:
finished_generating_at = message_row[10]
discussion["messages"].append(
{"sender": sender, "content": content, "type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at":finished_generating_at}
{"sender": sender, "content": content, "message_type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at":finished_generating_at}
)
discussions.append(discussion)
return discussions
@ -295,7 +295,7 @@ class DiscussionsDB:
finished_generating_at = message_row[9]
discussion["messages"].append(
{"sender": sender, "content": content, "type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at": finished_generating_at}
{"sender": sender, "content": content, "message_type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at": finished_generating_at}
)
discussions.append(discussion)
return discussions
@ -327,7 +327,7 @@ class DiscussionsDB:
finished_generating_at = message_row[9]
discussion["messages"].append(
{"sender": sender, "content": content, "type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at": finished_generating_at}
{"sender": sender, "content": content, "message_type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at": finished_generating_at}
)
discussions.append(discussion)
return discussions
@ -347,7 +347,7 @@ class DiscussionsDB:
for message_data in messages_data:
sender = message_data.get("sender")
content = message_data.get("content")
content_type = message_data.get("type")
content_type = message_data.get("message_type",message_data.get("type"))
rank = message_data.get("rank")
parent_message_id = message_data.get("parent_message_id")
binding = message_data.get("binding","")
@ -356,11 +356,11 @@ class DiscussionsDB:
created_at = message_data.get("created_at",datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
finished_generating_at = message_data.get("finished_generating_at",datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
discussion["messages"].append(
{"sender": sender, "content": content, "type": content_type, "rank": rank, "binding": binding, "model": model, "personality": personality, "created_at": created_at, "finished_generating_at": finished_generating_at}
{"sender": sender, "content": content, "message_type": content_type, "rank": rank, "binding": binding, "model": model, "personality": personality, "created_at": created_at, "finished_generating_at": finished_generating_at}
)
# Insert message into the database
self.insert("INSERT INTO message (sender, content, type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
self.insert("INSERT INTO message (sender, content, message_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(sender, content, content_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id))
discussions.append(discussion)
@ -460,13 +460,13 @@ class Message:
def insert_into_db(self):
self.message_id = self.discussions_db.insert(
"INSERT INTO message (sender, content, metadata, ui, type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO message (sender, content, metadata, ui, message_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(self.sender, self.content, self.metadata, self.ui, self.message_type, self.rank, self.parent_message_id, self.binding, self.model, self.personality, self.created_at, self.finished_generating_at, self.discussion_id)
)
def update_db(self):
self.message_id = self.discussions_db.insert(
"INSERT INTO message (sender, content, metadata, ui, type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO message (sender, content, metadata, ui, message_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(self.sender, self.content, self.metadata, self.ui, self.message_type, self.rank, self.parent_message_id, self.binding, self.model, self.personality, self.created_at, self.finished_generating_at, self.discussion_id)
)
def update(self, new_content, new_metadata=None, new_ui=None, commit=True):
@ -513,7 +513,7 @@ class Discussion:
"""Gets a list of messages information
Returns:
list: List of entries in the format {"id":message id, "sender":sender name, "content":message content, "type":message type, "rank": message rank}
list: List of entries in the format {"id":message id, "sender":sender name, "content":message content, "message_type":message type, "rank": message rank}
"""
self.current_message = Message.from_db(self.discussions_db, id)
return self.current_message
@ -596,7 +596,7 @@ class Discussion:
"""Gets a list of messages information
Returns:
list: List of entries in the format {"id":message id, "sender":sender name, "content":message content, "type":message type, "rank": message rank}
list: List of entries in the format {"id":message id, "sender":sender name, "content":message content, "message_type":message type, "rank": message rank}
"""
columns = Message.get_fields()

24
app.py
View File

@ -14,7 +14,7 @@ __github__ = "https://github.com/ParisNeo/lollms-webui"
__copyright__ = "Copyright 2023, "
__license__ = "Apache 2.0"
__version__ ="6.7Alpha2"
__version__ ="6.7Beta1"
main_repo = "https://github.com/ParisNeo/lollms-webui.git"
import os
@ -26,6 +26,8 @@ import time
import traceback
import webbrowser
from pathlib import Path
import os
from api.db import DiscussionsDB, Discussion
def run_update_script(args=None):
update_script = Path(__file__).parent/"update_script.py"
@ -313,6 +315,7 @@ class LoLLMsWebUI(LoLLMsAPPI):
self.add_endpoint(
"/list_databases", "list_databases", self.list_databases, methods=["GET"]
)
self.add_endpoint("/select_database", "select_database", self.select_database, methods=["POST"])
self.add_endpoint(
@ -1135,6 +1138,21 @@ class LoLLMsWebUI(LoLLMsAPPI):
databases = [f.name for f in self.lollms_paths.personal_databases_path.iterdir() if f.suffix==".db"]
return jsonify(databases)
def select_database(self):
data = request.get_json()
self.config.db_path = data["name"]
print(f'Selecting database {data["name"]}')
# Create database object
self.db = DiscussionsDB(self.lollms_paths.personal_databases_path/data["name"])
ASCIIColors.info("Checking discussions database... ",end="")
self.db.create_tables()
self.db.add_missing_columns()
ASCIIColors.success("ok")
if self.config.auto_save:
self.config.save_config()
return jsonify({"status":True})
def list_extensions_categories(self):
extensions_categories_dir = self.lollms_paths.extensions_zoo_path # replace with the actual path to the models folder
@ -1812,8 +1830,6 @@ class LoLLMsWebUI(LoLLMsAPPI):
category = data['category']
name = data['folder']
language = data.get('language', None)
package_path = f"{category}/{name}"
package_full_path = self.lollms_paths.extensions_zoo_path/package_path
config_file = package_full_path / "config.yaml"
@ -1830,7 +1846,7 @@ class LoLLMsWebUI(LoLLMsAPPI):
})
else:
pth = str(config_file).replace('\\','/')
ASCIIColors.error(f"nok : Personality not found @ {pth}")
ASCIIColors.error(f"nok : Extension not found @ {pth}")
ASCIIColors.yellow(f"Available personalities: {[p.name for p in self.mounted_personalities]}")
return jsonify({"status": False, "error":f"Personality not found @ {pth}"})

2
configs/config.yaml
View File

@ -59,7 +59,7 @@ audio_silenceTimer: 5000
use_discussions_history: false # Activate vectorizing previous conversations
use_files: true # Activate using files
data_vectorization_activate: true # To activate/deactivate data vectorization
data_vectorization_method: "ftidf_vectorizer" #"model_embedding" or "ftidf_vectorizer"
data_vectorization_method: "tfidf_vectorizer" #"model_embedding" or "tfidf_vectorizer"
data_visualization_method: "PCA" #"PCA" or "TSNE"
data_vectorization_save_db: False # For each new session, new files
data_vectorization_chunk_size: 512 # chunk size

108
web/dist/assets/index-a7ee745c.js → web/dist/assets/index-97b789f4.js vendored
View File

File diff suppressed because one or more lines are too long

2
web/dist/assets/index-a2b38a7a.css → web/dist/assets/index-da85d9b9.css vendored
View File

File diff suppressed because one or more lines are too long

4
web/dist/index.html vendored
View File

@ -6,8 +6,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LoLLMS WebUI - Welcome</title>
<script type="module" crossorigin src="/assets/index-a7ee745c.js"></script>
<link rel="stylesheet" href="/assets/index-a2b38a7a.css">
<script type="module" crossorigin src="/assets/index-97b789f4.js"></script>
<link rel="stylesheet" href="/assets/index-da85d9b9.css">
</head>
<body>
<div id="app"></div>

51
web/src/components/ChoiceDialog.vue
View File

@ -13,11 +13,20 @@
:class="{'selected-choice': choice === selectedChoice}"
class="py-2 px-4 cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-700"
>
<span class="font-bold"> {{ displayName(choice) }} </span><br>
<span class="text-xs text-gray-500" v-if="choice.size"> {{ this.formatSize(choice.size) }}</span>
<span class="font-bold">{{ displayName(choice) }}</span><br>
<span class="text-xs text-gray-500" v-if="choice.size">{{ formatSize(choice.size) }}</span>
<button v-if="can_remove" @click="removeChoice(choice, index)" class="ml-2 text-red-500 hover:text-red-600">
X
</button>
</li>
</ul>
</div>
<div class="mt-4" v-if="showInput">
<input v-model="newFilename" placeholder="Enter a filename" class="border border-gray-300 p-2 rounded-lg w-full">
<button @click="addNewFilename" class="mt-2 py-2 px-4 bg-green-500 hover:bg-green-600 text-white rounded-lg transition duration-300">
Add
</button>
</div>
<div class="flex justify-end mt-4">
<button
@click="closeDialog"
@ -38,6 +47,12 @@
>
Validate
</button>
<button
@click="toggleInput"
class="py-2 px-4 ml-2 bg-blue-500 hover:bg-blue-600 text-white rounded-lg transition duration-300"
>
Add New
</button>
</div>
</div>
</div>
@ -49,7 +64,12 @@ export default {
props: {
show: {
type: Boolean,
required: true,
default: false,
required: false,
},
can_remove: {
type: Boolean,
default: false,
},
title: {
type: String,
@ -63,6 +83,8 @@ export default {
data() {
return {
selectedChoice: null,
showInput: false, // Control the visibility of the input box
newFilename: '', // Store the new filename
};
},
methods: {
@ -85,7 +107,7 @@ export default {
},
validateChoice() {
// Perform validation if needed
this.$emit("choice-validated");
this.$emit("choice-validated", this.selectedChoice);
},
formatSize(size) {
if (size < 1024) {
@ -98,6 +120,27 @@ export default {
return (size / (1024 * 1024 * 1024)).toFixed(2) + " GB";
}
},
toggleInput() {
// Toggle the visibility of the input box
this.showInput = !this.showInput;
},
addNewFilename() {
// Add the new filename to the choices array
const newChoice = this.newFilename.trim();
if (newChoice !== '') {
this.choices.push(newChoice);
this.newFilename = ''; // Clear the input field
this.selectChoice(newChoice); // Select the newly added filename
}
this.showInput = false;
},
removeChoice(choice, index) {
this.choices.splice(index, 1); // Remove the choice from the array
if (choice === this.selectedChoice) {
this.selectedChoice = null; // Clear the selected choice if it was removed
}
this.$emit("choice-removed", choice); // Emit a custom event to notify the parent component
},
},
};
</script>

4
web/src/components/Message.vue
View File

@ -77,11 +77,11 @@
<i data-feather="copy"></i>
</div>
<div v-if="message.sender!=this.$store.state.mountedPers.name" class="text-lg hover:text-secondary duration-75 active:scale-90 p-2" title="Resend message"
@click.stop="resendMessage()">
@click.stop="resendMessage()" :class="{ 'disabled': editMsgMode }">
<i data-feather="refresh-cw"></i>
</div>
<div v-if="message.sender==this.$store.state.mountedPers.name" class="text-lg hover:text-secondary duration-75 active:scale-90 p-2" title="Resend message"
@click.stop="continueMessage()">
@click.stop="continueMessage()" :class="{ 'disabled': editMsgMode }">
<i data-feather="fast-forward"></i>
</div>
<!-- DELETE CONFIRMATION -->

20
web/src/views/DiscussionsView.vue
View File

@ -419,12 +419,26 @@ export default {
}
},
methods: {
ondatabase_selectorDialogSelected(choice){
async ondatabase_selectorDialogSelected(choice){
console.log("Selected:",choice)
this.$store.state.config.database=choice;
},
onclosedatabase_selectorDialog(){this.database_selectorDialogVisible=false;},
onvalidatedatabase_selectorChoice(){this.database_selectorDialogVisible=false;},
async onvalidatedatabase_selectorChoice(choice){
this.database_selectorDialogVisible=false;
const res = await axios.post("/select_database",{"name": choice});
if(res.status){
console.log("Selected database")
this.$store.state.config = await axios.get("/get_config");
console.log("new config loaded :",this.$store.state.config)
let dbs = await axios.get("/list_databases")["data"];
console.log("New list of database: ",dbs)
this.$store.state.databases = dbs
console.log("New list of database: ",this.$store.state.databases)
location.reload();
}
},
save_configuration() {
this.showConfirmation = false
axios.post('/save_settings', {})

4
web/src/views/SettingsView.vue
View File

@ -843,7 +843,7 @@
@change="settingsChanged=true"
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
<option value="ftidf_vectorizer">ftidf Vectorizer</option>
<option value="tfidf_vectorizer">tfidf Vectorizer</option>
<option value="model_embedding">Model Embedding</option>
</select>
</td>
@ -2218,7 +2218,7 @@ export default {
oncloseVariantChoiceDialog(){
this.variantSelectionDialogVisible=false;
},
onvalidateVariantChoice(){
onvalidateVariantChoice(choice){
this.variantSelectionDialogVisible=false;
this.currenModelToInstall.installing=true;
let model_object = this.currenModelToInstall;