ExternalVendorCode/lollms-webui
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms-webui.git synced 2024-12-18 20:17:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fixed icons error

Browse Source
This commit is contained in:
Saifeddine ALOUI 2023-10-07 23:16:27 +02:00
parent 3157922fbf
commit 8f438edd5c
13 changed files with 221 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
api/__init__.py
View File

@ -19,7 +19,8 @@ from lollms.binding import LOLLMSConfig, BindingBuilder, LLMBinding, ModelBuilde
from lollms.paths import LollmsPaths
from lollms.helpers import ASCIIColors, trace_exception
from lollms.app import LollmsApplication
from lollms.utilities import File64BitsManager, PromptReshaper
from lollms.utilities import File64BitsManager, PromptReshaper, TextVectorizer
import threading
from tqdm import tqdm
import traceback
@ -121,6 +122,9 @@ class LoLLMsAPPI(LollmsApplication):
self.config_file_path = config_file_path
self.cancel_gen = False
self.discussion_store = None
# Keeping track of current discussion and message
self._current_user_message_id = 0
self._current_ai_message_id = 0
@ -1154,13 +1158,37 @@ class LoLLMsAPPI(LollmsApplication):
if self.config["override_personality_model_parameters"]:
conditionning = conditionning+ "\n!@>user description:\nName:"+self.config["user_name"]+"\n"+self.config["user_description"]+"\n"
if len(self.personality.files)>0 and self.personality.vectorizer:
str_docs = ""
if self.config.use_discussions_history:
if self.discussion_store is None:
self.discussion_store = TextVectorizer(self.config.data_vectorization_method, # supported "model_embedding" or "ftidf_vectorizer"
model=self.model, #needed in case of using model_embedding
database_path=self.lollms_paths.personal_databases_path/"discussionsdb.json",
save_db=self.config.data_vectorization_save_db,
visualize_data_at_startup=False,
visualize_data_at_add_file=False,
visualize_data_at_generate=False,
data_visualization_method="PCA",
database_dict=None)
corpus = self.db.export_all_as_markdown()
self.discussion_store.add_document("discussions", corpus, self.config.data_vectorization_chunk_size, self.config.data_vectorization_overlap_size )
pr = PromptReshaper("{{conditionning}}\n!@>document chunks:\n{{doc}}\n{{content}}")
emb = self.personality.vectorizer.embed_query(message.content)
docs, sorted_similarities = self.personality.vectorizer.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)
str_docs = ""
emb = self.discussion_store.embed_query(message.content)
docs, sorted_similarities = self.discussion_store.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)
for doc, infos in zip(docs, sorted_similarities):
str_docs+=f"document chunk:\nchunk path: {infos[0]}\nchunk content:{doc}"
if len(self.personality.files)>0 and self.personality.vectorizer:
emb = self.personality.vectorizer.embed_query(message.content)
docs, sorted_similarities = self.personality.vectorizer.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)
for doc, infos in zip(docs, sorted_similarities):
str_docs+=f"document chunk:\nchunk path: {infos[0]}\nchunk content:{doc}"
if str_docs!="":
pr = PromptReshaper("{{conditionning}}\n!@>document chunks:\n{{doc}}\n{{content}}")
discussion_messages = pr.build({
"doc":str_docs,
"conditionning":conditionning,
@ -1494,7 +1522,7 @@ class LoLLMsAPPI(LollmsApplication):
self.connections[client_id]["current_discussion"].load_message(message_id)
self.connections[client_id]["generated_text"] = message.content
else:
self.new_message(client_id, self.personality.name, "✍ please stand by ...")
self.new_message(client_id, self.personality.name, "✍ warming up ...")
self.socketio.sleep(0.01)
# prepare query and reception

50
api/db.py
View File

@ -249,6 +249,56 @@ class DiscussionsDB:
)
discussions.append(discussion)
return discussions
def export_all_as_markdown(self):
data = self.export_all_discussions_to_json()
# Initialize an empty result string
result = ''
# Iterate through discussions in the JSON data
for discussion in data:
# Extract the title
title = discussion['title']
# Append the title with '#' as Markdown heading
result += f'#{title}\n'
# Iterate through messages in the discussion
for message in discussion['messages']:
sender = message['sender']
content = message['content']
# Append the sender and content in a Markdown format
result += f'{sender}: {content}\n'
return result
def export_all_discussions_to_json(self):
# Convert the list of discussion IDs to a tuple
db_discussions = self.select(
f"SELECT * FROM discussion"
)
discussions = []
for row in db_discussions:
discussion_id = row[0]
discussion_title = row[1]
discussion = {"id": discussion_id, "title":discussion_title, "messages": []}
rows = self.select(f"SELECT sender, content, message_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at FROM message WHERE discussion_id=?",(discussion_id,))
for message_row in rows:
sender = message_row[0]
content = message_row[1]
content_type = message_row[2]
rank = message_row[3]
parent_message_id = message_row[4]
binding = message_row[5]
model = message_row[6]
personality = message_row[7]
created_at = message_row[8]
finished_generating_at = message_row[9]
discussion["messages"].append(
{"sender": sender, "content": content, "type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at": finished_generating_at}
)
discussions.append(discussion)
return discussions
def export_discussions_to_json(self, discussions_ids:list):
# Convert the list of discussion IDs to a tuple

10
app.py
View File

@ -14,7 +14,7 @@ __github__ = "https://github.com/ParisNeo/lollms-webui"
__copyright__ = "Copyright 2023, "
__license__ = "Apache 2.0"
__version__ ="6.7Alpha1"
__version__ ="6.7Alpha2"
main_repo = "https://github.com/ParisNeo/lollms-webui.git"
import os
@ -310,6 +310,10 @@ class LoLLMsWebUI(LoLLMsAPPI):
self.add_endpoint(
"/list_personalities", "list_personalities", self.list_personalities, methods=["GET"]
)
self.add_endpoint(
"/list_databases", "list_databases", self.list_databases, methods=["GET"]
)
self.add_endpoint(
"/list_extensions_categories", "list_extensions_categories", self.list_extensions_categories, methods=["GET"]
@ -1126,6 +1130,10 @@ class LoLLMsWebUI(LoLLMsAPPI):
personalities=[]
ASCIIColors.error(f"No personalities found. Using default one {ex}")
return jsonify(personalities)
def list_databases(self):
databases = [f.name for f in self.lollms_paths.personal_databases_path.iterdir() if f.suffix==".db"]
return jsonify(databases)
def list_extensions_categories(self):

4
configs/config.yaml
View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Models Configuration file ===========================
version: 24
version: 25
binding_name: null
model_name: null
@ -29,7 +29,6 @@ override_personality_model_parameters: false #if true the personality parameters
extensions: []
user_name: user
user_description: ""
use_user_name_in_discussions: false
@ -57,6 +56,7 @@ audio_auto_send_input: true
audio_silenceTimer: 5000
# Data vectorization
use_discussions_history: false # Activate vectorizing previous conversations
use_files: true # Activate using files
data_vectorization_activate: true # To activate/deactivate data vectorization
data_vectorization_method: "ftidf_vectorizer" #"model_embedding" or "ftidf_vectorizer"

2
web/dist/assets/index-1e156293.css → web/dist/assets/index-a2b38a7a.css vendored
View File

File diff suppressed because one or more lines are too long

80
web/dist/assets/index-5dd957eb.js → web/dist/assets/index-a7ee745c.js vendored
View File

File diff suppressed because one or more lines are too long

4
web/dist/index.html vendored
View File

@ -6,8 +6,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LoLLMS WebUI - Welcome</title>
<script type="module" crossorigin src="/assets/index-5dd957eb.js"></script>
<link rel="stylesheet" href="/assets/index-1e156293.css">
<script type="module" crossorigin src="/assets/index-a7ee745c.js"></script>
<link rel="stylesheet" href="/assets/index-a2b38a7a.css">
</head>
<body>
<div id="app"></div>

14
web/src/components/ChoiceDialog.vue
View File

@ -13,8 +13,8 @@
:class="{'selected-choice': choice === selectedChoice}"
class="py-2 px-4 cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-700"
>
<span class="font-bold"> {{ choice.name }} </span><br>
<span class="text-xs text-gray-500"> {{ this.formatSize(choice.size) }}</span>
<span class="font-bold"> {{ displayName(choice) }} </span><br>
<span class="text-xs text-gray-500" v-if="choice.size"> {{ this.formatSize(choice.size) }}</span>
</li>
</ul>
</div>
@ -66,6 +66,16 @@ export default {
};
},
methods: {
displayName(choice) {
console.log("choice:",choice)
if (typeof choice === 'string') {
return choice;
} else if (choice && choice.name) {
return choice.name;
} else {
return '';
}
},
selectChoice(choice) {
this.selectedChoice = choice; // Update the selectedChoice when a choice is clicked
this.$emit("choice-selected", choice);

1
web/src/components/ExtensionEntry.vue
View File

@ -6,7 +6,6 @@
:title="!extension.installed ? 'Not installed' : ''">
<div :class="!extension.installed ? 'border-red-500' : ''">
<div class="flex flex-row items-center flex-shrink-0 gap-3">
<img @click="toggleSelected" ref="imgElement" :src="getImgUrl()" @error="defaultImg($event)"
class="w-10 h-10 rounded-full object-fill text-red-700 cursor-pointer">

4
web/src/components/Message.vue
View File

@ -192,7 +192,7 @@
}
</style>
<script>
import botImgPlaceholder from "../assets/logo.svg"
import botImgPlaceholder from "../assets/logo.png"
import userImgPlaceholder from "../assets/default_user.svg"
const bUrl = import.meta.env.VITE_LOLLMS_API_BASEURL
import { nextTick } from 'vue'
@ -368,8 +368,10 @@ export default {
},
getImgUrl() {
if (this.avatar) {
console.log("Avatar:",bUrl + this.avatar)
return bUrl + this.avatar
}
console.log("No avatar found")
return botImgPlaceholder;
},

22
web/src/main.js
View File

@ -40,7 +40,7 @@ export const store = createStore({
ramUsage:null,
vramUsage:null,
extensionsZoo:[],
activeExtensions:[],
databases:[],
}
},
mutations: {
@ -84,12 +84,13 @@ export const store = createStore({
state.vramUsage = vramUsage;
},
setActiveExtensions(state, activeExtensions) {
state.activeExtensions = activeExtensions;
},
setExtensionsZoo(state, extensionsZoo) {
state.extensionsZoo = extensionsZoo;
},
setDatabases(state, databases) {
state.databases = databases;
},
// increment (state) {
// state.count++
// }
@ -129,8 +130,8 @@ export const store = createStore({
return state.vramUsage;
},
getActiveExtensions(state) {
return state.activeExtensions;
getDatabasesList(state){
return state.databases;
},
getExtensionsZoo(state) {
return state.extensionsZoo;
@ -172,6 +173,13 @@ export const store = createStore({
// Handle error
}
},
async refreshDatabase({ commit }) {
let databases = await api_get_req("list_databases")
console.log("databases:",databases)
commit('setDatabases', databases);
},
async refreshPersonalitiesZoo({ commit }) {
let personalities = []
const catdictionary = await api_get_req("get_all_personalities")
@ -433,6 +441,8 @@ app.mixin({
actionsExecuted = true;
console.log("Calling")
await this.$store.dispatch('refreshConfig');
await this.$store.dispatch('refreshDatabase');
console.log("recovered config : ${}");
await this.$store.dispatch('getVersion');
console.log("recovered version");

34
web/src/views/DiscussionsView.vue
View File

@ -69,7 +69,7 @@
<i data-feather="refresh-ccw"></i>
</button>
<button class="text-2xl hover:text-secondary duration-75 active:scale-90" title="Export database"
type="button">
type="button" @click.stop="database_selectorDialogVisible=true">
<i data-feather="database"></i>
</button>
<input type="file" ref="fileDialog" style="display: none" @change="importDiscussions" />
@ -271,6 +271,13 @@
<Toast ref="toast">
</Toast>
<MessageBox ref="messageBox" />
<ChoiceDialog reference="database_selector" class="z-20"
:show="database_selectorDialogVisible"
:choices="databases"
@choice-selected="ondatabase_selectorDialogSelected"
@close-dialog="onclosedatabase_selectorDialog"
@choice-validated="onvalidatedatabase_selectorChoice"
/>
</template>
@ -404,6 +411,7 @@ export default {
isDiscussionBottom: false,
personalityAvatars: [], // object array of personality name: and avatar: props
fileList: [],
database_selectorDialogVisible:false,
isDragOverDiscussion: false,
isDragOverChat: false,
panelCollapsed: false, // left panel collapse
@ -411,6 +419,12 @@ export default {
}
},
methods: {
ondatabase_selectorDialogSelected(choice){
console.log("Selected:",choice)
this.$store.state.config.database=choice;
},
onclosedatabase_selectorDialog(){this.database_selectorDialogVisible=false;},
onvalidatedatabase_selectorChoice(){this.database_selectorDialogVisible=false;},
save_configuration() {
this.showConfirmation = false
axios.post('/save_settings', {})
@ -1626,7 +1640,11 @@ export default {
async activated() {
//console.log('settings changed acc', this.$store.state.settingsChanged)
// await this.getPersonalityAvatars()
while (this.isReady === false) {
await new Promise((resolve) => setTimeout(resolve, 100)); // Wait for 100ms
}
await this.getPersonalityAvatars()
console.log("Avatars found:",this.personalityAvatars)
if (this.isCreated) {
// this.loadLastUsedDiscussion()
nextTick(() => {
@ -1644,7 +1662,8 @@ export default {
ChatBox,
WelcomeComponent,
Toast,
DragDrop
DragDrop,
ChoiceDialog
},
watch: {
filterTitle(newVal) {
@ -1680,6 +1699,15 @@ export default {
},
computed: {
isReady:{
get() {
return this.$store.state.ready;
},
},
databases(){
return this.$store.state.databases;
},
client_id() {
return socket.id
},
@ -1731,6 +1759,8 @@ import { onMounted } from 'vue'
import { initFlowbite } from 'flowbite'
import { store } from '../main'
import ChoiceDialog from '@/components/ChoiceDialog.vue'
// initialize components based on data attribute selectors
onMounted(() => {
initFlowbite()

23
web/src/views/SettingsView.vue
View File

@ -759,7 +759,7 @@
</td>
</tr>
</table>
</Card>
</Card>
<Card title="Files Vectorization" :is_subcard="true" class="pb-2 m-2">
<table class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500">
<tr>
@ -778,7 +778,24 @@
>
</div>
</td>
</tr>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="use_discussions_history" class="text-sm font-bold" style="margin-right: 1rem;">Activate discussion vectorization:</label>
</td>
<td>
<div class="flex flex-row">
<input
type="checkbox"
id="use_discussions_history"
required
v-model="configFile.use_discussions_history"
@change="settingsChanged=true"
class="mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
</div>
</td>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="data_vectorization_activate" class="text-sm font-bold" style="margin-right: 1rem;">Activate files vectorization:</label>
@ -4043,7 +4060,7 @@ export default {
},
mountedExtensions:{
get() {
return this.$store.state.activeExtensions;
return this.$store.state.config.extensions;
},
set(value) {
this.$store.commit('setActiveExtensions', value);