ExternalVendorCode/lollms-webui
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms-webui.git synced 2024-12-18 20:17:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

new rag system

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-06-28 02:45:21 +02:00
parent 403ff16141
commit b1b0fccbac
8 changed files with 207 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/config.yaml
View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 122
version: 123
binding_name: null
model_name: null
model_variant: null
@ -247,6 +247,8 @@ rag_vectorizer: bert # possible values bert, tfidf, word2vec
rag_vectorizer_model: bert-base-nli-mean-tokens # The model name if applicable
rag_vectorizer_parameters: null # Parameters of the model in json format
rag_chunk_size: 512 # number of tokens per chunk
rag_overlap: 0 # number of tokens of overlap
rag_n_chunks: 4 #Number of chunks to recover from the database
rag_clean_chunks: true #Removed all uinecessary spaces and line returns
rag_follow_subfolders: true #if true the vectorizer will vectorize the content of subfolders too

3
events/lollms_generation_events.py
View File

@ -147,9 +147,6 @@ def add_events(sio:socketio):
else:
lollmsElfServer.error("I am busy. Come back later.", client_id=client_id)
@sio.on('generate_msg_from')
def handle_generate_msg_from(sid, data):
client_id = sid

2
lollms_core

@ -1 +1 @@
Subproject commit 9cbcfbb2824896e9f745166892d79a3786c1d79e
Subproject commit d670770464ca5024e046e10a61ab82827924c5d3

8
lollms_webui.py
View File

@ -995,15 +995,17 @@ class LOLLMSWebUI(LOLLMSElfServer):
ASCIIColors.success("--> Step ended:"+chunk)
else:
ASCIIColors.error("--> Step ended:"+chunk)
if message_type == MSG_TYPE.MSG_TYPE_EXCEPTION:
self.error(chunk, client_id=client_id)
ASCIIColors.error("--> Exception from personality:"+chunk)
if message_type == MSG_TYPE.MSG_TYPE_WARNING:
self.warning(chunk,client_id=client_id)
ASCIIColors.error("--> Exception from personality:"+chunk)
if message_type == MSG_TYPE.MSG_TYPE_EXCEPTION:
self.error(chunk, client_id=client_id)
ASCIIColors.error("--> Exception from personality:"+chunk)
return
if message_type == MSG_TYPE.MSG_TYPE_INFO:
self.info(chunk, client_id=client_id)
ASCIIColors.info("--> Info:"+chunk)
return
if message_type == MSG_TYPE.MSG_TYPE_UI:
self.update_message(client_id, "", parameters, metadata, chunk, MSG_TYPE.MSG_TYPE_UI)

348
web/dist/assets/index-85e70ccc.js → web/dist/assets/index-2028504d.js vendored
View File

File diff suppressed because one or more lines are too long

2
web/dist/assets/index-5340f2b4.css → web/dist/assets/index-90b8fd3b.css vendored
View File

File diff suppressed because one or more lines are too long

4
web/dist/index.html vendored
View File

@ -6,8 +6,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LoLLMS WebUI - Welcome</title>
<script type="module" crossorigin src="/assets/index-85e70ccc.js"></script>
<link rel="stylesheet" href="/assets/index-5340f2b4.css">
<script type="module" crossorigin src="/assets/index-2028504d.js"></script>
<link rel="stylesheet" href="/assets/index-90b8fd3b.css">
</head>
<body>
<div id="app"></div>

24
web/src/views/SettingsView.vue
View File

@ -1042,8 +1042,8 @@
<option value="bert-base-nli-mean-tokens">bert-base-nli-mean-tokens</option>
</select>
</td>
</tr>
<tr>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="rag_chunk_size" class="text-sm font-bold" style="margin-right: 1rem;">RAG chunk size:</label>
</td>
@ -1059,7 +1059,25 @@
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
</td>
</tr>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="rag_overlap" class="text-sm font-bold" style="margin-right: 1rem;">RAG overlap size:</label>
</td>
<td>
<input id="rag_overlap" v-model="configFile.rag_overlap"
@change="settingsChanged=true"
type="range" min="0" max="64000" step="1"
class="flex-none h-2 mt-14 mb-2 w-full bg-gray-200 rounded-lg appearance-none cursor-pointer dark:bg-gray-700 focus:ring-blue-500 focus:border-blue-500 dark:border-gray-600 dark:placeholder-gray-400 dark:focus:ring-blue-500 dark:focus:border-blue-500">
<input v-model="configFile.rag_overlap"
type="number"
@change="settingsChanged=true"
class="w-full mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
</td>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="rag_n_chunks" class="text-sm font-bold" style="margin-right: 1rem;">RAG number of chunks:</label>