ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-26 07:41:06 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

upgraded

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-03-19 00:23:28 +01:00
parent 3193a46658
commit 517ffa0c58
1 changed files with 1 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
lollms/server/events/lollms_rag_events.py
View File

@ -29,40 +29,4 @@ from tqdm import tqdm
lollmsElfServer = LOLLMSElfServer.get_instance() lollmsElfServer = LOLLMSElfServer.get_instance()
# ----------------------------------- events ----------------------------------------- # ----------------------------------- events -----------------------------------------
def add_events(sio:socketio):
@sio.on('upgrade_vectorization')
def upgrade_vectorization():
if lollmsElfServer.config.data_vectorization_activate and lollmsElfServer.config.activate_skills_lib:
try:
run_async(partial(sio.emit,'show_progress'))
lollmsElfServer.sio.sleep(0)
ASCIIColors.yellow("0- Detected discussion vectorization request")
folder = lollmsElfServer.lollms_paths.personal_discussions_path/"vectorized_dbs"
folder.mkdir(parents=True, exist_ok=True)
lollmsElfServer.build_long_term_skills_memory()
ASCIIColors.yellow("1- Exporting discussions")
discussions = lollmsElfServer.db.export_all_as_markdown_list_for_vectorization()
ASCIIColors.yellow("2- Adding discussions to vectorizer")
index = 0
nb_discussions = len(discussions)
for (title,discussion) in tqdm(discussions):
run_async(partial(sio.emit,'update_progress',{'value':int(100*(index/nb_discussions))}))
lollmsElfServer.sio.sleep(0)
index += 1
if discussion!='':
skill = lollmsElfServer.learn_from_discussion(title, discussion)
lollmsElfServer.long_term_memory.add_document(title, skill, chunk_size=lollmsElfServer.config.data_vectorization_chunk_size, overlap_size=lollmsElfServer.config.data_vectorization_overlap_size, force_vectorize=False, add_as_a_bloc=False)
ASCIIColors.yellow("3- Indexing database")
lollmsElfServer.long_term_memory.index()
ASCIIColors.yellow("4- Saving database")
lollmsElfServer.long_term_memory.save_to_json()
if lollmsElfServer.config.data_vectorization_visualize_on_vectorization:
lollmsElfServer.long_term_memory.show_document(show_interactive_form=True)
ASCIIColors.yellow("Ready")
except Exception as ex:
ASCIIColors.error(f"Couldn't vectorize database:{ex}")
run_async(partial(sio.emit,'hide_progress'))
lollmsElfServer.sio.sleep(0)