ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2025-01-20 19:49:25 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update lollms_file_system.py

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-06-20 08:51:58 +02:00 committed by GitHub
parent 487e18d3ea
commit f3df6d3837
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
lollms/server/endpoints/lollms_file_system.py
View File

@ -135,8 +135,11 @@ def select_rag_database(client) -> Optional[Dict[str, Path]]:
if lollmsElfServer.config.rag_vectorizer == "bert": if lollmsElfServer.config.rag_vectorizer == "bert":
lollmsElfServer.backup_trust_store()
from lollmsvectordb.vectorizers.bert_vectorizer import BERTVectorizer from lollmsvectordb.vectorizers.bert_vectorizer import BERTVectorizer
v = BERTVectorizer() v = BERTVectorizer()
lollmsElfServer.restore_trust_store()
elif lollmsElfServer.config.rag_vectorizer == "tfidf": elif lollmsElfServer.config.rag_vectorizer == "tfidf":
from lollmsvectordb.vectorizers.tfidf_vectorizer import TFIDFVectorizer from lollmsvectordb.vectorizers.tfidf_vectorizer import TFIDFVectorizer
v = TFIDFVectorizer() v = TFIDFVectorizer()
@ -155,13 +158,11 @@ def select_rag_database(client) -> Optional[Dict[str, Path]]:
text = TextDocumentsLoader.read_file(fn) text = TextDocumentsLoader.read_file(fn)
title = fn.stem # Use the file name without extension as the title title = fn.stem # Use the file name without extension as the title
vdb.add_document(title, text, fn) vdb.add_document(title, text, fn)
lollmsElfServer.ShowBlockingMessage(f"Adding a new database.\nAdding {title}")
print(f"Added document: {title}") print(f"Added document: {title}")
except Exception as e: except Exception as e:
print(f"Failed to add document {fn}: {e}") print(f"Failed to add document {fn}: {e}")
if vdb.new_data: #New files are added, need reindexing if vdb.new_data: #New files are added, need reindexing
ASCIIColors.blue("Indexing database ...", end="", flush=True) ASCIIColors.blue("Indexing database ...", end="", flush=True)
lollmsElfServer.ShowBlockingMessage(f"Adding a new database.\nIndexing database")
vdb.build_index() vdb.build_index()
ASCIIColors.success("OK") ASCIIColors.success("OK")
lollmsElfServer.HideBlockingMessage() lollmsElfServer.HideBlockingMessage()
@ -258,8 +259,10 @@ def toggle_mount_rag_database(database_infos: MountDatabase):
from lollmsvectordb.tokenizers.tiktoken_tokenizer import TikTokenTokenizer from lollmsvectordb.tokenizers.tiktoken_tokenizer import TikTokenTokenizer
if lollmsElfServer.config.rag_vectorizer == "bert": if lollmsElfServer.config.rag_vectorizer == "bert":
lollmsElfServer.backup_trust_store()
from lollmsvectordb.vectorizers.bert_vectorizer import BERTVectorizer from lollmsvectordb.vectorizers.bert_vectorizer import BERTVectorizer
v = BERTVectorizer() v = BERTVectorizer()
lollmsElfServer.restore_trust_store()
elif lollmsElfServer.config.rag_vectorizer == "tfidf": elif lollmsElfServer.config.rag_vectorizer == "tfidf":
from lollmsvectordb.vectorizers.tfidf_vectorizer import TFIDFVectorizer from lollmsvectordb.vectorizers.tfidf_vectorizer import TFIDFVectorizer
v = TFIDFVectorizer() v = TFIDFVectorizer()