ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-19 04:37:54 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update app.py

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-06-21 09:56:20 +02:00 committed by GitHub
parent cc7269869a
commit 95f59598fd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 36 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
lollms/app.py
View File

@ -90,40 +90,7 @@ class LollmsApplication(LoLLMsCom):
self.stt = None
self.ttm = None
self.ttv = None
try:
self.load_rag_dbs()
except Exception as ex:
trace_exception(ex)
for entry in self.config.rag_databases:
try:
if "mounted" in entry:
parts = entry.split("::")
database_name = parts[0]
database_path = parts[1]
if not PackageManager.check_package_installed_with_version("lollmsvectordb","0.5.1"):
PackageManager.install_or_update("lollmsvectordb")
from lollmsvectordb import VectorDatabase
from lollmsvectordb.text_document_loader import TextDocumentsLoader
from lollmsvectordb.lollms_tokenizers.tiktoken_tokenizer import TikTokenTokenizer
if self.config.rag_vectorizer == "bert":
self.backup_trust_store()
from lollmsvectordb.lollms_vectorizers.bert_vectorizer import BERTVectorizer
v = BERTVectorizer()
self.restore_trust_store()
elif self.config.rag_vectorizer == "tfidf":
from lollmsvectordb.lollms_vectorizers.tfidf_vectorizer import TFIDFVectorizer
v = TFIDFVectorizer()
vdb = VectorDatabase(Path(database_path)/"db_name.sqlite", v, self.model if self.model else TikTokenTokenizer(), n_neighbors=self.config.rag_n_chunks)
vdb.build_index()
self.active_rag_dbs.append({"name":database_name,"path":database_path,"vectorizer":vdb})
self.config.save_config()
except Exception as ex:
trace_exception(ex)
self.rt_com = None
if not free_mode:
try:
@ -190,6 +157,41 @@ class LollmsApplication(LoLLMsCom):
self.mount_personalities()
self.mount_extensions()
try:
self.load_rag_dbs()
except Exception as ex:
trace_exception(ex)
for entry in self.config.rag_databases:
try:
if "mounted" in entry:
parts = entry.split("::")
database_name = parts[0]
database_path = parts[1]
if not PackageManager.check_package_installed_with_version("lollmsvectordb","0.5.1"):
PackageManager.install_or_update("lollmsvectordb")
from lollmsvectordb import VectorDatabase
from lollmsvectordb.text_document_loader import TextDocumentsLoader
from lollmsvectordb.lollms_tokenizers.tiktoken_tokenizer import TikTokenTokenizer
if self.config.rag_vectorizer == "bert":
self.backup_trust_store()
from lollmsvectordb.lollms_vectorizers.bert_vectorizer import BERTVectorizer
v = BERTVectorizer()
self.restore_trust_store()
elif self.config.rag_vectorizer == "tfidf":
from lollmsvectordb.lollms_vectorizers.tfidf_vectorizer import TFIDFVectorizer
v = TFIDFVectorizer()
vdb = VectorDatabase(Path(database_path)/"db_name.sqlite", v, self.model if self.model else TikTokenTokenizer(), n_neighbors=self.config.rag_n_chunks)
vdb.build_index()
self.active_rag_dbs.append({"name":database_name,"path":database_path,"vectorizer":vdb})
self.config.save_config()
except Exception as ex:
trace_exception(ex)
def backup_trust_store(self):
self.bk_store = None