ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-24 06:46:40 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Update discussions_database.py

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-05-03 14:33:20 +02:00 committed by GitHub
parent 58e777f9ab
commit 3ffaf28156
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lollms/databases/discussions_database.py
View File

@ -662,9 +662,14 @@ class Discussion:
if len(self.vectorizer.chunks)==0 and len(self.text_files)>0: if len(self.vectorizer.chunks)==0 and len(self.text_files)>0:
for path in self.text_files: for path in self.text_files:
data = GenericDataLoader.read_file(path) data = GenericDataLoader.read_file(path)
self.vectorizer.add_document(path, data, self.lollms.config.data_vectorization_chunk_size, self.lollms.config.data_vectorization_overlap_size, add_first_line_to_all_chunks=True if path.suffix==".csv" else False) try:
self.vectorizer.add_document(path, data, self.lollms.config.data_vectorization_chunk_size, self.lollms.config.data_vectorization_overlap_size, add_first_line_to_all_chunks=True if path.suffix==".csv" else False)
except Exception as ex:
trace_exception(ex)
try:
self.vectorizer.index() self.vectorizer.index()
except Exception as ex:
trace_exception(ex)
def update_file_lists(self): def update_file_lists(self):
self.text_files = [Path(file) for file in self.discussion_text_folder.glob('*')] self.text_files = [Path(file) for file in self.discussion_text_folder.glob('*')]