ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2025-02-01 00:45:29 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

upgraded

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-04-01 23:36:32 +02:00
parent 9046f541a9
commit 52c3568439
1 changed files with 16 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lollms/personality.py
View File

@ -879,8 +879,22 @@ class AIPersonality:
database_dict=None)
ASCIIColors.green("Ok")
else:
self.persona_data_vectorizer = None
self._data = None
files = [f for f in self.data_path.iterdir() if f.suffix.lower() in [".txt", ".pdf", ".docx", ".pptx", ".md", ".py", ".c", ".cpp"] ]
if len(files>0):
dl = GenericDataLoader()
self.persona_data_vectorizer = TextVectorizer(
"tfidf_vectorizer", # self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
save_db=True,
database_path=self.database_path,
data_visualization_method=VisualizationMethod.PCA,
database_dict=None)
for f in files:
text = dl.read_file(f)
else:
self.persona_data_vectorizer = None
self._data = None
if self.run_scripts: