ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-21 13:37:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-04-01 23:40:21 +02:00
parent 52c3568439
commit f810f0d0cc
1 changed files with 25 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
lollms/personality.py
View File

@ -867,21 +867,10 @@ class AIPersonality:
self.audio_samples = [f for f in self.audio_path.iterdir()]
# Verify if the persona has a data folder
self.database_path = self.data_path / "db.json"
if self.database_path.exists():
ASCIIColors.info("Loading database ...",end="")
self.persona_data_vectorizer = TextVectorizer(
"tfidf_vectorizer", # self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
save_db=True,
database_path=self.database_path,
data_visualization_method=VisualizationMethod.PCA,
database_dict=None)
ASCIIColors.green("Ok")
else:
files = [f for f in self.data_path.iterdir() if f.suffix.lower() in [".txt", ".pdf", ".docx", ".pptx", ".md", ".py", ".c", ".cpp"] ]
if len(files>0):
dl = GenericDataLoader()
if self.data_path.exists():
self.database_path = self.data_path / "db.json"
if self.database_path.exists():
ASCIIColors.info("Loading database ...",end="")
self.persona_data_vectorizer = TextVectorizer(
"tfidf_vectorizer", # self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
@ -889,13 +878,28 @@ class AIPersonality:
database_path=self.database_path,
data_visualization_method=VisualizationMethod.PCA,
database_dict=None)
for f in files:
text = dl.read_file(f)
ASCIIColors.green("Ok")
else:
self.persona_data_vectorizer = None
self._data = None
files = [f for f in self.data_path.iterdir() if f.suffix.lower() in [".txt", ".pdf", ".docx", ".pptx", ".md", ".py", ".c", ".cpp"] ]
if len(files>0):
dl = GenericDataLoader()
self.persona_data_vectorizer = TextVectorizer(
"tfidf_vectorizer", # self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
save_db=True,
database_path=self.database_path,
data_visualization_method=VisualizationMethod.PCA,
database_dict=None)
for f in files:
text = dl.read_file(f)
else:
self.persona_data_vectorizer = None
self._data = None
else:
self.persona_data_vectorizer = None
self._data = None
if self.run_scripts:
# Search for any processor code