ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-18 20:27:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added persona data

Browse Source
This commit is contained in:
Saifeddine ALOUI 2023-11-12 00:42:14 +01:00
parent 9ea8ea114c
commit a3f9c83513
1 changed files with 35 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
lollms/personality.py
View File

@ -135,6 +135,9 @@ Date: {{date}}
self._logo: Optional[Image.Image] = None self._logo: Optional[Image.Image] = None
self._processor = None self._processor = None
self._data = None
if personality_package_path is None: if personality_package_path is None:
self.config = {} self.config = {}
@ -261,13 +264,42 @@ Date: {{date}}
self.assets_path = self.personality_package_path / "assets" self.assets_path = self.personality_package_path / "assets"
# Get the scripts folder path # Get the scripts folder path
self.scripts_path = self.personality_package_path / "scripts" self.scripts_path = self.personality_package_path / "scripts"
# Get the languages folder path
self.languages_path = self.personality_package_path / "languages"
# Get the data folder path
self.data_path = self.personality_package_path / "data"
# If not exist recreate # If not exist recreate
self.assets_path.mkdir(parents=True, exist_ok=True) self.assets_path.mkdir(parents=True, exist_ok=True)
# If not exist recreate # If not exist recreate
self.scripts_path.mkdir(parents=True, exist_ok=True) self.scripts_path.mkdir(parents=True, exist_ok=True)
# Verify if the persona has a data folder
if self.data_path.exists():
text = []
text_files = [file if file.exists() else "" for file in self.data_path.glob("*.txt")]
for file in text_files:
with open(str(file),"r") as f:
text.append(f.read())
# Replace 'example_dir' with your desired directory containing .txt files
self._data = "\n".join(map((lambda x: f"\n{x}"), text))
print(self._data)
ASCIIColors.info("Building data ...",end="")
self.persona_data_vectorizer = TextVectorizer(
self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
save_db=False,
data_visualization_method=VisualizationMethod.PCA,
database_dict=None)
self.persona_data_vectorizer.add_document("persona_data", self._data,512,0)
self.persona_data_vectorizer.index()
ASCIIColors.green("Ok")
else:
self.persona_data_vectorizer = None
self._data = None
if self.run_scripts: if self.run_scripts:
# Search for any processor code # Search for any processor code
@ -289,6 +321,7 @@ Date: {{date}}
self._assets_list = contents self._assets_list = contents
return config return config
def remove_file(self, path, callback=None): def remove_file(self, path, callback=None):
try: try:
if path in self.text_files: if path in self.text_files:
@ -1063,6 +1096,7 @@ class APScript(StateMachine):
self.personality_config.config.file_path = self.configuration_file_path self.personality_config.config.file_path = self.configuration_file_path
self.callback = callback self.callback = callback
# Installation # Installation
if (not self.configuration_file_path.exists() or self.installation_option==InstallOption.FORCE_INSTALL) and self.installation_option!=InstallOption.NEVER_INSTALL: if (not self.configuration_file_path.exists() or self.installation_option==InstallOption.FORCE_INSTALL) and self.installation_option!=InstallOption.NEVER_INSTALL:
self.install() self.install()