enhanced

lollms/app.py

@@ -44,8 +44,11 @@ class LollmsApplication:
                 ASCIIColors.info("Personalities zoo found in your personal space.\nPulling last personalities zoo")
                 subprocess.run(["git", "-C", self.lollms_paths.personalities_zoo_path, "pull"])            
                 # Pull the repository if it already exists
-                ASCIIColors.info("Extensions zoo found in your personal space.\nPulling last personalities zoo")
+                ASCIIColors.info("Extensions zoo found in your personal space.\nPulling last Extensions zoo")
                 subprocess.run(["git", "-C", self.lollms_paths.extensions_zoo_path, "pull"])            
+                # Pull the repository if it already exists
+                ASCIIColors.info("Qlora found in your personal space.\nPulling last qlora code")
+                subprocess.run(["git", "-C", self.lollms_paths.gptqlora_path, "pull"])            
         except Exception as ex:
             ASCIIColors.error("Couldn't pull zoos. Please contact the main dev on our discord channel and report the problem.")
             trace_exception(ex)

lollms/paths.py

@@ -14,6 +14,7 @@ lollms_extensions_zoo_path = lollms_path / "extensions_zoo"
 personalities_zoo_repo = "https://github.com/ParisNeo/lollms_personalities_zoo.git"
 bindings_zoo_repo = "https://github.com/ParisNeo/lollms_bindings_zoo.git"
 extensions_zoo_repo = "https://github.com/ParisNeo/lollms_extensions_zoo.git"
+gptqlora_repo = "https://github.com/ParisNeo/gptqlora.git"
 
 # Now we speify the personal folders
 class LollmsPaths:
@@ -44,6 +45,9 @@ class LollmsPaths:
         self.personal_outputs_path          = self.personal_path / "outputs"
         self.personal_user_infos_path       = self.personal_path / "user_infos"
 
+        self.personal_trainers_path         = self.personal_path / "trainers"
+        self.gptqlora_path            = self.personal_trainers_path / "gptqlora"
+
 
         self.bindings_zoo_path              = self.personal_path / "bindings_zoo"
         self.personalities_zoo_path         = self.personal_path / "personalities_zoo"
@@ -60,6 +64,10 @@ class LollmsPaths:
         ASCIIColors.yellow(f"{self.personal_models_path}")
         ASCIIColors.yellow("personal_user_infos_path:",end="")
         ASCIIColors.yellow(f"{self.personal_user_infos_path}")
+        ASCIIColors.yellow("personal_trainers_path:",end="")
+        ASCIIColors.yellow(f"{self.personal_trainers_path}")
+        ASCIIColors.yellow("personal_trainers_path:",end="")
+        ASCIIColors.yellow(f"{self.gptqlora_path}")
         ASCIIColors.yellow("personal_data_path:",end="")
         ASCIIColors.yellow(f"{self.personal_data_path}")
         ASCIIColors.green("-------------------------------------------------------------")
@@ -79,7 +87,9 @@ class LollmsPaths:
             "Bindings Zoo Path": self.bindings_zoo_path,
             "Personalities Zoo Path": self.personalities_zoo_path,
             "Extensions zoo path": self.extensions_zoo_path,
-            "Personal user infos path": self.personal_user_infos_path
+            "Personal user infos path": self.personal_user_infos_path,
+            "Personal trainers path": self.personal_trainers_path,
+            "Personal gptqlora trainer path": self.gptqlora_path,
         }
         return "\n".join([f"{category}: {path}" for category, path in directories.items()])
 
@@ -96,6 +106,8 @@ class LollmsPaths:
         self.personal_outputs_path.mkdir(parents=True, exist_ok=True)
         self.personal_uploads_path.mkdir(parents=True, exist_ok=True)
         self.personal_user_infos_path.mkdir(parents=True, exist_ok=True)
+        self.personal_trainers_path.mkdir(parents=True, exist_ok=True)
+        
         
         if not self.bindings_zoo_path.exists():
             # Clone the repository to the target path
@@ -112,6 +124,12 @@ class LollmsPaths:
             ASCIIColors.info("No extensions found in your personal space.\nCloning the extensions zoo")
             subprocess.run(["git", "clone", extensions_zoo_repo, self.extensions_zoo_path])
 
+        if not self.gptqlora_path.exists():
+            # Clone the repository to the target path
+            ASCIIColors.info("No gptqlora found in your personal space.\nCloning the gptqlora repo")
+            subprocess.run(["git", "clone", gptqlora_repo, self.gptqlora_path])
+            subprocess.run(["pip", "install", "-r", "requirements.txt"], cwd=self.gptqlora_path)
+
 
     def copy_default_config(self):
         local_config_path = self.personal_configuration_path / f"{self.tool_prefix}local_config.yaml"

lollms/utilities.py

@@ -725,10 +725,15 @@ class GenericDataLoader:
 
 
 class PromptReshaper:
-    def __init__(self, template):
+    def __init__(self, template:str):
         self.template = template
-
-    def build(self, placeholders, tokenize, detokenize, max_nb_tokens, place_holders_to_sacrifice=[]):
+    def replace(self, placeholders:dict)->str:
+        template = self.template
+        # Calculate the number of tokens for each placeholder
+        for placeholder, text in placeholders.items():
+            template = template.replace(placeholder, text)
+        return template
+    def build(self, placeholders:dict, tokenize, detokenize, max_nb_tokens:int, place_holders_to_sacrifice:list=[])->str:
         # Tokenize the template without placeholders
         template_text = self.template
         for placeholder in placeholders:

setup.py

@@ -26,7 +26,7 @@ def get_all_files(path):
 
 setuptools.setup(
     name="lollms",
-    version="5.1.1",
+    version="5.2.0",
     author="Saifeddine ALOUI",
     author_email="aloui.saifeddine@gmail.com",
     description="A python library for AI personality definition",