V5: new personalities structuring

LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2023 Saifeddine ALOUI (aka: ParisNeo)
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

api/__init__.py

@@ -437,9 +437,9 @@ class LoLLMsAPPI(LollmsApplication):
                 else:
                     self.connections[client_id]["current_discussion"] = self.db.create_discussion()
             messages = self.connections[client_id]["current_discussion"].get_messages()
-
+            jsons = [m.to_json() for m in messages]
             self.socketio.emit('discussion',
-                        [m.to_json() for m in messages],
+                        jsons,
                         room=client_id
             )
 
@@ -473,6 +473,7 @@ class LoLLMsAPPI(LollmsApplication):
             terminate_thread(self.connections[client_id]['generation_thread'])
             ASCIIColors.error(f'Client {request.sid} canceled generation')
             self.cancel_gen = False
+            self.buzy=False
 
         @socketio.on('send_file')
         def send_file(data):
@@ -596,7 +597,7 @@ class LoLLMsAPPI(LollmsApplication):
             self.mounted_personalities=[]
 
         loaded = self.mounted_personalities
-        loaded_names = [f"{p.language}/{p.category}/{p.personality_folder_name}" for p in loaded]
+        loaded_names = [f"{p.category}/{p.personality_folder_name}" for p in loaded]
         mounted_personalities=[]
         ASCIIColors.success(f" ╔══════════════════════════════════════════════════╗ ")
         ASCIIColors.success(f" ║           Building mounted Personalities         ║ ")
@@ -817,7 +818,8 @@ class LoLLMsAPPI(LollmsApplication):
                             client_id, 
                             sender, 
                             content,
-                            metadata=None, 
+                            parameters=None,
+                            metadata=[],
                             message_type:MSG_TYPE=MSG_TYPE.MSG_TYPE_FULL, 
                             sender_type:SENDER_TYPES=SENDER_TYPES.SENDER_TYPES_AI
                         ):
@@ -827,7 +829,7 @@ class LoLLMsAPPI(LollmsApplication):
             sender_type         = sender_type.value,
             sender              = sender,
             content             = content,
-            metadata            = json.dumps(metadata, indent=4) if metadata is not None and type(metadata)== dict else metadata,
+            metadata            = json.dumps(metadata, indent=4) if metadata is not None and type(metadata) == list else metadata,
             rank                = 0,
             parent_message_id   = self.connections[client_id]["current_discussion"].current_message.id,
             binding             = self.config["binding_name"],
@@ -841,7 +843,8 @@ class LoLLMsAPPI(LollmsApplication):
                     "message_type":             message_type.value,
                     "sender_type":              SENDER_TYPES.SENDER_TYPES_AI.value,
                     "content":                  content,
-                    "metadata":                 json.dumps(metadata, indent=4) if metadata is not None and type(metadata)== dict else metadata,
+                    "parameters":               parameters,
+                    "metadata":                 json.dumps(metadata, indent=4) if metadata is not None and type(metadata)== list else metadata,
                     "id":                       msg.id,
                     "parent_message_id":        msg.parent_message_id,
 
@@ -854,8 +857,13 @@ class LoLLMsAPPI(LollmsApplication):
                 }, room=client_id
         )
 
-    def update_message(self, client_id, chunk, metadata, msg_type:MSG_TYPE=None):
+    def update_message(self, client_id, chunk,                             
+                            parameters=None,
+                            metadata=[], 
+                            msg_type:MSG_TYPE=None
+                        ):
         self.connections[client_id]["current_discussion"].current_message.finished_generating_at=datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+        mtdt = json.dumps(metadata, indent=4) if metadata is not None and type(metadata)== list else metadata
         self.socketio.emit('update_message', {
                                         "sender": self.personality.name,
                                         'id':self.connections[client_id]["current_discussion"].current_message.id, 
@@ -863,11 +871,12 @@ class LoLLMsAPPI(LollmsApplication):
                                         'discussion_id':self.connections[client_id]["current_discussion"].discussion_id,
                                         'message_type': msg_type.value if msg_type is not None else MSG_TYPE.MSG_TYPE_CHUNK.value if self.nb_received_tokens>1 else MSG_TYPE.MSG_TYPE_FULL.value,
                                         'finished_generating_at': self.connections[client_id]["current_discussion"].current_message.finished_generating_at,
-                                        'metadata':json.dumps(metadata, indent=4) if metadata is not None and type(metadata)== dict else metadata
+                                        'parameters':parameters,
+                                        'metadata':mtdt
                                     }, room=client_id
                             )
         self.socketio.sleep(0.01)
-        self.connections[client_id]["current_discussion"].update_message(self.connections[client_id]["generated_text"])
+        self.connections[client_id]["current_discussion"].update_message(self.connections[client_id]["generated_text"], new_metadata=mtdt)
 
     def close_message(self, client_id):
         # Send final message
@@ -886,7 +895,15 @@ class LoLLMsAPPI(LollmsApplication):
 
                                     }, room=client_id
                             )
-    def process_chunk(self, chunk, message_type:MSG_TYPE, metadata:dict={}, client_id:int=0):
+    def process_chunk(
+                        self, 
+                        chunk, 
+                        message_type:MSG_TYPE,
+
+                        parameters=None, 
+                        metadata:list=[], 
+                        client_id:int=0
+                    ):
         """
         Processes a chunk of generated text
         """
@@ -896,7 +913,7 @@ class LoLLMsAPPI(LollmsApplication):
         if message_type == MSG_TYPE.MSG_TYPE_STEP_START:
             ASCIIColors.info("--> Step started:"+chunk)
         if message_type == MSG_TYPE.MSG_TYPE_STEP_END:
-            if metadata['status']:
+            if parameters['status']:
                 ASCIIColors.success("--> Step ended:"+chunk)
             else:
                 ASCIIColors.error("--> Step ended:"+chunk)
@@ -912,7 +929,12 @@ class LoLLMsAPPI(LollmsApplication):
 
         if message_type == MSG_TYPE.MSG_TYPE_NEW_MESSAGE:
             self.nb_received_tokens = 0
-            self.new_message(client_id, self.personality.name, chunk, metadata = metadata["metadata"], message_type= MSG_TYPE(metadata["type"]))
+            self.new_message(
+                                client_id, 
+                                self.personality.name, 
+                                chunk, 
+                                metadata = parameters["metadata"], 
+                                message_type= MSG_TYPE(parameters["type"]))
 
         elif message_type == MSG_TYPE.MSG_TYPE_FINISHED_MESSAGE:
             self.close_message(client_id)
@@ -927,7 +949,7 @@ class LoLLMsAPPI(LollmsApplication):
             if antiprompt:
                 ASCIIColors.warning(f"\nDetected hallucination with antiprompt: {antiprompt}")
                 self.connections[client_id]["generated_text"] = self.remove_text_from_string(self.connections[client_id]["generated_text"],antiprompt)
-                self.update_message(client_id, self.connections[client_id]["generated_text"], metadata,MSG_TYPE.MSG_TYPE_FULL)
+                self.update_message(client_id, self.connections[client_id]["generated_text"], parameters, metadata,MSG_TYPE.MSG_TYPE_FULL)
                 return False
             else:
                 self.update_message(client_id, chunk, metadata)
@@ -944,11 +966,11 @@ class LoLLMsAPPI(LollmsApplication):
             self.connections[client_id]["generated_text"] = chunk
             self.nb_received_tokens += 1
             ASCIIColors.green(f"Received {self.nb_received_tokens} tokens",end="\r",flush=True)
-            self.update_message(client_id, chunk, metadata, msg_type=message_type)
+            self.update_message(client_id, chunk,  parameters, metadata, msg_type=message_type)
             return True
         # Stream the generated text to the frontend
         else:
-            self.update_message(client_id, chunk, metadata, message_type)
+            self.update_message(client_id, chunk, parameters, metadata, message_type)
         return True
 
 

api/db.py

@@ -414,19 +414,29 @@ class Message:
             "INSERT INTO message (sender, content, type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at, discussion_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", 
             (self.sender, self.content, self.message_type, self.rank, self.parent_message_id, self.binding, self.model, self.personality, self.created_at, self.finished_generating_at, self.discussion_id)
         )
-    def update(self, new_content, commit=True):
+    def update(self, new_content, new_metadata=None, commit=True):
         self.finished_generating_at = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
         
         # print(f"{current_date_time}")
+        if new_metadata is None:
             self.discussions_db.update(
                 f"UPDATE message SET content = ?, finished_generating_at = ? WHERE id = ?",(new_content, self.finished_generating_at,self.id)
             )
-
+        else:
+            self.discussions_db.update(
+                f"UPDATE message SET content = ?, metadata = ?, finished_generating_at = ? WHERE id = ?",(new_content, new_metadata, self.finished_generating_at,self.id)
+            )
     def to_json(self):
         attributes = Message.get_fields()
         msgJson = {}
         for attribute_name in attributes:
             attribute_value = getattr(self, attribute_name, None)
+            if attribute_name=="metadata":
+                if type(attribute_value) == str:
+                    msgJson[attribute_name] = json.loads(attribute_value)
+                else:
+                    msgJson[attribute_name] = attribute_value
+            else:
                 msgJson[attribute_name] = attribute_value
         return msgJson
 
@@ -545,14 +555,14 @@ class Discussion:
                 return message
         return None
 
-    def update_message(self, new_content):
+    def update_message(self, new_content, new_metadata=None):
         """Updates the content of a message
 
         Args:
             message_id (int): The id of the message to be changed
             new_content (str): The nex message content
         """
-        self.current_message.update(new_content)
+        self.current_message.update(new_content, new_metadata)
     
     def message_rank_up(self, message_id):
         """Increments the rank of the message

app.py

@@ -14,7 +14,7 @@ __github__ = "https://github.com/ParisNeo/lollms-webui"
 __copyright__ = "Copyright 2023, "
 __license__ = "Apache 2.0"
 
-__version__ ="4.1"
+__version__ ="5.0"
 
 main_repo = "https://github.com/ParisNeo/lollms-webui.git"
 import os
@@ -176,7 +176,7 @@ class LoLLMsWebUI(LoLLMsAPPI):
     def __init__(self, args, _app, _socketio, config:LOLLMSConfig, config_file_path:Path|str, lollms_paths:LollmsPaths) -> None:
         self.args = args
         if len(config.personalities)==0:
-            config.personalities.append("english/generic/lollms")
+            config.personalities.append("generic/lollms")
             config["active_personality_id"] = 0
             config.save_config()
 
@@ -190,11 +190,9 @@ class LoLLMsWebUI(LoLLMsAPPI):
         app.template_folder = "web/dist"
 
         if len(config["personalities"])>0:
-            self.personality_language= config["personalities"][config["active_personality_id"]].split("/")[0]
+            self.personality_category= config["personalities"][config["active_personality_id"]].split("/")[0]
-            self.personality_category= config["personalities"][config["active_personality_id"]].split("/")[1]
+            self.personality_name= config["personalities"][config["active_personality_id"]].split("/")[1]
-            self.personality_name= config["personalities"][config["active_personality_id"]].split("/")[2]
         else:
-            self.personality_language = "english"
             self.personality_category = "generic"
             self.personality_name = "lollms"
 
@@ -272,9 +270,6 @@ class LoLLMsWebUI(LoLLMsAPPI):
         self.add_endpoint(
             "/list_models", "list_models", self.list_models, methods=["GET"]
         )
-        self.add_endpoint(
-            "/list_personalities_languages", "list_personalities_languages", self.list_personalities_languages, methods=["GET"]
-        )        
         self.add_endpoint(
             "/list_personalities_categories", "list_personalities_categories", self.list_personalities_categories, methods=["GET"]
         )
@@ -282,10 +277,6 @@ class LoLLMsWebUI(LoLLMsAPPI):
             "/list_personalities", "list_personalities", self.list_personalities, methods=["GET"]
         )
         
-        self.add_endpoint(
-            "/list_languages", "list_languages", self.list_languages, methods=["GET"]
-        )
-        
         self.add_endpoint(
             "/list_discussions", "list_discussions", self.list_discussions, methods=["GET"]
         )
@@ -452,14 +443,11 @@ class LoLLMsWebUI(LoLLMsAPPI):
     def get_all_personalities(self):
         personalities_folder = self.lollms_paths.personalities_zoo_path
         personalities = {}
-        for language_folder in personalities_folder.iterdir():
+
-            lang = language_folder.stem
+        for category_folder in  personalities_folder.iterdir():
-            if language_folder.is_dir() and not language_folder.stem.startswith('.'):
-                personalities[language_folder.name] = {}
-                for category_folder in  language_folder.iterdir():
             cat = category_folder.stem
             if category_folder.is_dir() and not category_folder.stem.startswith('.'):
-                        personalities[language_folder.name][category_folder.name] = []
+                personalities[category_folder.name] = []
                 for personality_folder in category_folder.iterdir():
                     pers = personality_folder.stem
                     if personality_folder.is_dir() and not personality_folder.stem.startswith('.'):
@@ -487,7 +475,7 @@ class LoLLMsWebUI(LoLLMsAPPI):
                                 personality_info['help'] = config_data.get('help', '')
                                 personality_info['commands'] = config_data.get('commands', '')
                             real_assets_path = personality_folder/ 'assets'
-                                    assets_path = Path("personalities") / lang / cat / pers / 'assets'
+                            assets_path = Path("personalities") / cat / pers / 'assets'
                             gif_logo_path = assets_path / 'logo.gif'
                             webp_logo_path = assets_path / 'logo.webp'
                             png_logo_path = assets_path / 'logo.png'
@@ -518,19 +506,15 @@ class LoLLMsWebUI(LoLLMsAPPI):
                                 personality_info['avatar'] = str(bmp_logo_path).replace("\\","/")
                             else:
                                 personality_info['avatar'] = ""
-                                    personalities[language_folder.name][category_folder.name].append(personality_info)
+                            personalities[category_folder.name].append(personality_info)
                         except Exception as ex:
                             print(f"Couldn't load personality from {personality_folder} [{ex}]")
         return json.dumps(personalities)
     
     def get_personality(self):
-        lang = request.args.get('language')
         category = request.args.get('category')
         name = request.args.get('name')
-        if category!="personal":
+        personality_folder = self.lollms_paths.personalities_zoo_path/f"{category}"/f"{name}"
-            personality_folder = self.lollms_paths.personalities_zoo_path/f"{lang}"/f"{category}"/f"{name}"
-        else:
-            personality_folder = self.lollms_paths.personal_personalities_path/f"{lang}"/f"{category}"/f"{name}"
         personality_path = personality_folder/f"config.yaml"
         personality_info = {}
         with open(personality_path) as config_file:
@@ -591,18 +575,14 @@ class LoLLMsWebUI(LoLLMsAPPI):
         elif setting_name== "ctx_size":
             self.config["ctx_size"]=int(data['setting_value'])
 
-
-        elif setting_name== "language":
-            self.config["language"]=data['setting_value']
-
         elif setting_name== "personality_folder":
             self.personality_name=data['setting_value']
             if len(self.config["personalities"])>0:
                 if self.config["active_personality_id"]<len(self.config["personalities"]):
-                    self.config["personalities"][self.config["active_personality_id"]] = f"{self.personality_language}/{self.personality_category}/{self.personality_name}"
+                    self.config["personalities"][self.config["active_personality_id"]] = f"{self.personality_category}/{self.personality_name}"
                 else:
                     self.config["active_personality_id"] = 0
-                    self.config["personalities"][self.config["active_personality_id"]] = f"{self.personality_language}/{self.personality_category}/{self.personality_name}"
+                    self.config["personalities"][self.config["active_personality_id"]] = f"{self.personality_category}/{self.personality_name}"
                 
                 if self.personality_category!="Custom":
                     personality_fn = self.lollms_paths.personalities_zoo_path/self.config["personalities"][self.config["active_personality_id"]]
@@ -610,7 +590,7 @@ class LoLLMsWebUI(LoLLMsAPPI):
                     personality_fn = self.lollms_paths.personal_personalities_path/self.config["personalities"][self.config["active_personality_id"]].split("/")[-1]
                 self.personality.load_personality(personality_fn)
             else:
-                self.config["personalities"].append(f"{self.personality_language}/{self.personality_category}/{self.personality_name}")
+                self.config["personalities"].append(f"{self.personality_category}/{self.personality_name}")
         elif setting_name== "override_personality_model_parameters":
             self.config["override_personality_model_parameters"]=bool(data['setting_value'])
 
@@ -784,46 +764,24 @@ class LoLLMsWebUI(LoLLMsAPPI):
             return jsonify([])
     
 
-    def list_personalities_languages(self):
-        personalities_languages_dir = self.lollms_paths.personalities_zoo_path  # replace with the actual path to the models folder
-        personalities_languages = [f.stem for f in personalities_languages_dir.iterdir() if f.is_dir() and not f.name.startswith(".")]
-        return jsonify(personalities_languages)
-
     def list_personalities_categories(self):
-        language = request.args.get('language')
+        personalities_categories_dir = self.lollms_paths.personalities_zoo_path  # replace with the actual path to the models folder
-        if language is None:
-            language = 'english'
-        personalities_categories_dir = self.lollms_paths.personalities_zoo_path/f'{language}'  # replace with the actual path to the models folder
         personalities_categories = [f.stem for f in personalities_categories_dir.iterdir() if f.is_dir() and not f.name.startswith(".")]
         return jsonify(personalities_categories)
     
     def list_personalities(self):
-        language = request.args.get('language')
-        if language is None:
-            language = 'english'
         category = request.args.get('category')
         if not category:
             return jsonify([])
             
         try:
-            personalities_dir = self.lollms_paths.personalities_zoo_path/f'{language}/{category}'  # replace with the actual path to the models folder
+            personalities_dir = self.lollms_paths.personalities_zoo_path/f'{category}'  # replace with the actual path to the models folder
             personalities = [f.stem for f in personalities_dir.iterdir() if f.is_dir() and not f.name.startswith(".")]
         except Exception as ex:
             personalities=[]
             ASCIIColors.error(f"No personalities found. Using default one {ex}")
         return jsonify(personalities)
 
-    def list_languages(self):
-        lanuguages= [
-        { "value": "en-US", "label": "English" },
-        { "value": "fr-FR", "label": "Français" },
-        { "value": "ar-AR", "label": "العربية" },
-        { "value": "it-IT", "label": "Italiano" },
-        { "value": "de-DE", "label": "Deutsch" },
-        { "value": "nl-XX", "label": "Dutch" },
-        { "value": "zh-CN", "label": "中國人" }
-        ]
-        return jsonify(lanuguages)
 
 
     def list_discussions(self):
@@ -832,10 +790,9 @@ class LoLLMsWebUI(LoLLMsAPPI):
 
 
     def delete_personality(self):
-        lang = request.args.get('language')
         category = request.args.get('category')
         name = request.args.get('name')
-        path = Path("personalities")/lang/category/name
+        path = Path("personalities")/category/name
         try:
             shutil.rmtree(path)
             return jsonify({'status':True})
@@ -1220,11 +1177,10 @@ class LoLLMsWebUI(LoLLMsAPPI):
         except Exception as e:
             print(f"Error occurred while parsing JSON: {e}")
             return
-        language = data['language']
         category = data['category']
         name = data['folder']
 
-        package_path = f"{language}/{category}/{name}"
+        package_path = f"{category}/{name}"
         package_full_path = self.lollms_paths.personalities_zoo_path/package_path
         config_file = package_full_path / "config.yaml"
         if config_file.exists():
@@ -1256,27 +1212,26 @@ class LoLLMsWebUI(LoLLMsAPPI):
         except Exception as e:
             print(f"Error occurred while parsing JSON: {e}")
             return
-        language = data['language']
         category = data['category']
         name = data['folder']
 
 
 
 
-        package_path = f"{language}/{category}/{name}"
+        package_path = f"{category}/{name}"
         package_full_path = self.lollms_paths.personalities_zoo_path/package_path
         config_file = package_full_path / "config.yaml"
         if config_file.exists():
             ASCIIColors.info(f"Unmounting personality {package_path}")
-            index = self.config["personalities"].index(f"{language}/{category}/{name}")
+            index = self.config["personalities"].index(f"{category}/{name}")
-            self.config["personalities"].remove(f"{language}/{category}/{name}")
+            self.config["personalities"].remove(f"{category}/{name}")
             if self.config["active_personality_id"]>=index:
                 self.config["active_personality_id"]=0
             if len(self.config["personalities"])>0:
                 self.mounted_personalities = self.rebuild_personalities()
                 self.personality = self.mounted_personalities[self.config["active_personality_id"]]
             else:
-                self.personalities = ["english/generic/lollms"]
+                self.personalities = ["generic/lollms"]
                 self.mounted_personalities = self.rebuild_personalities()
                 self.personality = self.mounted_personalities[self.config["active_personality_id"]]
 
@@ -1310,19 +1265,18 @@ class LoLLMsWebUI(LoLLMsAPPI):
         except Exception as e:
             print(f"Error occurred while parsing JSON: {e}")
             return
-        language    = data['language']
         category    = data['category']
         name        = data['folder']
         try:
-            index = self.config["personalities"].index(f"{language}/{category}/{name}")
+            index = self.config["personalities"].index(f"{category}/{name}")
-            self.config["personalities"].remove(f"{language}/{category}/{name}")
+            self.config["personalities"].remove(f"{category}/{name}")
             if self.config["active_personality_id"]>=index:
                 self.config["active_personality_id"]=0
             if len(self.config["personalities"])>0:
                 self.mounted_personalities = self.rebuild_personalities()
                 self.personality = self.mounted_personalities[self.config["active_personality_id"]]
             else:
-                self.personalities = ["english/generic/lollms"]
+                self.personalities = ["generic/lollms"]
                 self.mounted_personalities = self.rebuild_personalities()
                 self.personality = self.mounted_personalities[self.config["active_personality_id"]]
             self.apply_settings()
@@ -1333,7 +1287,7 @@ class LoLLMsWebUI(LoLLMsAPPI):
                         "active_personality_id":self.config["active_personality_id"]
                         })         
         except:
-            ASCIIColors.error(f"nok : Personality not found @ {language}/{category}/{name}")
+            ASCIIColors.error(f"nok : Personality not found @ {category}/{name}")
             return jsonify({"status": False, "error":"Couldn't unmount personality"})         
          
     def get_active_personality_settings(self):
@@ -1422,14 +1376,13 @@ class LoLLMsWebUI(LoLLMsAPPI):
         except Exception as e:
             print(f"Error occurred while parsing JSON: {e}")
             return
-        language = data['language']
         category = data['category']
         name = data['folder']
 
         if category.startswith("personal"):
-            personality_folder = self.lollms_paths.personal_personalities_path/f"{language}"/f"{category}"/f"{name}"
+            personality_folder = self.lollms_paths.personal_personalities_path/f"{category}"/f"{name}"
         else:
-            personality_folder = self.lollms_paths.personalities_zoo_path/f"{language}"/f"{category}"/f"{name}"
+            personality_folder = self.lollms_paths.personalities_zoo_path/f"{category}"/f"{name}"
 
         personality = AIPersonality(personality_folder,
                                     self.lollms_paths, 
@@ -1675,13 +1628,11 @@ class LoLLMsWebUI(LoLLMsAPPI):
     def get_current_personality_path_infos(self):
         if self.personality is None:
             return jsonify({
-                "personality_language":"",
                 "personality_category":"", 
                 "personality_name":""
             })
         else:
             return jsonify({
-                "personality_language":self.personality_language,
                 "personality_category":self.personality_category, 
                 "personality_name":self.personality_name
             })

configs/config.yaml

@@ -24,7 +24,7 @@ repeat_penalty: 1.2
 n_threads: 8
 
 #Personality parameters
-personalities: ["english/generic/lollms"]
+personalities: ["generic/lollms"]
 active_personality_id: 0
 override_personality_model_parameters: false #if true the personality parameters are overriden by those of the configuration (may affect personality behaviour) 
 
@@ -38,7 +38,7 @@ debug: False
 db_path: database.db
 
 # Automatic update
-auto_update: false
+auto_update: true
 
 
 # Audio

docs/dev/server_endpoints.md

@@ -180,7 +180,6 @@ This Flask server provides various endpoints to manage and interact with the cha
   "override_personality_model_parameters": false,
   "personality": "Computing advisor",
   "personality_category": "Helpers",
-  "personality_language": "english",
   "port": 9600,
   "repeat_last_n": 40,
   "repeat_penalty": 1.2,

docs/tutorials/noobs_tutorial.md

@@ -1,7 +1,7 @@
-# GPT4ALL WebUI Tutorial
+# LOLLMS WebUI Tutorial
 
 ## Introduction
-Welcome to the GPT4ALL WebUI tutorial! In this tutorial, we will walk you through the steps to effectively use this powerful tool. GPT4ALL WebUI is designed to provide access to a variety of language models (LLMs) and offers a range of functionalities to enhance your tasks.
+Welcome to the LOLLMS WebUI tutorial! In this tutorial, we will walk you through the steps to effectively use this powerful tool. LOLLMS WebUI is designed to provide access to a variety of language models (LLMs) and offers a range of functionalities to enhance your tasks.
 
 ## Installation
 1. Ensure that you have Python 3.10 or a higher version, as well as Git, installed on your system. Confirm that the Python installation is in your system's path and can be accessed via the terminal. You can check your Python version by typing `python --version` in the terminal. If the version is lower than 3.10 or you encounter an error, please install a newer version.
@@ -50,6 +50,6 @@ Welcome to the GPT4ALL WebUI tutorial! In this tutorial, we will walk you throug
 6. Note: In the release version, it will be possible to change the database path directly from the UI.
 
 ## Conclusion
-Congratulations! You have learned how to install and use GPT4ALL WebUI effectively. Experiment with different bindings, models, and personalities to find the best fit for your needs. Remember to report any bugs or issues you encounter, as this project is developed by volunteers in their free time. Please support the project by liking, subscribing, and sharing this video to help it reach more people.
+Congratulations! You have learned how to install and use LOLLMS WebUI effectively. Experiment with different bindings, models, and personalities to find the best fit for your needs. Remember to report any bugs or issues you encounter, as this project is developed by volunteers in their free time. Please support the project by liking, subscribing, and sharing this video to help it reach more people.
 
 See you in the next tutorial!

docs/youtube/script_install.md

@@ -1,4 +1,4 @@
-Hi there. Today I finally make a presentation of the new alpha version of GPT4ALL webui.
+Hi there. Today I finally make a presentation of the new alpha version of LOLLMS webui.
 Join us as we dive into the features and functionalities of this new version.
 
 First, I want to thank all of you for your support. We have reached 2.1k stars on github And I hope you continue spreading the word about this tool to give it more visibility.

setup.py

@@ -16,8 +16,8 @@ requirements = read_requirements("requirements.txt")
 requirements_dev = read_requirements("requirements_dev.txt")
 
 setuptools.setup(
-    name="GPT4Allui",
+    name="Lollms-webui",
-    version="0.0.5",
+    version="5.0.0",
     author="Saifeddine ALOUI",
     author_email="aloui.saifeddine@gmail.com",
     description="A web ui for running chat models with different bindings. Supports multiple personalities and extensions.",

templates/index.html

@@ -21,7 +21,7 @@
         </button>
       </div>
     </div>
-    <h1 class="text-2xl text-black dark:text-white">GPT4ALL - WEBUI</h1>
+    <h1 class="text-2xl text-black dark:text-white">LOLLMS - WEBUI</h1>
   </div>
   <div class="content-center items-center mt-4">
     <ul class="flex

web/dist/index.html

@@ -6,8 +6,8 @@
     
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LoLLMS WebUI - Welcome</title>
-    <script type="module" crossorigin src="/assets/index-bf13a6d6.js"></script>
+    <script type="module" crossorigin src="/assets/index-7801f2bd.js"></script>
-    <link rel="stylesheet" href="/assets/index-362f6f68.css">
+    <link rel="stylesheet" href="/assets/index-3e7ad0ad.css">
   </head>
   <body>
     <div id="app"></div>

web/src/components/JsonViewer.vue

@@ -41,9 +41,25 @@ export default {
   },
   computed: {
     formattedJson() {
-      return  this.jsonData.replace(/\n/g, '<br>');
+      console.log(typeof this.jsonData)
+      if (typeof this.jsonData==='string'){
+        let jsonData = JSON.stringify(JSON.parse(this.jsonData), null, '\t').replace(/\n/g, '<br>')
+        console.log(jsonData)
+        console.log(this.jsonFormText)
+        return  jsonData;
+
+      }
+      else{
+        let jsonData = JSON.stringify(this.jsonData, null, '\t').replace(/\n/g, '<br>')
+        console.log(jsonData)
+        console.log(this.jsonFormText)
+        return  jsonData;
+
+      }
     },
     isObject() {
+      console.log(typeof this.jsonData)
+      console.log(this.jsonData)
       return typeof this.jsonData === 'object' && this.jsonData !== null;
     },
     isContentPresent() {
@@ -58,9 +74,7 @@ export default {
       this.collapsed = !this.collapsed;
     },
     toggleCollapsible() {
-      if (!this.isObject) {
         this.collapsed = !this.collapsed;
-      }
     },
   },
 };

web/src/components/Message.vue

@@ -140,13 +140,17 @@
                         </div>
                     </div>
                     
-                    <MarkdownRenderer ref="mdRender" v-if="!editMsgMode && !message.metadata" :markdown-text="message.content">
+                    <MarkdownRenderer ref="mdRender" v-if="!editMsgMode" :markdown-text="message.content">
                     </MarkdownRenderer>
-                    <textarea v-if="editMsgMode && !message.metadata" ref="mdTextarea" :rows="4"
+                    <textarea v-if="editMsgMode" ref="mdTextarea" :rows="4"
                         class="block p-2.5 w-full text-sm text-gray-900 bg-gray-50 rounded-lg border border-gray-300 focus:ring-blue-500 focus:border-blue-500 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500"
                         :style="{ minHeight: mdRenderHeight + `px` }" placeholder="Enter message here..."
                         v-model="this.message.content"></textarea>
-                    <JsonViewer :jsonFormText="message.content" :jsonData="message.metadata" />
+                    <div  v-if="message.metadata !== null">
+                        <div v-for="(metadata, index) in message.metadata" :key="'json-' + message.id + '-' + index" class="json font-bold">
+                            <JsonViewer :jsonFormText="metadata.title" :jsonData="metadata.content" />
+                        </div>
+                    </div> 
                         
 
                 </div>

web/src/components/MountedPersonalitiesList.vue

@@ -333,7 +333,6 @@ export default {
 
             try {
                 const obj = {
-                    language: pers.language,
                     category: pers.category,
                     folder: pers.folder
                 }
@@ -355,7 +354,6 @@ export default {
 
             try {
                 const obj = {
-                    language: pers.language,
                     category: pers.category,
                     folder: pers.folder
                 }
@@ -376,7 +374,6 @@ export default {
             if (!pers) { return { 'status': false, 'error': 'no personality - unmount_personality' } }
 
             const obj = {
-                language: pers.language,
                 category: pers.category,
                 folder: pers.folder
             }

web/src/components/PersonalityEntry.vue

@@ -64,12 +64,6 @@
 
             {{ personality.author }}
           </div>
-          <div class="flex items-center">
-            <i data-feather="globe" class="w-5 m-1"></i>
-            <b>Language:&nbsp;</b>
-
-            {{ personality.language }}
-          </div>
           <div class="flex items-center">
             <i data-feather="bookmark" class="w-5 m-1"></i>
             <b>Category:&nbsp;</b>

web/src/components/PersonalityViewer.vue

@@ -6,7 +6,6 @@
       </div>
       <p><strong>Author:</strong> {{ personalityAuthor }}</p>
       <p><strong>Description:</strong> {{ personalityDescription }}</p>
-      <p><strong>Language:</strong> {{ personalityLanguage }}</p>
       <p><strong>Category:</strong> {{ personalityCategory }}</p>
       <p v-if="disclaimer"><strong>Disclaimer:</strong> {{ disclaimer }}</p>
       <p><strong>Conditioning Text:</strong> {{ conditioningText }}</p>
@@ -39,7 +38,6 @@
         personalityName: 'Personality Name',
         personalityAuthor: 'Author Name',
         personalityDescription: 'Personality Description',
-        personalityLanguage: 'English',
         personalityCategory: 'Category',
         disclaimer: 'Disclaimer text',
         conditioningText: 'Conditioning Text',

web/src/main.js

@@ -118,9 +118,8 @@ export const store = createStore({
           const configFile = await api_get_req('get_config')
           let personality_path_infos = configFile.personalities[configFile.active_personality_id].split("/")
           //let personality_path_infos = await this.api_get_req("get_current_personality_path_infos")
-          configFile.personality_language = personality_path_infos[0]
+          configFile.personality_category = personality_path_infos[0]
-          configFile.personality_category = personality_path_infos[1]
+          configFile.personality_folder = personality_path_infos[1]
-          configFile.personality_folder = personality_path_infos[2]
 
           commit('setConfig', configFile);
         } catch (error) {
@@ -130,11 +129,7 @@ export const store = createStore({
       },
       async refreshPersonalitiesArr({ commit }) {
           let personalities = []
-          const dictionary = await api_get_req("get_all_personalities")
+          const catdictionary = await api_get_req("get_all_personalities")
-          const langkeys = Object.keys(dictionary); // returns languages folder names
-          for (let i = 0; i < langkeys.length; i++) {
-              const langkey = langkeys[i];
-              const catdictionary = dictionary[langkey];
           const catkeys = Object.keys(catdictionary); // returns categories
 
           for (let j = 0; j < catkeys.length; j++) {
@@ -142,15 +137,14 @@ export const store = createStore({
               const personalitiesArray = catdictionary[catkey];
               const modPersArr = personalitiesArray.map((item) => {
 
-                      const isMounted = this.state.config.personalities.includes(langkey + '/' + catkey + '/' + item.folder)
+                  const isMounted = this.state.config.personalities.includes(catkey + '/' + item.folder)
                   // if (isMounted) {
                   //     console.log(item)
                   // }
                   let newItem = {}
                   newItem = item
                   newItem.category = catkey // add new props to items
-                      newItem.language = langkey // add new props to items
+                  newItem.full_path = catkey + '/' + item.folder // add new props to items
-                      newItem.full_path = langkey + '/' + catkey + '/' + item.folder // add new props to items
                   newItem.isMounted = isMounted // add new props to items
                   return newItem
               })
@@ -163,8 +157,6 @@ export const store = createStore({
               }
           }
 
-          }
-
           personalities.sort((a, b) => a.name.localeCompare(b.name))
 
           commit('setPersonalities', personalities);

web/src/views/DiscussionsView.vue

@@ -384,7 +384,7 @@ export default {
                 SENDER_TYPES_AI                 : 1, // Sent by ai
                 SENDER_TYPES_SYSTEM             : 2, // Sent by athe system
             },
-            version                             : "4.0",
+            version                             : "5.0",
             list                                : [], // Discussion list
             tempList                            : [], // Copy of Discussion list (used for keeping the original list during filtering discussions/searching action)
             currentDiscussion                   : {}, // Current/selected discussion id
@@ -884,6 +884,7 @@ export default {
                 rank:                   0,
 
                 steps                   : [],
+                parameters              : msgObj.parameters,
                 metadata                : msgObj.metadata
             }
             console.log(responseMessage)
@@ -979,7 +980,8 @@ export default {
                             rank:                   0,
 
                             steps:                  [],
-                            metadata:               {}
+                            parameters:             null,
+                            metadata:               []
 
                         };
                         this.createUserMsg(usrMessage);
@@ -1030,15 +1032,18 @@ export default {
                     if (matchingStep) {
                         matchingStep.done = true;
                         try {
-                            const metadata = JSON.parse(msgObj.metadata);
+                            console.log(msgObj.parameters)
-                            matchingStep.status=metadata.status
+                            const parameters = msgObj.parameters;
-                            console.log(metadata);
+                            matchingStep.status=parameters.status
+                            console.log(parameters);
+                            
                         } catch (error) {
                             console.error('Error parsing JSON:', error.message);
                         }
                     }
                 } else if (msgObj.message_type == this.msgTypes.MSG_TYPE_JSON_INFOS) {
                     console.log("JSON message")
+                    console.log(msgObj.metadata)
                     messageItem.metadata = msgObj.metadata
                 } else if (msgObj.message_type == this.msgTypes.MSG_TYPE_EXCEPTION) {
                     this.$refs.toast.showToast(msgObj.content, 5, false)

web/src/views/SettingsView.vue

@@ -1291,21 +1291,6 @@
                             </div>
 
                     </div>
-                    <div class="mx-2 mb-4" v-if="!searchPersonality">
-                        <label for="persLang" class="block mb-2 text-sm font-medium text-gray-900 dark:text-white">
-                            Personalities Languages: ({{ persLangArr.length }})
-                        </label>
-                        <select id="persLang" @change="update_personality_language($event.target.value, refresh)"
-                            class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500">
-
-                            <option v-for="item in persLangArr" :selected="item === this.configFile.personality_language">{{
-                                item
-                            }}
-
-                            </option>
-
-                        </select>
-                    </div>
                     <div class="mx-2 mb-4" v-if="!searchPersonality">
                         <label for="persCat" class="block  mb-2 text-sm font-medium text-gray-900 dark:text-white">
                             Personalities Category: ({{ persCatgArr.length }})
@@ -1656,8 +1641,6 @@ export default {
             currenModelToInstall:null,
             // Loading text
             loading_text:"",
-            // Current personality language
-            personality_language:null,
             // Current personality category
             personality_category:null,
             // install custom model
@@ -1683,10 +1666,8 @@ export default {
             pzl_collapsed: false,
             bzl_collapsed: false,
             // Settings stuff
-            persLangArr: [],
             persCatgArr: [],
             persArr: [],
-            langArr: [],
             showConfirmation: false,
             showToast: false,
             isLoading: false,
@@ -1851,23 +1832,18 @@ export default {
             if (this.configFile.model_name) {
                 this.isModelSelected = true
             }
-            this.persLangArr = await this.api_get_req("list_personalities_languages")
+            this.persCatgArr = await this.api_get_req("list_personalities_categories")
-            this.persCatgArr = await this.api_get_req("list_personalities_categories?language="+this.configFile.personality_language)
+            this.persArr = await this.api_get_req("list_personalities?category="+this.configFile.personality_category)
-            this.persArr = await this.api_get_req("list_personalities?language="+this.configFile.personality_language+"&category"+this.configFile.personality_category)
-            this.langArr = await this.api_get_req("list_languages")
 
             this.bindingsArr.sort((a, b) => a.name.localeCompare(b.name))
             this.modelsArr.sort()
-            this.persLangArr.sort()
             this.persCatgArr.sort()
             this.persArr.sort()
-            this.langArr.sort()
 
 
             //await this.getPersonalitiesArr()
-            this.personality_language = this.configFile.personality_language
             this.personality_category = this.configFile.personality_category
-            this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category && item.language === this.configFile.personality_language)
+            this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category)
             this.personalitiesFiltered.sort()
             //mountedPersArr
             this.modelsFiltered = this.models
@@ -2485,10 +2461,6 @@ export default {
         onMessageBoxOk() {
             console.log("OK button clicked");
         },
-        update_personality_language(lang, next){
-            this.personality_language = lang
-            next()
-        },
 
         update_personality_category(cat, next){
             this.personality_category = cat
@@ -2498,13 +2470,12 @@ export default {
         refresh() {
             console.log("Refreshing")
             this.$store.dispatch('refreshConfig').then(() => {
-                console.log(this.personality_language)
                 console.log(this.personality_category)
 
-                this.api_get_req("list_personalities_categories?language="+this.personality_language).then((cats)=>{
+                this.api_get_req("list_personalities_categories").then((cats)=>{
                     console.log("cats",cats)
                     this.persCatgArr = cats
-                    this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.personality_category && item.language === this.personality_language)
+                    this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.personality_category)
                     this.personalitiesFiltered.sort()
                 })
 
@@ -2728,7 +2699,7 @@ export default {
             }
 
             this.personalities.sort((a, b) => a.name.localeCompare(b.name))
-            this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category && item.language === this.configFile.personality_language)
+            this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category)
             this.personalitiesFiltered.sort()
             console.log('per filtered', this.personalitiesFiltered)
             this.isLoading = false
@@ -2736,7 +2707,7 @@ export default {
         },
         async filterPersonalities() {
             if (!this.searchPersonality) {
-                this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category && item.language === this.configFile.personality_language)
+                this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category )
                 this.personalitiesFiltered.sort()
                 this.searchPersonalityInProgress = false
                 return
@@ -2756,7 +2727,7 @@ export default {
 
                 this.personalitiesFiltered = seachedPersonalities.sort()
             } else {
-                this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category && item.language === this.configFile.personality_language)
+                this.personalitiesFiltered = this.personalities.filter((item) => item.category === this.configFile.personality_category)
                 this.personalitiesFiltered.sort()
             }
             this.searchPersonalityInProgress = false