Upgraded backend

lollms/server/endpoints/lollms_discussion.py

@@ -25,6 +25,7 @@ class GenerateRequest(BaseModel):
     text: str
 
 class DatabaseSelectionParameters(BaseModel):
+    client_id: str
     name: str
 
 class EditTitleParameters(BaseModel):
@@ -63,6 +64,9 @@ async def list_databases():
 @router.post("/select_database")
 def select_database(data:DatabaseSelectionParameters):
     sanitize_path(data.name)
+    client = lollmsElfServer.session.get_client(data.client_id)
+    
+
     print(f'Selecting database {data.name}')
     # Create database object
     lollmsElfServer.db = DiscussionsDB(lollmsElfServer.lollms_paths, data.name)

lollms/server/endpoints/lollms_generator.py

@@ -45,6 +45,22 @@ def get_generation_status():
 
 
 # ----------------------------------- Generation -----------------------------------------
+class LollmsTokenizeRequest(BaseModel):
+    prompt: str
+
+@router.post("/lollms_tokenize")
+async def lollms_tokenize(request: LollmsTokenizeRequest):
+    try:
+        tokens = elf_server.model.tokenize(request.prompt)
+        named_tokens=[]
+        for token in tokens:
+            detoken = elf_server.model.detokenize([token])
+            named_tokens.append([detoken,token])
+        tokens = elf_server.model.tokenize(request.prompt)
+        return {"status":True,"raw_tokens":tokens, "named_tokens":named_tokens}
+    except Exception as ex:
+        return {"status":False,"error":str(ex)}
+
 class LollmsGenerateRequest(BaseModel):
     prompt: str
     model_name: Optional[str] = None

lollms/server/endpoints/lollms_personalities_infos.py

@@ -581,8 +581,8 @@ async def copy_to_custom_personas(data: PersonalityInfos):
         return {"status":False}
     else:
         personality_folder = lollmsElfServer.lollms_paths.personalities_zoo_path/f"{category}"/f"{name}"
-        destination_folder = lollmsElfServer.lollms_paths.personal_personalities_path
-        shutil.copy(personality_folder, destination_folder)
+        destination_folder = lollmsElfServer.lollms_paths.custom_personalities_path
+        shutil.copytree(personality_folder, destination_folder)
         return {"status":True}
 
 # ------------------------------------------- Interaction with personas ------------------------------------------------

lollms/server/events/lollms_personality_events.py

@@ -50,7 +50,7 @@ def add_events(sio:socketio):
 
     ALLOWED_EXTENSIONS = {
         'txt', 'csv', 'py', 'html', 'js', 'jpg', 'jpeg', 'png', 'gif', 'bmp', 'tiff', 'ico', 'svg', 'mp4', 'mp3', 'avi', 'mov',
-        'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'pdf', 'js'
+        'doc', 'docx', 'ppt', 'pptx', 'xls', 'xlsx', 'pdf', 'js', "md"
     }
 
     def allowed_file(filename):
@@ -71,6 +71,7 @@ def add_events(sio:socketio):
 
         if not allowed_file(filename):
             print(f"Invalid file type: {filename}")
+            lollmsElfServer.InfoMessage(f"Invalid file type: {filename}")
             return
         ext = filename.split(".")[-1].lower()
         if ext in ["wav", "mp3"]: