added integrated lightrag services

lollms/app.py

@@ -28,9 +28,10 @@ import platform
 import gc
 import yaml
 import time
-from lollms.utilities import PackageManager
+from lollms.utilities import run_with_current_interpreter
 import socket
 import json
+import pipmaster as pm
 class LollmsApplication(LoLLMsCom):
     def __init__(
                     self, 
@@ -375,8 +376,36 @@ class LollmsApplication(LoLLMsCom):
                             rag_db | {"binding": lr}
                     )
     def start_servers(self):
-
         ASCIIColors.yellow("* - * - * - Starting services - * - * - *")
+        def start_local_services(*args, **kwargs):
+            for rag_server in self.config.rag_local_services:
+                try:
+                    # - alias: datalake
+                    #     key: ''
+                    #     path: ''
+                    #     start_at_startup: false
+                    #     type: lightrag
+                    #     url: http://localhost:9621/
+
+                    if rag_server["start_at_startup"]:
+                        if rag_server["type"]=="lightrag":
+                            try:
+                                if not pm.is_installed("lightrag-hku"):
+                                    pm.install("lightrag-hku[api]")
+                                subprocess.Popen(
+                                ["lightrag-server", "--llm-binding", "lollms", "--embedding-binding", "lollms", "--input-dir", rag_server["input_path"], "--working-dir", rag_server["working_path"]],
+                                text=True,
+                                stdout=None, # This will make the output go directly to console
+                                stderr=None  # This will make the errors go directly to console
+                                )
+                            except Exception as ex:
+                                trace_exception(ex)
+                except Exception as ex:
+                    trace_exception(ex)
+                    self.warning(f"Couldn't start lightrag")
+
+        ASCIIColors.execute_with_animation("Loading RAG servers", start_local_services,ASCIIColors.color_blue)
+        
         tts_services = []
         stt_services = []
         def start_ttt(*args, **kwargs):
@@ -399,7 +428,7 @@ class LollmsApplication(LoLLMsCom):
                     trace_exception(ex)
                     self.warning(f"Couldn't load vllm")
         ASCIIColors.execute_with_animation("Loading TTT services", start_ttt,ASCIIColors.color_blue)
-        print("OK")
+
         def start_stt(*args, **kwargs):
             if self.config.whisper_activate or self.config.active_stt_service == "whisper":
                 try:
@@ -416,7 +445,6 @@ class LollmsApplication(LoLLMsCom):
                 self.stt = LollmsWhisper(self, self.config.whisper_model)
 
         ASCIIColors.execute_with_animation("Loading STT services", start_stt, ASCIIColors.color_blue)
-        print("OK")
 
         def start_tts(*args, **kwargs):
             if self.config.active_tts_service == "xtts":
@@ -450,7 +478,6 @@ class LollmsApplication(LoLLMsCom):
                 self.tts = self.xtts
 
         ASCIIColors.execute_with_animation("Loading TTS services", start_tts, ASCIIColors.color_blue)
-        print("OK")
 
         def start_tti(*args, **kwargs):
             if self.config.enable_sd_service:
@@ -493,7 +520,7 @@ class LollmsApplication(LoLLMsCom):
                     self.tti = LollmsComfyUI(self, comfyui_base_url=self.config.comfyui_base_url)
 
         ASCIIColors.execute_with_animation("Loading loacal TTI services", start_tti, ASCIIColors.color_blue)
-        print("OK")
+
         def start_ttv(*args, **kwargs):
             if self.config.active_ttv_service == "lumalabs" and (self.ttv is None or self.tti.name!="lumalabs"):
                 try:

lollms/server/endpoints/lollms_file_system.py

@@ -91,8 +91,87 @@ def open_file(file_types: List[str]) -> Optional[Path]:
         return None
     
 
+def select_lightrag_input_folder_(client) -> Optional[Dict[str, Path]]:
+    """
+    Opens a folder selection dialog and then a string input dialog to get the database name using PyQt5.
+    
+    Returns:
+        Optional[Dict[str, Path]]: A dictionary with the database name and the database path, or None if no folder was selected.
+    """
+    try:
+        # Create a QApplication instance
+        app = QApplication.instance()
+        if not app:
+            app = QApplication(sys.argv)
 
-def select_rag_database(client) -> Optional[Dict[str, Path]]:
+        # Open the folder selection dialog
+        dialog = QFileDialog()
+        # dialog.setOption(QFileDialog.DontUseNativeDialog, True)
+        dialog.setWindowFlag(Qt.WindowStaysOnTopHint, True)
+        dialog.setWindowModality(Qt.ApplicationModal)
+        dialog.raise_()
+        dialog.activateWindow()
+
+        # Add a custom filter to show network folders
+        dialog.setFileMode(QFileDialog.Directory)
+        
+        # Show the dialog modally
+        if dialog.exec_() == QFileDialog.Accepted:
+            folder_path = dialog.selectedFiles()[0]  # Get the selected folder path
+            if folder_path:
+                try:
+                    run_async(partial(lollmsElfServer.sio.emit,'lightrag_input_folder_added', {"path": str(folder_path)}, to=client.client_id))
+                except Exception as ex:
+                    trace_exception(ex)
+                return {"database_path": Path(folder_path)}
+
+            else:
+                return None
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return None
+
+def select_lightrag_output_folder_(client) -> Optional[Dict[str, Path]]:
+    """
+    Opens a folder selection dialog and then a string input dialog to get the database name using PyQt5.
+    
+    Returns:
+        Optional[Dict[str, Path]]: A dictionary with the database name and the database path, or None if no folder was selected.
+    """
+    try:
+        # Create a QApplication instance
+        app = QApplication.instance()
+        if not app:
+            app = QApplication(sys.argv)
+
+        # Open the folder selection dialog
+        dialog = QFileDialog()
+        # dialog.setOption(QFileDialog.DontUseNativeDialog, True)
+        dialog.setWindowFlag(Qt.WindowStaysOnTopHint, True)
+        dialog.setWindowModality(Qt.ApplicationModal)
+        dialog.raise_()
+        dialog.activateWindow()
+
+        # Add a custom filter to show network folders
+        dialog.setFileMode(QFileDialog.Directory)
+        
+        # Show the dialog modally
+        if dialog.exec_() == QFileDialog.Accepted:
+            folder_path = dialog.selectedFiles()[0]  # Get the selected folder path
+            if folder_path:
+                try:
+                    run_async(partial(lollmsElfServer.sio.emit,'lightrag_output_folder_added', {"path": str(folder_path)}, to=client.client_id))
+                except Exception as ex:
+                    trace_exception(ex)
+                return {"database_path": Path(folder_path)}
+            else:
+                return None
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return None
+
+
+def select_lollmsvectordb_input_folder_(client) -> Optional[Dict[str, Path]]:
     """
     Opens a folder selection dialog and then a string input dialog to get the database name using PyQt5.
     
@@ -175,7 +254,7 @@ def select_rag_database(client) -> Optional[Dict[str, Path]]:
                             vdb.build_index()
                             ASCIIColors.success("OK")
                         lollmsElfServer.HideBlockingMessage()
-                        run_async(partial(lollmsElfServer.sio.emit,'rag_db_added', {"datalake_name": db_name, "path": str(folder_path)}, to=client.client_id))
+                        run_async(partial(lollmsElfServer.sio.emit,'lollmsvectordb_datalake_added', {"datalake_name": db_name, "path": str(folder_path)}, to=client.client_id))
 
                     except Exception as ex:
                         trace_exception(ex)
@@ -265,13 +344,34 @@ def get_file(file_infos: FileOpenRequest):
     return open_file(file_infos.file_types)
 
 
-@router.post("/add_rag_database")
-async def add_rag_database(database_infos: SelectDatabase):
+@router.post("/select_lollmsvectordb_input_folder")
+async def select_lollmsvectordb_input_folder(database_infos: SelectDatabase):
     """
     Selects and names a database 
     """ 
     client = check_access(lollmsElfServer, database_infos.client_id)
-    lollmsElfServer.rag_thread = threading.Thread(target=select_rag_database, args=[client])
+    lollmsElfServer.rag_thread = threading.Thread(target=select_lollmsvectordb_input_folder_, args=[client])
+    lollmsElfServer.rag_thread.start()
+    return True
+
+
+@router.post("/select_lightrag_input_folder")
+async def select_lightrag_input_folder(database_infos: SelectDatabase):
+    """
+    Selects and names a database 
+    """ 
+    client = check_access(lollmsElfServer, database_infos.client_id)
+    lollmsElfServer.rag_thread = threading.Thread(target=select_lightrag_input_folder_, args=[client])
+    lollmsElfServer.rag_thread.start()
+    return True
+
+@router.post("/select_lightrag_output_folder")
+async def select_lightrag_output_folder(database_infos: SelectDatabase):
+    """
+    Selects and names a database 
+    """ 
+    client = check_access(lollmsElfServer, database_infos.client_id)
+    lollmsElfServer.rag_thread = threading.Thread(target=select_lightrag_output_folder_, args=[client])
     lollmsElfServer.rag_thread.start()
     return True
 
@@ -450,7 +550,7 @@ async def vectorize_folder(database_infos: FolderInfos):
                 vdb.build_index()
                 ASCIIColors.success("OK")
             lollmsElfServer.HideBlockingMessage()
-            run_async(partial(lollmsElfServer.sio.emit,'rag_db_added', {"datalake_name": db_name, "path": str(folder_path)}, to=client.client_id))
+            run_async(partial(lollmsElfServer.sio.emit,'lollmsvectordb_datalake_added', {"datalake_name": db_name, "path": str(folder_path)}, to=client.client_id))
 
         except Exception as ex:
             trace_exception(ex)

lollms/server/endpoints/lollms_models_infos.py

@@ -40,7 +40,7 @@ async def list_models():
         List[str]: A list of model names.
     """
     if lollmsElfServer.binding is not None:
-        ASCIIColors.yellow("Listing models")
+        ASCIIColors.yellow("Listing models", end="")
         models = lollmsElfServer.binding.list_models()
         ASCIIColors.green("ok")
         return models

lollms/server/endpoints/lollms_personalities_infos.py

@@ -125,7 +125,7 @@ def get_personality():
 
 @router.get("/get_all_personalities")
 def get_all_personalities():
-    ASCIIColors.yellow("Listing all personalities")
+    ASCIIColors.yellow("Listing all personalities", end="")
     personalities_folder = lollmsElfServer.lollms_paths.personalities_zoo_path
     personalities = {}
 

lollms/utilities.py

@@ -60,6 +60,94 @@ from enum import Enum
 from pathlib import Path
 import shutil
 
+from pathlib import Path
+import sys
+import subprocess
+from typing import Union, List
+
+def run_with_current_interpreter(
+    script_path: Union[str, Path], 
+    args: List[str] = None
+) -> subprocess.CompletedProcess:
+    """
+    Runs a Python script using the current interpreter.
+    
+    Args:
+        script_path: Path to the Python script to execute
+        args: Optional list of arguments to pass to the script
+        
+    Returns:
+        subprocess.CompletedProcess object containing the execution result
+        
+    Example:
+        result = run_with_current_interpreter(Path("my_script.py"), ["arg1", "arg2"])
+    """
+        
+    # Get current Python interpreter path
+    interpreter_path = sys.executable
+    
+    # Prepare command
+    command = [interpreter_path, str(script_path)]
+    if args:
+        command.extend(args)
+        
+    # Run the script and return the result
+    return subprocess.run(
+        command,
+        text=True,
+        check=True,  # Raises CalledProcessError if return code != 0
+        stdout=None, # This will make the output go directly to console
+        stderr=None  # This will make the errors go directly to console
+    )
+
+from pathlib import Path
+import sys
+import subprocess
+from typing import Union, List, Optional
+
+def run_module(
+    module_name: str,
+    args: Optional[List[str]] = None,
+) -> subprocess.CompletedProcess:
+    """
+    Runs a Python module using the current interpreter with the -m flag and outputs to console.
+    
+    Args:
+        module_name: Name of the module to run (e.g. 'pip', 'http.server')
+        args: Optional list of arguments to pass to the module
+        
+    Returns:
+        subprocess.CompletedProcess object containing the execution result
+        
+    Example:
+        # Run pip list
+        result = run_module("pip", ["list"])
+        # Run http.server on port 8000
+        result = run_module("http.server", ["8000"])
+    """
+    # Get current Python interpreter path
+    interpreter_path = sys.executable
+    
+    # Prepare command
+    command = [interpreter_path, "-m", module_name]
+    if args:
+        command.extend(args)
+        
+    try:
+        # Run the module with direct console output
+        return subprocess.run(
+            command,
+            text=True,
+            check=True,  # Raises CalledProcessError if return code != 0
+            stdout=None, # This will make the output go directly to console
+            stderr=None  # This will make the errors go directly to console
+        )
+    except subprocess.CalledProcessError as e:
+        print(f"Error running module {module_name}")
+        raise
+
+
+
 class EnvManager(Enum):
     CONDA = 'conda'
     VENV = 'venv'