From ce1219bc208ec6ee52e2dd3890d0167d3a09cf60 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Fri, 20 Dec 2024 01:05:17 +0100 Subject: [PATCH] upgraded --- configs/config.yaml | 5 +- lollms/configs/config.yaml | 5 +- lollms/databases/remote_databases/__init__.py | 0 .../remote_databases/light_rag_database.py | 81 ++++++++++++ .../remote_databases/lollms_database.py | 123 ++++++++++++++++++ 5 files changed, 212 insertions(+), 2 deletions(-) create mode 100644 lollms/databases/remote_databases/__init__.py create mode 100644 lollms/databases/remote_databases/light_rag_database.py create mode 100644 lollms/databases/remote_databases/lollms_database.py diff --git a/configs/config.yaml b/configs/config.yaml index 759692a..882651c 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 146 +version: 147 # video viewing and news recovering last_viewed_video: null @@ -278,6 +278,9 @@ audio_pitch: 1 audio_auto_send_input: true audio_silenceTimer: 5000 +# relmote databases +remote_databases: [] # This is the list of remote databases addresses in form database_type::database url + # Data vectorization rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data rag_vectorizer: semantic # possible values semantic, tfidf, openai, ollama diff --git a/lollms/configs/config.yaml b/lollms/configs/config.yaml index 759692a..882651c 100644 --- a/lollms/configs/config.yaml +++ b/lollms/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 146 +version: 147 # video viewing and news recovering last_viewed_video: null @@ -278,6 +278,9 @@ audio_pitch: 1 audio_auto_send_input: true audio_silenceTimer: 5000 +# relmote databases +remote_databases: [] # This is the list of remote databases addresses in form database_type::database url + # Data vectorization rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data rag_vectorizer: semantic # possible values semantic, tfidf, openai, ollama diff --git a/lollms/databases/remote_databases/__init__.py b/lollms/databases/remote_databases/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/lollms/databases/remote_databases/light_rag_database.py b/lollms/databases/remote_databases/light_rag_database.py new file mode 100644 index 0000000..adde5f0 --- /dev/null +++ b/lollms/databases/remote_databases/light_rag_database.py @@ -0,0 +1,81 @@ +import requests +from typing import Optional, Any, List, Dict +from enum import Enum +from lollms.databases.remote_databases.lollms_database import LollmsDatabase + +class SearchMode(str, Enum): + naive = "naive" + local = "local" + global_ = "global" + hybrid = "hybrid" + +class LollmsLightRag(LollmsDatabase): + def __init__(self, server_url: str, name: str, description: Optional[str] = None): + """ + Initialize LightRAG client + + Args: + server_url (str): Full URL of the LightRAG server + name (str): Name of the database + description (Optional[str]): Description of the database + """ + super().__init__(name, description) + self.server_url = server_url.rstrip('/') + + def add_document(self, content: str, metadata: Optional[Dict[str, Any]] = None) -> bool: + try: + response = requests.post( + f"{self.server_url}/documents/text", + json={"text": content} + ) + response.raise_for_status() + self.update_last_modified() + return True + except requests.RequestException: + return False + + def query(self, text: str, **kwargs) -> str: + try: + response = requests.post( + f"{self.server_url}/query", + json={ + "query": text, + **kwargs + } + ) + response.raise_for_status() + return response.json()["response"] + except requests.RequestException as e: + raise ConnectionError(f"Failed to query: {str(e)}") + + def clear(self) -> bool: + try: + response = requests.delete(f"{self.server_url}/documents") + response.raise_for_status() + self.update_last_modified() + return True + except requests.RequestException: + return False + + def get_stats(self) -> Dict[str, Any]: + health_info = self.health_check() + return { + "indexed_files": health_info.get("indexed_files", 0), + "configuration": health_info.get("configuration", {}) + } + + def health_check(self) -> Dict[str, Any]: + try: + response = requests.get(f"{self.server_url}/health") + response.raise_for_status() + return response.json() + except requests.RequestException as e: + raise ConnectionError(f"Failed to check health: {str(e)}") + + def backup(self, path: str) -> bool: + # Implementation would depend on LightRAG backup capabilities + raise NotImplementedError("Backup not implemented for LightRAG") + + def restore(self, path: str) -> bool: + # Implementation would depend on LightRAG restore capabilities + raise NotImplementedError("Restore not implemented for LightRAG") \ No newline at end of file diff --git a/lollms/databases/remote_databases/lollms_database.py b/lollms/databases/remote_databases/lollms_database.py new file mode 100644 index 0000000..e9dfb21 --- /dev/null +++ b/lollms/databases/remote_databases/lollms_database.py @@ -0,0 +1,123 @@ +from abc import ABC, abstractmethod +from typing import Optional, Any, List, Dict +from datetime import datetime + +class LollmsDatabase(ABC): + """ + Base class for all Lollms database implementations + """ + def __init__(self, name: str, description: Optional[str] = None): + """ + Initialize the database + + Args: + name (str): Name of the database + description (Optional[str]): Description of the database + """ + self.name = name + self.description = description + self.created_at = datetime.now() + self.last_modified = datetime.now() + + @abstractmethod + def add_document(self, content: str, metadata: Optional[Dict[str, Any]] = None) -> bool: + """ + Add a document to the database + + Args: + content (str): The content to add + metadata (Optional[Dict[str, Any]]): Additional metadata about the document + + Returns: + bool: True if successful, False otherwise + """ + pass + + @abstractmethod + def query(self, text: str, **kwargs) -> str: + """ + Query the database + + Args: + text (str): Query text + **kwargs: Additional query parameters + + Returns: + str: Query response + """ + pass + + @abstractmethod + def clear(self) -> bool: + """ + Clear all documents from the database + + Returns: + bool: True if successful, False otherwise + """ + pass + + @abstractmethod + def get_stats(self) -> Dict[str, Any]: + """ + Get database statistics + + Returns: + Dict[str, Any]: Database statistics + """ + pass + + @abstractmethod + def health_check(self) -> Dict[str, Any]: + """ + Check database health + + Returns: + Dict[str, Any]: Health status information + """ + pass + + def get_info(self) -> Dict[str, Any]: + """ + Get database information + + Returns: + Dict[str, Any]: Database information + """ + return { + "name": self.name, + "description": self.description, + "created_at": self.created_at, + "last_modified": self.last_modified, + "type": self.__class__.__name__ + } + + @abstractmethod + def backup(self, path: str) -> bool: + """ + Backup the database + + Args: + path (str): Backup destination path + + Returns: + bool: True if successful, False otherwise + """ + pass + + @abstractmethod + def restore(self, path: str) -> bool: + """ + Restore the database from backup + + Args: + path (str): Backup source path + + Returns: + bool: True if successful, False otherwise + """ + pass + + def update_last_modified(self): + """Update the last modified timestamp""" + self.last_modified = datetime.now() \ No newline at end of file