From 3f9987e439cdd13608436a6ac404618090a921b8 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Mon, 13 Jan 2025 01:58:56 +0100 Subject: [PATCH] upgraded rag system --- configs/config.yaml | 10 +- lollms/app.py | 512 ++++++------------ lollms/configs/config.yaml | 10 +- .../remote_databases/light_rag_database.py | 2 +- lollms/personality.py | 7 - lollms/server/endpoints/lollms_file_system.py | 141 ++--- 6 files changed, 249 insertions(+), 433 deletions(-) diff --git a/configs/config.yaml b/configs/config.yaml index 882651c..0c7fb04 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 147 +version: 148 # video viewing and news recovering last_viewed_video: null @@ -279,10 +279,14 @@ audio_auto_send_input: true audio_silenceTimer: 5000 # relmote databases -remote_databases: [] # This is the list of remote databases addresses in form database_type::database url +# This is the list of datalakes to be used for RAG +# Datalakes hae the following entries +# +datalakes: [] # Data vectorization -rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data +rag_local_services: [] # This is the list of rag services served locally + rag_vectorizer: semantic # possible values semantic, tfidf, openai, ollama rag_service_url: "http://localhost:11434" # rag service url for ollama rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable diff --git a/lollms/app.py b/lollms/app.py index 694bfd6..843203d 100644 --- a/lollms/app.py +++ b/lollms/app.py @@ -319,54 +319,61 @@ class LollmsApplication(LoLLMsCom): def get_uploads_path(self, client_id): return self.lollms_paths.personal_uploads_path + def load_rag_dbs(self): - self.active_rag_dbs = [] - for rag_db in self.config.rag_databases: + ASCIIColors.info("Loading RAG datalakes") + self.active_datalakes = [] + for rag_db in self.config.datalakes: if rag_db['mounted']: - try: - from lollmsvectordb import VectorDatabase - from lollmsvectordb.text_document_loader import TextDocumentsLoader - from lollmsvectordb.lollms_tokenizers.tiktoken_tokenizer import TikTokenTokenizer + if rag_db['type']=='lollmsvectordb': + try: + from lollmsvectordb import VectorDatabase + from lollmsvectordb.text_document_loader import TextDocumentsLoader + from lollmsvectordb.lollms_tokenizers.tiktoken_tokenizer import TikTokenTokenizer - # Vectorizer selection - if self.config.rag_vectorizer == "semantic": - from lollmsvectordb.lollms_vectorizers.semantic_vectorizer import SemanticVectorizer - vectorizer = SemanticVectorizer(self.config.rag_vectorizer_model) - elif self.config.rag_vectorizer == "tfidf": - from lollmsvectordb.lollms_vectorizers.tfidf_vectorizer import TFIDFVectorizer - vectorizer = TFIDFVectorizer() - elif self.config.rag_vectorizer == "openai": - from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer - vectorizer = OpenAIVectorizer( - self.config.rag_vectorizer_model, - self.config.rag_vectorizer_openai_key - ) - elif self.config.rag_vectorizer == "ollama": - from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer - vectorizer = OllamaVectorizer( - self.config.rag_vectorizer_model, - self.config.rag_service_url + # Vectorizer selection + if self.config.rag_vectorizer == "semantic": + from lollmsvectordb.lollms_vectorizers.semantic_vectorizer import SemanticVectorizer + vectorizer = SemanticVectorizer(self.config.rag_vectorizer_model) + elif self.config.rag_vectorizer == "tfidf": + from lollmsvectordb.lollms_vectorizers.tfidf_vectorizer import TFIDFVectorizer + vectorizer = TFIDFVectorizer() + elif self.config.rag_vectorizer == "openai": + from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer + vectorizer = OpenAIVectorizer( + self.config.rag_vectorizer_model, + self.config.rag_vectorizer_openai_key + ) + elif self.config.rag_vectorizer == "ollama": + from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer + vectorizer = OllamaVectorizer( + self.config.rag_vectorizer_model, + self.config.rag_service_url + ) + + # Create database path and initialize VectorDatabase + db_path = Path(rag_db['path']) / f"{rag_db['alias']}.sqlite" + vdb = VectorDatabase( + db_path, + vectorizer, + None if self.config.rag_vectorizer == "semantic" else self.model if self.model else TikTokenTokenizer(), + n_neighbors=self.config.rag_n_chunks + ) + + # Add to active databases + self.active_datalakes.append( + rag_db | {"binding": vdb} ) - # Create database path and initialize VectorDatabase - db_path = Path(rag_db['path']) / f"{rag_db['alias']}.sqlite" - vdb = VectorDatabase( - db_path, - vectorizer, - None if self.config.rag_vectorizer == "semantic" else self.model if self.model else TikTokenTokenizer(), - n_neighbors=self.config.rag_n_chunks - ) - - # Add to active databases - self.active_rag_dbs.append({ - "name": rag_db['alias'], - "path": rag_db['path'], - "vectorizer": vdb - }) - - except Exception as ex: - trace_exception(ex) - ASCIIColors.error(f"Couldn't load {db_path} consider revectorizing it") + except Exception as ex: + trace_exception(ex) + ASCIIColors.error(f"Couldn't load {db_path} consider revectorizing it") + elif rag_db['type']=='lightrag': + from lollmsvectordb.database_clients.lightrag_client import LollmsLightRagConnector + lr = LollmsLightRagConnector(rag_db['url'], rag_db['key']) + self.active_datalakes.append( + rag_db | {"binding": lr} + ) def start_servers(self): ASCIIColors.yellow("* - * - * - Starting services - * - * - *") @@ -928,7 +935,6 @@ class LollmsApplication(LoLLMsCom): start_ai_header_id_template = self.config.start_ai_header_id_template end_ai_header_id_template = self.config.end_ai_header_id_template - system_message_template = self.config.system_message_template if self.personality.callback is None: self.personality.callback = partial(self.process_data, client_id=client_id) @@ -977,8 +983,6 @@ class LollmsApplication(LoLLMsCom): internet_search_results = "" internet_search_infos = [] documentation = "" - knowledge = "" - knowledge_infos = {"titles":[],"contents":[]} # boosting information @@ -1006,6 +1010,114 @@ class LollmsApplication(LoLLMsCom): discussion = None if generation_type != "simple_question": + # Standard RAG + if not self.personality.ignore_discussion_documents_rag: + if self.personality.persona_data_vectorizer or len(self.active_datalakes) > 0 or ((len(client.discussion.text_files) > 0) and client.discussion.vectorizer is not None) or self.config.activate_skills_lib: + #Prepare query + + # Recover or initialize discussion + if discussion is None: + discussion = self.recover_discussion(client_id) + + # Build documentation if empty + if documentation == "": + documentation = f"{self.separator_template}".join([ + f"{self.system_custom_header('important information')}", + "Utilize Documentation Data: Always refer to the provided documentation to answer user questions accurately.", + "Absence of Information: If the required information is not available in the documentation, inform the user that the requested information is not present in the documentation section.", + "Strict Adherence to Documentation: It is strictly prohibited to provide answers without concrete evidence from the documentation.", + "Cite Your Sources: After providing an answer, include the full path to the document where the information was found.", + f"{self.system_custom_header('Documentation')}" + ]) + documentation += f"{self.separator_template}" + + # Process query + if self.config.rag_build_keys_words: + self.personality.step_start("Building vector store query") + query = self.personality.fast_gen( + f"{self.separator_template}" + f"{self.system_custom_header('instruction')} Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\n" + f"Do not answer the prompt. Do not add explanations." + f"{self.separator_template}" + f"{self.system_custom_header('discussion')}'\n{discussion[-2048:]}" + f"{self.separator_template}" + f"{self.ai_custom_header('enhanced query')}", + max_generation_size=256, + show_progress=True, + callback=self.personality.sink + ) + self.personality.step_end("Building vector store query") + ASCIIColors.cyan(f"Query: {query}") + else: + query = current_message.content + + # RAGs + if len(self.active_datalakes) > 0: + recovered_ids=[[] for _ in range(len(self.active_datalakes))] + for i,db in enumerate(self.active_datalakes): + if db['mounted']: + try: + if db["type"]=="lollmsvectordb": + from lollmsvectordb.vector_database import VectorDatabase + binding:VectorDatabase = db["binding"] + + r=binding.search(query, self.config.rag_n_chunks, recovered_ids[i]) + recovered_ids[i]+=[rg.chunk_id for rg in r] + if self.config.rag_activate_multi_hops: + r = [rg for rg in r if self.personality.verify_rag_entry(query, rg.text)] + documentation += "\n".join(["## chunk" + research_result.text for research_result in r])+"\n" + elif db["type"]=="lightrag": + try: + from lollmsvectordb.database_clients.lightrag_client import LollmsLightRagConnector + lc:LollmsLightRagConnector = db["binding"] + documentation += lc.query(query) + + except Exception as ex: + trace_exception(ex) + except Exception as ex: + trace_exception(ex) + self.personality.error(f"Couldn't recover information from Datalake {db['alias']}") + + if self.personality.persona_data_vectorizer: + chunks:List[Chunk] = self.personality.persona_data_vectorizer.search(query, int(self.config.rag_n_chunks)) + for chunk in chunks: + if self.config.rag_put_chunk_informations_into_context: + documentation += f"{self.system_custom_header('document chunk')}\n## document title: {chunk.doc.title}\n## chunk content:\n{chunk.text}\n" + else: + documentation += f"{self.system_custom_header('chunk')}\n{chunk.text}\n" + + if (len(client.discussion.text_files) > 0) and client.discussion.vectorizer is not None: + chunks:List[Chunk] = client.discussion.vectorizer.search(query, int(self.config.rag_n_chunks)) + for chunk in chunks: + if self.config.rag_put_chunk_informations_into_context: + documentation += f"{self.system_custom_header('document chunk')}\n## document title: {chunk.doc.title}\n## chunk content:\n{chunk.text}\n" + else: + documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" + # Check if there is discussion knowledge to add to the prompt + if self.config.activate_skills_lib: + try: + # skills = self.skills_library.query_entry(query) + self.personality.step_start("Adding skills") + if self.config.debug: + ASCIIColors.info(f"Query : {query}") + skill_titles, skills, similarities = self.skills_library.query_vector_db(query, top_k=3, min_similarity=self.config.rag_min_correspondance)#query_entry_fts(query) + skills_detials=[{"title": title, "content":content, "similarity":similarity} for title, content, similarity in zip(skill_titles, skills, similarities)] + + if len(skills)>0: + if documentation=="": + documentation=f"{self.system_custom_header('skills library knowledges')}\n" + for i,skill in enumerate(skills_detials): + documentation += "---\n"+ self.system_custom_header(f"knowledge {i}") +f"\ntitle:\n{skill['title']}\ncontent:\n{skill['content']}\n---\n" + self.personality.step_end("Adding skills") + except Exception as ex: + trace_exception(ex) + self.warning("Couldn't add long term memory information to the context. Please verify the vector database") # Add information about the user + self.personality.step_end("Adding skills") + + documentation += f"{self.separator_template}{self.system_custom_header('important information')}Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.\n" + + + # Internet if self.config.activate_internet_search or force_using_internet or generation_type == "full_context_with_internet": if discussion is None: discussion = self.recover_discussion(client_id) @@ -1062,303 +1174,8 @@ class LollmsApplication(LoLLMsCom): else: self.personality.step_end("Performing Internet search (advanced mode: slower but more advanced)") - if self.personality.persona_data_vectorizer: - if documentation=="": - documentation=f"{self.separator_template}{self.start_header_id_template}Documentation:\n" - - if not self.config.rag_deactivate: - if self.config.rag_build_keys_words: - if discussion is None: - discussion = self.recover_discussion(client_id) - query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True) - ASCIIColors.cyan(f"Query:{query}") - else: - query = current_message.content - try: - chunks:List[Chunk] = self.personality.persona_data_vectorizer.search(query, int(self.config.rag_n_chunks)) - for chunk in chunks: - if self.config.rag_put_chunk_informations_into_context: - documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n" - else: - documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" - - documentation += f"{self.separator_template}{self.start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.\n" - - except Exception as ex: - trace_exception(ex) - self.warning("Couldn't add documentation to the context. Please verify the vector database") - else: - docs = self.personality.persona_data_vectorizer.list_documents() - for doc in docs: - documentation += self.personality.persona_data_vectorizer.get_document(doc['title']) - - if not self.personality.ignore_discussion_documents_rag: - query = None - if len(self.active_rag_dbs) > 0 : - if discussion is None: - discussion = self.recover_discussion(client_id) - - if self.config.rag_build_keys_words: - self.personality.step_start("Building vector store query") - q = f"{self.separator_template}".join([ - "make a RAG vector database query from the last user prompt given this discussion.", - "--- discussion --", - f"{discussion[-2048:]}", - "---", - "Make sure to write the RAG vector database query in your output json code." - ]) - template = """{ -"query": "[the rag query deduced from the last messge in the discussion]" -} -""" - query = self.personality.generate_code(q, self.personality.image_files, template, callback=self.personality.sink) - if query is None: - query = current_message.content - else: - try: - query = json.loads(query) - query = query["query"] - except Exception as ex: - ASCIIColors.error("failed to generate the query") - query = current_message.content - self.personality.step_end("Building vector store query") - ASCIIColors.magenta(f"Query: {query}") - self.personality.step(f"Query: {query}") - else: - query = current_message.content - if documentation=="": - documentation=f"{self.separator_template}".join([ - f"{self.system_custom_header('important information')}", - "Always refer to the provided documentation to answer user questions accurately.", - "Absence of Information: If the required information is not available in the documentation, inform the user that the requested information is not present in the documentation section.", - "Strict Adherence to Documentation: It is strictly prohibited to provide answers without concrete evidence from the documentation.", - "Cite Your Sources: After providing an answer, include the full path to the document where the information was found.", - self.system_custom_header("Documentation")]) - documentation += f"{self.separator_template}" - full_documentation="" - if self.config.contextual_summary: - for db in self.active_rag_dbs: - v:VectorDatabase = db["vectorizer"] - docs = v.list_documents() - for doc in docs: - document=v.get_document(document_path = doc["path"]) - self.personality.step_start(f"Summaryzing document {doc['path']}") - def post_process(summary): - return summary - summary = self.personality.summarize_text(document, - f"Extract information from the following text chunk to answer this request.\n{self.system_custom_header('query')}{query}", chunk_summary_post_processing=post_process, callback=self.personality.sink) - self.personality.step_end(f"Summaryzing document {doc['path']}") - document_infos = f"{self.separator_template}".join([ - self.system_custom_header('document contextual summary'), - f"source_document_title:{doc['title']}", - f"source_document_path:{doc['path']}", - f"content:\n{summary}\n" - ]) - documentation_entries.append({ - "document_title":doc['title'], - "document_path":doc['path'], - "chunk_content":summary, - "chunk_size":0, - "similarity":0, - }) - if summary!="": - v.add_summaries(doc['path'],[{"context":query, "summary":summary}]) - full_documentation += document_infos - documentation += self.personality.summarize_text(full_documentation, f"Extract information from the current text chunk and previous text chunks to answer the query. If there is no information about the query, just return an empty string.\n{self.system_custom_header('query')}{query}", callback=self.personality.sink) - else: - results = [] - recovered_ids=[[] for _ in range(len(self.active_rag_dbs))] - hop_id = 0 - while( len(results)=self.config.rag_n_chunks: - break - n_neighbors = self.active_rag_dbs[0]["vectorizer"].n_neighbors - sorted_results = sorted(results, key=lambda x: x.distance)[:n_neighbors] - - for chunk in sorted_results: - document_infos = f"{self.separator_template}".join([ - f"{self.start_header_id_template}document chunk{self.end_header_id_template}", - f"source_document_title:{chunk.doc.title}", - f"source_document_path:{chunk.doc.path}", - f"content:\n{chunk.text}\n" - ]) - documentation_entries.append({ - "document_title":chunk.doc.title, - "document_path":chunk.doc.path, - "chunk_content":chunk.text, - "chunk_size":chunk.nb_tokens, - "similarity":1-chunk.distance, - }) - documentation += document_infos - - if (len(self.config.remote_databases) > 0) and not self.config.rag_deactivate: - for db in self.config.remote_databases: - # Check if database is mounted - if db['mounted']: - try: - if db['type'] == "lightrag": - from lollmsvectordb.database_clients.lightrag_client import LollmsLightRagConnector - lc = LollmsLightRagConnector(db['url'], db['key']) - - # Recover or initialize discussion - if discussion is None: - discussion = self.recover_discussion(client_id) - - # Build documentation if empty - if documentation == "": - documentation = f"{self.separator_template}".join([ - f"{self.system_custom_header('important information')}", - "Utilize Documentation Data: Always refer to the provided documentation to answer user questions accurately.", - "Absence of Information: If the required information is not available in the documentation, inform the user that the requested information is not present in the documentation section.", - "Strict Adherence to Documentation: It is strictly prohibited to provide answers without concrete evidence from the documentation.", - "Cite Your Sources: After providing an answer, include the full path to the document where the information was found.", - f"{self.system_custom_header('Documentation')}" - ]) - documentation += f"{self.separator_template}" - - # Process query - if query is None: - if self.config.rag_build_keys_words: - self.personality.step_start("Building vector store query") - query = self.personality.fast_gen( - f"{self.separator_template}" - f"{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\n" - f"Do not answer the prompt. Do not add explanations." - f"{self.separator_template}" - f"{self.start_header_id_template}discussion:\n{discussion[-2048:]}" - f"{self.separator_template}" - f"{self.start_header_id_template}enhanced query: ", - max_generation_size=256, - show_progress=True, - callback=self.personality.sink - ) - self.personality.step_end("Building vector store query") - ASCIIColors.cyan(f"Query: {query}") - else: - query = current_message.content - - documentation += lc.query(query) - - except Exception as ex: - trace_exception(ex) - - - if (len(client.discussion.text_files) > 0) and client.discussion.vectorizer is not None: - if not self.config.rag_deactivate: - if discussion is None: - discussion = self.recover_discussion(client_id) - - if documentation=="": - documentation=f"{self.separator_template}".join([ - f"{self.system_custom_header('important information')}", - "Utilize Documentation Data: Always refer to the provided documentation to answer user questions accurately.", - "Absence of Information: If the required information is not available in the documentation, inform the user that the requested information is not present in the documentation section.", - "Strict Adherence to Documentation: It is strictly prohibited to provide answers without concrete evidence from the documentation.", - "Cite Your Sources: After providing an answer, include the full path to the document where the information was found.", - f"{self.system_custom_header('Documentation')}"]) - - documentation += f"{self.separator_template}" - - if query is None: - if self.config.rag_build_keys_words: - self.personality.step_start("Building vector store query") - query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink) - self.personality.step_end("Building vector store query") - ASCIIColors.cyan(f"Query: {query}") - else: - query = current_message.content - - - full_documentation="" - if self.config.contextual_summary: - v = client.discussion.vectorizer - docs = v.list_documents() - for doc in docs: - document=v.get_document(document_path = doc["path"]) - self.personality.step_start(f"Summeryzing document {doc['path']}") - summary = self.personality.summarize_text(document, f"Extract information from the following text chunk to answer this request. If there is no information about the query, just return an empty string.\n{self.system_custom_header('query')}{query}", callback=self.personality.sink) - self.personality.step_end(f"Summeryzing document {doc['path']}") - document_infos = f"{self.separator_template}".join([ - self.system_custom_header('document contextual summary'), - f"source_document_title:{doc['title']}", - f"source_document_path:{doc['path']}", - f"content:\n{summary}\n" - ]) - documentation_entries.append({ - "document_title":doc['title'], - "document_path":doc['path'], - "chunk_content":summary, - "chunk_size":0, - "similarity":0, - }) - if summary!="": - v.add_summaries(doc['path'],[{"context":query, "summary":summary}]) - full_documentation += document_infos - documentation += self.personality.summarize_text(full_documentation, f"Extract information from the current text chunk and previous text chunks to answer the query. If there is no information about the query, just return an empty string.\n{self.system_custom_header('query')}{query}", callback=self.personality.sink) - else: - try: - chunks:List[Chunk] = client.discussion.vectorizer.search(query, int(self.config.rag_n_chunks)) - for chunk in chunks: - if self.config.rag_put_chunk_informations_into_context: - documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n" - else: - documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" - - documentation += f"{self.separator_template}{self.start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.\n" - except Exception as ex: - trace_exception(ex) - self.warning("Couldn't add documentation to the context. Please verify the vector database") - else: - docs = client.discussion.vectorizer.get_all_documents() - documentation += "\n\n".join(docs) + "\n" - - # Check if there is discussion knowledge to add to the prompt - if self.config.activate_skills_lib: - try: - self.personality.step_start("Querying skills library") - if discussion is None: - discussion = self.recover_discussion(client_id) - self.personality.step_start("Building query") - query = self.personality.generate_code(f"""Your task is to carefully read the provided discussion and reformulate {self.config.user_name}'s request concisely. -{self.system_custom_header("discussion")} -{discussion[-2048:]} -""", template="""{ - "request": "the reformulated request" -}""", callback=self.personality.sink) - query_code = json.loads(query) - query = query_code["request"] - self.personality.step_end("Building query") - self.personality.step(f"query: {query}") - # skills = self.skills_library.query_entry(query) - self.personality.step_start("Adding skills") - if self.config.debug: - ASCIIColors.info(f"Query : {query}") - skill_titles, skills, similarities = self.skills_library.query_vector_db(query, top_k=3, min_similarity=self.config.rag_min_correspondance)#query_entry_fts(query) - skills_detials=[{"title": title, "content":content, "similarity":similarity} for title, content, similarity in zip(skill_titles, skills, similarities)] - - if len(skills)>0: - if knowledge=="": - knowledge=f"{self.system_custom_header('skills library knowledges')}\n" - for i,skill in enumerate(skills_detials): - knowledge += "---\n"+ self.system_custom_header(f"knowledge {i}") +f"\ntitle:\n{skill['title']}\ncontent:\n{skill['content']}\n---\n" - self.personality.step_end("Adding skills") - self.personality.step_end("Querying skills library") - except Exception as ex: - trace_exception(ex) - self.warning("Couldn't add long term memory information to the context. Please verify the vector database") # Add information about the user - self.personality.step_end("Adding skills") - self.personality.step_end("Querying skills library",False) + #User description user_description="" if self.config.use_user_informations_in_discussion: user_description=f"{self.start_header_id_template}User description{self.end_header_id_template}\n"+self.config.user_description+"\n" @@ -1396,13 +1213,6 @@ class LollmsApplication(LoLLMsCom): tokens_documentation = [] n_doc_tk = 0 - # Tokenize the knowledge text and calculate its number of tokens - if len(knowledge)>0: - tokens_history = self.model.tokenize(knowledge) - n_history_tk = len(tokens_history) - else: - tokens_history = [] - n_history_tk = 0 # Tokenize user description @@ -1415,7 +1225,7 @@ class LollmsApplication(LoLLMsCom): # Calculate the total number of tokens between conditionning, documentation, and knowledge - total_tokens = n_cond_tk + n_isearch_tk + n_doc_tk + n_history_tk + n_user_description_tk + n_positive_boost + n_negative_boost + n_fun_mode + total_tokens = n_cond_tk + n_isearch_tk + n_doc_tk + n_user_description_tk + n_positive_boost + n_negative_boost + n_fun_mode # Calculate the available space for the messages available_space = self.config.ctx_size - n_tokens - total_tokens @@ -1530,8 +1340,6 @@ class LollmsApplication(LoLLMsCom): "internet_search_results":internet_search_results, "documentation":documentation, "documentation_entries":documentation_entries, - "knowledge":knowledge, - "knowledge_infos":knowledge_infos, "user_description":user_description, "discussion_messages":discussion_messages, "positive_boost":positive_boost, diff --git a/lollms/configs/config.yaml b/lollms/configs/config.yaml index 882651c..0c7fb04 100644 --- a/lollms/configs/config.yaml +++ b/lollms/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 147 +version: 148 # video viewing and news recovering last_viewed_video: null @@ -279,10 +279,14 @@ audio_auto_send_input: true audio_silenceTimer: 5000 # relmote databases -remote_databases: [] # This is the list of remote databases addresses in form database_type::database url +# This is the list of datalakes to be used for RAG +# Datalakes hae the following entries +# +datalakes: [] # Data vectorization -rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data +rag_local_services: [] # This is the list of rag services served locally + rag_vectorizer: semantic # possible values semantic, tfidf, openai, ollama rag_service_url: "http://localhost:11434" # rag service url for ollama rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable diff --git a/lollms/databases/remote_databases/light_rag_database.py b/lollms/databases/remote_databases/light_rag_database.py index adde5f0..b277bdc 100644 --- a/lollms/databases/remote_databases/light_rag_database.py +++ b/lollms/databases/remote_databases/light_rag_database.py @@ -1,7 +1,7 @@ import requests from typing import Optional, Any, List, Dict from enum import Enum -from lollms.databases.remote_databases.lollms_database import LollmsDatabase +from lollms.databases.datalakes.lollms_database import LollmsDatabase class SearchMode(str, Enum): naive = "naive" diff --git a/lollms/personality.py b/lollms/personality.py index dc960f5..387be9c 100644 --- a/lollms/personality.py +++ b/lollms/personality.py @@ -262,7 +262,6 @@ class AIPersonality: context_details["conditionning"], context_details["internet_search_results"], context_details["documentation"], - context_details["knowledge"], context_details["user_description"], context_details["discussion_messages"], context_details["positive_boost"], @@ -3759,12 +3758,6 @@ class APScript(StateMachine): ])) sacrifice_id += 1 - if context_details["knowledge"] and "knowledge" not in suppress: - full_context.append( self.separator_template.join([ - self.system_custom_header("knowledge"), - context_details["knowledge"] - ])) - sacrifice_id += 1 if context_details["user_description"] and "user_description" not in suppress: full_context.append( self.separator_template.join([ diff --git a/lollms/server/endpoints/lollms_file_system.py b/lollms/server/endpoints/lollms_file_system.py index 93b896e..80b8f91 100644 --- a/lollms/server/endpoints/lollms_file_system.py +++ b/lollms/server/endpoints/lollms_file_system.py @@ -175,13 +175,13 @@ def select_rag_database(client) -> Optional[Dict[str, Path]]: vdb.build_index() ASCIIColors.success("OK") lollmsElfServer.HideBlockingMessage() - run_async(partial(lollmsElfServer.sio.emit,'rag_db_added', {"database_name": db_name, "database_path": str(folder_path)}, to=client.client_id)) + run_async(partial(lollmsElfServer.sio.emit,'rag_db_added', {"datalake_name": db_name, "path": str(folder_path)}, to=client.client_id)) except Exception as ex: trace_exception(ex) lollmsElfServer.HideBlockingMessage() - return {"database_name": db_name, "database_path": Path(folder_path)} + return {"datalake_name": db_name, "database_path": Path(folder_path)} else: return None else: @@ -221,7 +221,7 @@ def find_rag_database_by_name(entries: List[Dict[str, Union[str, bool]]], name: if entry['alias'] == name: # For remote databases, return the URL, for local ones return the type - return i, entry.get('url', '') or entry.get('type', '') + return i, entry return -1, "" @@ -237,7 +237,7 @@ class FolderInfos(BaseModel): class MountDatabase(BaseModel): client_id: str - database_name:str + datalake_name:str class FolderOpenRequest(BaseModel): @@ -281,65 +281,66 @@ def toggle_mount_rag_database(database_infos: MountDatabase): Selects and names a database """ client = check_access(lollmsElfServer, database_infos.client_id) - index, path = find_rag_database_by_name(lollmsElfServer.config.rag_databases, database_infos.database_name) + index, db_entry = find_rag_database_by_name(lollmsElfServer.config.datalakes, database_infos.datalake_name) - if index < 0: - # Check remote databases - index, path = find_rag_database_by_name(lollmsElfServer.config.remote_databases, database_infos.database_name) - db_entry = lollmsElfServer.config.remote_databases[index] - else: - # Local database - db_entry = lollmsElfServer.config.rag_databases[index] if not db_entry['mounted']: def process(): try: - if not db_entry['is_local']: # Remote database + if db_entry['type']=="lightrag": lollmsElfServer.ShowBlockingMessage(f"Mounting database {db_entry['alias']}") - lollmsElfServer.config.remote_databases[index]['mounted'] = True - lollmsElfServer.config.save_config() - lollmsElfServer.info(f"Database {database_infos.database_name} mounted successfully") - lollmsElfServer.HideBlockingMessage() - else: # Local database - lollmsElfServer.ShowBlockingMessage(f"Mounting database {db_entry['alias']}") - lollmsElfServer.config.rag_databases[index]['mounted'] = True - - from lollmsvectordb import VectorDatabase - from lollmsvectordb.text_document_loader import TextDocumentsLoader - from lollmsvectordb.lollms_tokenizers.tiktoken_tokenizer import TikTokenTokenizer - - # Vectorizer selection logic - if lollmsElfServer.config.rag_vectorizer == "semantic": - from lollmsvectordb.lollms_vectorizers.semantic_vectorizer import SemanticVectorizer - v = SemanticVectorizer(lollmsElfServer.config.rag_vectorizer_model, - lollmsElfServer.config.rag_vectorizer_execute_remote_code) - elif lollmsElfServer.config.rag_vectorizer == "tfidf": - from lollmsvectordb.lollms_vectorizers.tfidf_vectorizer import TFIDFVectorizer - v = TFIDFVectorizer() - elif lollmsElfServer.config.rag_vectorizer == "openai": - from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer - v = OpenAIVectorizer(lollmsElfServer.config.rag_vectorizer_openai_key) - elif lollmsElfServer.config.rag_vectorizer == "ollama": - from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer - v = OllamaVectorizer(lollmsElfServer.config.rag_vectorizer_model, - lollmsElfServer.config.rag_service_url) - - vdb = VectorDatabase( - Path(path)/f"{database_infos.database_name}.sqlite", - v, - lollmsElfServer.model if lollmsElfServer.model else TikTokenTokenizer(), - chunk_size=lollmsElfServer.config.rag_chunk_size, - clean_chunks=lollmsElfServer.config.rag_clean_chunks, - n_neighbors=lollmsElfServer.config.rag_n_chunks - ) - lollmsElfServer.active_rag_dbs.append({ - "name": database_infos.database_name, - "path": path, - "vectorizer": vdb + from lollmsvectordb.database_clients.lightrag_client import LollmsLightRagConnector + lr = LollmsLightRagConnector(db_entry['url'], db_entry['key']) + lollmsElfServer.config.datalakes[index]['mounted'] = True + lollmsElfServer.active_datalakes.append(lollmsElfServer.config.datalakes[index] | { + "binding": lr }) lollmsElfServer.config.save_config() - lollmsElfServer.info(f"Database {database_infos.database_name} mounted successfully") + lollmsElfServer.info(f"Datalake {database_infos.datalake_name} mounted successfully") lollmsElfServer.HideBlockingMessage() + if db_entry['type']=="lollmsvectordb": + lollmsElfServer.ShowBlockingMessage(f"Mounting database {db_entry['alias']}") + try: + from lollmsvectordb import VectorDatabase + from lollmsvectordb.text_document_loader import TextDocumentsLoader + from lollmsvectordb.lollms_tokenizers.tiktoken_tokenizer import TikTokenTokenizer + + # Vectorizer selection logic + if lollmsElfServer.config.rag_vectorizer == "semantic": + from lollmsvectordb.lollms_vectorizers.semantic_vectorizer import SemanticVectorizer + v = SemanticVectorizer(lollmsElfServer.config.rag_vectorizer_model, + lollmsElfServer.config.rag_vectorizer_execute_remote_code) + elif lollmsElfServer.config.rag_vectorizer == "tfidf": + from lollmsvectordb.lollms_vectorizers.tfidf_vectorizer import TFIDFVectorizer + v = TFIDFVectorizer() + elif lollmsElfServer.config.rag_vectorizer == "openai": + from lollmsvectordb.lollms_vectorizers.openai_vectorizer import OpenAIVectorizer + v = OpenAIVectorizer(lollmsElfServer.config.rag_vectorizer_openai_key) + elif lollmsElfServer.config.rag_vectorizer == "ollama": + from lollmsvectordb.lollms_vectorizers.ollama_vectorizer import OllamaVectorizer + v = OllamaVectorizer(lollmsElfServer.config.rag_vectorizer_model, + lollmsElfServer.config.rag_service_url) + + vdb = VectorDatabase( + Path(db_entry['path'])/f"{database_infos.datalake_name}.sqlite", + v, + lollmsElfServer.model if lollmsElfServer.model else TikTokenTokenizer(), + chunk_size=lollmsElfServer.config.rag_chunk_size, + clean_chunks=lollmsElfServer.config.rag_clean_chunks, + n_neighbors=lollmsElfServer.config.rag_n_chunks + ) + lollmsElfServer.config.datalakes[index]['mounted'] = True + lollmsElfServer.active_datalakes.append(lollmsElfServer.config.datalakes[index] | { + "binding": vdb + }) + lollmsElfServer.config.save_config() + lollmsElfServer.info(f"Database {database_infos.datalake_name} mounted successfully") + lollmsElfServer.HideBlockingMessage() + except Exception as ex: + trace_exception(ex) + lollmsElfServer.error(f"Database {database_infos.datalake_name} couldn't be mounted!!\n{ex}\nTry reindexing the database.") + lollmsElfServer.HideBlockingMessage() + except Exception as ex: trace_exception(ex) lollmsElfServer.HideBlockingMessage() @@ -348,30 +349,36 @@ def toggle_mount_rag_database(database_infos: MountDatabase): lollmsElfServer.rag_thread.start() else: # Unmounting logic - if not db_entry['is_local']: # Remote database - lollmsElfServer.config.remote_databases[index]['mounted'] = False - lollmsElfServer.config.save_config() - else: # Local database - lollmsElfServer.config.rag_databases[index]['mounted'] = False - lollmsElfServer.active_rag_dbs = [ - db for db in lollmsElfServer.active_rag_dbs - if db["name"] != database_infos.database_name + if db_entry['type']=="lightrag": + lollmsElfServer.config.datalakes[index]['mounted'] = False + lollmsElfServer.active_datalakes = [ + db for db in lollmsElfServer.active_datalakes + if db["alias"] != database_infos.datalake_name ] lollmsElfServer.config.save_config() + lollmsElfServer.info(f"Datalake {database_infos.datalake_name} unmounted successfully") + elif db_entry['type']=="lollmsvectordb": + lollmsElfServer.config.datalakes[index]['mounted'] = False + lollmsElfServer.active_datalakes = [ + db for db in lollmsElfServer.active_datalakes + if db["alias"] != database_infos.datalake_name + ] + lollmsElfServer.config.save_config() + lollmsElfServer.info(f"Datalake {database_infos.datalake_name} unmounted successfully") @router.post("/upload_files_2_rag_db") async def upload_files_2_rag_db(database_infos: FolderInfos): client = check_access(lollmsElfServer, database_infos.client_id) - index, path = find_rag_database_by_name(lollmsElfServer.config.rag_databases, database_infos.database_name) + index, path = find_rag_database_by_name(lollmsElfServer.config.datalakes, database_infos.datalake_name) if index < 0: # Check remote databases - index, path = find_rag_database_by_name(lollmsElfServer.config.remote_databases, database_infos.database_name) - db_entry = lollmsElfServer.config.remote_databases[index] + index, path = find_rag_database_by_name(lollmsElfServer.config.datalakes, database_infos.datalake_name) + db_entry = lollmsElfServer.config.datalakes[index] else: # Local database - db_entry = lollmsElfServer.config.rag_databases[index] + db_entry = lollmsElfServer.config.datalakes[index] @router.post("/vectorize_folder") async def vectorize_folder(database_infos: FolderInfos): @@ -443,7 +450,7 @@ async def vectorize_folder(database_infos: FolderInfos): vdb.build_index() ASCIIColors.success("OK") lollmsElfServer.HideBlockingMessage() - run_async(partial(lollmsElfServer.sio.emit,'rag_db_added', {"database_name": db_name, "database_path": str(folder_path)}, to=client.client_id)) + run_async(partial(lollmsElfServer.sio.emit,'rag_db_added', {"datalake_name": db_name, "path": str(folder_path)}, to=client.client_id)) except Exception as ex: trace_exception(ex)