mirror of
https://github.com/ParisNeo/lollms.git
synced 2024-12-18 20:27:58 +00:00
upgraded rag
This commit is contained in:
Saifeddine ALOUI
parent
25c9ee57da
commit
8ca1e3c2c0
@ -1,5 +1,5 @@
|
||||
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
||||
version: 113
|
||||
version: 114
|
||||
binding_name: null
|
||||
model_name: null
|
||||
model_variant: null
|
||||
@ -239,7 +239,7 @@ audio_auto_send_input: true
|
||||
audio_silenceTimer: 5000
|
||||
|
||||
# Data vectorization
|
||||
data_sources: [] # This is the list of paths to database sources. Each database is a folder containing data
|
||||
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
|
||||
|
||||
activate_skills_lib: false # Activate vectorizing previous conversations
|
||||
skills_lib_database_name: "default" # Default skills database
|
||||
|
||||
176
lollms/app.py
176
lollms/app.py
@ -27,6 +27,8 @@ import platform
|
||||
import gc
|
||||
import yaml
|
||||
import time
|
||||
from lollms.utilities import PackageManager
|
||||
|
||||
class LollmsApplication(LoLLMsCom):
|
||||
def __init__(
|
||||
self,
|
||||
@ -87,6 +89,20 @@ class LollmsApplication(LoLLMsCom):
|
||||
self.ttm = None
|
||||
self.ttv = None
|
||||
|
||||
self.active_rag_dbs = []
|
||||
for entry in self.config.rag_databases:
|
||||
if "mounted" in entry:
|
||||
parts = entry.split("::")
|
||||
if not PackageManager.check_package_installed("lollmsvectordb"):
|
||||
PackageManager.install_package("lollmsvectordb")
|
||||
|
||||
from lollmsvectordb.vectorizers.bert_vectorizer import BERTVectorizer
|
||||
from lollmsvectordb import VectorDatabase
|
||||
from lollmsvectordb.text_document_loader import TextDocumentsLoader
|
||||
v = BERTVectorizer()
|
||||
vdb = VectorDatabase(Path(parts[1])/"db_name.sqlite", v)
|
||||
self.active_rag_dbs.append({"name":parts[0],"path":parts[1],"vectorizer":vdb})
|
||||
|
||||
self.rt_com = None
|
||||
if not free_mode:
|
||||
try:
|
||||
@ -790,6 +806,14 @@ class LollmsApplication(LoLLMsCom):
|
||||
self.config.save_config()
|
||||
return True
|
||||
|
||||
def recover_discussion(self,client_id, message_index=-1):
|
||||
messages = self.session.get_client(client_id).discussion.get_messages()
|
||||
discussion=""
|
||||
for msg in messages:
|
||||
if message_index!=-1 and msg>message_index:
|
||||
break
|
||||
discussion += "\n" + self.config.discussion_prompt_separator + msg.sender + ": " + msg.content.strip()
|
||||
return discussion
|
||||
# -------------------------------------- Prompt preparing
|
||||
def prepare_query(self, client_id: str, message_id: int = -1, is_continue: bool = False, n_tokens: int = 0, generation_type = None, force_using_internet=False) -> Tuple[str, str, List[str]]:
|
||||
"""
|
||||
@ -966,74 +990,96 @@ class LollmsApplication(LoLLMsCom):
|
||||
except Exception as ex:
|
||||
trace_exception(ex)
|
||||
self.warning("Couldn't add documentation to the context. Please verify the vector database")
|
||||
|
||||
if not self.personality.ignore_discussion_documents_rag and (len(client.discussion.text_files) > 0) and client.discussion.vectorizer is not None:
|
||||
if discussion is None:
|
||||
discussion = self.recover_discussion(client_id)
|
||||
if not self.personality.ignore_discussion_documents_rag:
|
||||
query = None
|
||||
if len(self.active_rag_dbs) > 0 :
|
||||
if self.config.data_vectorization_build_keys_words:
|
||||
self.personality.step_start("Building vector store query")
|
||||
query = self.personality.fast_gen(f"{separator_template}{start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{separator_template}{start_header_id_template}discussion:\n{discussion[-2048:]}{separator_template}{start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink)
|
||||
self.personality.step_end("Building vector store query")
|
||||
ASCIIColors.cyan(f"Query: {query}")
|
||||
else:
|
||||
query = current_message.content
|
||||
if documentation=="":
|
||||
documentation=f"{separator_template}{start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.{separator_template}{start_header_id_template}Documentation:\n"
|
||||
results = []
|
||||
for db in self.active_rag_dbs:
|
||||
v = db["vectorizer"]
|
||||
r=v.search(query)
|
||||
results+=r
|
||||
n_neighbors = self.active_rag_dbs[0]["vectorizer"].n_neighbors
|
||||
sorted_results = sorted(results, key=lambda x: x[3])[:n_neighbors]
|
||||
for vector, text, title, distance in sorted_results:
|
||||
documentation += f"{start_header_id_template}document chunk{end_header_id_template}\nsource_document_title:{title}\ncontent:{text}\n"
|
||||
|
||||
if documentation=="":
|
||||
documentation=f"{separator_template}{start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.{separator_template}{start_header_id_template}Documentation:\n"
|
||||
|
||||
if self.config.data_vectorization_build_keys_words:
|
||||
self.personality.step_start("Building vector store query")
|
||||
query = self.personality.fast_gen(f"{separator_template}{start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{separator_template}{start_header_id_template}discussion:\n{discussion[-2048:]}{separator_template}{start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink)
|
||||
self.personality.step_end("Building vector store query")
|
||||
ASCIIColors.cyan(f"Query: {query}")
|
||||
else:
|
||||
query = current_message.content
|
||||
|
||||
try:
|
||||
if self.config.data_vectorization_force_first_chunk and len(client.discussion.vectorizer.chunks)>0:
|
||||
doc_index = list(client.discussion.vectorizer.chunks.keys())[0]
|
||||
|
||||
doc_id = client.discussion.vectorizer.chunks[doc_index]['document_id']
|
||||
content = client.discussion.vectorizer.chunks[doc_index]['chunk_text']
|
||||
|
||||
if self.config.data_vectorization_put_chunk_informations_into_context:
|
||||
documentation += f"{start_header_id_template}document chunk{end_header_id_template}\nchunk_infos:{doc_id}\ncontent:{content}\n"
|
||||
else:
|
||||
documentation += f"{start_header_id_template}chunk{end_header_id_template}\n{content}\n"
|
||||
|
||||
docs, sorted_similarities, document_ids = client.discussion.vectorizer.recover_text(query, top_k=int(self.config.data_vectorization_nb_chunks))
|
||||
for doc, infos in zip(docs, sorted_similarities):
|
||||
if self.config.data_vectorization_force_first_chunk and len(client.discussion.vectorizer.chunks)>0 and infos[0]==doc_id:
|
||||
continue
|
||||
if self.config.data_vectorization_put_chunk_informations_into_context:
|
||||
documentation += f"{start_header_id_template}document chunk{end_header_id_template}\nchunk path: {infos[0]}\nchunk content:\n{doc}\n"
|
||||
else:
|
||||
documentation += f"{start_header_id_template}chunk{end_header_id_template}\n{doc}\n"
|
||||
|
||||
documentation += f"{separator_template}{start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.\n"
|
||||
except Exception as ex:
|
||||
trace_exception(ex)
|
||||
self.warning("Couldn't add documentation to the context. Please verify the vector database")
|
||||
# Check if there is discussion knowledge to add to the prompt
|
||||
if self.config.activate_skills_lib:
|
||||
try:
|
||||
self.personality.step_start("Querying skills library")
|
||||
if (len(client.discussion.text_files) > 0) and client.discussion.vectorizer is not None:
|
||||
if discussion is None:
|
||||
discussion = self.recover_discussion(client_id)
|
||||
self.personality.step_start("Building query")
|
||||
query = self.personality.fast_gen(f"{start_header_id_template}{system_message_template}{end_header_id_template}Your task is to carefully read the provided discussion and reformulate {self.config.user_name}'s request concisely. Return only the reformulated request without any additional explanations, commentary, or output.{separator_template}{start_header_id_template}discussion:\n{discussion[-2048:]}{separator_template}{start_header_id_template}search query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink)
|
||||
self.personality.step_end("Building query")
|
||||
# skills = self.skills_library.query_entry(query)
|
||||
self.personality.step_start("Adding skills")
|
||||
if self.config.debug:
|
||||
ASCIIColors.info(f"Query : {query}")
|
||||
skill_titles, skills = self.skills_library.query_vector_db(query, top_k=3, max_dist=1000)#query_entry_fts(query)
|
||||
knowledge_infos={"titles":skill_titles,"contents":skills}
|
||||
if len(skills)>0:
|
||||
if knowledge=="":
|
||||
knowledge=f"{start_header_id_template}knowledge{end_header_id_template}\n"
|
||||
for i,(title, content) in enumerate(zip(skill_titles,skills)):
|
||||
knowledge += f"{start_header_id_template}knowledge {i}{end_header_id_template}\ntitle:\n{title}\ncontent:\n{content}\n"
|
||||
self.personality.step_end("Adding skills")
|
||||
self.personality.step_end("Querying skills library")
|
||||
except Exception as ex:
|
||||
ASCIIColors.error(ex)
|
||||
self.warning("Couldn't add long term memory information to the context. Please verify the vector database") # Add information about the user
|
||||
self.personality.step_end("Adding skills")
|
||||
self.personality.step_end("Querying skills library",False)
|
||||
|
||||
if documentation=="":
|
||||
documentation=f"{separator_template}{start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.{separator_template}{start_header_id_template}Documentation:\n"
|
||||
|
||||
if query is None:
|
||||
if self.config.data_vectorization_build_keys_words:
|
||||
self.personality.step_start("Building vector store query")
|
||||
query = self.personality.fast_gen(f"{separator_template}{start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{separator_template}{start_header_id_template}discussion:\n{discussion[-2048:]}{separator_template}{start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink)
|
||||
self.personality.step_end("Building vector store query")
|
||||
ASCIIColors.cyan(f"Query: {query}")
|
||||
else:
|
||||
query = current_message.content
|
||||
|
||||
try:
|
||||
if self.config.data_vectorization_force_first_chunk and len(client.discussion.vectorizer.chunks)>0:
|
||||
doc_index = list(client.discussion.vectorizer.chunks.keys())[0]
|
||||
|
||||
doc_id = client.discussion.vectorizer.chunks[doc_index]['document_id']
|
||||
content = client.discussion.vectorizer.chunks[doc_index]['chunk_text']
|
||||
|
||||
if self.config.data_vectorization_put_chunk_informations_into_context:
|
||||
documentation += f"{start_header_id_template}document chunk{end_header_id_template}\nchunk_infos:{doc_id}\ncontent:{content}\n"
|
||||
else:
|
||||
documentation += f"{start_header_id_template}chunk{end_header_id_template}\n{content}\n"
|
||||
|
||||
docs, sorted_similarities, document_ids = client.discussion.vectorizer.recover_text(query, top_k=int(self.config.data_vectorization_nb_chunks))
|
||||
for doc, infos in zip(docs, sorted_similarities):
|
||||
if self.config.data_vectorization_force_first_chunk and len(client.discussion.vectorizer.chunks)>0 and infos[0]==doc_id:
|
||||
continue
|
||||
if self.config.data_vectorization_put_chunk_informations_into_context:
|
||||
documentation += f"{start_header_id_template}document chunk{end_header_id_template}\nchunk path: {infos[0]}\nchunk content:\n{doc}\n"
|
||||
else:
|
||||
documentation += f"{start_header_id_template}chunk{end_header_id_template}\n{doc}\n"
|
||||
|
||||
documentation += f"{separator_template}{start_header_id_template}important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.\n"
|
||||
except Exception as ex:
|
||||
trace_exception(ex)
|
||||
self.warning("Couldn't add documentation to the context. Please verify the vector database")
|
||||
# Check if there is discussion knowledge to add to the prompt
|
||||
if self.config.activate_skills_lib:
|
||||
try:
|
||||
self.personality.step_start("Querying skills library")
|
||||
if discussion is None:
|
||||
discussion = self.recover_discussion(client_id)
|
||||
self.personality.step_start("Building query")
|
||||
query = self.personality.fast_gen(f"{start_header_id_template}{system_message_template}{end_header_id_template}Your task is to carefully read the provided discussion and reformulate {self.config.user_name}'s request concisely. Return only the reformulated request without any additional explanations, commentary, or output.{separator_template}{start_header_id_template}discussion:\n{discussion[-2048:]}{separator_template}{start_header_id_template}search query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink)
|
||||
self.personality.step_end("Building query")
|
||||
# skills = self.skills_library.query_entry(query)
|
||||
self.personality.step_start("Adding skills")
|
||||
if self.config.debug:
|
||||
ASCIIColors.info(f"Query : {query}")
|
||||
skill_titles, skills = self.skills_library.query_vector_db(query, top_k=3, max_dist=1000)#query_entry_fts(query)
|
||||
knowledge_infos={"titles":skill_titles,"contents":skills}
|
||||
if len(skills)>0:
|
||||
if knowledge=="":
|
||||
knowledge=f"{start_header_id_template}knowledge{end_header_id_template}\n"
|
||||
for i,(title, content) in enumerate(zip(skill_titles,skills)):
|
||||
knowledge += f"{start_header_id_template}knowledge {i}{end_header_id_template}\ntitle:\n{title}\ncontent:\n{content}\n"
|
||||
self.personality.step_end("Adding skills")
|
||||
self.personality.step_end("Querying skills library")
|
||||
except Exception as ex:
|
||||
ASCIIColors.error(ex)
|
||||
self.warning("Couldn't add long term memory information to the context. Please verify the vector database") # Add information about the user
|
||||
self.personality.step_end("Adding skills")
|
||||
self.personality.step_end("Querying skills library",False)
|
||||
user_description=""
|
||||
if self.config.use_user_informations_in_discussion:
|
||||
user_description=f"{start_header_id_template}User description{end_header_id_template}\n"+self.config.user_description+"\n"
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
||||
version: 113
|
||||
version: 114
|
||||
binding_name: null
|
||||
model_name: null
|
||||
model_variant: null
|
||||
@ -239,7 +239,7 @@ audio_auto_send_input: true
|
||||
audio_silenceTimer: 5000
|
||||
|
||||
# Data vectorization
|
||||
data_sources: [] # This is the list of paths to database sources. Each database is a folder containing data
|
||||
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
|
||||
|
||||
activate_skills_lib: false # Activate vectorizing previous conversations
|
||||
skills_lib_database_name: "default" # Default skills database
|
||||
|
||||
@ -1,5 +1,5 @@
|
||||
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
||||
version: 106
|
||||
version: 114
|
||||
binding_name: null
|
||||
model_name: null
|
||||
model_variant: null
|
||||
@ -31,7 +31,17 @@ app_custom_logo: ""
|
||||
discussion_prompt_separator: "!@>"
|
||||
start_header_id_template: "!@>"
|
||||
end_header_id_template: ": "
|
||||
|
||||
separator_template: "\n"
|
||||
|
||||
start_user_header_id_template: "!@>"
|
||||
end_user_header_id_template: ": "
|
||||
end_user_message_id_template: ""
|
||||
|
||||
start_ai_header_id_template: "!@>"
|
||||
end_ai_header_id_template: ": "
|
||||
end_ai_message_id_template: ""
|
||||
|
||||
system_message_template: "system"
|
||||
|
||||
seed: -1
|
||||
@ -152,10 +162,19 @@ openai_tts_voice: "alloy"
|
||||
|
||||
# ***************** TTI *****************
|
||||
|
||||
use_negative_prompt: true
|
||||
use_ai_generated_negative_prompt: false
|
||||
negative_prompt_generation_prompt: Generate negative prompt for the following prompt. negative prompt is a set of words that describe things we do not want to have in the generated image.
|
||||
default_negative_prompt: (((text))), (((ugly))), (((duplicate))), ((morbid)), ((mutilated)), out of frame, extra fingers, mutated hands, ((poorly drawn hands)), ((poorly drawn face)), (((mutation))), (((deformed))), blurry, ((bad anatomy)), (((bad proportions))), ((extra limbs)), cloned face, (((disfigured))), ((extra arms)), (((extra legs))), mutated hands, (fused fingers), (too many fingers), (((long neck))), ((watermark)), ((robot eyes))
|
||||
|
||||
# Image generation service
|
||||
enable_sd_service: false
|
||||
sd_base_url: http://localhost:7860
|
||||
|
||||
# Image generation service
|
||||
enable_fooocus_service: false
|
||||
fooocus_base_url: http://localhost:7860
|
||||
|
||||
# diffuser
|
||||
diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload
|
||||
diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
|
||||
@ -170,6 +189,7 @@ midjourney_key: ""
|
||||
# Image generation service comfyui
|
||||
enable_comfyui_service: false
|
||||
comfyui_base_url: http://127.0.0.1:8188/
|
||||
comfyui_model: v1-5-pruned-emaonly.ckpt
|
||||
|
||||
# Motion control service
|
||||
enable_motion_ctrl_service: false
|
||||
@ -219,6 +239,8 @@ audio_auto_send_input: true
|
||||
audio_silenceTimer: 5000
|
||||
|
||||
# Data vectorization
|
||||
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
|
||||
|
||||
activate_skills_lib: false # Activate vectorizing previous conversations
|
||||
skills_lib_database_name: "default" # Default skills database
|
||||
|
||||
@ -240,10 +262,11 @@ data_vectorization_make_persistance: false # If true, the data will be persistan
|
||||
|
||||
# Activate internet search
|
||||
activate_internet_search: false
|
||||
activate_internet_pages_judgement: true
|
||||
internet_vectorization_chunk_size: 512 # chunk size
|
||||
internet_vectorization_overlap_size: 128 # overlap between chunks size
|
||||
internet_vectorization_nb_chunks: 2 # number of chunks to use
|
||||
internet_nb_search_pages: 3 # number of pages to select
|
||||
internet_vectorization_overlap_size: 0 # overlap between chunks size
|
||||
internet_vectorization_nb_chunks: 4 # number of chunks to use
|
||||
internet_nb_search_pages: 8 # number of pages to select
|
||||
internet_quick_search: false # If active the search engine will not load and read the webpages
|
||||
internet_activate_search_decision: false # If active the ai decides by itself if it needs to do search
|
||||
# Helpers
|
||||
|
||||
@ -161,6 +161,23 @@ def select_rag_database() -> Optional[Dict[str, Path]]:
|
||||
except Exception as e:
|
||||
print(f"An error occurred: {e}")
|
||||
return None
|
||||
|
||||
def find_rag_database_by_name(entries: List[str], name: str) -> Optional[str]:
|
||||
"""
|
||||
Finds an entry in the list by its name.
|
||||
|
||||
Args:
|
||||
entries (List[str]): The list of entries in the form 'name::path'.
|
||||
name (str): The name to search for.
|
||||
|
||||
Returns:
|
||||
Optional[str]: The entry if found, otherwise None.
|
||||
"""
|
||||
for i, entry in enumerate(entries):
|
||||
entry_name, entry_path = entry.split('::')
|
||||
if entry_name == name:
|
||||
return i, entry_path
|
||||
return None
|
||||
# ----------------------------------- Personal files -----------------------------------------
|
||||
class SelectDatabase(BaseModel):
|
||||
client_id: str
|
||||
@ -209,6 +226,16 @@ def mount_rag_database(database_infos: MountDatabase):
|
||||
Selects and names a database
|
||||
"""
|
||||
client = check_access(lollmsElfServer, database_infos.client_id)
|
||||
client.rag_databases.append(database_infos.database_name)
|
||||
return select_rag_database()
|
||||
index, path = find_rag_database_by_name(lollmsElfServer.config.rag_databases,database_infos.database_name)
|
||||
if not lollmsElfServer.config.rag_databases[index].split("::")[-1]=="mounted":
|
||||
lollmsElfServer.config.rag_databases[index] = lollmsElfServer.config.rag_databases[index] + "::mounted"
|
||||
if not PackageManager.check_package_installed("lollmsvectordb"):
|
||||
PackageManager.install_package("lollmsvectordb")
|
||||
|
||||
from lollmsvectordb.vectorizers.bert_vectorizer import BERTVectorizer
|
||||
from lollmsvectordb import VectorDatabase
|
||||
from lollmsvectordb.text_document_loader import TextDocumentsLoader
|
||||
v = BERTVectorizer()
|
||||
vdb = VectorDatabase(Path(path)/"db_name.sqlite", v)
|
||||
lollmsElfServer.active_rag_dbs.append({"name":database_infos.database_name,"path":path,"vectorizer":vdb})
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user