From 1fa9b0b37b2d4d6e710d677776334e00ed1333b8 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Sun, 8 Dec 2024 02:19:28 +0100 Subject: [PATCH] upgraded naming and configurations --- configs/config.yaml | 17 +++-------------- .../personal/configs/lollms_elf_config.yaml | 4 ++-- .../personal/configs/lollms_elf_config.yaml | 4 ++-- .../configs/lollms_elf_local_config.yaml | 4 ++-- lollms/app.py | 10 +++++----- lollms/configs/config.yaml | 17 +++-------------- .../functions/knowledge/build_knowledge_db.py | 2 +- lollms/personality.py | 3 --- lollms/server/configs/config.yaml | 4 ++-- .../configs/lollms_discord_local_config.yaml | 4 ++-- 10 files changed, 22 insertions(+), 47 deletions(-) diff --git a/configs/config.yaml b/configs/config.yaml index 3eff8d3..ca4467a 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -281,7 +281,7 @@ audio_silenceTimer: 5000 # Data vectorization rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data rag_vectorizer: semantic # possible values semantic, tfidf, openai -rag_vectorizer_model: sentence-transformers/bert-base-nli-mean-tokens # The model name if applicable +rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable rag_vectorizer_parameters: null # Parameters of the model in json format rag_chunk_size: 512 # number of tokens per chunk rag_overlap: 0 # number of tokens of overlap @@ -305,20 +305,9 @@ activate_skills_lib: false # Activate vectorizing previous conversations skills_lib_database_name: "default" # Default skills database max_summary_size: 512 # in tokens -data_vectorization_visualize_on_vectorization: false -data_vectorization_activate: true # To activate/deactivate data vectorization -data_vectorization_method: "tfidf_vectorizer" #"model_embedding" or "tfidf_vectorizer" -data_visualization_method: "PCA" #"PCA" or "TSNE" -data_vectorization_sentense_transformer_model: "BAAI/bge-m3" # you can use another model by setting its name here or its path -data_vectorization_save_db: true # For each new session, new files -data_vectorization_chunk_size: 512 # chunk size -data_vectorization_overlap_size: 128 # overlap between chunks size -data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. -data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used -data_vectorization_make_persistance: false # If true, the data will be persistant webween runs +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. # Activate internet search activate_internet_search: false diff --git a/elf_docker_cfg/personal/configs/lollms_elf_config.yaml b/elf_docker_cfg/personal/configs/lollms_elf_config.yaml index 725f0af..b8e7228 100644 --- a/elf_docker_cfg/personal/configs/lollms_elf_config.yaml +++ b/elf_docker_cfg/personal/configs/lollms_elf_config.yaml @@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files data_vectorization_chunk_size: 512 # chunk size data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_make_persistance: false # If true, the data will be persistant webween runs diff --git a/elf_test_cfg/personal/configs/lollms_elf_config.yaml b/elf_test_cfg/personal/configs/lollms_elf_config.yaml index 725f0af..b8e7228 100644 --- a/elf_test_cfg/personal/configs/lollms_elf_config.yaml +++ b/elf_test_cfg/personal/configs/lollms_elf_config.yaml @@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files data_vectorization_chunk_size: 512 # chunk size data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_make_persistance: false # If true, the data will be persistant webween runs diff --git a/elf_test_cfg/personal/configs/lollms_elf_local_config.yaml b/elf_test_cfg/personal/configs/lollms_elf_local_config.yaml index 725f0af..b8e7228 100644 --- a/elf_test_cfg/personal/configs/lollms_elf_local_config.yaml +++ b/elf_test_cfg/personal/configs/lollms_elf_local_config.yaml @@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files data_vectorization_chunk_size: 512 # chunk size data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_make_persistance: false # If true, the data will be persistant webween runs diff --git a/lollms/app.py b/lollms/app.py index da7701d..5192a81 100644 --- a/lollms/app.py +++ b/lollms/app.py @@ -1058,7 +1058,7 @@ class LollmsApplication(LoLLMsCom): if not self.config.rag_deactivate: - if self.config.data_vectorization_build_keys_words: + if self.config.rag_build_keys_words: if discussion is None: discussion = self.recover_discussion(client_id) query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True) @@ -1068,7 +1068,7 @@ class LollmsApplication(LoLLMsCom): try: chunks:List[Chunk] = self.personality.persona_data_vectorizer.search(query, int(self.config.rag_n_chunks)) for chunk in chunks: - if self.config.data_vectorization_put_chunk_informations_into_context: + if self.config.rag_put_chunk_informations_into_context: documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n" else: documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" @@ -1089,7 +1089,7 @@ class LollmsApplication(LoLLMsCom): if discussion is None: discussion = self.recover_discussion(client_id) - if self.config.data_vectorization_build_keys_words: + if self.config.rag_build_keys_words: self.personality.step_start("Building vector store query") q = f"{self.separator_template}".join([ f"{self.system_custom_header('instruction')}Read the entire discussion and rewrite the last prompt for someone who hasn't read the discussion.", @@ -1194,7 +1194,7 @@ class LollmsApplication(LoLLMsCom): documentation += f"{self.separator_template}" if query is None: - if self.config.data_vectorization_build_keys_words: + if self.config.rag_build_keys_words: self.personality.step_start("Building vector store query") query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink) self.personality.step_end("Building vector store query") @@ -1233,7 +1233,7 @@ class LollmsApplication(LoLLMsCom): try: chunks:List[Chunk] = client.discussion.vectorizer.search(query, int(self.config.rag_n_chunks)) for chunk in chunks: - if self.config.data_vectorization_put_chunk_informations_into_context: + if self.config.rag_put_chunk_informations_into_context: documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n" else: documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" diff --git a/lollms/configs/config.yaml b/lollms/configs/config.yaml index 3eff8d3..ca4467a 100644 --- a/lollms/configs/config.yaml +++ b/lollms/configs/config.yaml @@ -281,7 +281,7 @@ audio_silenceTimer: 5000 # Data vectorization rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data rag_vectorizer: semantic # possible values semantic, tfidf, openai -rag_vectorizer_model: sentence-transformers/bert-base-nli-mean-tokens # The model name if applicable +rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable rag_vectorizer_parameters: null # Parameters of the model in json format rag_chunk_size: 512 # number of tokens per chunk rag_overlap: 0 # number of tokens of overlap @@ -305,20 +305,9 @@ activate_skills_lib: false # Activate vectorizing previous conversations skills_lib_database_name: "default" # Default skills database max_summary_size: 512 # in tokens -data_vectorization_visualize_on_vectorization: false -data_vectorization_activate: true # To activate/deactivate data vectorization -data_vectorization_method: "tfidf_vectorizer" #"model_embedding" or "tfidf_vectorizer" -data_visualization_method: "PCA" #"PCA" or "TSNE" -data_vectorization_sentense_transformer_model: "BAAI/bge-m3" # you can use another model by setting its name here or its path -data_vectorization_save_db: true # For each new session, new files -data_vectorization_chunk_size: 512 # chunk size -data_vectorization_overlap_size: 128 # overlap between chunks size -data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. -data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used -data_vectorization_make_persistance: false # If true, the data will be persistant webween runs +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. # Activate internet search activate_internet_search: false diff --git a/lollms/functions/knowledge/build_knowledge_db.py b/lollms/functions/knowledge/build_knowledge_db.py index 2953dad..8f9d9df 100644 --- a/lollms/functions/knowledge/build_knowledge_db.py +++ b/lollms/functions/knowledge/build_knowledge_db.py @@ -90,7 +90,7 @@ def buildKnowledgeDB(llm:APScript, data_store:VectorDatabase, data_folder_path:s qna_list=[] # Perform further processing with questions_vector for index, question in enumerate(questions_vector): - docs, sorted_similarities, document_ids = data_store.search(question, n_results=int(llm.personality_config.data_vectorization_nb_chunks)) + docs, sorted_similarities, document_ids = data_store.search(question, n_results=int(llm.personality_config.rag_n_chunks)) if llm.personality_config.use_enhanced_mode: llm.step_start(f"Verifying RAG data_{index}") prompt_text = """{llm.config.start_header_id_template}chunk: {{chunk}} diff --git a/lollms/personality.py b/lollms/personality.py index 0a929c6..f3dd587 100644 --- a/lollms/personality.py +++ b/lollms/personality.py @@ -1391,9 +1391,6 @@ Use this structure: for f in files: text = dl.read_file(f) self.persona_data_vectorizer.add_document(f.name, text, f) - # data_vectorization_chunk_size: 512 # chunk size - # data_vectorization_overlap_size: 128 # overlap between chunks size - # data_vectorization_nb_chunks: 2 # number of chunks to use self.persona_data_vectorizer.build_index() else: diff --git a/lollms/server/configs/config.yaml b/lollms/server/configs/config.yaml index 2001ad4..ca67065 100644 --- a/lollms/server/configs/config.yaml +++ b/lollms/server/configs/config.yaml @@ -314,8 +314,8 @@ data_vectorization_save_db: true # For each new session, new files data_vectorization_chunk_size: 512 # chunk size data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_make_persistance: false # If true, the data will be persistant webween runs diff --git a/personal_data/configs/lollms_discord_local_config.yaml b/personal_data/configs/lollms_discord_local_config.yaml index e4b3ace..6e7f497 100644 --- a/personal_data/configs/lollms_discord_local_config.yaml +++ b/personal_data/configs/lollms_discord_local_config.yaml @@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files data_vectorization_chunk_size: 512 # chunk size data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_nb_chunks: 2 # number of chunks to use -data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation -data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. +rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation +rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_make_persistance: false # If true, the data will be persistant webween runs