ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-18 20:27:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

upgraded naming and configurations

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-12-08 02:19:28 +01:00
parent 1a26259e86
commit 1fa9b0b37b
10 changed files with 22 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
configs/config.yaml
View File

@ -281,7 +281,7 @@ audio_silenceTimer: 5000
# Data vectorization # Data vectorization
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
rag_vectorizer: semantic # possible values semantic, tfidf, openai rag_vectorizer: semantic # possible values semantic, tfidf, openai
rag_vectorizer_model: sentence-transformers/bert-base-nli-mean-tokens # The model name if applicable rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable
rag_vectorizer_parameters: null # Parameters of the model in json format rag_vectorizer_parameters: null # Parameters of the model in json format
rag_chunk_size: 512 # number of tokens per chunk rag_chunk_size: 512 # number of tokens per chunk
rag_overlap: 0 # number of tokens of overlap rag_overlap: 0 # number of tokens of overlap
@ -305,20 +305,9 @@ activate_skills_lib: false # Activate vectorizing previous conversations
skills_lib_database_name: "default" # Default skills database skills_lib_database_name: "default" # Default skills database
max_summary_size: 512 # in tokens max_summary_size: 512 # in tokens
data_vectorization_visualize_on_vectorization: false
data_vectorization_activate: true # To activate/deactivate data vectorization
data_vectorization_method: "tfidf_vectorizer" #"model_embedding" or "tfidf_vectorizer"
data_visualization_method: "PCA" #"PCA" or "TSNE"
data_vectorization_sentense_transformer_model: "BAAI/bge-m3" # you can use another model by setting its name here or its path
data_vectorization_save_db: true # For each new session, new files rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_chunk_size: 512 # chunk size rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs
# Activate internet search # Activate internet search
activate_internet_search: false activate_internet_search: false

4
elf_docker_cfg/personal/configs/lollms_elf_config.yaml
View File

@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files
data_vectorization_chunk_size: 512 # chunk size data_vectorization_chunk_size: 512 # chunk size
data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs data_vectorization_make_persistance: false # If true, the data will be persistant webween runs

4
elf_test_cfg/personal/configs/lollms_elf_config.yaml
View File

@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files
data_vectorization_chunk_size: 512 # chunk size data_vectorization_chunk_size: 512 # chunk size
data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs data_vectorization_make_persistance: false # If true, the data will be persistant webween runs

4
elf_test_cfg/personal/configs/lollms_elf_local_config.yaml
View File

@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files
data_vectorization_chunk_size: 512 # chunk size data_vectorization_chunk_size: 512 # chunk size
data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs data_vectorization_make_persistance: false # If true, the data will be persistant webween runs

10
lollms/app.py
View File

@ -1058,7 +1058,7 @@ class LollmsApplication(LoLLMsCom):
if not self.config.rag_deactivate: if not self.config.rag_deactivate:
if self.config.data_vectorization_build_keys_words: if self.config.rag_build_keys_words:
if discussion is None: if discussion is None:
discussion = self.recover_discussion(client_id) discussion = self.recover_discussion(client_id)
query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True) query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True)
@ -1068,7 +1068,7 @@ class LollmsApplication(LoLLMsCom):
try: try:
chunks:List[Chunk] = self.personality.persona_data_vectorizer.search(query, int(self.config.rag_n_chunks)) chunks:List[Chunk] = self.personality.persona_data_vectorizer.search(query, int(self.config.rag_n_chunks))
for chunk in chunks: for chunk in chunks:
if self.config.data_vectorization_put_chunk_informations_into_context: if self.config.rag_put_chunk_informations_into_context:
documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n" documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n"
else: else:
documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n"
@ -1089,7 +1089,7 @@ class LollmsApplication(LoLLMsCom):
if discussion is None: if discussion is None:
discussion = self.recover_discussion(client_id) discussion = self.recover_discussion(client_id)
if self.config.data_vectorization_build_keys_words: if self.config.rag_build_keys_words:
self.personality.step_start("Building vector store query") self.personality.step_start("Building vector store query")
q = f"{self.separator_template}".join([ q = f"{self.separator_template}".join([
f"{self.system_custom_header('instruction')}Read the entire discussion and rewrite the last prompt for someone who hasn't read the discussion.", f"{self.system_custom_header('instruction')}Read the entire discussion and rewrite the last prompt for someone who hasn't read the discussion.",
@ -1194,7 +1194,7 @@ class LollmsApplication(LoLLMsCom):
documentation += f"{self.separator_template}" documentation += f"{self.separator_template}"
if query is None: if query is None:
if self.config.data_vectorization_build_keys_words: if self.config.rag_build_keys_words:
self.personality.step_start("Building vector store query") self.personality.step_start("Building vector store query")
query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink) query = self.personality.fast_gen(f"{self.separator_template}{self.start_header_id_template}instruction: Read the discussion and rewrite the last prompt for someone who didn't read the entire discussion.\nDo not answer the prompt. Do not add explanations.{self.separator_template}{self.start_header_id_template}discussion:\n{discussion[-2048:]}{self.separator_template}{self.start_header_id_template}enhanced query: ", max_generation_size=256, show_progress=True, callback=self.personality.sink)
self.personality.step_end("Building vector store query") self.personality.step_end("Building vector store query")
@ -1233,7 +1233,7 @@ class LollmsApplication(LoLLMsCom):
try: try:
chunks:List[Chunk] = client.discussion.vectorizer.search(query, int(self.config.rag_n_chunks)) chunks:List[Chunk] = client.discussion.vectorizer.search(query, int(self.config.rag_n_chunks))
for chunk in chunks: for chunk in chunks:
if self.config.data_vectorization_put_chunk_informations_into_context: if self.config.rag_put_chunk_informations_into_context:
documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n" documentation += f"{self.start_header_id_template}document chunk{self.end_header_id_template}\ndocument title: {chunk.doc.title}\nchunk content:\n{chunk.text}\n"
else: else:
documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n" documentation += f"{self.start_header_id_template}chunk{self.end_header_id_template}\n{chunk.text}\n"

17
lollms/configs/config.yaml
View File

@ -281,7 +281,7 @@ audio_silenceTimer: 5000
# Data vectorization # Data vectorization
rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data rag_databases: [] # This is the list of paths to database sources. Each database is a folder containing data
rag_vectorizer: semantic # possible values semantic, tfidf, openai rag_vectorizer: semantic # possible values semantic, tfidf, openai
rag_vectorizer_model: sentence-transformers/bert-base-nli-mean-tokens # The model name if applicable rag_vectorizer_model: "BAAI/bge-m3" # The model name if applicable
rag_vectorizer_parameters: null # Parameters of the model in json format rag_vectorizer_parameters: null # Parameters of the model in json format
rag_chunk_size: 512 # number of tokens per chunk rag_chunk_size: 512 # number of tokens per chunk
rag_overlap: 0 # number of tokens of overlap rag_overlap: 0 # number of tokens of overlap
@ -305,20 +305,9 @@ activate_skills_lib: false # Activate vectorizing previous conversations
skills_lib_database_name: "default" # Default skills database skills_lib_database_name: "default" # Default skills database
max_summary_size: 512 # in tokens max_summary_size: 512 # in tokens
data_vectorization_visualize_on_vectorization: false
data_vectorization_activate: true # To activate/deactivate data vectorization
data_vectorization_method: "tfidf_vectorizer" #"model_embedding" or "tfidf_vectorizer"
data_visualization_method: "PCA" #"PCA" or "TSNE"
data_vectorization_sentense_transformer_model: "BAAI/bge-m3" # you can use another model by setting its name here or its path
data_vectorization_save_db: true # For each new session, new files rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_chunk_size: 512 # chunk size rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs
# Activate internet search # Activate internet search
activate_internet_search: false activate_internet_search: false

2
lollms/functions/knowledge/build_knowledge_db.py
View File

@ -90,7 +90,7 @@ def buildKnowledgeDB(llm:APScript, data_store:VectorDatabase, data_folder_path:s
qna_list=[] qna_list=[]
# Perform further processing with questions_vector # Perform further processing with questions_vector
for index, question in enumerate(questions_vector): for index, question in enumerate(questions_vector):
docs, sorted_similarities, document_ids = data_store.search(question, n_results=int(llm.personality_config.data_vectorization_nb_chunks)) docs, sorted_similarities, document_ids = data_store.search(question, n_results=int(llm.personality_config.rag_n_chunks))
if llm.personality_config.use_enhanced_mode: if llm.personality_config.use_enhanced_mode:
llm.step_start(f"Verifying RAG data_{index}") llm.step_start(f"Verifying RAG data_{index}")
prompt_text = """{llm.config.start_header_id_template}chunk: {{chunk}} prompt_text = """{llm.config.start_header_id_template}chunk: {{chunk}}

3
lollms/personality.py
View File

@ -1391,9 +1391,6 @@ Use this structure:
for f in files: for f in files:
text = dl.read_file(f) text = dl.read_file(f)
self.persona_data_vectorizer.add_document(f.name, text, f) self.persona_data_vectorizer.add_document(f.name, text, f)
# data_vectorization_chunk_size: 512 # chunk size
# data_vectorization_overlap_size: 128 # overlap between chunks size
# data_vectorization_nb_chunks: 2 # number of chunks to use
self.persona_data_vectorizer.build_index() self.persona_data_vectorizer.build_index()
else: else:

4
lollms/server/configs/config.yaml
View File

@ -314,8 +314,8 @@ data_vectorization_save_db: true # For each new session, new files
data_vectorization_chunk_size: 512 # chunk size data_vectorization_chunk_size: 512 # chunk size
data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs data_vectorization_make_persistance: false # If true, the data will be persistant webween runs

4
personal_data/configs/lollms_discord_local_config.yaml
View File

@ -279,8 +279,8 @@ data_vectorization_save_db: true # For each new session, new files
data_vectorization_chunk_size: 512 # chunk size data_vectorization_chunk_size: 512 # chunk size
data_vectorization_overlap_size: 128 # overlap between chunks size data_vectorization_overlap_size: 128 # overlap between chunks size
data_vectorization_nb_chunks: 2 # number of chunks to use data_vectorization_nb_chunks: 2 # number of chunks to use
data_vectorization_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation rag_put_chunk_informations_into_context: false # if true then each chunk will be preceded by its information which may waste some context space but allow the ai to point where it found th einformation
data_vectorization_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself. rag_build_keys_words: true # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
data_vectorization_make_persistance: false # If true, the data will be persistant webween runs data_vectorization_make_persistance: false # If true, the data will be persistant webween runs