upgraded

lollms/app.py

@@ -636,8 +636,9 @@ class LollmsApplication(LoLLMsCom):
                 try:
                     docs, sorted_similarities, document_ids = self.personality.persona_data_vectorizer.recover_text(query, top_k=self.config.data_vectorization_nb_chunks)
                     for doc, infos, doc_id in zip(docs, sorted_similarities, document_ids):
-                        documentation += f"document chunk:\nchunk_infos:{infos}\ncontent:{doc}"
-                except:
+                        documentation += f"!@>document chunk:\nchunk_infos:{infos}\ncontent:{doc}\n"
+                except Exception as ex:
+                    trace_exception(ex)
                     self.warning("Couldn't add documentation to the context. Please verify the vector database")
             
             if len(self.personality.text_files) > 0 and self.personality.vectorizer:
@@ -654,11 +655,21 @@ class LollmsApplication(LoLLMsCom):
                     query = current_message.content
 
                 try:
+                    if self.config.data_vectorization_force_first_chunk and len(self.personality.vectorizer.chunks)>0:
+                        doc_index = list(self.personality.vectorizer.chunks.keys())[0]
+
+                        doc_id = self.personality.vectorizer.chunks[doc_index]['document_id']
+                        content = self.personality.vectorizer.chunks[doc_index]['chunk_text']
+                        documentation += f"!@>document chunk:\nchunk_infos:{doc_id}\ncontent:{content}\n"
+
                     docs, sorted_similarities, document_ids = self.personality.vectorizer.recover_text(query, top_k=self.config.data_vectorization_nb_chunks)
                     for doc, infos in zip(docs, sorted_similarities):
-                        documentation += f"document chunk:\nchunk path: {infos[0]}\nchunk content:\n{doc}"
-                    documentation += "\n!@>important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation."
-                except:
+                        if self.config.data_vectorization_force_first_chunk and len(self.personality.vectorizer.chunks)>0 and infos[0]==doc_id:
+                            continue
+                        documentation += f"!@>document chunk:\nchunk path: {infos[0]}\nchunk content:\n{doc}\n"
+                    documentation += "\n!@>important information: Use the documentation data to answer the user questions. If the data is not present in the documentation, please tell the user that the information he is asking for does not exist in the documentation section. It is strictly forbidden to give the user an answer without having actual proof from the documentation.\n"
+                except Exception as ex:
+                    trace_exception(ex)
                     self.warning("Couldn't add documentation to the context. Please verify the vector database")
             # Check if there is discussion knowledge to add to the prompt
             if self.config.activate_skills_lib:

lollms/personality.py

@@ -1046,6 +1046,7 @@ class AIPersonality:
                 # self.ShowBlockingMessage("Adding file to vector store.\nPlease stand by")
                 self.text_files.append(path)
                 ASCIIColors.info("Received text compatible file")
+                self.ShowBlockingMessage("Processing file\nPlease wait ...")
                 if process:
                     if self.vectorizer is None:
                         self.vectorizer = TextVectorizer(

lollms/server/events/lollms_personality_events.py

@@ -104,7 +104,8 @@ def add_events(sio:socketio):
 
         if is_last_chunk:
             lollmsElfServer.success('File received and saved successfully')
-            lollmsElfServer.ShowBlockingMessage(f"File received {file_path.name}.\nProcessing ...")
+            lollmsElfServer.HideBlockingMessage()
+            lollmsElfServer.ShowBlockingMessage(f"File received {file_path.name}.\nVectorizing the data ...")
 
             if lollmsElfServer.personality.processor:
                 result = lollmsElfServer.personality.processor.add_file(file_path, client, partial(lollmsElfServer.process_chunk, client_id=client_id))