fixed icons error

api/__init__.py

@@ -19,7 +19,8 @@ from lollms.binding import LOLLMSConfig, BindingBuilder, LLMBinding, ModelBuilde
 from lollms.paths import LollmsPaths
 from lollms.helpers import ASCIIColors, trace_exception
 from lollms.app import LollmsApplication
-from lollms.utilities import File64BitsManager, PromptReshaper
+from lollms.utilities import File64BitsManager, PromptReshaper, TextVectorizer
+
 import threading
 from tqdm import tqdm 
 import traceback
@@ -121,6 +122,9 @@ class LoLLMsAPPI(LollmsApplication):
         self.config_file_path = config_file_path
         self.cancel_gen = False
 
+        
+        self.discussion_store = None
+
         # Keeping track of current discussion and message
         self._current_user_message_id = 0
         self._current_ai_message_id = 0
@@ -1154,13 +1158,37 @@ class LoLLMsAPPI(LollmsApplication):
         if self.config["override_personality_model_parameters"]:
             conditionning = conditionning+ "\n!@>user description:\nName:"+self.config["user_name"]+"\n"+self.config["user_description"]+"\n"
 
-        if len(self.personality.files)>0 and self.personality.vectorizer:
+        str_docs = ""
+
+        if self.config.use_discussions_history:
+            if self.discussion_store is None:
+                self.discussion_store = TextVectorizer(self.config.data_vectorization_method, # supported "model_embedding" or "ftidf_vectorizer"
+                        model=self.model, #needed in case of using model_embedding
+                        database_path=self.lollms_paths.personal_databases_path/"discussionsdb.json",
+                        save_db=self.config.data_vectorization_save_db,
+                        visualize_data_at_startup=False,
+                        visualize_data_at_add_file=False,
+                        visualize_data_at_generate=False,
+                        data_visualization_method="PCA",
+                        database_dict=None)
+                corpus = self.db.export_all_as_markdown()
+                self.discussion_store.add_document("discussions", corpus, self.config.data_vectorization_chunk_size, self.config.data_vectorization_overlap_size )
+
             pr = PromptReshaper("{{conditionning}}\n!@>document chunks:\n{{doc}}\n{{content}}")
-            emb = self.personality.vectorizer.embed_query(message.content)
+            emb = self.discussion_store.embed_query(message.content)
-            docs, sorted_similarities = self.personality.vectorizer.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)     
+            docs, sorted_similarities = self.discussion_store.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)     
-            str_docs = ""
             for doc, infos in zip(docs, sorted_similarities):
                 str_docs+=f"document chunk:\nchunk path: {infos[0]}\nchunk content:{doc}"
+
+
+        if len(self.personality.files)>0 and self.personality.vectorizer:
+            emb = self.personality.vectorizer.embed_query(message.content)
+            docs, sorted_similarities = self.personality.vectorizer.recover_text(emb, top_k=self.config.data_vectorization_nb_chunks)
+            for doc, infos in zip(docs, sorted_similarities):
+                str_docs+=f"document chunk:\nchunk path: {infos[0]}\nchunk content:{doc}"
+
+        if str_docs!="":                
+            pr = PromptReshaper("{{conditionning}}\n!@>document chunks:\n{{doc}}\n{{content}}")
             discussion_messages = pr.build({
                                     "doc":str_docs,
                                     "conditionning":conditionning,
@@ -1494,7 +1522,7 @@ class LoLLMsAPPI(LollmsApplication):
                 self.connections[client_id]["current_discussion"].load_message(message_id)
                 self.connections[client_id]["generated_text"] = message.content
             else:
-                self.new_message(client_id, self.personality.name, "✍ please stand by ...")
+                self.new_message(client_id, self.personality.name, "✍ warming up ...")
             self.socketio.sleep(0.01)
 
             # prepare query and reception

api/db.py

@@ -249,6 +249,56 @@ class DiscussionsDB:
                 )
             discussions.append(discussion)
         return discussions
+    def export_all_as_markdown(self):
+        data = self.export_all_discussions_to_json()
+
+        # Initialize an empty result string
+        result = ''
+
+        # Iterate through discussions in the JSON data
+        for discussion in data:
+            # Extract the title
+            title = discussion['title']
+            # Append the title with '#' as Markdown heading
+            result += f'#{title}\n'
+
+            # Iterate through messages in the discussion
+            for message in discussion['messages']:
+                sender = message['sender']
+                content = message['content']
+                # Append the sender and content in a Markdown format
+                result += f'{sender}: {content}\n'
+
+        return result
+
+    def export_all_discussions_to_json(self):
+        # Convert the list of discussion IDs to a tuple
+        db_discussions = self.select(
+            f"SELECT * FROM discussion"
+        )
+        discussions = []
+        for row in db_discussions:
+            discussion_id = row[0]
+            discussion_title = row[1]
+            discussion = {"id": discussion_id, "title":discussion_title, "messages": []}
+            rows = self.select(f"SELECT sender, content, message_type, rank, parent_message_id, binding, model, personality, created_at, finished_generating_at FROM message WHERE discussion_id=?",(discussion_id,))
+            for message_row in rows:
+                sender = message_row[0]
+                content = message_row[1]
+                content_type = message_row[2]
+                rank = message_row[3]
+                parent_message_id = message_row[4]
+                binding = message_row[5]
+                model = message_row[6]
+                personality = message_row[7]
+                created_at = message_row[8]
+                finished_generating_at = message_row[9]
+                
+                discussion["messages"].append(
+                    {"sender": sender, "content": content, "type": content_type, "rank": rank, "parent_message_id": parent_message_id, "binding": binding, "model":model, "personality":personality, "created_at":created_at, "finished_generating_at": finished_generating_at}
+                )
+            discussions.append(discussion)
+        return discussions
 
     def export_discussions_to_json(self, discussions_ids:list):
         # Convert the list of discussion IDs to a tuple

app.py

@@ -14,7 +14,7 @@ __github__ = "https://github.com/ParisNeo/lollms-webui"
 __copyright__ = "Copyright 2023, "
 __license__ = "Apache 2.0"
 
-__version__ ="6.7Alpha1"
+__version__ ="6.7Alpha2"
 
 main_repo = "https://github.com/ParisNeo/lollms-webui.git"
 import os
@@ -310,6 +310,10 @@ class LoLLMsWebUI(LoLLMsAPPI):
         self.add_endpoint(
             "/list_personalities", "list_personalities", self.list_personalities, methods=["GET"]
         )
+        self.add_endpoint(
+            "/list_databases", "list_databases", self.list_databases, methods=["GET"]
+        )
+        
         
         self.add_endpoint(
             "/list_extensions_categories", "list_extensions_categories", self.list_extensions_categories, methods=["GET"]
@@ -1127,6 +1131,10 @@ class LoLLMsWebUI(LoLLMsAPPI):
             ASCIIColors.error(f"No personalities found. Using default one {ex}")
         return jsonify(personalities)
     
+    def list_databases(self):
+        databases = [f.name for f in self.lollms_paths.personal_databases_path.iterdir() if f.suffix==".db"]
+        return jsonify(databases)
+
 
     def list_extensions_categories(self):
         extensions_categories_dir = self.lollms_paths.extensions_zoo_path  # replace with the actual path to the models folder

configs/config.yaml

@@ -1,5 +1,5 @@
 # =================== Lord Of Large Language Models Configuration file =========================== 
-version: 24
+version: 25
 binding_name: null
 model_name: null
 
@@ -29,7 +29,6 @@ override_personality_model_parameters: false #if true the personality parameters
 
 extensions: []
 
-
 user_name: user
 user_description: ""
 use_user_name_in_discussions: false
@@ -57,6 +56,7 @@ audio_auto_send_input: true
 audio_silenceTimer: 5000
 
 # Data vectorization
+use_discussions_history: false # Activate vectorizing previous conversations
 use_files: true # Activate using files
 data_vectorization_activate: true # To activate/deactivate data vectorization
 data_vectorization_method: "ftidf_vectorizer" #"model_embedding" or "ftidf_vectorizer"

web/dist/index.html

@@ -6,8 +6,8 @@
     
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LoLLMS WebUI - Welcome</title>
-    <script type="module" crossorigin src="/assets/index-5dd957eb.js"></script>
+    <script type="module" crossorigin src="/assets/index-a7ee745c.js"></script>
-    <link rel="stylesheet" href="/assets/index-1e156293.css">
+    <link rel="stylesheet" href="/assets/index-a2b38a7a.css">
   </head>
   <body>
     <div id="app"></div>

web/src/components/ChoiceDialog.vue

@@ -13,8 +13,8 @@
               :class="{'selected-choice': choice === selectedChoice}"
               class="py-2 px-4 cursor-pointer hover:bg-gray-200 dark:hover:bg-gray-700"
             >
-              <span class="font-bold"> {{ choice.name }} </span><br>
+              <span class="font-bold"> {{ displayName(choice) }} </span><br>
-              <span class="text-xs text-gray-500"> {{ this.formatSize(choice.size) }}</span>
+              <span class="text-xs text-gray-500" v-if="choice.size"> {{ this.formatSize(choice.size) }}</span>
             </li>
           </ul>
         </div>
@@ -66,6 +66,16 @@ export default {
     };
   },
   methods: {
+    displayName(choice) {
+      console.log("choice:",choice)
+      if (typeof choice === 'string') {
+        return choice;
+      } else if (choice && choice.name) {
+        return choice.name;
+      } else {
+        return ''; 
+      }
+    },
     selectChoice(choice) {
       this.selectedChoice = choice; // Update the selectedChoice when a choice is clicked
       this.$emit("choice-selected", choice);

web/src/components/ExtensionEntry.vue

@@ -6,7 +6,6 @@
     :title="!extension.installed ? 'Not installed' : ''">
 
     <div :class="!extension.installed ? 'border-red-500' : ''">
-
       <div class="flex flex-row items-center  flex-shrink-0 gap-3">
         <img @click="toggleSelected" ref="imgElement" :src="getImgUrl()" @error="defaultImg($event)"
           class="w-10 h-10 rounded-full object-fill text-red-700 cursor-pointer">

web/src/components/Message.vue

@@ -192,7 +192,7 @@
 }
 </style>
 <script>
-import botImgPlaceholder from "../assets/logo.svg"
+import botImgPlaceholder from "../assets/logo.png"
 import userImgPlaceholder from "../assets/default_user.svg"
 const bUrl = import.meta.env.VITE_LOLLMS_API_BASEURL
 import { nextTick } from 'vue'
@@ -368,8 +368,10 @@ export default {
         },
         getImgUrl() {
             if (this.avatar) {
+                console.log("Avatar:",bUrl + this.avatar)
                 return bUrl + this.avatar
             }
+            console.log("No avatar found")
             return botImgPlaceholder;
 
         },

web/src/main.js

@@ -40,7 +40,7 @@ export const store = createStore({
         ramUsage:null,
         vramUsage:null,
         extensionsZoo:[],
-        activeExtensions:[],
+        databases:[],
       }
     },
     mutations: {      
@@ -84,12 +84,13 @@ export const store = createStore({
         state.vramUsage = vramUsage;
       },
       
-      setActiveExtensions(state, activeExtensions) {
-        state.activeExtensions = activeExtensions;
-      },
       setExtensionsZoo(state, extensionsZoo) {
         state.extensionsZoo = extensionsZoo;
       },
+      setDatabases(state, databases) {
+        state.databases = databases;
+      },
+      
       // increment (state) {
       //   state.count++
       // }    
@@ -129,8 +130,8 @@ export const store = createStore({
         return state.vramUsage;
       },
       
-      getActiveExtensions(state) {
+      getDatabasesList(state){
-        return state.activeExtensions;
+        return state.databases;
       },
       getExtensionsZoo(state) {
         return state.extensionsZoo;
@@ -172,6 +173,13 @@ export const store = createStore({
           // Handle error
         }
       },
+      async refreshDatabase({ commit }) {
+        let databases = await api_get_req("list_databases")
+        console.log("databases:",databases)
+        commit('setDatabases', databases);
+      },
+
+
       async refreshPersonalitiesZoo({ commit }) {
           let personalities = []
           const catdictionary = await api_get_req("get_all_personalities")
@@ -433,6 +441,8 @@ app.mixin({
       actionsExecuted = true;
       console.log("Calling")
       await this.$store.dispatch('refreshConfig');
+      await this.$store.dispatch('refreshDatabase');
+      
       console.log("recovered config : ${}");
       await this.$store.dispatch('getVersion');
       console.log("recovered version");          

web/src/views/DiscussionsView.vue

@@ -69,7 +69,7 @@
                         <i data-feather="refresh-ccw"></i>
                     </button>
                     <button class="text-2xl hover:text-secondary duration-75 active:scale-90" title="Export database"
-                        type="button">
+                        type="button" @click.stop="database_selectorDialogVisible=true">
                         <i data-feather="database"></i>
                     </button>
                     <input type="file" ref="fileDialog" style="display: none" @change="importDiscussions" />
@@ -271,6 +271,13 @@
     <Toast ref="toast">
     </Toast>
     <MessageBox ref="messageBox" />
+    <ChoiceDialog reference="database_selector" class="z-20"
+      :show="database_selectorDialogVisible"
+      :choices="databases"
+      @choice-selected="ondatabase_selectorDialogSelected"
+      @close-dialog="onclosedatabase_selectorDialog"
+      @choice-validated="onvalidatedatabase_selectorChoice"
+    />      
 </template>
 
 
@@ -404,6 +411,7 @@ export default {
             isDiscussionBottom: false,
             personalityAvatars: [], // object array of personality name: and avatar: props
             fileList: [],
+            database_selectorDialogVisible:false,
             isDragOverDiscussion: false,
             isDragOverChat: false,
             panelCollapsed: false, // left panel collapse
@@ -411,6 +419,12 @@ export default {
         }
     },
     methods: {
+        ondatabase_selectorDialogSelected(choice){
+            console.log("Selected:",choice)
+            this.$store.state.config.database=choice;
+        },
+        onclosedatabase_selectorDialog(){this.database_selectorDialogVisible=false;},
+        onvalidatedatabase_selectorChoice(){this.database_selectorDialogVisible=false;},
         save_configuration() {
             this.showConfirmation = false
             axios.post('/save_settings', {})
@@ -1626,7 +1640,11 @@ export default {
     async activated() {
         //console.log('settings changed acc', this.$store.state.settingsChanged)
         // await this.getPersonalityAvatars()
+        while (this.isReady === false) {
+                await new Promise((resolve) => setTimeout(resolve, 100)); // Wait for 100ms
+            }  
         await this.getPersonalityAvatars()
+        console.log("Avatars found:",this.personalityAvatars)
         if (this.isCreated) {
            // this.loadLastUsedDiscussion()
             nextTick(() => {
@@ -1644,7 +1662,8 @@ export default {
         ChatBox,
         WelcomeComponent,
         Toast,
-        DragDrop
+        DragDrop,       
+        ChoiceDialog
     },
     watch: {
         filterTitle(newVal) {
@@ -1680,6 +1699,15 @@ export default {
         
     },
     computed: {
+        isReady:{
+            
+            get() {
+                return this.$store.state.ready;
+            },
+        },
+        databases(){            
+            return this.$store.state.databases;
+        },
         client_id() {
             return socket.id
         },
@@ -1731,6 +1759,8 @@ import { onMounted } from 'vue'
 import { initFlowbite } from 'flowbite'
 import { store } from '../main'
 
+import ChoiceDialog from '@/components/ChoiceDialog.vue'
+
 // initialize components based on data attribute selectors
 onMounted(() => {
     initFlowbite()

web/src/views/SettingsView.vue

@@ -780,6 +780,23 @@
                                         </td>
                                         </tr>
                                         <tr>
+                                        <td style="min-width: 200px;">
+                                            <label for="use_discussions_history" class="text-sm font-bold" style="margin-right: 1rem;">Activate discussion vectorization:</label>
+                                        </td>
+                                        <td>
+                                            <div class="flex flex-row">
+                                            <input
+                                            type="checkbox"
+                                            id="use_discussions_history"
+                                            required
+                                            v-model="configFile.use_discussions_history"
+                                            @change="settingsChanged=true"
+                                            class="mt-1 px-2 py-1 border border-gray-300 rounded  dark:bg-gray-600"
+                                            >
+                                            </div>
+                                        </td>
+                                        </tr>                                                                                 
+                                        <tr>
                                         <td style="min-width: 200px;">
                                             <label for="data_vectorization_activate" class="text-sm font-bold" style="margin-right: 1rem;">Activate files vectorization:</label>
                                         </td>
@@ -4043,7 +4060,7 @@ export default {
         },
         mountedExtensions:{
             get() {
-                return this.$store.state.activeExtensions;
+                return this.$store.state.config.extensions;
             },
             set(value) {
                 this.$store.commit('setActiveExtensions', value);