upgraded vector db

configs/config.yaml

@@ -1,5 +1,5 @@
 # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== 
-version: 125
+version: 127
 binding_name: null
 model_name: null
 model_variant: null
@@ -153,6 +153,7 @@ xtts_top_k: 50
 xtts_top_p: 0.85
 xtts_speed: 1
 xtts_enable_text_splitting: true
+xtts_freq: 22050
 
 # openai_whisper configuration
 openai_tts_key: ""

endpoints/chat_bar.py

@@ -21,7 +21,6 @@ from lollms.databases.discussions_database import DiscussionsDB
 from lollms.types import SENDER_TYPES
 from typing import List
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil

endpoints/lollms_advanced.py

@@ -18,7 +18,6 @@ from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from lollms.client_session import Client
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil

endpoints/lollms_help.py

@@ -20,7 +20,6 @@ from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from lollms.client_session import Client
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil

endpoints/lollms_message.py

@@ -17,7 +17,6 @@ from lollms.utilities import detect_antiprompt, remove_text_from_string, trace_e
 from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from lollms.security import forbid_remote_access
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from typing import Any, Optional
 from pydantic import BaseModel, ValidationError

lollms_webui.py

@@ -27,7 +27,6 @@ import asyncio
 import os
 
 
-from safe_store import TextVectorizer, VectorizationMethod, VisualizationMethod
 import threading
 from tqdm import tqdm 
 import traceback
@@ -193,21 +192,6 @@ class LOLLMSWebUI(LOLLMSElfServer):
         self.db.add_missing_columns()
         ASCIIColors.success("ok")
 
-        # prepare vectorization
-        if self.config.data_vectorization_activate and self.config.activate_skills_lib:
-            try:
-                ASCIIColors.yellow("Loading long term memory")
-                folder = self.lollms_paths.personal_discussions_path/"vectorized_dbs"
-                folder.mkdir(parents=True, exist_ok=True)
-                self.build_long_term_skills_memory()
-                ASCIIColors.yellow("Ready")
-
-            except Exception as ex:
-                trace_exception(ex)
-                self.long_term_memory = None
-        else:
-            self.long_term_memory = None
-
         # This is used to keep track of messages 
         self.download_infos={}
 
@@ -274,9 +258,6 @@ class LOLLMSWebUI(LOLLMSElfServer):
             if self.check_module_update_(repo_path, branch_name):
                 return True
             repo_path = str(Path(__file__).parent/"lollms_core")
-            if self.check_module_update_(repo_path, branch_name):
-                return True
-            repo_path = str(Path(__file__).parent/"utilities/safe_store")
             if self.check_module_update_(repo_path, branch_name):
                 return True
             return False
@@ -451,6 +432,7 @@ class LOLLMSWebUI(LOLLMSElfServer):
                                 self.tts = LollmsXTTS(
                                                         self, 
                                                         voices_folders=[voices_folder, Path(__file__).parent.parent.parent/"services/xtts/voices"],
+                                                        freq=self.config.xtts_freq
                                                         )
                             
                         except Exception as ex:

scripts/linux/linux_install.sh

@@ -103,9 +103,6 @@ if [ -d "lollms-webui" ]; then
   git submodule update --init --recursive
   cd lollms_core 
   pip install -e .
-  cd ..
-  cd utilities/safe_store
-  pip install -e .
   cd ../..
   cd utilities/pipmaster
   pip install -e .
@@ -116,9 +113,6 @@ else
   git submodule update --init --recursive
   cd lollms-webui/lollms_core
   pip install -e .
-  cd ..
-  cd utilities/safe_store
-  pip install -e .
   cd ../..
 fi
 

scripts/macos/macos_install.sh

@@ -159,16 +159,12 @@ if [ -d "lollms-webui" ]; then
   git submodule update --init --recursive
   cd $INSTALL_DIR/lollms-webui/lollms_core 
   pip install -e .
-  cd $INSTALL_DIR/lollms-webui/utilities/safe_store
-  pip install -e .
 
 else
   git clone --depth 1  --recurse-submodules "$REPO_URL"
   git submodule update --init --recursive
   cd $INSTALL_DIR/lollms-webui/lollms_core
   pip install -e .
-  cd $INSTALL_DIR/lollms-webui/utilities/safe_store
-  pip install -e .
   cd $INSTALL_DIR/lollms-webui/utilities/pipmaster
   pip install -e .
 

scripts/windows/ui_installer/first_install.bat

@@ -100,9 +100,6 @@ git clone --depth 1  --recurse-submodules https://github.com/ParisNeo/lollms-web
 git submodule update --init --recursive
 cd lollms-webui\lollms_core
 pip install -e .
-cd ..
-cd utilities\safe_store
-pip install -e .
 cd ..\..
 
 pip install -r requirements.txt

scripts/windows/win_install.bat

@@ -92,18 +92,12 @@ if exist lollms-webui\ (
   cd
   cd lollms_core 
   pip install -e .
-  cd ..
-  cd utilities\safe_store
-  pip install -e .
   cd ..\..
 ) else (
   git clone --depth 1  --recurse-submodules https://github.com/ParisNeo/lollms-webui.git
   git submodule update --init --recursive
   cd lollms-webui\lollms_core
   pip install -e .
-  cd ..
-  cd utilities\safe_store
-  pip install -e .
   cd ..\..
   cd utilities\pipmaster
   pip install -e .

scripts/wsl/install_script.sh

@@ -51,9 +51,6 @@ echo "Initializing submodules"
 git submodule update --init --recursive
 cd lollms-webui\lollms_core
 pip install -e .
-cd ..
-cd utilities\safe_store
-pip install -e .
 cd ..\..
 
 # Install requirements

update_script.py

@@ -35,7 +35,6 @@ def run_git_pull():
             # Clone the repository to the target path
             ASCIIColors.info("Lollms_core found in the app space.\nPulling last lollms_core")
             subprocess.run(["git", "-C", str(execution_path/"lollms_core"), "pull"])            
-            subprocess.run(["git", "-C", str(execution_path/"utilities/safe_store"), "pull"])         
               
                     
         except Exception as ex:
@@ -50,7 +49,6 @@ def install_requirements():
     try:
         subprocess.check_call(["pip", "install", "--upgrade", "-r", "requirements.txt"])
         subprocess.check_call(["pip", "install", "--upgrade", "-e", "lollms_core"])
-        subprocess.check_call(["pip", "install", "--upgrade", "-e", "utilities/safe_store"])
     except subprocess.CalledProcessError as e:
         print(f"Error during pip install: {e}")
 

utilities/execution_engines/latex_execution_engine.py

@@ -16,7 +16,6 @@ from ascii_colors import get_trace_exception, trace_exception
 from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil

web/dist/index.html

@@ -6,8 +6,8 @@
     
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LoLLMS WebUI</title>
-    <script type="module" crossorigin src="/assets/index-e244d07f.js"></script>
+    <script type="module" crossorigin src="/assets/index-156a33f2.js"></script>
-    <link rel="stylesheet" href="/assets/index-5cbc5b69.css">
+    <link rel="stylesheet" href="/assets/index-4cfc40d0.css">
   </head>
   <body>
     <div id="app"></div>

web/src/views/SettingsView.vue

@@ -2293,6 +2293,25 @@
                                     </div>
                                 </td>
                                 </tr>
+                                <tr>
+                                <td style="min-width: 200px;">
+                                    <label for="xtts_freq" class="text-sm font-bold" style="margin-right: 1rem;">Frequency (controls the tone):</label>
+                                </td>
+                                <td>
+                                    <div class="flex flex-row">
+                                        <input
+                                            type="number"
+                                            id="xtts_freq"
+                                            required
+                                            v-model.number="configFile.xtts_freq"
+                                            @change="settingsChanged=true"
+                                            class="mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
+                                            step="0.01"
+                                        >
+                                    </div>
+                                </td>
+                                </tr>
+
                                 <tr>
                                 <td style="min-width: 200px;">
                                     <label for="auto_read" class="text-sm font-bold" style="margin-right: 1rem;">Enable auto read:</label>