upgraded vector db

2024-12-18 20:17:50 +00:00 · 2024-07-18 01:32:20 +02:00 · 2024-07-18 01:32:20 +02:00 · a631f73da1
commit a631f73da1
parent 6f4e38f51f
21 changed files with 218 additions and 245 deletions
--- a/configs/config.yaml
+++ b/configs/config.yaml
@ -1,5 +1,5 @@
 # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== 
-version: 125
+version: 127
 binding_name: null
 model_name: null
 model_variant: null
@ -153,6 +153,7 @@ xtts_top_k: 50
 xtts_top_p: 0.85
 xtts_speed: 1
 xtts_enable_text_splitting: true
+xtts_freq: 22050

 # openai_whisper configuration
 openai_tts_key: ""
--- a/endpoints/chat_bar.py
+++ b/endpoints/chat_bar.py
@ -21,7 +21,6 @@ from lollms.databases.discussions_database import DiscussionsDB
 from lollms.types import SENDER_TYPES
 from typing import List
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil
--- a/endpoints/lollms_advanced.py
+++ b/endpoints/lollms_advanced.py
@ -18,7 +18,6 @@ from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from lollms.client_session import Client
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil
--- a/endpoints/lollms_help.py
+++ b/endpoints/lollms_help.py
@ -20,7 +20,6 @@ from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from lollms.client_session import Client
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil
--- a/endpoints/lollms_message.py
+++ b/endpoints/lollms_message.py
@ -17,7 +17,6 @@ from lollms.utilities import detect_antiprompt, remove_text_from_string, trace_e
 from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from lollms.security import forbid_remote_access
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from typing import Any, Optional
 from pydantic import BaseModel, ValidationError
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 6f409816515813f8b302cdfec9b9276ad16f1367
+Subproject commit 1437b2c40d28e40b2e9965fac399e30f46f1e1a2
--- a/lollms_webui.py
+++ b/lollms_webui.py
@ -27,7 +27,6 @@ import asyncio
 import os


-from safe_store import TextVectorizer, VectorizationMethod, VisualizationMethod
 import threading
 from tqdm import tqdm 
 import traceback
@ -193,21 +192,6 @@ class LOLLMSWebUI(LOLLMSElfServer):
        self.db.add_missing_columns()
        ASCIIColors.success("ok")

-        # prepare vectorization
-        if self.config.data_vectorization_activate and self.config.activate_skills_lib:
-            try:
-                ASCIIColors.yellow("Loading long term memory")
-                folder = self.lollms_paths.personal_discussions_path/"vectorized_dbs"
-                folder.mkdir(parents=True, exist_ok=True)
-                self.build_long_term_skills_memory()
-                ASCIIColors.yellow("Ready")
-
-            except Exception as ex:
-                trace_exception(ex)
-                self.long_term_memory = None
-        else:
-            self.long_term_memory = None
-
        # This is used to keep track of messages 
        self.download_infos={}

@ -274,9 +258,6 @@ class LOLLMSWebUI(LOLLMSElfServer):
            if self.check_module_update_(repo_path, branch_name):
                return True
            repo_path = str(Path(__file__).parent/"lollms_core")
-            if self.check_module_update_(repo_path, branch_name):
-                return True
-            repo_path = str(Path(__file__).parent/"utilities/safe_store")
            if self.check_module_update_(repo_path, branch_name):
                return True
            return False
@ -451,6 +432,7 @@ class LOLLMSWebUI(LOLLMSElfServer):
                                self.tts = LollmsXTTS(
                                                        self, 
                                                        voices_folders=[voices_folder, Path(__file__).parent.parent.parent/"services/xtts/voices"],
+                                                        freq=self.config.xtts_freq
                                                        )
                            
                        except Exception as ex:
--- a/scripts/linux/linux_install.sh
+++ b/scripts/linux/linux_install.sh
@ -103,9 +103,6 @@ if [ -d "lollms-webui" ]; then
  git submodule update --init --recursive
  cd lollms_core 
  pip install -e .
-  cd ..
-  cd utilities/safe_store
-  pip install -e .
  cd ../..
  cd utilities/pipmaster
  pip install -e .
@ -116,9 +113,6 @@ else
  git submodule update --init --recursive
  cd lollms-webui/lollms_core
  pip install -e .
-  cd ..
-  cd utilities/safe_store
-  pip install -e .
  cd ../..
 fi

--- a/scripts/macos/macos_install.sh
+++ b/scripts/macos/macos_install.sh
@ -159,16 +159,12 @@ if [ -d "lollms-webui" ]; then
  git submodule update --init --recursive
  cd $INSTALL_DIR/lollms-webui/lollms_core 
  pip install -e .
-  cd $INSTALL_DIR/lollms-webui/utilities/safe_store
-  pip install -e .

 else
  git clone --depth 1  --recurse-submodules "$REPO_URL"
  git submodule update --init --recursive
  cd $INSTALL_DIR/lollms-webui/lollms_core
  pip install -e .
-  cd $INSTALL_DIR/lollms-webui/utilities/safe_store
-  pip install -e .
  cd $INSTALL_DIR/lollms-webui/utilities/pipmaster
  pip install -e .

--- a/scripts/windows/ui_installer/first_install.bat
+++ b/scripts/windows/ui_installer/first_install.bat
@ -100,9 +100,6 @@ git clone --depth 1  --recurse-submodules https://github.com/ParisNeo/lollms-web
 git submodule update --init --recursive
 cd lollms-webui\lollms_core
 pip install -e .
-cd ..
-cd utilities\safe_store
-pip install -e .
 cd ..\..

 pip install -r requirements.txt
--- a/scripts/windows/win_install.bat
+++ b/scripts/windows/win_install.bat
@ -92,18 +92,12 @@ if exist lollms-webui\ (
  cd
  cd lollms_core 
  pip install -e .
-  cd ..
-  cd utilities\safe_store
-  pip install -e .
  cd ..\..
 ) else (
  git clone --depth 1  --recurse-submodules https://github.com/ParisNeo/lollms-webui.git
  git submodule update --init --recursive
  cd lollms-webui\lollms_core
  pip install -e .
-  cd ..
-  cd utilities\safe_store
-  pip install -e .
  cd ..\..
  cd utilities\pipmaster
  pip install -e .
--- a/scripts/wsl/install_script.sh
+++ b/scripts/wsl/install_script.sh
@ -51,9 +51,6 @@ echo "Initializing submodules"
 git submodule update --init --recursive
 cd lollms-webui\lollms_core
 pip install -e .
-cd ..
-cd utilities\safe_store
-pip install -e .
 cd ..\..

 # Install requirements
--- a/update_script.py
+++ b/update_script.py
@ -35,7 +35,6 @@ def run_git_pull():
            # Clone the repository to the target path
            ASCIIColors.info("Lollms_core found in the app space.\nPulling last lollms_core")
            subprocess.run(["git", "-C", str(execution_path/"lollms_core"), "pull"])            
-            subprocess.run(["git", "-C", str(execution_path/"utilities/safe_store"), "pull"])         
              
                    
        except Exception as ex:
@ -50,7 +49,6 @@ def install_requirements():
    try:
        subprocess.check_call(["pip", "install", "--upgrade", "-r", "requirements.txt"])
        subprocess.check_call(["pip", "install", "--upgrade", "-e", "lollms_core"])
-        subprocess.check_call(["pip", "install", "--upgrade", "-e", "utilities/safe_store"])
    except subprocess.CalledProcessError as e:
        print(f"Error during pip install: {e}")

--- a/utilities/execution_engines/latex_execution_engine.py
+++ b/utilities/execution_engines/latex_execution_engine.py
@ -16,7 +16,6 @@ from ascii_colors import get_trace_exception, trace_exception
 from ascii_colors import ASCIIColors
 from lollms.databases.discussions_database import DiscussionsDB
 from pathlib import Path
-from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
 import tqdm
 from fastapi import FastAPI, UploadFile, File
 import shutil
--- a/utilities/safe_store
+++ b/utilities/safe_store
@ -1 +1 @@
-Subproject commit 28f48c5e0ba1f2c92922f7474d7ce3d72764cba1
+Subproject commit 2a1aca125fffb71c7a98a49222f89080ce6daf13
--- a/web/dist/assets/index-156a33f2.js
+++ b/web/dist/assets/index-156a33f2.js
--- a/web/dist/assets/index-4cfc40d0.css
+++ b/web/dist/assets/index-4cfc40d0.css
--- a/web/dist/index.html
+++ b/web/dist/index.html
@ -6,8 +6,8 @@
    
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>LoLLMS WebUI</title>
-    <script type="module" crossorigin src="/assets/index-e244d07f.js"></script>
-    <link rel="stylesheet" href="/assets/index-5cbc5b69.css">
+    <script type="module" crossorigin src="/assets/index-156a33f2.js"></script>
+    <link rel="stylesheet" href="/assets/index-4cfc40d0.css">
  </head>
  <body>
    <div id="app"></div>
--- a/web/src/views/SettingsView.vue
+++ b/web/src/views/SettingsView.vue
@ -2293,6 +2293,25 @@
                                    </div>
                                </td>
                                </tr>
+                                <tr>
+                                <td style="min-width: 200px;">
+                                    <label for="xtts_freq" class="text-sm font-bold" style="margin-right: 1rem;">Frequency (controls the tone):</label>
+                                </td>
+                                <td>
+                                    <div class="flex flex-row">
+                                        <input
+                                            type="number"
+                                            id="xtts_freq"
+                                            required
+                                            v-model.number="configFile.xtts_freq"
+                                            @change="settingsChanged=true"
+                                            class="mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
+                                            step="0.01"
+                                        >
+                                    </div>
+                                </td>
+                                </tr>
+
                                <tr>
                                <td style="min-width: 200px;">
                                    <label for="auto_read" class="text-sm font-bold" style="margin-right: 1rem;">Enable auto read:</label>
--- a/zoos/extensions_zoo
+++ b/zoos/extensions_zoo
@ -1 +1 @@
-Subproject commit 7f6b99c1e5fc01050b6c60b30706dd141a18d734
+Subproject commit 803b608354940b14642705d8dcb94b196b2a610d
--- a/zoos/personalities_zoo
+++ b/zoos/personalities_zoo
@ -1 +1 @@
-Subproject commit 3d24a21b17fb39af78f2cfc2f22bb1535a042b74
+Subproject commit babf95044dc34e46d74843ae17bc87e06f5cecd8