Enhanced tool, added audio

lollms/app.py

@@ -46,6 +46,8 @@ class LollmsApplication(LoLLMsCom):
         self.model:LLMBinding       = None
         self.long_term_memory       = None
 
+        self.tts                    = None
+
         try:
             if config.auto_update:
                 # Clone the repository to the target path

lollms/audio_gen_modules/lollms_xtts.py

@@ -0,0 +1,193 @@
+# Title LollmsXTTS
+# Licence: MIT
+# Author : Paris Neo
+# Adapted from the work of daswer123's xtts-api-server
+# check it out : https://github.com/daswer123/xtts-api-server
+# Here is a copy of the LICENCE https://github.com/daswer123/xtts-api-server/blob/main/LICENSE
+# All rights are reserved
+
+from pathlib import Path
+import os
+import sys
+from lollms.app import LollmsApplication
+from lollms.paths import LollmsPaths
+from lollms.config import TypedConfig, ConfigTemplate, BaseConfig
+from lollms.utilities import PackageManager
+import time
+import io
+import sys
+import requests
+import os
+import base64
+import subprocess
+import time
+import json
+import platform
+from dataclasses import dataclass
+from PIL import Image, PngImagePlugin
+from enum import Enum
+from typing import List, Dict, Any
+
+from ascii_colors import ASCIIColors, trace_exception
+from lollms.paths import LollmsPaths
+from lollms.utilities import git_pull
+import subprocess
+
+
+def verify_xtts(lollms_paths:LollmsPaths):
+    # Clone repository
+    root_dir = lollms_paths.personal_path
+    shared_folder = root_dir/"shared"
+    xtts_folder = shared_folder / "xtts"
+    return xtts_folder.exists()
+    
+def install_xtts(lollms_paths:LollmsPaths):
+    root_dir = lollms_paths.personal_path
+    shared_folder = root_dir/"shared"
+    xtts_folder = shared_folder / "xtts"
+    if not PackageManager.check_package_installed("xtts-api-server"):
+        PackageManager.install_package("xtts-api-server")
+
+    xtts_folder.mkdir(exist_ok=True,parents=True)
+    ASCIIColors.green("XTTS server installed successfully")
+
+
+def get_xtts(lollms_paths:LollmsPaths):
+    root_dir = lollms_paths.personal_path
+    shared_folder = root_dir/"shared"
+    xtts_folder = shared_folder / "xtts"
+    xtts_script_path = xtts_folder / "lollms_xtts.py"
+    git_pull(xtts_folder)
+    
+    if xtts_script_path.exists():
+        ASCIIColors.success("lollms_xtts found.")
+        ASCIIColors.success("Loading source file...",end="")
+        # use importlib to load the module from the file path
+        from lollms.audio_gen_modules.lollms_xtts import LollmsXTTS
+        ASCIIColors.success("ok")
+        return LollmsXTTS
+
+class LollmsXTTS:
+    has_controlnet = False
+    def __init__(
+                    self, 
+                    app:LollmsApplication, 
+                    auto_xtts_base_url=None,
+                    share=False,
+                    max_retries=10,
+                    voice_samples_path=""
+                    ):
+        if auto_xtts_base_url=="" or auto_xtts_base_url=="http://127.0.0.1:8020":
+            auto_xtts_base_url = None
+        # Get the current directory
+        lollms_paths = app.lollms_paths
+        self.app = app
+        root_dir = lollms_paths.personal_path
+        self.voice_samples_path = voice_samples_path
+        
+        # Store the path to the script
+        if auto_xtts_base_url is None:
+            self.auto_xtts_base_url = "http://127.0.0.1:8020"
+            if not verify_xtts(lollms_paths):
+                install_xtts(app.lollms_paths)
+        else:
+            self.auto_xtts_base_url = auto_xtts_base_url
+
+        self.auto_xtts_url = self.auto_xtts_base_url+"/sdapi/v1"
+        shared_folder = root_dir/"shared"
+        self.xtts_folder = shared_folder / "xtts"
+        self.output_dir = root_dir / "outputs/sd"
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+       
+        ASCIIColors.red(".____    ________  .____    .____       _____    _________         ____  __________________________________ ")
+        ASCIIColors.red("|    |   \_____  \ |    |   |    |     /     \  /   _____/         \   \/  /\__    ___/\__    ___/   _____/ ")
+        ASCIIColors.red("|    |    /   |   \|    |   |    |    /  \ /  \ \_____  \   ______  \     /   |    |     |    |  \_____  \  ")
+        ASCIIColors.red("|    |___/    |    \    |___|    |___/    Y    \/        \ /_____/  /     \   |    |     |    |  /        \ ")
+        ASCIIColors.red("|_______ \_______  /_______ \_______ \____|__  /_______  /         /___/\  \  |____|     |____| /_______  / ")
+        ASCIIColors.red("        \/       \/        \/       \/       \/        \/                \_/                            \/  ")
+
+        ASCIIColors.red(" Forked from daswer123's XTTS server")
+        ASCIIColors.red(" Integration in lollms by ParisNeo using daswer123's webapi ")
+
+        if not self.wait_for_service(1,False) and auto_xtts_base_url is None:
+            ASCIIColors.info("Loading lollms_xtts")
+            os.environ['xtts_WEBUI_RESTARTING'] = '1' # To forbid sd webui from showing on the browser automatically
+            # Launch the Flask service using the appropriate script for the platform
+            ASCIIColors.info("Running on windows")
+            self.output_folder = app.lollms_paths.personal_outputs_path/"audio_out"
+            self.output_folder.mkdir(parents=True, exist_ok=True)
+            subprocess.Popen(["python", "-m", "xtts_api_server", "-o", f"{self.output_folder}", "-sf", f"{self.voice_samples_path}"])
+
+        # Wait until the service is available at http://127.0.0.1:7860/
+        self.wait_for_service(max_retries=max_retries)
+
+
+    def wait_for_service(self, max_retries = 150, show_warning=True):
+        url = f"{self.auto_xtts_base_url}/languages"
+        # Adjust this value as needed
+        retries = 0
+
+        while retries < max_retries or max_retries<0:
+            try:
+                response = requests.get(url)
+                if response.status_code == 200:
+                    print("Service is available.")
+                    if self.app is not None:
+                        self.app.success("XTTS Service is now available.")
+                    return True
+            except requests.exceptions.RequestException:
+                pass
+
+            retries += 1
+            time.sleep(3)
+        if show_warning:
+            print("Service did not become available within the given time.")
+            if self.app is not None:
+                self.app.error("XTTS Service did not become available within the given time.")
+        return False
+
+    def set_speaker_folder(self, speaker_folder):
+        url = f"{self.auto_xtts_base_url}/set_speaker_folder"
+
+        # Define the request body
+        payload = {
+            "speaker_folder": str(speaker_folder)
+        }
+
+        # Send the POST request
+        response = requests.post(url, json=payload)
+
+        # Check the response status code
+        if response.status_code == 200:
+            print("Request successful")
+            return True
+            # You can access the response data using response.json()
+        else:
+            print("Request failed with status code:", response.status_code)
+            return False
+
+    def tts_to_file(self, text, speaker_wav, file_name_or_path, language="en"):
+        url = f"{self.auto_xtts_base_url}/tts_to_file"
+
+        # Define the request body
+        payload = {
+            "text": text,
+            "speaker_wav": speaker_wav,
+            "language": language,
+            "file_name_or_path": file_name_or_path
+        }
+        headers = {
+            'accept': 'application/json',
+            'Content-Type': 'application/json'
+        }
+
+        # Send the POST request
+        response =  requests.post(url, headers=headers, data=json.dumps(payload))
+
+        # Check the response status code
+        if response.status_code == 200:
+            print("Request successful")
+            # You can access the response data using response.json()
+        else:
+            print("Request failed with status code:", response.status_code)

lollms/configs/config.yaml

@@ -1,5 +1,5 @@
 # =================== Lord Of Large Language Models Configuration file =========================== 
-version: 30
+version: 31
 binding_name: null
 model_name: null
 
@@ -14,7 +14,7 @@ discussion_prompt_separator: "!@>"
 seed: -1
 n_predict: 1024
 ctx_size: 4084
-min_n_predict: 256
+min_n_predict: 512
 temperature: 0.9
 top_k: 50
 top_p: 0.95
@@ -53,6 +53,7 @@ auto_show_browser: true
 audio_in_language: 'en-US'
 audio_out_voice: null
 auto_speak: false
+auto_read: false
 audio_pitch: 1
 audio_auto_send_input: true
 audio_silenceTimer: 5000
@@ -71,3 +72,5 @@ data_vectorization_chunk_size: 512 # chunk size
 data_vectorization_overlap_size: 128 # overlap between chunks size
 data_vectorization_nb_chunks: 2 # number of chunks to use
 data_vectorization_build_keys_words: false # If true, when querrying the database, we use keywords generated from the user prompt instead of the prompt itself.
+data_vectorization_force_first_chunk: false # If true, the first chunk of the document will systematically be used
+data_vectorization_make_persistance: false # If true, the data will be persistant webween runs

lollms/media.py

@@ -8,6 +8,7 @@ License: Apache 2.0
 """
 from lollms.utilities import PackageManager
 from lollms.com import LoLLMsCom
+from ascii_colors import ASCIIColors
 import platform
 import subprocess
 
@@ -22,8 +23,11 @@ if not PackageManager.check_package_installed("cv2"):
     else:
         os.system('sudo apt-get update')
         os.system('sudo apt-get install libgl1-mesa-glx python3-opencv -y')
+        os.system('pip install opencv-python')
+try:
     import cv2
-
+except:
+    ASCIIColors.error("Couldn't install opencv!")
 
 
 if not PackageManager.check_package_installed("scipy"):
@@ -58,6 +62,7 @@ except:
     PackageManager.install_package("wave")
 
 import sounddevice as sd
+import wave
 
 class AudioRecorder:
     def __init__(self, socketio, filename, channels=1, sample_rate=16000, chunk_size=24678, silence_threshold=150.0, silence_duration=2, callback=None, lollmsCom=None):

lollms/personality.py

@@ -117,6 +117,7 @@ class AIPersonality:
         self._name: str = "lollms"
         self._user_name: str = "user"
         self._category: str = "General"
+        self._category_desc: str = "General"
         self._supported_languages: str = []
         self._selected_language: str = selected_language
 
@@ -170,6 +171,7 @@ Date: {{date}}
             return
         else:
             parts = str(personality_package_path).split("/")
+            self._category = parts[0]
             if parts[0] == "custom_personalities":
                 self.personality_package_path = self.lollms_paths.custom_personalities_path/parts[1]
             else:
@@ -563,7 +565,7 @@ Date: {{date}}
         self._author = config.get("author", self._author)
         self._name = config.get("name", self._name)
         self._user_name = config.get("user_name", self._user_name)
-        self._category = config.get("category", self._category)
+        self._category_desc = config.get("category", self._category)
 
         self._personality_description = config.get("personality_description", self._personality_description)
         self._personality_conditioning = config.get("personality_conditioning", self._personality_conditioning)
@@ -605,7 +607,8 @@ Date: {{date}}
         self.languages_path = self.personality_package_path / "languages"
         # Get the data folder path
         self.data_path = self.personality_package_path / "data"
-
+        # Get the data folder path
+        self.audio_path = self.personality_package_path / "audio"
 
         # If not exist recreate
         self.assets_path.mkdir(parents=True, exist_ok=True)
@@ -613,6 +616,12 @@ Date: {{date}}
         # If not exist recreate
         self.scripts_path.mkdir(parents=True, exist_ok=True)
 
+        # If not exist recreate
+        self.audio_path.mkdir(parents=True, exist_ok=True)
+
+        # samples
+        self.audio_samples = [f for f in self.audio_path.iterdir()]
+
         # Verify if the persona has a data folder
         self.database_path = self.data_path / "db.json"
         if self.database_path.exists():
@@ -920,11 +929,21 @@ Date: {{date}}
         """Get the category."""
         return self._category
 
+    @property
+    def category_desc(self) -> str:
+        """Get the category."""
+        return self._category_desc
+
     @category.setter
     def category(self, value: str):
         """Set the category."""
         self._category = value
 
+    @category_desc.setter
+    def category_desc(self, value: str):
+        """Set the category."""
+        self._category_desc = value
+
 
     @property
     def supported_languages(self) -> str: