ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-18 20:27:58 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Moved to new discussion database format

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-02-19 00:22:48 +01:00
parent d6532fbaf6
commit da2d2d9a63
17 changed files with 117 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
configs/config.yaml
View File

@ -1,11 +1,11 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 64
version: 66
binding_name: null
model_name: null
show_news_panel: True
# Execution protection
# Security measures
turn_on_code_execution: True
turn_on_code_validation: True
turn_on_open_file_validation: False
@ -49,7 +49,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: default
# Automatic updates
debug: False

2
elf_docker_cfg/personal/configs/lollms_elf_config.yaml
View File

@ -37,7 +37,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: database.db
# Automatic updates
debug: False

2
elf_test_cfg/personal/configs/lollms_elf_config.yaml
View File

@ -37,7 +37,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: database.db
# Automatic updates
debug: False

2
elf_test_cfg/personal/configs/lollms_elf_local_config.yaml
View File

@ -37,7 +37,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: database.db
# Automatic updates
debug: False

61
lollms/app.py
View File

@ -28,8 +28,6 @@ class LollmsApplication(LoLLMsCom):
lollms_paths:LollmsPaths,
load_binding=True,
load_model=True,
load_voice_service=True,
load_sd_service=True,
try_select_binding=False,
try_select_model=False,
callback=None,
@ -59,29 +57,6 @@ class LollmsApplication(LoLLMsCom):
self.tts = None
if not free_mode:
if self.config.enable_ollama_service:
try:
from lollms.services.ollama.lollms_ollama import Service
self.ollama = Service(self, base_url=self.config.ollama_base_url)
except Exception as ex:
trace_exception(ex)
self.warning(f"Couldn't load Ollama")
if self.config.enable_voice_service and load_voice_service:
try:
from lollms.services.xtts.lollms_xtts import LollmsXTTS
self.tts = LollmsXTTS(self, voice_samples_path=lollms_paths.custom_voices_path, xtts_base_url=self.config.xtts_base_url, wait_for_service=False)
except:
self.warning(f"Couldn't load XTTS")
if self.config.enable_sd_service and load_sd_service:
try:
from lollms.services.sd.lollms_sd import LollmsSD
self.sd = LollmsSD(self, auto_sd_base_url=self.config.sd_base_url)
except:
self.warning(f"Couldn't load SD")
try:
if config.auto_update:
# Clone the repository to the target path
@ -147,13 +122,43 @@ class LollmsApplication(LoLLMsCom):
self.mount_personalities()
self.mount_extensions()
def get_uploads_path(self, client_id):
return self.lollms_paths.personal_uploads_path
def start_servers( self,
load_voice_service=True,
load_sd_service=True,
):
if self.config.enable_ollama_service:
try:
from lollms.services.ollama.lollms_ollama import Service
self.ollama = Service(self, base_url=self.config.ollama_base_url)
except Exception as ex:
trace_exception(ex)
self.warning(f"Couldn't load Ollama")
if self.config.enable_voice_service and load_voice_service:
try:
from lollms.services.xtts.lollms_xtts import LollmsXTTS
self.tts = LollmsXTTS(self, voice_samples_path=lollms_paths.custom_voices_path, xtts_base_url=self.config.xtts_base_url, wait_for_service=False)
except:
self.warning(f"Couldn't load XTTS")
if self.config.enable_sd_service and load_sd_service:
try:
from lollms.services.sd.lollms_sd import LollmsSD
self.sd = LollmsSD(self, auto_sd_base_url=self.config.sd_base_url)
except:
self.warning(f"Couldn't load SD")
def build_long_term_skills_memory(self):
db_path:Path = self.lollms_paths.personal_databases_path/self.config.db_path.split(".")[0]
db_path.mkdir(exist_ok=True, parents=True)
discussion_db_name:Path = self.lollms_paths.personal_discussions_path/self.config.discussion_db_name.split(".")[0]
discussion_db_name.mkdir(exist_ok=True, parents=True)
self.long_term_memory = TextVectorizer(
vectorization_method=VectorizationMethod.TFIDF_VECTORIZER,
model=self.model,
database_path=db_path/"skills_memory.json",
database_path=discussion_db_name/"skills_memory.json",
save_db=True,
data_visualization_method=VisualizationMethod.PCA,
)

2
lollms/binding.py
View File

@ -86,8 +86,10 @@ class LLMBinding:
self.sync_configuration(self.binding_config, lollms_paths)
# Installation
if (not self.configuration_file_path.exists() or installation_option==InstallOption.FORCE_INSTALL) and installation_option!=InstallOption.NEVER_INSTALL:
self.ShowBlockingMessage("Installing Binding")
self.install()
self.binding_config.config.save_config()
self.HideBlockingMessage()
else:
self.load_binding_config()

8
lollms/configs/config.yaml
View File

@ -1,16 +1,18 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 63
version: 66
binding_name: null
model_name: null
show_news_panel: True
# Execution protection
# Security measures
turn_on_code_execution: True
turn_on_code_validation: True
turn_on_open_file_validation: False
turn_on_send_file_validation: False
force_accept_remote_access: false
# Server information
headless_server_mode: False
allowed_origins: []
@ -47,7 +49,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: default
# Automatic updates
debug: False

11
lollms/paths.py
View File

@ -62,7 +62,8 @@ class LollmsPaths:
self.personal_path = personal_path.resolve()
self.personal_configuration_path = self.personal_path / "configs"
self.personal_data_path = self.personal_path / "data"
self.personal_databases_path = self.personal_path / "databases"
self.personal_memory_path = self.personal_path / "memory"
self.personal_discussions_path = self.personal_path / "discussion_databases"
self.personal_models_path = self.personal_path / "models"
self.personal_uploads_path = self.personal_path / "uploads"
self.personal_log_path = self.personal_path / "logs"
@ -106,8 +107,8 @@ class LollmsPaths:
ASCIIColors.yellow(f"{self.personal_path}")
ASCIIColors.red("personal_configuration_path:",end="")
ASCIIColors.yellow(f"{self.personal_configuration_path}")
ASCIIColors.red("personal_databases_path:",end="")
ASCIIColors.yellow(f"{self.personal_databases_path}")
ASCIIColors.red("personal_discussions_path:",end="")
ASCIIColors.yellow(f"{self.personal_discussions_path}")
ASCIIColors.red("personal_models_path:",end="")
ASCIIColors.yellow(f"{self.personal_models_path}")
ASCIIColors.red("personal_user_infos_path:",end="")
@ -145,7 +146,7 @@ class LollmsPaths:
"Global paths configuration Path": self.global_paths_cfg_path,
"Personal Configuration Path": self.personal_configuration_path,
"Personal Data Path": self.personal_data_path,
"Personal Databases Path": self.personal_databases_path,
"Personal Databases Path": self.personal_discussions_path,
"Personal Models Path": self.personal_models_path,
"Personal Uploads Path": self.personal_uploads_path,
"Personal Log Path": self.personal_log_path,
@ -167,7 +168,7 @@ class LollmsPaths:
self.personal_configuration_path.mkdir(parents=True, exist_ok=True)
self.personal_models_path.mkdir(parents=True, exist_ok=True)
self.personal_data_path.mkdir(parents=True, exist_ok=True)
self.personal_databases_path.mkdir(parents=True, exist_ok=True)
self.personal_discussions_path.mkdir(parents=True, exist_ok=True)
self.personal_log_path.mkdir(parents=True, exist_ok=True)
self.personal_certificates.mkdir(parents=True, exist_ok=True)
self.personal_outputs_path.mkdir(parents=True, exist_ok=True)

13
lollms/personality.py
View File

@ -13,7 +13,7 @@ from lollms.config import InstallOption, TypedConfig, BaseConfig
from lollms.main_config import LOLLMSConfig
from lollms.paths import LollmsPaths
from lollms.binding import LLMBinding, BindingType
from lollms.utilities import PromptReshaper, PackageManager
from lollms.utilities import PromptReshaper, PackageManager, discussion_path_to_url
from lollms.com import NotificationType, NotificationDisplayType
import pkg_resources
@ -930,8 +930,8 @@ Date: {{date}}
def add_file(self, path, callback=None):
if not self.callback:
self.callback = callback
db_path = self.lollms_paths.personal_databases_path / "personalities" / self.name / "db.json"
db_path.parent.mkdir(parents=True, exist_ok=True)
discussion_db_name = self.lollms_paths.personal_discussions_path / "personalities" / self.name / "db.json"
discussion_db_name.parent.mkdir(parents=True, exist_ok=True)
path = Path(path)
if path.suffix in [".wav",".mp3"]:
self.new_message("")
@ -961,9 +961,8 @@ Date: {{date}}
if self.callback:
try:
pth = str(path).replace("\\","/").split('/')
if "uploads" in pth:
idx = pth.index("uploads")
pth = "/".join(pth[idx:])
if "discussion_databases" in pth:
pth = discussion_path_to_url(path)
self.new_message("",MSG_TYPE.MSG_TYPE_FULL)
output = f'<img src="{pth}" width="800">\n\n'
self.full(output)
@ -1001,7 +1000,7 @@ Date: {{date}}
self.vectorizer = TextVectorizer(
self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
model=self.model, #needed in case of using model_embedding
database_path=db_path,
database_path=discussion_db_name,
save_db=self.config.data_vectorization_save_db,
data_visualization_method=VisualizationMethod.PCA,
database_dict=None)

2
lollms/server/configs/config.yaml
View File

@ -49,7 +49,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: database.db
# Automatic updates
debug: False

4
lollms/server/elf_server.py
View File

@ -56,8 +56,6 @@ class LOLLMSElfServer(LollmsApplication):
lollms_paths: LollmsPaths,
load_binding=True,
load_model=True,
load_voice_service=True,
load_sd_service=True,
try_select_binding=False,
try_select_model=False,
callback=None,
@ -69,8 +67,6 @@ class LOLLMSElfServer(LollmsApplication):
lollms_paths,
load_binding=load_binding,
load_model=load_model,
load_sd_service=load_sd_service,
load_voice_service=load_voice_service,
try_select_binding=try_select_binding,
try_select_model=try_select_model,
callback=callback,

27
lollms/server/endpoints/lollms_binding_files_server.py
View File

@ -273,3 +273,30 @@ async def serve_uploads(path: str):
return FileResponse(str(file_path))
@router.get("/discussions/{path:path}")
async def serve_discussions(path: str):
"""
Serve discussion file.
Args:
filename (str): The name of the uploads file to serve.
Returns:
FileResponse: The file response containing the requested uploads file.
"""
if ".." in path:
ASCIIColors.error("A suspected LFI attack detected. The path sent to the server has .. in it!")
raise HTTPException(status_code=400, detail="Invalid path!")
root_dir = lollmsElfServer.lollms_paths.personal_discussions_path
root_dir.mkdir(exist_ok=True, parents=True)
file_path = root_dir / path
if not Path(file_path).exists():
raise HTTPException(status_code=404, detail="File not found")
return FileResponse(str(file_path))

21
lollms/server/endpoints/lollms_binding_infos.py
View File

@ -245,12 +245,14 @@ def get_active_binding_settings():
else:
return {}
class BindingSettingsRequest(BaseModel):
type: str = Field(..., min_length=1, max_length=50)
value: Any
# class BindingSettingsRequest(BaseModel):
# value: list
# @router.post("/set_active_binding_settings")
# async def set_active_binding_settings(request: BindingSettingsRequest):
@router.post("/set_active_binding_settings")
async def set_active_binding_settings(request: BindingSettingsRequest):
async def set_active_binding_settings(request: Request):
data = await request.json()
"""
Sets the active binding settings.
@ -263,16 +265,7 @@ async def set_active_binding_settings(request: BindingSettingsRequest):
if lollmsElfServer.binding is not None:
if hasattr(lollmsElfServer.binding,"binding_config"):
if request.type=="list" and type(request.value)==str:
try:
v = json.loads(request.value)
except:
v= ""
if type(v)==list:
request.value = v
else:
request.value = [request.value]
lollmsElfServer.binding.binding_config.update_template([{"type": request.type, "value": request.value}])
lollmsElfServer.binding.binding_config.update_template(data)
lollmsElfServer.binding.binding_config.config.save_config()
lollmsElfServer.binding.settings_updated()
if lollmsElfServer.config.auto_save:

2
lollms/server/events/lollms_personality_events.py
View File

@ -69,7 +69,7 @@ def add_events(sio:socketio):
print(f"Invalid file type: {filename}")
return
path:Path = lollmsElfServer.lollms_paths.personal_uploads_path / lollmsElfServer.personality.personality_folder_name
path:Path = lollmsElfServer.get_uploads_path(client_id) / lollmsElfServer.personality.personality_folder_name
path.mkdir(parents=True, exist_ok=True)
file_path = path / filename

2
lollms/server/events/lollms_rag_events.py
View File

@ -38,7 +38,7 @@ def add_events(sio:socketio):
run_async(partial(sio.emit,'show_progress'))
lollmsElfServer.sio.sleep(0)
ASCIIColors.yellow("0- Detected discussion vectorization request")
folder = lollmsElfServer.lollms_paths.personal_databases_path/"vectorized_dbs"
folder = lollmsElfServer.lollms_paths.personal_discussions_path/"vectorized_dbs"
folder.mkdir(parents=True, exist_ok=True)
lollmsElfServer.build_long_term_skills_memory()

21
lollms/utilities.py
View File

@ -35,6 +35,16 @@ import os
import sys
def yes_or_no_input(prompt):
while True:
user_input = input(prompt + " (yes/no): ").lower()
if user_input == 'yes':
return True
elif user_input == 'no':
return False
else:
print("Please enter 'yes' or 'no'.")
def show_yes_no_dialog(title, text):
import tkinter as tk
from tkinter import messagebox
@ -98,6 +108,17 @@ def file_path_to_url(file_path):
return "/".join([urllib.parse.quote(p, safe="") for p in url.split("/")])
def discussion_path_to_url(file_path:str|Path)->str:
"""
This function takes a file path as an argument and converts it into a URL format. It first removes the initial part of the file path until the "outputs" string is reached, then replaces backslashes with forward slashes and quotes each segment with urllib.parse.quote() before joining them with forward slashes to form the final URL.
:param file_path: str, the file path in the format of a Windows system
:return: str, the converted URL format of the given file path
"""
file_path = str(file_path)
url = "/"+file_path[file_path.index("discussion_databases"):].replace("\\","/").replace("discussion_databases","discussions")
return "/".join([urllib.parse.quote(p, safe="") for p in url.split("/")])
def is_asyncio_loop_running():
"""

2
personal_data/configs/lollms_discord_local_config.yaml
View File

@ -37,7 +37,7 @@ user_avatar: default_user.svg
use_user_informations_in_discussion: false
# UI parameters
db_path: database.db
discussion_db_name: database.db
# Automatic updates
debug: False