added new entry to discussion database

This commit is contained in:
Saifeddine ALOUI 2024-08-11 20:10:59 +02:00
parent fa70a95766
commit 1b4c8ca293
6 changed files with 164 additions and 33 deletions

View File

@ -30,6 +30,7 @@ import gc
import yaml import yaml
import time import time
from lollms.utilities import PackageManager from lollms.utilities import PackageManager
import socket
class LollmsApplication(LoLLMsCom): class LollmsApplication(LoLLMsCom):
def __init__( def __init__(
@ -68,7 +69,6 @@ class LollmsApplication(LoLLMsCom):
self.tts = None self.tts = None
self.session = Session(lollms_paths) self.session = Session(lollms_paths)
self.skills_library = SkillsLibrary(self.lollms_paths.personal_skills_path/(self.config.skills_lib_database_name+".db")) self.skills_library = SkillsLibrary(self.lollms_paths.personal_skills_path/(self.config.skills_lib_database_name+".db"))
self.tasks_library = TasksLibrary(self) self.tasks_library = TasksLibrary(self)
self.handle_generate_msg: Callable[[str, Dict], None] = None self.handle_generate_msg: Callable[[str, Dict], None] = None
@ -94,29 +94,35 @@ class LollmsApplication(LoLLMsCom):
self.ttv = None self.ttv = None
self.rt_com = None self.rt_com = None
self.is_internet_available = self.check_internet_connection()
if not free_mode: if not free_mode:
try: try:
if config.auto_update: if config.auto_update and self.is_internet_available:
# Clone the repository to the target path # Clone the repository to the target path
if self.lollms_paths.lollms_core_path.exists(): if self.lollms_paths.lollms_core_path.exists():
ASCIIColors.info("Lollms_core found in the app space.\nPulling last lollms_core") def check_lollms_core():
subprocess.run(["git", "-C", self.lollms_paths.lollms_core_path, "pull"]) subprocess.run(["git", "-C", self.lollms_paths.lollms_core_path, "pull"])
if self.lollms_paths.safe_store_path.exists(): ASCIIColors.blue("Lollms_core found in the app space.")
ASCIIColors.info("safe_store_path found in the app space.\nPulling last safe_store_path") ASCIIColors.execute_with_animation("Pulling last lollms_core", check_lollms_core)
subprocess.run(["git", "-C", self.lollms_paths.safe_store_path, "pull"])
def check_lollms_bindings_zoo():
subprocess.run(["git", "-C", self.lollms_paths.bindings_zoo_path, "pull"])
ASCIIColors.blue("Bindings zoo found in your personal space.")
ASCIIColors.execute_with_animation("Pulling last bindings zoo", check_lollms_bindings_zoo)
# Pull the repository if it already exists # Pull the repository if it already exists
def check_lollms_personalities_zoo():
ASCIIColors.info("Bindings zoo found in your personal space.\nPulling last bindings zoo") subprocess.run(["git", "-C", self.lollms_paths.personalities_zoo_path, "pull"])
subprocess.run(["git", "-C", self.lollms_paths.bindings_zoo_path, "pull"]) ASCIIColors.blue("Personalities zoo found in your personal space.")
ASCIIColors.execute_with_animation("Pulling last personalities zoo", check_lollms_personalities_zoo)
# Pull the repository if it already exists # Pull the repository if it already exists
ASCIIColors.info("Personalities zoo found in your personal space.\nPulling last personalities zoo") def check_lollms_models_zoo():
subprocess.run(["git", "-C", self.lollms_paths.personalities_zoo_path, "pull"]) subprocess.run(["git", "-C", self.lollms_paths.models_zoo_path, "pull"])
# Pull the repository if it already exists ASCIIColors.blue("Models zoo found in your personal space.")
ASCIIColors.info("Extensions zoo found in your personal space.\nPulling last Extensions zoo") ASCIIColors.execute_with_animation("Pulling last Models zoo", check_lollms_models_zoo)
subprocess.run(["git", "-C", self.lollms_paths.extensions_zoo_path, "pull"])
# Pull the repository if it already exists
ASCIIColors.info("Models zoo found in your personal space.\nPulling last Models zoo")
subprocess.run(["git", "-C", self.lollms_paths.models_zoo_path, "pull"])
except Exception as ex: except Exception as ex:
ASCIIColors.error("Couldn't pull zoos. Please contact the main dev on our discord channel and report the problem.") ASCIIColors.error("Couldn't pull zoos. Please contact the main dev on our discord channel and report the problem.")
trace_exception(ex) trace_exception(ex)
@ -130,7 +136,7 @@ class LollmsApplication(LoLLMsCom):
if load_binding: if load_binding:
try: try:
ASCIIColors.info(f">Loading binding {self.config.binding_name}. Please wait ...") ASCIIColors.info(f">Loading binding {self.config.binding_name}. Please wait ...")
self.binding = self.load_binding() self.binding = self.load_binding()
except Exception as ex: except Exception as ex:
ASCIIColors.error(f"Failed to load binding.\nReturned exception: {ex}") ASCIIColors.error(f"Failed to load binding.\nReturned exception: {ex}")
trace_exception(ex) trace_exception(ex)
@ -165,6 +171,17 @@ class LollmsApplication(LoLLMsCom):
except Exception as ex: except Exception as ex:
trace_exception(ex) trace_exception(ex)
@staticmethod
def check_internet_connection():
global is_internet_available
try:
# Attempt to connect to a reliable server (in this case, Google's DNS)
socket.create_connection(("8.8.8.8", 53), timeout=3)
is_internet_available = True
return True
except OSError:
is_internet_available = False
return False
def backup_trust_store(self): def backup_trust_store(self):

View File

@ -24,10 +24,9 @@ import subprocess
from lollms.config import TypedConfig, InstallOption from lollms.config import TypedConfig, InstallOption
from lollms.main_config import LOLLMSConfig from lollms.main_config import LOLLMSConfig
from lollms.com import NotificationType, NotificationDisplayType, LoLLMsCom from lollms.com import NotificationType, NotificationDisplayType, LoLLMsCom
from lollms.security import sanitize_path
from lollms.utilities import show_message_dialog from lollms.utilities import show_message_dialog
from lollms.types import BindingType from lollms.types import BindingType
from lollms.client_session import Client
import urllib import urllib
import inspect import inspect
from datetime import datetime from datetime import datetime
@ -469,12 +468,13 @@ class LLMBinding:
""" """
pass pass
async def handle_request(self, request: Request) -> Dict[str, Any]: async def handle_request(self, data: dict, client:Client=None) -> Dict[str, Any]:
""" """
Handle client requests. Handle client requests.
Args: Args:
data (dict): A dictionary containing the request data. data (dict): A dictionary containing the request data.
client (Client): A refertence to the client asking for this request.
Returns: Returns:
dict: A dictionary containing the response, including at least a "status" key. dict: A dictionary containing the response, including at least a "status" key.
@ -484,8 +484,9 @@ class LLMBinding:
Example usage: Example usage:
``` ```
handler = YourHandlerClass() handler = YourHandlerClass()
client = checkaccess(lollmsServer, client_id)
request_data = {"command": "some_command", "parameters": {...}} request_data = {"command": "some_command", "parameters": {...}}
response = handler.handle_request(request_data) response = handler.handle_request(request_data, client)
``` ```
""" """
return {"status":True} return {"status":True}

View File

@ -17,6 +17,9 @@ import gc
import json import json
import shutil import shutil
from lollms.tasks import TasksLibrary from lollms.tasks import TasksLibrary
import json
from typing import Dict, Any
__author__ = "parisneo" __author__ = "parisneo"
__github__ = "https://github.com/ParisNeo/lollms-webui" __github__ = "https://github.com/ParisNeo/lollms-webui"
__copyright__ = "Copyright 2023, " __copyright__ = "Copyright 2023, "
@ -37,7 +40,7 @@ class DiscussionsDB:
self.discussion_db_file_path = self.discussion_db_path/"database.db" self.discussion_db_file_path = self.discussion_db_path/"database.db"
def create_tables(self): def create_tables(self):
db_version = 12 db_version = 13
with sqlite3.connect(self.discussion_db_file_path) as conn: with sqlite3.connect(self.discussion_db_file_path) as conn:
cursor = conn.cursor() cursor = conn.cursor()
@ -52,6 +55,7 @@ class DiscussionsDB:
CREATE TABLE IF NOT EXISTS discussion ( CREATE TABLE IF NOT EXISTS discussion (
id INTEGER PRIMARY KEY AUTOINCREMENT, id INTEGER PRIMARY KEY AUTOINCREMENT,
title TEXT, title TEXT,
metadata TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
) )
""") """)
@ -98,6 +102,7 @@ class DiscussionsDB:
'discussion': [ 'discussion': [
'id', 'id',
'title', 'title',
'metadata',
'created_at' 'created_at'
], ],
'message': [ 'message': [
@ -931,7 +936,7 @@ class Discussion:
) )
def title(self): def title(self):
"""Renames the discussion """Recovers the discussion title
Args: Args:
new_title (str): The nex discussion name new_title (str): The nex discussion name
@ -941,6 +946,67 @@ class Discussion:
) )
return rows[0][0] return rows[0][0]
def set_metadata(self, new_metadata: Dict[str, Any]) -> None:
"""
Sets the metadata for the discussion.
Args:
new_metadata (Dict[str, Any]): The new metadata as a dictionary.
"""
metadata_json = json.dumps(new_metadata)
self.discussions_db.update(
"UPDATE discussion SET metadata=? WHERE id=?",
(metadata_json, self.discussion_id)
)
def get_metadata(self) -> Dict[str, Any]:
"""
Retrieves the discussion metadata.
Returns:
Dict[str, Any]: The metadata as a dictionary. Returns an empty dictionary if metadata is None or invalid JSON.
"""
rows = self.discussions_db.select(
f"SELECT metadata FROM discussion WHERE id=?",
(self.discussion_id,)
)
metadata_json = rows[0][0] if rows else None
if metadata_json is None:
return {}
try:
return json.loads(metadata_json)
except json.JSONDecodeError:
return {}
def update_metadata(self, key: str, value: Any) -> None:
"""
Updates a specific key in the metadata.
Args:
key (str): The key to update or add.
value (Any): The value to set for the key.
"""
current_metadata = self.get_metadata()
current_metadata[key] = value
self.set_metadata(current_metadata)
def delete_metadata_key(self, key: str) -> None:
"""
Deletes a specific key from the metadata.
Args:
key (str): The key to delete.
"""
current_metadata = self.get_metadata()
if key in current_metadata:
del current_metadata[key]
self.set_metadata(current_metadata)
def delete_discussion(self): def delete_discussion(self):
"""Deletes the discussion """Deletes the discussion
""" """

View File

@ -7,6 +7,8 @@ if not PackageManager.check_package_installed("PyQt5"):
PackageManager.install_package("PyQt5") PackageManager.install_package("PyQt5")
from ascii_colors import trace_exception from ascii_colors import trace_exception
from functools import partial from functools import partial
from lollms.functions.prompting.image_gen_prompts import get_image_gen_prompt, get_random_image_gen_prompt
def build_negative_prompt(image_generation_prompt, llm): def build_negative_prompt(image_generation_prompt, llm):
start_header_id_template = llm.config.start_header_id_template start_header_id_template = llm.config.start_header_id_template
@ -21,7 +23,7 @@ def build_negative_prompt(image_generation_prompt, llm):
f"{start_header_id_template}negative_prompt{end_header_id_template}", f"{start_header_id_template}negative_prompt{end_header_id_template}",
]) ])
def build_image(prompt, negative_prompt, width, height, processor:APScript, client:Client): def build_image(prompt, negative_prompt, width, height, processor:APScript, client:Client, return_format="markdown"):
try: try:
if processor.personality.config.active_tti_service=="diffusers": if processor.personality.config.active_tti_service=="diffusers":
if not processor.personality.app.tti: if not processor.personality.app.tti:
@ -89,12 +91,50 @@ def build_image(prompt, negative_prompt, width, height, processor:APScript, clie
file = str(file) file = str(file)
escaped_url = discussion_path_to_url(file) escaped_url = discussion_path_to_url(file)
return f'\nRespond with this link in markdown format:\n![]({escaped_url})'
if return_format == "markdown":
return f'\nRespond with this link in markdown format:\n![]({escaped_url})'
elif return_format == "url":
return escaped_url
elif return_format == "path":
return file
elif return_format == "url_and_path":
return {"url": escaped_url, "path": file}
else:
return f"Invalid return_format: {return_format}. Supported formats are 'markdown', 'url', 'path', and 'url_and_path'."
except Exception as ex: except Exception as ex:
trace_exception(ex) trace_exception(ex)
return f"Couldn't generate image. Make sure {processor.personality.config.active_tti_service} service is installed" return f"Couldn't generate image. Make sure {processor.personality.config.active_tti_service} service is installed"
def build_image_from_simple_prompt(prompt, processor:APScript, client:Client, width=1024, height=1024, examples_extraction_mathod="random", number_of_examples_to_recover=3, production_type="artwork", max_generation_prompt_size=1024):
examples = ""
expmls = []
if examples_extraction_mathod=="random":
expmls = get_random_image_gen_prompt(number_of_examples_to_recover)
elif examples_extraction_mathod=="rag_based":
expmls = get_image_gen_prompt(prompt, number_of_examples_to_recover)
for i,expml in enumerate(expmls):
examples += f"example {i}:"+expml+"\n"
prompt = processor.build_prompt([
processor.system_full_header,
f"Act as artbot, the art prompt generation AI.",
"Use the discussion information to come up with an image generation prompt without referring to it.",
f"Be precise and describe the style as well as the {production_type} description details.", #conditionning
"Do not explain the prompt, just answer with the prompt in the right prompting style.",
processor.system_custom_header("discussion"),
processor.system_custom_header("Production type") + f"{production_type}",
processor.system_custom_header("Instruction") + f"Use the following as examples and follow their format to build the special prompt." if examples!="" else "",
processor.system_custom_header("Prompt examples") if examples!="" else "",
processor.system_custom_header("Examples") + f"{examples}",
processor.system_custom_header("Prompt"),
],2)
positive_prompt = processor.generate(prompt, max_generation_prompt_size, callback=processor.sink).strip().replace("</s>","").replace("<s>","")
return build_image(positive_prompt, "", width, height, processor, client, "url_and_path")
def build_image_function(processor, client): def build_image_function(processor, client):
if processor.config.use_negative_prompt: if processor.config.use_negative_prompt:
if processor.config.use_ai_generated_negative_prompt: if processor.config.use_ai_generated_negative_prompt:

View File

@ -2258,12 +2258,13 @@ class APScript(StateMachine):
trace_exception(ex) trace_exception(ex)
self.warning(f"Couldn't execute command {command}") self.warning(f"Couldn't execute command {command}")
async def handle_request(self, request: Request) -> Dict[str, Any]: async def handle_request(self, data: dict, client:Client=None) -> Dict[str, Any]:
""" """
Handle client requests. Handle client requests.
Args: Args:
data (dict): A dictionary containing the request data. data (dict): A dictionary containing the request data.
client (Client): A refertence to the client asking for this request.
Returns: Returns:
dict: A dictionary containing the response, including at least a "status" key. dict: A dictionary containing the response, including at least a "status" key.
@ -2273,13 +2274,15 @@ class APScript(StateMachine):
Example usage: Example usage:
``` ```
handler = YourHandlerClass() handler = YourHandlerClass()
client = checkaccess(lollmsServer, client_id)
request_data = {"command": "some_command", "parameters": {...}} request_data = {"command": "some_command", "parameters": {...}}
response = await handler.handle_request(request_data) response = handler.handle_request(request_data, client)
``` ```
""" """
return {"status":True} return {"status":True}
def load_personality_config(self): def load_personality_config(self):
""" """
Load the content of local_config.yaml file. Load the content of local_config.yaml file.

View File

@ -683,6 +683,10 @@ class PersonalityInfos(BaseModel):
category:str category:str
name:str name:str
class PersonalityRequest(BaseModel):
client_id:str
data:dict
@router.post("/copy_to_custom_personas") @router.post("/copy_to_custom_personas")
async def copy_to_custom_personas(data: PersonalityInfos): async def copy_to_custom_personas(data: PersonalityInfos):
""" """
@ -706,12 +710,12 @@ async def copy_to_custom_personas(data: PersonalityInfos):
# ------------------------------------------- Interaction with personas ------------------------------------------------ # ------------------------------------------- Interaction with personas ------------------------------------------------
@router.post("/post_to_personality") @router.post("/post_to_personality")
async def post_to_personality(request: Request): async def post_to_personality(request: PersonalityRequest):
"""Post data to a personality""" """Post data to a personality"""
client =check_access(lollmsElfServer, request.client_id)
try: try:
if hasattr(lollmsElfServer.personality.processor,'handle_request'): if hasattr(lollmsElfServer.personality.processor,'handle_request'):
return await lollmsElfServer.personality.processor.handle_request(request) return await lollmsElfServer.personality.processor.handle_request(request.data, client)
else: else:
return {} return {}
except Exception as ex: except Exception as ex: