From 104293d5e922d0631f307e23da30069efcc8e2de Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Sun, 19 May 2024 16:56:47 +0200 Subject: [PATCH] changed function calls --- lollms/functions/generate_image.py | 48 ++++++++++++++++----------- lollms/functions/select_image_file.py | 43 ++++++++++++++++++++++++ lollms/functions/take_a_photo.py | 10 ++++++ lollms/functions/take_screen_shot.py | 17 +++++++++- lollms/media.py | 2 +- lollms/personality.py | 33 +++++++++++++++++- lollms/utilities.py | 4 +++ 7 files changed, 135 insertions(+), 22 deletions(-) create mode 100644 lollms/functions/select_image_file.py diff --git a/lollms/functions/generate_image.py b/lollms/functions/generate_image.py index e04fe96..22653ef 100644 --- a/lollms/functions/generate_image.py +++ b/lollms/functions/generate_image.py @@ -10,43 +10,53 @@ import cv2 import time from PyQt5 import QtWidgets, QtGui, QtCore import sys +from functools import partial -def build_image(prompt, width, height, self:APScript, client:Client): +def build_image(prompt, width, height, processor:APScript, client:Client): try: - if self.personality.config.active_tti_service=="autosd": - if not self.personality.app.tti: + if processor.personality.config.active_tti_service=="autosd": + if not processor.personality.app.tti: from lollms.services.sd.lollms_sd import LollmsSD - self.step_start("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") - self.personality.app.tti = LollmsSD(self.personality.app, self.personality.name, max_retries=-1,auto_sd_base_url=self.personality.config.sd_base_url) - self.personality.app.sd = self.personality.app.tti - self.step_end("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") - file, infos = self.personality.app.tti.paint( + processor.step_start("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") + processor.personality.app.tti = LollmsSD(processor.personality.app, processor.personality.name, max_retries=-1,auto_sd_base_url=processor.personality.config.sd_base_url) + processor.personality.app.sd = processor.personality.app.tti + processor.step_end("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") + file, infos = processor.personality.app.tti.paint( prompt, "", - self.personality.image_files, + processor.personality.image_files, width = width, height = height, output_path=client.discussion.discussion_folder ) - elif self.personality_config.image_generation_engine in ["dall-e-2", "dall-e-3"]: - if not self.personality.app.tti: + elif processor.personality_config.image_generation_engine in ["dall-e-2", "dall-e-3"]: + if not processor.personality.app.tti: from lollms.services.dalle.lollms_dalle import LollmsDalle - self.step_start("Loading dalle service") - self.personality.app.tti = LollmsDalle(self.personality.app, self.personality.config.dall_e_key, self.personality_config.image_generation_engine) - self.personality.app.dalle = self.personality.app.tti - self.step_end("Loading dalle service") - self.step_start("Painting") - file = self.personality.app.tti.paint( + processor.step_start("Loading dalle service") + processor.personality.app.tti = LollmsDalle(processor.personality.app, processor.personality.config.dall_e_key, processor.personality_config.image_generation_engine) + processor.personality.app.dalle = processor.personality.app.tti + processor.step_end("Loading dalle service") + processor.step_start("Painting") + file = processor.personality.app.tti.paint( prompt, width = width, height = height, output_path=client.discussion.discussion_folder ) - self.step_end("Painting") + processor.step_end("Painting") file = str(file) escaped_url = discussion_path_to_url(file) return f'\n![]({escaped_url})' except Exception as ex: trace_exception(ex) - return "Couldn't generate image. Make sure Auto1111's stable diffusion service is installed" \ No newline at end of file + return "Couldn't generate image. Make sure Auto1111's stable diffusion service is installed" + + +def build_image_function(processor, client): + return { + "function_name": "build_image", + "function": partial(build_image, processor=processor, client=client), + "function_description": "Builds and shows an image from a prompt and width and height parameters. A square 1024x1024, a portrait woudl be 1024x1820 or landscape 1820x1024.", + "function_parameters": [{"name": "prompt", "type": "str"}, {"name": "width", "type": "int"}, {"name": "height", "type": "int"}] + } diff --git a/lollms/functions/select_image_file.py b/lollms/functions/select_image_file.py new file mode 100644 index 0000000..9c946e9 --- /dev/null +++ b/lollms/functions/select_image_file.py @@ -0,0 +1,43 @@ +from lollms.utilities import PackageManager, find_first_available_file_index, discussion_path_to_url +from lollms.client_session import Client +if not PackageManager.check_package_installed("pyautogui"): + PackageManager.install_package("pyautogui") +if not PackageManager.check_package_installed("PyQt5"): + PackageManager.install_package("PyQt5") +import cv2 +import time +from PyQt5 import QtWidgets, QtGui, QtCore +import sys +from functools import partial + + +def select_image_file(client): + app = QtWidgets.QApplication(sys.argv) + options = QtWidgets.QFileDialog.Options() + options |= QtWidgets.QFileDialog.ReadOnly + file_name, _ = QtWidgets.QFileDialog.getOpenFileName(None, "Select Image File", "", "Image Files (*.png *.jpg *.jpeg *.bmp);;All Files (*)", options=options) + if file_name: + frame = cv2.imread(file_name) + + if frame is None: + return "Failed to read the image file." + + view_image = client.discussion.discussion_folder/"view_images" + image = client.discussion.discussion_folder/"images" + index = find_first_available_file_index(view_image,"selected_image_",".png") + fn_view = view_image/f"selected_image_{index}.png" + cv2.imwrite(str(fn_view), frame) + fn = image/f"selected_image_{index}.png" + cv2.imwrite(str(fn), frame) + client.discussion.image_files.append(fn) + return f'' + else: + return "No file selected." + +def select_image_file_function(processor, client): + return { + "function_name": "select_image_file", + "function": partial(select_image_file, processor=processor, client=client), + "function_description": "Opens a file dialog to select an image file and displays it.", + "function_parameters": [] + } \ No newline at end of file diff --git a/lollms/functions/take_a_photo.py b/lollms/functions/take_a_photo.py index d5ab4ea..923ea82 100644 --- a/lollms/functions/take_a_photo.py +++ b/lollms/functions/take_a_photo.py @@ -9,6 +9,7 @@ import cv2 import time from PyQt5 import QtWidgets, QtGui, QtCore import sys +from functools import partial class CameraWindow(QtWidgets.QWidget): def __init__(self): @@ -113,3 +114,12 @@ def take_photo(client, use_ui=False): cv2.imwrite(str(fn), frame) client.discussion.image_files.append(fn) return f'' + + +def take_a_photo_function(processor, client): + return { + "function_name": "build_image", + "function": partial(take_photo, processor=processor, client=client), + "function_description": "Builds and shows an image from a prompt and width and height parameters. A square 1024x1024, a portrait woudl be 1024x1820 or landscape 1820x1024.", + "function_parameters": [{"name": "prompt", "type": "str"}, {"name": "width", "type": "int"}, {"name": "height", "type": "int"}] + } diff --git a/lollms/functions/take_screen_shot.py b/lollms/functions/take_screen_shot.py index 5746888..683b4c6 100644 --- a/lollms/functions/take_screen_shot.py +++ b/lollms/functions/take_screen_shot.py @@ -9,6 +9,10 @@ import pyautogui from PyQt5 import QtWidgets, QtGui from PyQt5.QtCore import Qt import sys + +from functools import partial + + class ScreenshotWindow(QtWidgets.QWidget): def __init__(self, client, screenshot, fn_view, fn): super().__init__() @@ -59,4 +63,15 @@ def take_screenshot(self, client: Client, use_ui: bool = False): screenshot.save(fn_view) screenshot.save(fn) client.discussion.image_files.append(fn) - return f'' \ No newline at end of file + return f'' + +def take_screenshot_function(client): + return { + "function_name": "take_screenshot", + "function": partial(take_screenshot, client=client), + "function_description": "Takes a screenshot of the current screen. Optionally allows user interface for image cropping and saving.", + "function_parameters": [ + {"name": "client", "type": "Client", "description": "The client object managing the discussion and images."}, + {"name": "use_ui", "type": "bool", "default": False, "description": "Flag to determine if a user interface should be used for editing the screenshot."} + ] + } \ No newline at end of file diff --git a/lollms/media.py b/lollms/media.py index 8e5a7fb..28afbaa 100644 --- a/lollms/media.py +++ b/lollms/media.py @@ -354,7 +354,7 @@ class RTCom: # self.transcription_signal.update_status.emit("Transcribing") self.lc.info("Transcribing") ASCIIColors.green("<>") - result = self.whisper.transcribe(str(Path(self.logs_folder)/filename)) + result = self.lc.tts.transcribe(str(Path(self.logs_folder)/filename)) transcription_fn = str(Path(self.logs_folder)/filename) + ".txt" with open(transcription_fn, "w", encoding="utf-8") as f: f.write(result["text"]) diff --git a/lollms/personality.py b/lollms/personality.py index aa0030f..7c5daeb 100644 --- a/lollms/personality.py +++ b/lollms/personality.py @@ -2408,7 +2408,21 @@ class APScript(StateMachine): self.step_end(f" Summary of {doc_name} - Processing chunk : {i+1}/{len(chunks)}") return "\n".join(summeries) - + def build_prompt_from_context_details(self, context_details:dict, custom_entries): + return self.build_prompt([ + context_details["conditionning"] if context_details["conditionning"] else "", + "!@>documentation:\n"+context_details["documentation"] if context_details["documentation"] else "", + "!@>knowledge:\n"+context_details["knowledge"] if context_details["knowledge"] else "", + context_details["user_description"] if context_details["user_description"] else "", + "!@>positive_boost:\n"+context_details["positive_boost"] if context_details["positive_boost"] else "", + "!@>negative_boost:\n"+context_details["negative_boost"] if context_details["negative_boost"] else "", + "!@>current_language:\n"+context_details["current_language"] if context_details["current_language"] else "", + "!@>fun_mode:\n"+context_details["fun_mode"] if context_details["fun_mode"] else "", + "!@>discussion_window:\n"+context_details["discussion_messages"] if context_details["discussion_messages"] else "", + custom_entries, + "!@>"+context_details["ai_prefix"].replace("!@>","").replace(":","")+":" + ], + 8) def build_prompt(self, prompt_parts:List[str], sacrifice_id:int=-1, context_size:int=None, minimum_spare_context_size:int=None): """ Builds the prompt for code generation. @@ -3293,6 +3307,7 @@ The AI should respond in this format using data from actions_list: return generated_text, function_calls + def generate_with_function_calls_and_images(self, prompt: str, images:list, functions: List[Dict[str, Any]], max_answer_length: Optional[int] = None) -> List[Dict[str, Any]]: """ Performs text generation with function calls. @@ -3419,6 +3434,22 @@ The AI should respond in this format using data from actions_list: return function_calls + + def interact_with_function_call(self, prompt, function_definitions, prompt_after_execution=True): + if len(self.personality.image_files)>0: + out, function_calls = self.generate_with_function_calls_and_images(prompt, self.personality.image_files, function_definitions) + else: + out, function_calls = self.generate_with_function_calls(prompt, function_definitions) + if len(function_calls)>0: + outputs = self.execute_function_calls(function_calls,function_definitions) + out += "\n!@>function calls results:\n" + "\n".join([str(o) for o in outputs]) + prompt += out + "!@>"+self.personality.name+":" + if len(self.personality.image_files)>0: + out, function_calls = self.generate_with_function_calls_and_images(prompt, self.personality.image_files, function_definitions) + else: + out, function_calls = self.generate_with_function_calls(prompt, function_definitions) + return out + #Helper method to convert outputs path to url def path2url(file): file = str(file).replace("\\","/") diff --git a/lollms/utilities.py b/lollms/utilities.py index cf10464..1e53004 100644 --- a/lollms/utilities.py +++ b/lollms/utilities.py @@ -954,6 +954,10 @@ class PackageManager: return True except ImportError: return False + except Exception as ex: + trace_exception(ex) + ASCIIColors.error("Something is wrong with your library.\nIt looks installed, but I am not able to call it.\nTry to reinstall it.") + return False @staticmethod def safe_import(module_name, library_name=None):