changed function calls

lollms/functions/generate_image.py

@@ -10,43 +10,53 @@ import cv2
 import time
 from PyQt5 import QtWidgets, QtGui, QtCore
 import sys
+from functools import partial
 
-def build_image(prompt, width, height, self:APScript, client:Client):
+def build_image(prompt, width, height, processor:APScript, client:Client):
     try:
-        if self.personality.config.active_tti_service=="autosd":
-            if not self.personality.app.tti:
+        if processor.personality.config.active_tti_service=="autosd":
+            if not processor.personality.app.tti:
                 from lollms.services.sd.lollms_sd import LollmsSD
-                self.step_start("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service")
-                self.personality.app.tti = LollmsSD(self.personality.app, self.personality.name, max_retries=-1,auto_sd_base_url=self.personality.config.sd_base_url)
-                self.personality.app.sd = self.personality.app.tti
-                self.step_end("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service")
-            file, infos = self.personality.app.tti.paint(
+                processor.step_start("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service")
+                processor.personality.app.tti = LollmsSD(processor.personality.app, processor.personality.name, max_retries=-1,auto_sd_base_url=processor.personality.config.sd_base_url)
+                processor.personality.app.sd = processor.personality.app.tti
+                processor.step_end("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service")
+            file, infos = processor.personality.app.tti.paint(
                             prompt, 
                             "",
-                            self.personality.image_files,
+                            processor.personality.image_files,
                             width = width,
                             height = height,
                             output_path=client.discussion.discussion_folder
                         )
-        elif self.personality_config.image_generation_engine in ["dall-e-2", "dall-e-3"]:
-            if not self.personality.app.tti:
+        elif processor.personality_config.image_generation_engine in ["dall-e-2", "dall-e-3"]:
+            if not processor.personality.app.tti:
                 from lollms.services.dalle.lollms_dalle import LollmsDalle
-                self.step_start("Loading dalle service")
-                self.personality.app.tti = LollmsDalle(self.personality.app, self.personality.config.dall_e_key, self.personality_config.image_generation_engine)
-                self.personality.app.dalle = self.personality.app.tti
-                self.step_end("Loading dalle service")
-            self.step_start("Painting")
-            file = self.personality.app.tti.paint(
+                processor.step_start("Loading dalle service")
+                processor.personality.app.tti = LollmsDalle(processor.personality.app, processor.personality.config.dall_e_key, processor.personality_config.image_generation_engine)
+                processor.personality.app.dalle = processor.personality.app.tti
+                processor.step_end("Loading dalle service")
+            processor.step_start("Painting")
+            file = processor.personality.app.tti.paint(
                             prompt, 
                             width = width,
                             height = height,
                             output_path=client.discussion.discussion_folder
                         )
-            self.step_end("Painting")
+            processor.step_end("Painting")
 
         file = str(file)
         escaped_url =  discussion_path_to_url(file)
         return f'\n![]({escaped_url})'
     except Exception as ex:
         trace_exception(ex)
-        return "Couldn't generate image. Make sure Auto1111's stable diffusion service is installed"
+        return "Couldn't generate image. Make sure Auto1111's stable diffusion service is installed"
+
+
+def build_image_function(processor, client):
+        return {
+                "function_name": "build_image",
+                "function": partial(build_image, processor=processor, client=client),
+                "function_description": "Builds and shows an image from a prompt and width and height parameters. A square 1024x1024, a portrait woudl be 1024x1820 or landscape 1820x1024.",
+                "function_parameters": [{"name": "prompt", "type": "str"}, {"name": "width", "type": "int"}, {"name": "height", "type": "int"}]                
+            }

lollms/functions/select_image_file.py

@@ -0,0 +1,43 @@
+from lollms.utilities import PackageManager, find_first_available_file_index, discussion_path_to_url
+from lollms.client_session import Client
+if not PackageManager.check_package_installed("pyautogui"):
+    PackageManager.install_package("pyautogui")
+if not PackageManager.check_package_installed("PyQt5"):
+    PackageManager.install_package("PyQt5")
+import cv2
+import time
+from PyQt5 import QtWidgets, QtGui, QtCore
+import sys
+from functools import partial
+
+
+def select_image_file(client):
+    app = QtWidgets.QApplication(sys.argv)
+    options = QtWidgets.QFileDialog.Options()
+    options |= QtWidgets.QFileDialog.ReadOnly
+    file_name, _ = QtWidgets.QFileDialog.getOpenFileName(None, "Select Image File", "", "Image Files (*.png *.jpg *.jpeg *.bmp);;All Files (*)", options=options)
+    if file_name:
+        frame = cv2.imread(file_name)
+
+        if frame is None:
+            return "Failed to read the image file."
+
+        view_image = client.discussion.discussion_folder/"view_images"
+        image = client.discussion.discussion_folder/"images"
+        index = find_first_available_file_index(view_image,"selected_image_",".png")
+        fn_view = view_image/f"selected_image_{index}.png"
+        cv2.imwrite(str(fn_view), frame)
+        fn = image/f"selected_image_{index}.png"
+        cv2.imwrite(str(fn), frame)
+        client.discussion.image_files.append(fn)
+        return f'<img src="{discussion_path_to_url(fn_view)}" width="80%"></img>'
+    else:
+        return "No file selected."
+
+def select_image_file_function(processor, client):
+    return {
+            "function_name": "select_image_file",
+            "function": partial(select_image_file, processor=processor, client=client),
+            "function_description": "Opens a file dialog to select an image file and displays it.",
+            "function_parameters": []                
+        }

lollms/functions/take_a_photo.py

@@ -9,6 +9,7 @@ import cv2
 import time
 from PyQt5 import QtWidgets, QtGui, QtCore
 import sys
+from functools import partial
 
 class CameraWindow(QtWidgets.QWidget):
     def __init__(self):
@@ -113,3 +114,12 @@ def take_photo(client, use_ui=False):
     cv2.imwrite(str(fn), frame)
     client.discussion.image_files.append(fn)
     return f'<img src="{discussion_path_to_url(fn_view)}" width="80%"></img>'
+
+
+def take_a_photo_function(processor, client):
+    return {
+            "function_name": "build_image",
+            "function": partial(take_photo, processor=processor, client=client),
+            "function_description": "Builds and shows an image from a prompt and width and height parameters. A square 1024x1024, a portrait woudl be 1024x1820 or landscape 1820x1024.",
+            "function_parameters": [{"name": "prompt", "type": "str"}, {"name": "width", "type": "int"}, {"name": "height", "type": "int"}]                
+        }

lollms/functions/take_screen_shot.py

@@ -9,6 +9,10 @@ import pyautogui
 from PyQt5 import QtWidgets, QtGui
 from PyQt5.QtCore import Qt
 import sys
+
+from functools import partial
+
+
 class ScreenshotWindow(QtWidgets.QWidget):
     def __init__(self, client, screenshot, fn_view, fn):
         super().__init__()
@@ -59,4 +63,15 @@ def take_screenshot(self, client: Client, use_ui: bool = False):
         screenshot.save(fn_view)
         screenshot.save(fn)
         client.discussion.image_files.append(fn)
-        return f'<img src="{discussion_path_to_url(fn_view)}" width="80%"></img>'
+        return f'<img src="{discussion_path_to_url(fn_view)}" width="80%"></img>'
+
+def take_screenshot_function(client):
+    return {
+            "function_name": "take_screenshot",
+            "function": partial(take_screenshot, client=client),
+            "function_description": "Takes a screenshot of the current screen. Optionally allows user interface for image cropping and saving.",
+            "function_parameters": [
+                {"name": "client", "type": "Client", "description": "The client object managing the discussion and images."},
+                {"name": "use_ui", "type": "bool", "default": False, "description": "Flag to determine if a user interface should be used for editing the screenshot."}
+            ]                
+        }

lollms/media.py

@@ -354,7 +354,7 @@ class RTCom:
                     # self.transcription_signal.update_status.emit("Transcribing")
                     self.lc.info("Transcribing")
                     ASCIIColors.green("<<TRANSCRIBING>>")
-                    result = self.whisper.transcribe(str(Path(self.logs_folder)/filename))
+                    result = self.lc.tts.transcribe(str(Path(self.logs_folder)/filename))
                     transcription_fn = str(Path(self.logs_folder)/filename) + ".txt"
                     with open(transcription_fn, "w", encoding="utf-8") as f:
                         f.write(result["text"])

lollms/personality.py

@@ -2408,7 +2408,21 @@ class APScript(StateMachine):
             self.step_end(f" Summary of {doc_name} - Processing chunk : {i+1}/{len(chunks)}")
         return "\n".join(summeries)
 
-
+    def build_prompt_from_context_details(self, context_details:dict, custom_entries):
+        return self.build_prompt([
+                    context_details["conditionning"] if context_details["conditionning"] else "",
+                    "!@>documentation:\n"+context_details["documentation"] if context_details["documentation"] else "",
+                    "!@>knowledge:\n"+context_details["knowledge"] if context_details["knowledge"] else "",
+                    context_details["user_description"] if context_details["user_description"] else "",
+                    "!@>positive_boost:\n"+context_details["positive_boost"] if context_details["positive_boost"] else "",
+                    "!@>negative_boost:\n"+context_details["negative_boost"] if context_details["negative_boost"] else "",
+                    "!@>current_language:\n"+context_details["current_language"] if context_details["current_language"] else "",
+                    "!@>fun_mode:\n"+context_details["fun_mode"] if context_details["fun_mode"] else "",
+                    "!@>discussion_window:\n"+context_details["discussion_messages"] if context_details["discussion_messages"] else "",
+                    custom_entries,
+                    "!@>"+context_details["ai_prefix"].replace("!@>","").replace(":","")+":"
+                ], 
+                8)
     def build_prompt(self, prompt_parts:List[str], sacrifice_id:int=-1, context_size:int=None, minimum_spare_context_size:int=None):
         """
         Builds the prompt for code generation.
@@ -3293,6 +3307,7 @@ The AI should respond in this format using data from actions_list:
 
         return generated_text, function_calls
 
+
     def generate_with_function_calls_and_images(self, prompt: str, images:list, functions: List[Dict[str, Any]], max_answer_length: Optional[int] = None) -> List[Dict[str, Any]]:
         """
         Performs text generation with function calls.
@@ -3419,6 +3434,22 @@ The AI should respond in this format using data from actions_list:
 
         return function_calls
 
+
+    def interact_with_function_call(self, prompt, function_definitions, prompt_after_execution=True):
+        if len(self.personality.image_files)>0:
+            out, function_calls = self.generate_with_function_calls_and_images(prompt, self.personality.image_files, function_definitions)
+        else:
+            out, function_calls = self.generate_with_function_calls(prompt, function_definitions)
+        if len(function_calls)>0:
+            outputs = self.execute_function_calls(function_calls,function_definitions)
+            out += "\n!@>function calls results:\n" + "\n".join([str(o) for o in outputs])
+            prompt += out + "!@>"+self.personality.name+":"
+            if len(self.personality.image_files)>0:
+                out, function_calls = self.generate_with_function_calls_and_images(prompt, self.personality.image_files, function_definitions)
+            else:
+                out, function_calls = self.generate_with_function_calls(prompt, function_definitions)
+        return out
+
     #Helper method to convert outputs path to url
     def path2url(file):
         file = str(file).replace("\\","/")

lollms/utilities.py

@@ -954,6 +954,10 @@ class PackageManager:
             return True
         except ImportError:
             return False
+        except Exception as ex:
+            trace_exception(ex)
+            ASCIIColors.error("Something is wrong with your library.\nIt looks installed, but I am not able to call it.\nTry to reinstall it.")
+            return False
         
     @staticmethod
     def safe_import(module_name, library_name=None):