From 7468f4414c81c9a880fcc2fd7542dbf17d80a0a2 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Mon, 27 May 2024 23:59:55 +0200 Subject: [PATCH] enhanced diffusers --- configs/config.yaml | 6 ++- lollms/configs/config.yaml | 6 ++- lollms/functions/roll_a_dice.py | 38 +++++++++++++++++++ lollms/server/configs/config.yaml | 6 ++- lollms/services/diffusers/lollms_diffusers.py | 31 +++++++++------ lollms/services/xtts/lollms_xtts.py | 5 ++- 6 files changed, 77 insertions(+), 15 deletions(-) create mode 100644 lollms/functions/roll_a_dice.py diff --git a/configs/config.yaml b/configs/config.yaml index 9f3dffb..e67efc4 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 102 +version: 104 binding_name: null model_name: null model_variant: null @@ -151,6 +151,10 @@ openai_tts_voice: "alloy" enable_sd_service: false sd_base_url: http://localhost:7860 +# diffuser +diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload +diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS + # Dall e service key dall_e_key: "" dall_e_generation_engine: "dall-e-3" diff --git a/lollms/configs/config.yaml b/lollms/configs/config.yaml index 9f3dffb..e67efc4 100644 --- a/lollms/configs/config.yaml +++ b/lollms/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 102 +version: 104 binding_name: null model_name: null model_variant: null @@ -151,6 +151,10 @@ openai_tts_voice: "alloy" enable_sd_service: false sd_base_url: http://localhost:7860 +# diffuser +diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload +diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS + # Dall e service key dall_e_key: "" dall_e_generation_engine: "dall-e-3" diff --git a/lollms/functions/roll_a_dice.py b/lollms/functions/roll_a_dice.py new file mode 100644 index 0000000..39a1506 --- /dev/null +++ b/lollms/functions/roll_a_dice.py @@ -0,0 +1,38 @@ +# Lollms function call definition file +# Here you need to import any necessary imports depending on the function requested by the user +import random + +# Partial is useful if we need to preset some parameters +from functools import partial + +# It is advised to import typing elements +# from typing import List + +# Import PackageManager if there are potential libraries that need to be installed +from lollms.utilities import PackageManager + +# ascii_colors offers advanced console coloring and bug tracing +from ascii_colors import trace_exception + +# here is the core of the function to be built +def roll_a_dice() -> int: + try: + # handle exceptions + + # Perform dice roll + result = random.randint(1, 6) + + # Return the dice roll result + return result + except Exception as e: + return trace_exception(e) + + +#Here is the metadata function that should have the name in format function_name_function +def roll_a_dice_function(): + return { + "function_name": "roll_a_dice", # The function name in string + "function": roll_a_dice, # The function to be called + "function_description": "Returns a random dice roll result between 1 and 6.", # Description of the function + "function_parameters": [] # No parameters needed for this function + } diff --git a/lollms/server/configs/config.yaml b/lollms/server/configs/config.yaml index 9f3dffb..e67efc4 100644 --- a/lollms/server/configs/config.yaml +++ b/lollms/server/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 102 +version: 104 binding_name: null model_name: null model_variant: null @@ -151,6 +151,10 @@ openai_tts_voice: "alloy" enable_sd_service: false sd_base_url: http://localhost:7860 +# diffuser +diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload +diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS + # Dall e service key dall_e_key: "" dall_e_generation_engine: "dall-e-3" diff --git a/lollms/services/diffusers/lollms_diffusers.py b/lollms/services/diffusers/lollms_diffusers.py index 02872f6..5048ef9 100644 --- a/lollms/services/diffusers/lollms_diffusers.py +++ b/lollms/services/diffusers/lollms_diffusers.py @@ -8,7 +8,7 @@ import sys from lollms.app import LollmsApplication from lollms.paths import LollmsPaths from lollms.config import TypedConfig, ConfigTemplate, BaseConfig -from lollms.utilities import PackageManager, check_and_install_torch +from lollms.utilities import PackageManager, check_and_install_torch, find_next_available_filename import time import io import sys @@ -73,14 +73,17 @@ def install_diffusers(lollms_app:LollmsApplication): root_dir = lollms_app.lollms_paths.personal_path shared_folder = root_dir/"shared" diffusers_folder = shared_folder / "diffusers" + diffusers_folder.mkdir(exist_ok=True, parents=True) if not PackageManager.check_package_installed("diffusers"): PackageManager.install_or_update("diffusers") - diffusers_folder.mkdir(exist_ok=True, parents=True) + PackageManager.install_or_update("xformers") + def upgrade_diffusers(lollms_app:LollmsApplication): PackageManager.install_or_update("diffusers") + PackageManager.install_or_update("xformers") class LollmsDiffusers(LollmsTTI): @@ -115,13 +118,17 @@ class LollmsDiffusers(LollmsTTI): ASCIIColors.red(" ______ ") ASCIIColors.red(" |______| ") - import torch + import torch from diffusers import PixArtSigmaPipeline self.model = PixArtSigmaPipeline.from_pretrained( - "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", torch_dtype=torch.float16, cache_dir=self.models_dir + app.config.diffusers_model, torch_dtype=torch.float16, cache_dir=self.models_dir, + use_safetensors=True, ) # Enable memory optimizations. - self.model.enable_model_cpu_offload() + if app.config.diffusers_offloading_mode=="sequential_cpu_offload": + self.model.enable_sequential_cpu_offload() + elif app.coinfig.diffusers_offloading_mode=="model_cpu_offload": + self.model.enable_model_cpu_offload() @staticmethod def verify(app:LollmsApplication): @@ -162,11 +169,13 @@ class LollmsDiffusers(LollmsTTI): restore_faces=True, output_path=None ): - - array = self.model(diffusers_positive_prompt).images[0] - image = Image.fromarray(array) - + if output_path is None: + output_path = self.output_dir + from diffusers.utils.pil_utils import pt_to_pil + image = self.model(diffusers_positive_prompt, negative_prompt=diffusers_negative_prompt, guidance_scale=scale, num_inference_steps=steps,).images[0] + output_path = Path(output_path) + fn = find_next_available_filename(output_path,"diff_img_") # Save the image - image.save('output_image.png') - return None, None + image.save(fn) + return fn, {"prompt":diffusers_positive_prompt, "negative_prompt":diffusers_negative_prompt} diff --git a/lollms/services/xtts/lollms_xtts.py b/lollms/services/xtts/lollms_xtts.py index 12fbe15..5ea0016 100644 --- a/lollms/services/xtts/lollms_xtts.py +++ b/lollms/services/xtts/lollms_xtts.py @@ -199,7 +199,10 @@ class LollmsXTTS(LollmsTTS): "top_k": self.app.config.xtts_top_k, "enable_text_splitting": self.app.config.xtts_enable_text_splitting } - response = requests.post(f"{self.xtts_base_url}/set_tts_settings", settings) + response = requests.post(f"{self.xtts_base_url}/set_tts_settings", settings,headers={ + 'accept': 'application/json', + 'Content-Type': 'application/json' + }) if response.status_code == 200: ASCIIColors.success("XTTS updated successfully") except Exception as ex: