From dad67273b53a94395766bc20a2c54203719a6d86 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Mon, 27 May 2024 22:38:15 +0200 Subject: [PATCH] added new text to image tool (diffusers) --- lollms/app.py | 12 +- lollms/functions/generate_image.py | 17 +- lollms/server/endpoints/lollms_diffusers.py | 82 +++++++++ lollms/services/diffusers/lollms_diffusers.py | 172 ++++++++++++++++++ lollms/services/sd/lollms_sd.py | 16 -- 5 files changed, 279 insertions(+), 20 deletions(-) create mode 100644 lollms/server/endpoints/lollms_diffusers.py create mode 100644 lollms/services/diffusers/lollms_diffusers.py diff --git a/lollms/app.py b/lollms/app.py index b64b245..84abbb6 100644 --- a/lollms/app.py +++ b/lollms/app.py @@ -315,8 +315,11 @@ class LollmsApplication(LoLLMsCom): trace_exception(ex) self.warning(f"Couldn't load Motion control") - - if self.config.active_tti_service == "autosd": + + if self.config.active_tti_service == "diffusers": + from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers + self.tti = LollmsDiffusers(self) + elif self.config.active_tti_service == "autosd": from lollms.services.sd.lollms_sd import LollmsSD self.tti = LollmsSD(self) elif self.config.active_tti_service == "dall-e": @@ -410,7 +413,10 @@ class LollmsApplication(LoLLMsCom): trace_exception(ex) self.warning(f"Couldn't load Motion control") - if self.config.active_tti_service == "autosd": + if self.config.active_tti_service == "diffusers": + from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers + self.tti = LollmsDiffusers(self) + elif self.config.active_tti_service == "autosd": from lollms.services.sd.lollms_sd import LollmsSD self.tti = LollmsSD(self) elif self.config.active_tti_service == "dall-e": diff --git a/lollms/functions/generate_image.py b/lollms/functions/generate_image.py index 85bd0b8..99a63de 100644 --- a/lollms/functions/generate_image.py +++ b/lollms/functions/generate_image.py @@ -14,7 +14,22 @@ from functools import partial def build_image(prompt, antiprompt, width, height, processor:APScript, client:Client): try: - if processor.personality.config.active_tti_service=="autosd": + if processor.personality.config.active_tti_service=="diffusers": + if not processor.personality.app.tti: + from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers + processor.step_start("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") + processor.personality.app.tti = LollmsDiffusers(processor.personality.app, processor.personality.name, max_retries=-1,auto_sd_base_url=processor.personality.config.sd_base_url) + processor.personality.app.sd = processor.personality.app.tti + processor.step_end("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") + file, infos = processor.personality.app.tti.paint( + prompt, + antiprompt, + processor.personality.image_files, + width = width, + height = height, + output_path=client.discussion.discussion_folder + ) + elif processor.personality.config.active_tti_service=="autosd": if not processor.personality.app.tti: from lollms.services.sd.lollms_sd import LollmsSD processor.step_start("Loading ParisNeo's fork of AUTOMATIC1111's stable diffusion service") diff --git a/lollms/server/endpoints/lollms_diffusers.py b/lollms/server/endpoints/lollms_diffusers.py new file mode 100644 index 0000000..0580625 --- /dev/null +++ b/lollms/server/endpoints/lollms_diffusers.py @@ -0,0 +1,82 @@ +""" +project: lollms_webui +file: lollms_diffusers.py +author: ParisNeo +description: + This module is for diffusers installation and management + +""" +from fastapi import APIRouter, Request +from lollms_webui import LOLLMSWebUI +from pydantic import BaseModel +from starlette.responses import StreamingResponse +from lollms.types import MSG_TYPE +from lollms.main_config import BaseConfig +from lollms.utilities import detect_antiprompt, remove_text_from_string, trace_exception, find_first_available_file_index, add_period, PackageManager +from lollms.security import check_access +from pathlib import Path +from ascii_colors import ASCIIColors +import os +import platform + +# ----------------------- Defining router and main class ------------------------------ + +router = APIRouter() +lollmsElfServer:LOLLMSWebUI = LOLLMSWebUI.get_instance() + +class Identification(BaseModel): + client_id: str +class ModelPost(BaseModel): + client_id: str + model_url: str +# ----------------------- voice ------------------------------ + +@router.post("/install_diffusers") +# async def your_endpoint(request: Request): +# request_data = await request.json() +# print(request_data) # Use proper logging in real applications +def install_diffusers(data: Identification): + check_access(lollmsElfServer, data.client_id) + try: + if lollmsElfServer.config.headless_server_mode: + return {"status":False,"error":"Service installation is blocked when in headless mode for obvious security reasons!"} + + if lollmsElfServer.config.host!="localhost" and lollmsElfServer.config.host!="127.0.0.1": + return {"status":False,"error":"Service installation is blocked when the server is exposed outside for very obvious reasons!"} + + lollmsElfServer.ShowBlockingMessage("Installing Diffusers library\nPlease stand by") + from lollms.services.diffusers.lollms_diffusers import install_diffusers + install_diffusers(lollmsElfServer) + ASCIIColors.success("Done") + lollmsElfServer.HideBlockingMessage() + return {"status":True} + except Exception as ex: + lollmsElfServer.HideBlockingMessage() + lollmsElfServer.InfoMessage(f"It looks like I could not install SD because of this error:\n{ex}\nThis is commonly caused by a previous version that I couldn't delete. PLease remove {lollmsElfServer.lollms_paths.personal_path}/shared/auto_sd manually then try again") + return {"status":False, 'error':str(ex)} + + +@router.post("/upgrade_diffusers") +def upgrade_sd(data: Identification): + check_access(lollmsElfServer, data.client_id) + try: + if lollmsElfServer.config.headless_server_mode: + return {"status":False,"error":"Service installation is blocked when in headless mode for obvious security reasons!"} + + if lollmsElfServer.config.host!="localhost" and lollmsElfServer.config.host!="127.0.0.1": + return {"status":False,"error":"Service installation is blocked when the server is exposed outside for very obvious reasons!"} + + lollmsElfServer.ShowBlockingMessage("Upgrading Diffusers library\nPlease stand by") + from lollms.services.diffusers.lollms_diffusers import upgrade_diffusers + upgrade_diffusers(lollmsElfServer) + ASCIIColors.success("Done") + lollmsElfServer.HideBlockingMessage() + return {"status":True} + except Exception as ex: + lollmsElfServer.HideBlockingMessage() + lollmsElfServer.InfoMessage(f"It looks like I could not install SD because of this error:\n{ex}\nThis is commonly caused by a previous version that I couldn't delete. PLease remove {lollmsElfServer.lollms_paths.personal_path}/shared/auto_sd manually then try again") + return {"status":False, 'error':str(ex)} + +@router.post("/install_diffusers_model") +def install_model(data: ModelPost): + check_access(lollmsElfServer, data.client_id) diff --git a/lollms/services/diffusers/lollms_diffusers.py b/lollms/services/diffusers/lollms_diffusers.py new file mode 100644 index 0000000..02872f6 --- /dev/null +++ b/lollms/services/diffusers/lollms_diffusers.py @@ -0,0 +1,172 @@ +# Title LollmsDiffusers +# Licence: MIT +# Author : Paris Neo + # All rights are reserved + +from pathlib import Path +import sys +from lollms.app import LollmsApplication +from lollms.paths import LollmsPaths +from lollms.config import TypedConfig, ConfigTemplate, BaseConfig +from lollms.utilities import PackageManager, check_and_install_torch +import time +import io +import sys +import requests +import os +import base64 +import subprocess +import time +import json +import platform +from dataclasses import dataclass +from PIL import Image, PngImagePlugin +from enum import Enum +from typing import List, Dict, Any + +from ascii_colors import ASCIIColors, trace_exception +from lollms.paths import LollmsPaths +from lollms.tti import LollmsTTI +from lollms.utilities import git_pull, show_yes_no_dialog, run_script_in_env, create_conda_env +import subprocess +import shutil +from tqdm import tqdm +import threading + + + +def download_file(url, folder_path, local_filename): + # Make sure 'folder_path' exists + folder_path.mkdir(parents=True, exist_ok=True) + + with requests.get(url, stream=True) as r: + r.raise_for_status() + total_size = int(r.headers.get('content-length', 0)) + progress_bar = tqdm(total=total_size, unit='B', unit_scale=True) + with open(folder_path / local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + progress_bar.update(len(chunk)) + progress_bar.close() + + return local_filename + +def install_model(lollms_app:LollmsApplication, model_url): + root_dir = lollms_app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + if not PackageManager.check_package_installed("diffusers"): + PackageManager.install_or_update("diffusers") + if not PackageManager.check_package_installed("torch"): + check_and_install_torch(True) + + import torch + from diffusers import PixArtSigmaPipeline + + # You can replace the checkpoint id with "PixArt-alpha/PixArt-Sigma-XL-2-512-MS" too. + pipe = PixArtSigmaPipeline.from_pretrained( + "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", torch_dtype=torch.float16 + ) + + +def install_diffusers(lollms_app:LollmsApplication): + root_dir = lollms_app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + if not PackageManager.check_package_installed("diffusers"): + PackageManager.install_or_update("diffusers") + diffusers_folder.mkdir(exist_ok=True, parents=True) + + + +def upgrade_diffusers(lollms_app:LollmsApplication): + PackageManager.install_or_update("diffusers") + + +class LollmsDiffusers(LollmsTTI): + has_controlnet = False + def __init__( + self, + app:LollmsApplication, + wm = "Artbot", + ): + super().__init__(app) + self.ready = False + # Get the current directory + lollms_paths = app.lollms_paths + root_dir = lollms_paths.personal_path + + self.wm = wm + + shared_folder = root_dir/"shared" + self.diffusers_folder = shared_folder / "diffusers" + self.output_dir = root_dir / "outputs/diffusers" + self.models_dir = self.diffusers_folder / "models" + self.output_dir.mkdir(parents=True, exist_ok=True) + self.models_dir.mkdir(parents=True, exist_ok=True) + + + ASCIIColors.red(" _ _ _ _ _ __ __ ") + ASCIIColors.red(" | | | | | | (_)/ _|/ _| ") + ASCIIColors.red(" | | ___ | | |_ __ ___ ___ __| |_| |_| |_ _ _ ___ ___ _ __ ___ ") + ASCIIColors.red(" | | / _ \| | | '_ ` _ \/ __| / _` | | _| _| | | / __|/ _ \ '__/ __| ") + ASCIIColors.red(" | |___| (_) | | | | | | | \__ \| (_| | | | | | | |_| \__ \ __/ | \__ \ ") + ASCIIColors.red(" |______\___/|_|_|_| |_| |_|___/ \__,_|_|_| |_| \__,_|___/\___|_| |___/ ") + ASCIIColors.red(" ______ ") + ASCIIColors.red(" |______| ") + + import torch + from diffusers import PixArtSigmaPipeline + self.model = PixArtSigmaPipeline.from_pretrained( + "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", torch_dtype=torch.float16, cache_dir=self.models_dir + ) + # Enable memory optimizations. + self.model.enable_model_cpu_offload() + + @staticmethod + def verify(app:LollmsApplication): + # Clone repository + root_dir = app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + return diffusers_folder.exists() + + def get(app:LollmsApplication): + root_dir = app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + diffusers_script_path = diffusers_folder / "lollms_diffusers.py" + git_pull(diffusers_folder) + + if diffusers_script_path.exists(): + ASCIIColors.success("lollms_diffusers found.") + ASCIIColors.success("Loading source file...",end="") + # use importlib to load the module from the file path + from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers + ASCIIColors.success("ok") + return LollmsDiffusers + + + def paint( + self, + diffusers_positive_prompt, + diffusers_negative_prompt, + files=[], + sampler_name="Euler", + seed=-1, + scale=7.5, + steps=20, + img2img_denoising_strength=0.9, + width=512, + height=512, + restore_faces=True, + output_path=None + ): + + array = self.model(diffusers_positive_prompt).images[0] + image = Image.fromarray(array) + + # Save the image + image.save('output_image.png') + return None, None + diff --git a/lollms/services/sd/lollms_sd.py b/lollms/services/sd/lollms_sd.py index 6abb641..0852174 100644 --- a/lollms/services/sd/lollms_sd.py +++ b/lollms/services/sd/lollms_sd.py @@ -119,22 +119,6 @@ def upgrade_sd(lollms_app:LollmsApplication): subprocess.run(["git", "clone", "https://github.com/ParisNeo/SD-CN-Animation.git", str(sd_folder/"extensions/SD-CN-Animation")]) - -def upgrade_sd(lollms_app:LollmsApplication): - root_dir = lollms_app.lollms_paths.personal_path - shared_folder = root_dir/"shared" - sd_folder = shared_folder / "auto_sd" - if not sd_folder.exists(): - lollms_app.InfoMessage("Comfyui is not installed, install it first") - return - - subprocess.run(["git", "pull", str(sd_folder)]) - subprocess.run(["git", "pull", str(sd_folder/"extensions/SD-CN-Animation")]) - ASCIIColors.success("DONE") - - - - def raw_b64_img(image: Image) -> str: # XXX controlnet only accepts RAW base64 without headers with io.BytesIO() as output_bytes: