From 8b833fb615331e5728c6ae4bd2ddfb84fce4e335 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Fri, 31 May 2024 01:03:19 +0200 Subject: [PATCH] added stuff --- configs/config.yaml | 16 +- lollms/app.py | 38 +++-- lollms/configs/config.yaml | 16 +- lollms/functions/bibliography.py | 150 ++++++++++++++++++ lollms/personality.py | 6 +- lollms/services/fooocus/lollms_fooocus.py | 179 ++++++++++++++++++++++ lollms/tasks.py | 2 +- 7 files changed, 387 insertions(+), 20 deletions(-) create mode 100644 lollms/services/fooocus/lollms_fooocus.py diff --git a/configs/config.yaml b/configs/config.yaml index 5a540d3..b7c0dc5 100644 --- a/configs/config.yaml +++ b/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 106 +version: 108 binding_name: null model_name: null model_variant: null @@ -31,7 +31,17 @@ app_custom_logo: "" discussion_prompt_separator: "!@>" start_header_id_template: "!@>" end_header_id_template: ": " + separator_template: "\n" + +start_user_header_id_template: "!@>" +end_user_header_id_template: ": " +end_user_message_id_template: "" + +start_ai_header_id_template: "!@>" +end_ai_header_id_template: ": " +end_ai_message_id_template: "" + system_message_template: "system" seed: -1 @@ -156,6 +166,10 @@ openai_tts_voice: "alloy" enable_sd_service: false sd_base_url: http://localhost:7860 +# Image generation service +enable_fooocus_service: false +fooocus_base_url: http://localhost:7860 + # diffuser diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS diff --git a/lollms/app.py b/lollms/app.py index cb1d052..66d4f51 100644 --- a/lollms/app.py +++ b/lollms/app.py @@ -419,29 +419,29 @@ class LollmsApplication(LoLLMsCom): trace_exception(ex) self.warning(f"Couldn't load Motion control") - if self.config.active_tti_service == "diffusers": + if self.config.active_tti_service == "diffusers" and type(self.tti)!=LollmsDiffusers: from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers self.tti = LollmsDiffusers(self) - elif self.config.active_tti_service == "autosd": + elif self.config.active_tti_service == "autosd" and type(self.tti)!=LollmsSD: from lollms.services.sd.lollms_sd import LollmsSD self.tti = LollmsSD(self) - elif self.config.active_tti_service == "dall-e": + elif self.config.active_tti_service == "dall-e" and type(self.tti)!=LollmsDalle: from lollms.services.dalle.lollms_dalle import LollmsDalle self.tti = LollmsDalle(self, self.config.dall_e_key) - elif self.config.active_tti_service == "midjourney": + elif self.config.active_tti_service == "midjourney" and type(self.tti)!=LollmsMidjourney: from lollms.services.midjourney.lollms_midjourney import LollmsMidjourney self.tti = LollmsMidjourney(self, self.config.midjourney_key) - if self.config.active_tts_service == "openai_tts": + if self.config.active_tts_service == "openai_tts" and type(self.tts)!=LollmsOpenAITTS: from lollms.services.open_ai_tts.lollms_openai_tts import LollmsOpenAITTS self.tts = LollmsOpenAITTS(self, self.config.openai_tts_model, self.config.openai_tts_voice, self.config.openai_tts_key) elif self.config.active_tts_service == "xtts" and self.xtts: self.tts = self.xtts - if self.config.active_stt_service == "openai_whisper": + if self.config.active_stt_service == "openai_whisper" and type(self.stt)!=LollmsOpenAIWhisper: from lollms.services.openai_whisper.lollms_openai_whisper import LollmsOpenAIWhisper self.stt = LollmsOpenAIWhisper(self, self.config.openai_whisper_model, self.config.openai_whisper_key) - elif self.config.active_stt_service == "whisper": + elif self.config.active_stt_service == "whisper" and type(self.stt)!=LollmsWhisper : from lollms.services.whisper.lollms_whisper import LollmsWhisper self.stt = LollmsWhisper(self, self.config.whisper_model) @@ -771,11 +771,19 @@ class LollmsApplication(LoLLMsCom): """ start_header_id_template = self.config.start_header_id_template end_header_id_template = self.config.end_header_id_template + separator_template = self.config.separator_template + + start_user_header_id_template = self.config.start_user_header_id_template + end_user_header_id_template = self.config.end_user_header_id_template + end_user_message_id_template = self.config.end_user_message_id_template + + start_ai_header_id_template = self.config.start_ai_header_id_template + end_ai_header_id_template = self.config.end_ai_header_id_template + end_ai_message_id_template = self.config.end_ai_message_id_template + system_message_template = self.config.system_message_template - - if self.personality.callback is None: self.personality.callback = partial(self.process_chunk, client_id=client_id) # Get the list of messages @@ -1069,13 +1077,13 @@ class LollmsApplication(LoLLMsCom): # Tokenize the message content if self.config.use_model_name_in_discussions: if message.model: - msg = f"{separator_template}{start_header_id_template}{message.sender}({message.model}){end_header_id_template}" + message.content.strip() + msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}({message.model}){end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip() else: - msg = f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip() + msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip() message_tokenized = self.model.tokenize(msg) else: message_tokenized = self.model.tokenize( - f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip() + f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip() ) # Check if adding the message will exceed the available space if tokens_accumulated + len(message_tokenized) > available_space-n_tokens: @@ -1099,13 +1107,13 @@ class LollmsApplication(LoLLMsCom): if self.config.use_model_name_in_discussions: if message.model: - msg = f"{separator_template}{start_header_id_template}{message.sender}({message.model}){end_header_id_template}" + message.content.strip() + msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}({message.model}){end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip() else: - msg = f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip() + msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip() message_tokenized = self.model.tokenize(msg) else: message_tokenized = self.model.tokenize( - f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip() + f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip() ) # Add the tokenized message to the full_message_list diff --git a/lollms/configs/config.yaml b/lollms/configs/config.yaml index 5a540d3..b7c0dc5 100644 --- a/lollms/configs/config.yaml +++ b/lollms/configs/config.yaml @@ -1,5 +1,5 @@ # =================== Lord Of Large Language Multimodal Systems Configuration file =========================== -version: 106 +version: 108 binding_name: null model_name: null model_variant: null @@ -31,7 +31,17 @@ app_custom_logo: "" discussion_prompt_separator: "!@>" start_header_id_template: "!@>" end_header_id_template: ": " + separator_template: "\n" + +start_user_header_id_template: "!@>" +end_user_header_id_template: ": " +end_user_message_id_template: "" + +start_ai_header_id_template: "!@>" +end_ai_header_id_template: ": " +end_ai_message_id_template: "" + system_message_template: "system" seed: -1 @@ -156,6 +166,10 @@ openai_tts_voice: "alloy" enable_sd_service: false sd_base_url: http://localhost:7860 +# Image generation service +enable_fooocus_service: false +fooocus_base_url: http://localhost:7860 + # diffuser diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS diff --git a/lollms/functions/bibliography.py b/lollms/functions/bibliography.py index bab73af..20d32a4 100644 --- a/lollms/functions/bibliography.py +++ b/lollms/functions/bibliography.py @@ -206,3 +206,153 @@ if __name__ == "__main__": # Print the HTML output print(html_output) + + + +# Core function to search for PDFs on arXiv and download them to a specified directory +def search_and_rank(llm, search_subject:str, score_threshold:float, query: str, max_results: Optional[int] = 5, sort_by: Optional[str] = 'relevance', start_date: Optional[str] = None, end_date: Optional[str] = None, author: Optional[str] = None, client: Optional[Any] = None) -> (str, Dict[str, Any]): + try: + + if client is None: + download_to = Path("./pdf_search") + else: + download_to = client.discussion.discussion_folder / "pdf_search" + + # Construct the search URL with additional parameters + url = f'http://export.arxiv.org/api/query?search_query={query}&start=0&max_results={max_results}&sortBy={sort_by}' + if start_date: + url += f'&startDate={start_date}' + if end_date: + url += f'&endDate={end_date}' + if author: + url += f'&author={author}' + + response = requests.get(url) + response.raise_for_status() + + # Parse the response + soup = BeautifulSoup(response.content, 'xml') + entries = soup.find_all('entry') + + # Create the directory if it doesn't exist + download_to.mkdir(parents=True, exist_ok=True) + + # Extract PDF URLs and additional information + html_output = "" + report_content = "" + pdf_info = {} + for entry in entries: + pdf_url = entry.id.text.replace('abs', 'pdf') + '.pdf' + pdf_name = pdf_url.split('/')[-1] + pdf_path = download_to / pdf_name + + if client is None: + local_url = f'/discussions/pdf_search/{pdf_name}' + else: + local_url = discussion_path_to_url(pdf_path) + + # Extract additional information + title = entry.title.text + authors = ', '.join(author.find('name').text for author in entry.find_all('author')) + affiliations = ', '.join(affiliation.text for affiliation in entry.find_all('affiliation')) + abstract = entry.summary.text + published_date = entry.published.text + journal_ref = entry.find('journal_ref').text if entry.find('journal_ref') else 'N/A' + + # Use the LLM to rate the relevance of the text example compared to the search prompt + relevance_score = llm.fast_gen(f"Rate the relevance of the following text compared to the prompt on a scale of 0 to 1. Only respond with a number between 0 and 1:\nPrompt: {search_subject}\nText: {abstract}\nRelevance score:") + + # Extract the relevance score from the generated text + try: + score = float(relevance_score.strip()) + except ValueError: + score = 0.0 # Default score in case of parsing issues + + if score +

{title}

+

Authors: {authors}

+

Affiliations: {affiliations}

+

Abstract: {abstract}

+

Published Date: {published_date}

+

Journal/Conference: {journal_ref}

+

Relevance score: {score}

+

PDF Link

+

Local PDF

+ + """ + # Append to report content + report_content += f""" +Title: {title} +Authors: {authors} +Affiliations: {affiliations} +Abstract: {abstract} +Published Date: {published_date} +Journal/Conference: {journal_ref} +Relevance score: {score} +PDF Link: {pdf_url} +Local PDF: {local_url} +------------------------ + """ + # Append to pdf_info dict + pdf_info[pdf_name] = { + "title": title, + "authors": authors, + "affiliations": affiliations, + "abstract": abstract, + "published_date": published_date, + "journal_ref": journal_ref, + "pdf_url": pdf_url, + "local_url": local_url + } + + # Save the report to a text file + report_path = download_to / "pdf_search_report.txt" + with report_path.open('w', encoding='utf-8') as report_file: + report_file.write(report_content) + + html_output += "" + return html_output, pdf_info + + except Exception as e: + return trace_exception(e), {} + +# Metadata function +def search_and_rank_function(llm, score_threshold:float, client: Optional[Any] = None): + return { + "function_name": "arxiv_pdf_search", # The function name in string + "function": partial(arxiv_pdf_search, llm=llm, client=client, score_threshold= score_threshold), # The function to be called with partial to preset client + "function_description": "Searches for PDFs on arXiv based on a query, downloads them to a specified directory, rates them, sort them by rating and returns a HTML string containing article details and links, along with a dictionary containing detailed information about each PDF.", # Description of the function + "function_parameters": [ # The set of parameters + {"name": "search_subject", "type": "str", "description": "The search subject."}, + {"name": "query", "type": "str", "description": "The search query for arXiv."}, + {"name": "max_results", "type": "int", "description": "The maximum number of results to return. (Optional)", "optional": True, "default": 5}, + {"name": "sort_by", "type": "str", "description": "The sorting criteria for the search results (e.g., relevance, lastUpdatedDate). (Optional)", "optional": True, "default": "relevance"}, + {"name": "start_date", "type": "str", "description": "The start date for the search results in the format YYYY-MM-DD. (Optional)", "optional": True}, + {"name": "end_date", "type": "str", "description": "The end date for the search results in the format YYYY-MM-DD. (Optional)", "optional": True}, + {"name": "author", "type": "str", "description": "The author name for the search results. (Optional)", "optional": True}, + ] + } \ No newline at end of file diff --git a/lollms/personality.py b/lollms/personality.py index bbc6b29..086c423 100644 --- a/lollms/personality.py +++ b/lollms/personality.py @@ -3401,8 +3401,9 @@ The AI should respond in this format using data from actions_list: start_header_id_template = self.config.start_header_id_template end_header_id_template = self.config.end_header_id_template system_message_template = self.config.system_message_template + separator_template = self.config.separator_template - function_descriptions = [f"{start_header_id_template}{system_message_template}{end_header_id_template}If you need to call a function to fulfull the user request, use a function markdown tag with the function call as the following json format:", + function_descriptions = [f"{start_header_id_template}{system_message_template}{end_header_id_template}If you need to call a function to fulfill the user request, use a function markdown tag with the function call as the following json format:", "```function", "{", '"function_name":the name of the function to be called,', @@ -3412,8 +3413,9 @@ The AI should respond in this format using data from actions_list: "Only use available functions.", "You can call multiple functions in one generation.", "Each function call needs to be in a separate function markdown tag.", - "Do not add status of the execution as it will be added automatically by the system.", + f"Do not add status of the execution as it will be added automatically by the system.{separator_template}" f"{start_header_id_template}Available functions{end_header_id_template}\n"] + for function in functions: description = f"{function['function_name']}: {function['function_description']}\nparameters:{function['function_parameters']}" function_descriptions.append(description) diff --git a/lollms/services/fooocus/lollms_fooocus.py b/lollms/services/fooocus/lollms_fooocus.py new file mode 100644 index 0000000..77be3fe --- /dev/null +++ b/lollms/services/fooocus/lollms_fooocus.py @@ -0,0 +1,179 @@ +# Title LollmsFooocus +# Licence: MIT +# Author : Paris Neo + # All rights are reserved + +from pathlib import Path +import sys +from lollms.app import LollmsApplication +from lollms.paths import LollmsPaths +from lollms.config import TypedConfig, ConfigTemplate, BaseConfig +from lollms.utilities import PackageManager, check_and_install_torch, find_next_available_filename +import time +import io +import sys +import requests +import os +import base64 +import subprocess +import time +import json +import platform +from dataclasses import dataclass +from PIL import Image, PngImagePlugin +from enum import Enum +from typing import List, Dict, Any + +from ascii_colors import ASCIIColors, trace_exception +from lollms.paths import LollmsPaths +from lollms.tti import LollmsTTI +from lollms.utilities import git_pull, show_yes_no_dialog, run_script_in_env, create_conda_env +import subprocess +import shutil +from tqdm import tqdm +import threading + + + +def download_file(url, folder_path, local_filename): + # Make sure 'folder_path' exists + folder_path.mkdir(parents=True, exist_ok=True) + + with requests.get(url, stream=True) as r: + r.raise_for_status() + total_size = int(r.headers.get('content-length', 0)) + progress_bar = tqdm(total=total_size, unit='B', unit_scale=True) + with open(folder_path / local_filename, 'wb') as f: + for chunk in r.iter_content(chunk_size=8192): + f.write(chunk) + progress_bar.update(len(chunk)) + progress_bar.close() + + return local_filename + +def install_model(lollms_app:LollmsApplication, model_url): + root_dir = lollms_app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + if not PackageManager.check_package_installed("diffusers"): + PackageManager.install_or_update("diffusers") + if not PackageManager.check_package_installed("torch"): + check_and_install_torch(True) + + import torch + from diffusers import PixArtSigmaPipeline + + # You can replace the checkpoint id with "PixArt-alpha/PixArt-Sigma-XL-2-512-MS" too. + pipe = PixArtSigmaPipeline.from_pretrained( + "PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", torch_dtype=torch.float16 + ) + + +def install_diffusers(lollms_app:LollmsApplication): + root_dir = lollms_app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + diffusers_folder.mkdir(exist_ok=True, parents=True) + if not PackageManager.check_package_installed("diffusers"): + PackageManager.install_or_update("diffusers") + PackageManager.install_or_update("xformers") + + + + +def upgrade_diffusers(lollms_app:LollmsApplication): + PackageManager.install_or_update("diffusers") + PackageManager.install_or_update("xformers") + + +class LollmsFooocus(LollmsTTI): + has_controlnet = False + def __init__( + self, + app:LollmsApplication, + wm = "Artbot", + ): + super().__init__(app) + self.ready = False + # Get the current directory + lollms_paths = app.lollms_paths + root_dir = lollms_paths.personal_path + + self.wm = wm + + shared_folder = root_dir/"shared" + self.diffusers_folder = shared_folder / "diffusers" + self.output_dir = root_dir / "outputs/diffusers" + self.models_dir = self.diffusers_folder / "models" + self.output_dir.mkdir(parents=True, exist_ok=True) + self.models_dir.mkdir(parents=True, exist_ok=True) + + ASCIIColors.red(" _ _ _ ______ ") + ASCIIColors.red("| | | | | | ___| ") + ASCIIColors.red("| | ___ | | |_ __ ___ ___ | |_ ___ ___ ___ ___ _ _ ___ ") + ASCIIColors.red("| | / _ \| | | '_ ` _ \/ __| | _/ _ \ / _ \ / _ \ / __| | | / __|") + ASCIIColors.red("| |___| (_) | | | | | | | \__ \ | || (_) | (_) | (_) | (__| |_| \__ \ ") + ASCIIColors.red("\_____/\___/|_|_|_| |_| |_|___/ \_| \___/ \___/ \___/ \___|\__,_|___/") + ASCIIColors.red(" ______ ") + ASCIIColors.red(" |______| ") + import torch + from diffusers import PixArtSigmaPipeline + self.model = PixArtSigmaPipeline.from_pretrained( + app.config.diffusers_model, torch_dtype=torch.float16, cache_dir=self.models_dir, + use_safetensors=True, + ) + # Enable memory optimizations. + if app.config.diffusers_offloading_mode=="sequential_cpu_offload": + self.model.enable_sequential_cpu_offload() + elif app.coinfig.diffusers_offloading_mode=="model_cpu_offload": + self.model.enable_model_cpu_offload() + + @staticmethod + def verify(app:LollmsApplication): + # Clone repository + root_dir = app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + return diffusers_folder.exists() + + def get(app:LollmsApplication): + root_dir = app.lollms_paths.personal_path + shared_folder = root_dir/"shared" + diffusers_folder = shared_folder / "diffusers" + diffusers_script_path = diffusers_folder / "lollms_diffusers.py" + git_pull(diffusers_folder) + + if diffusers_script_path.exists(): + ASCIIColors.success("lollms_diffusers found.") + ASCIIColors.success("Loading source file...",end="") + # use importlib to load the module from the file path + from lollms.services.diffusers.lollms_diffusers import LollmsFooocus + ASCIIColors.success("ok") + return LollmsFooocus + + + def paint( + self, + positive_prompt, + negative_prompt, + files=[], + sampler_name="Euler", + seed=-1, + scale=7.5, + steps=20, + img2img_denoising_strength=0.9, + width=512, + height=512, + restore_faces=True, + output_path=None + ): + if output_path is None: + output_path = self.output_dir + from diffusers.utils.pil_utils import pt_to_pil + image = self.model(positive_prompt, negative_prompt=negative_prompt, guidance_scale=scale, num_inference_steps=steps,).images[0] + output_path = Path(output_path) + fn = find_next_available_filename(output_path,"diff_img_") + # Save the image + image.save(fn) + return fn, {"prompt":positive_prompt, "negative_prompt":negative_prompt} + diff --git a/lollms/tasks.py b/lollms/tasks.py index 5499fa7..4c15e36 100644 --- a/lollms/tasks.py +++ b/lollms/tasks.py @@ -767,7 +767,7 @@ class TasksLibrary: end_header_id_template = self.config.end_header_id_template system_message_template = self.config.system_message_template separator_template = self.config.separator_template - function_descriptions = [f"{start_header_id_template}information{end_header_id_template}If you need to call a function to fulfull the user request, use a function markdown tag with the function call as the following json format:", + function_descriptions = [f"{start_header_id_template}information{end_header_id_template}If you need to call a function to fulfill the user request, use a function markdown tag with the function call as the following json format:", "```function", "{", '"function_name":the name of the function to be called,',