ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2025-02-06 19:19:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added few needed stuff

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-02-27 17:06:04 +01:00
parent 11f823bb8a
commit 487aa45ce2
6 changed files with 165 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
configs/config.yaml
View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file =========================== # =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 73 version: 74
binding_name: null binding_name: null
model_name: null model_name: null
model_variant: null model_variant: null
@ -85,6 +85,10 @@ current_language: en
enable_sd_service: false enable_sd_service: false
sd_base_url: http://localhost:7860 sd_base_url: http://localhost:7860
# Motion control service
enable_motion_ctrl_service: false
motion_ctrl_base_url: http://localhost:7861
# ollama service # ollama service
enable_ollama_service: false enable_ollama_service: false
ollama_base_url: http://localhost:11434 ollama_base_url: http://localhost:11434

9
lollms/app.py
View File

@ -163,6 +163,15 @@ class LollmsApplication(LoLLMsCom):
except: except:
self.warning(f"Couldn't load SD") self.warning(f"Couldn't load SD")
if self.config.enable_motion_ctrl_service:
try:
from lollms.services.motion_ctrl.lollms_motion_ctrl import Service
self.motion_ctrl = Service(self, base_url=self.config.motion_ctrl_base_url)
except Exception as ex:
trace_exception(ex)
self.warning(f"Couldn't load Motion control")
def build_long_term_skills_memory(self): def build_long_term_skills_memory(self):
discussion_db_name:Path = self.lollms_paths.personal_discussions_path/self.config.discussion_db_name.split(".")[0] discussion_db_name:Path = self.lollms_paths.personal_discussions_path/self.config.discussion_db_name.split(".")[0]
discussion_db_name.mkdir(exist_ok=True, parents=True) discussion_db_name.mkdir(exist_ok=True, parents=True)

6
lollms/configs/config.yaml
View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file =========================== # =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 73 version: 74
binding_name: null binding_name: null
model_name: null model_name: null
model_variant: null model_variant: null
@ -85,6 +85,10 @@ current_language: en
enable_sd_service: false enable_sd_service: false
sd_base_url: http://localhost:7860 sd_base_url: http://localhost:7860
# Motion control service
enable_motion_ctrl_service: false
motion_ctrl_base_url: http://localhost:7861
# ollama service # ollama service
enable_ollama_service: false enable_ollama_service: false
ollama_base_url: http://localhost:11434 ollama_base_url: http://localhost:11434

141
lollms/personality.py
View File

@ -126,6 +126,7 @@ class AIPersonality:
self.text_files = [] self.text_files = []
self.image_files = [] self.image_files = []
self.audio_files = []
self.images_descriptions = [] self.images_descriptions = []
self.vectorizer = None self.vectorizer = None
@ -928,68 +929,69 @@ class AIPersonality:
self.vectorizer = None self.vectorizer = None
return True return True
def add_file(self, path, client:Client, callback=None): def add_file(self, path, client:Client, callback=None, process=True):
output = "" output = ""
if not self.callback: if not self.callback:
self.callback = callback self.callback = callback
path = Path(path) path = Path(path)
if path.suffix in [".wav",".mp3"]: if path.suffix in [".wav",".mp3"]:
self.new_message("") self.audio_files.append(path)
self.info(f"Transcribing ... ") if process:
self.step_start("Transcribing ... ") self.new_message("")
if self.whisper is None: self.info(f"Transcribing ... ")
if not PackageManager.check_package_installed("whisper"): self.step_start("Transcribing ... ")
PackageManager.install_package("openai-whisper") if self.whisper is None:
try: if not PackageManager.check_package_installed("whisper"):
import conda.cli PackageManager.install_package("openai-whisper")
conda.cli.main("install", "conda-forge::ffmpeg", "-y") try:
except: import conda.cli
ASCIIColors.bright_red("Couldn't install ffmpeg. whisper won't work. Please install it manually") conda.cli.main("install", "conda-forge::ffmpeg", "-y")
except:
ASCIIColors.bright_red("Couldn't install ffmpeg. whisper won't work. Please install it manually")
import whisper import whisper
self.whisper = whisper.load_model("base") self.whisper = whisper.load_model("base")
result = self.whisper.transcribe(str(path))
transcription_fn = str(path)+".txt"
with open(transcription_fn, "w", encoding="utf-8") as f:
f.write(result["text"])
self.info(f"File saved to {transcription_fn}")
result = self.whisper.transcribe(str(path)) self.full(result["text"])
transcription_fn = str(path)+".txt" self.step_end("Transcribing ... ")
with open(transcription_fn, "w", encoding="utf-8") as f:
f.write(result["text"])
self.info(f"File saved to {transcription_fn}")
self.full(result["text"])
self.step_end("Transcribing ... ")
elif path.suffix in [".png",".jpg",".jpeg",".gif",".bmp",".svg",".webp"]: elif path.suffix in [".png",".jpg",".jpeg",".gif",".bmp",".svg",".webp"]:
if self.callback:
try:
pth = str(path).replace("\\","/").split('/')
if "discussion_databases" in pth:
pth = discussion_path_to_url(path)
self.new_message("",MSG_TYPE.MSG_TYPE_FULL)
output = f'<img src="{pth}" width="800">\n\n'
self.full(output)
if self.model.binding_type not in [BindingType.TEXT_IMAGE, BindingType.TEXT_IMAGE_VIDEO]:
# self.ShowBlockingMessage("Understanding image (please wait)")
from PIL import Image
img = Image.open(str(path))
# Convert the image to RGB mode
img = img.convert("RGB")
output += "## image description :\n"+ self.model.interrogate_blip([img])[0]
# output += "## image description :\n"+ self.model.qna_blip([img],"q:Describe this photo with as much details as possible.\na:")[0]
self.full(output)
self.HideBlockingMessage("Understanding image (please wait)")
if self.config.debug:
ASCIIColors.yellow(output)
else:
# self.ShowBlockingMessage("Importing image (please wait)")
self.HideBlockingMessage("Importing image (please wait)")
except Exception as ex:
trace_exception(ex)
self.HideBlockingMessage("Understanding image (please wait)", False)
ASCIIColors.error("Couldn't create new message")
self.image_files.append(path) self.image_files.append(path)
if process:
if self.callback:
try:
pth = str(path).replace("\\","/").split('/')
if "discussion_databases" in pth:
pth = discussion_path_to_url(path)
self.new_message("",MSG_TYPE.MSG_TYPE_FULL)
output = f'<img src="{pth}" width="800">\n\n'
self.full(output)
if self.model.binding_type not in [BindingType.TEXT_IMAGE, BindingType.TEXT_IMAGE_VIDEO]:
# self.ShowBlockingMessage("Understanding image (please wait)")
from PIL import Image
img = Image.open(str(path))
# Convert the image to RGB mode
img = img.convert("RGB")
output += "## image description :\n"+ self.model.interrogate_blip([img])[0]
# output += "## image description :\n"+ self.model.qna_blip([img],"q:Describe this photo with as much details as possible.\na:")[0]
self.full(output)
self.HideBlockingMessage("Understanding image (please wait)")
if self.config.debug:
ASCIIColors.yellow(output)
else:
# self.ShowBlockingMessage("Importing image (please wait)")
self.HideBlockingMessage("Importing image (please wait)")
except Exception as ex:
trace_exception(ex)
self.HideBlockingMessage("Understanding image (please wait)", False)
ASCIIColors.error("Couldn't create new message")
ASCIIColors.info("Received image file") ASCIIColors.info("Received image file")
if callback is not None: if callback is not None:
callback("Image file added successfully", MSG_TYPE.MSG_TYPE_INFO) callback("Image file added successfully", MSG_TYPE.MSG_TYPE_INFO)
@ -998,21 +1000,22 @@ class AIPersonality:
# self.ShowBlockingMessage("Adding file to vector store.\nPlease stand by") # self.ShowBlockingMessage("Adding file to vector store.\nPlease stand by")
self.text_files.append(path) self.text_files.append(path)
ASCIIColors.info("Received text compatible file") ASCIIColors.info("Received text compatible file")
if self.vectorizer is None: if process:
self.vectorizer = TextVectorizer( if self.vectorizer is None:
self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer" self.vectorizer = TextVectorizer(
model=self.model, #needed in case of using model_embedding self.config.data_vectorization_method, # supported "model_embedding" or "tfidf_vectorizer"
database_path=client.discussion.discussion_rag_folder/"db.json", model=self.model, #needed in case of using model_embedding
save_db=self.config.data_vectorization_save_db, database_path=client.discussion.discussion_rag_folder/"db.json",
data_visualization_method=VisualizationMethod.PCA, save_db=self.config.data_vectorization_save_db,
database_dict=None) data_visualization_method=VisualizationMethod.PCA,
data = GenericDataLoader.read_file(path) database_dict=None)
self.vectorizer.add_document(path, data, self.config.data_vectorization_chunk_size, self.config.data_vectorization_overlap_size, add_first_line_to_all_chunks=True if path.suffix==".csv" else False) data = GenericDataLoader.read_file(path)
self.vectorizer.index() self.vectorizer.add_document(path, data, self.config.data_vectorization_chunk_size, self.config.data_vectorization_overlap_size, add_first_line_to_all_chunks=True if path.suffix==".csv" else False)
if callback is not None: self.vectorizer.index()
callback("File added successfully",MSG_TYPE.MSG_TYPE_INFO) if callback is not None:
self.HideBlockingMessage("Adding file to vector store.\nPlease stand by") callback("File added successfully",MSG_TYPE.MSG_TYPE_INFO)
return True self.HideBlockingMessage("Adding file to vector store.\nPlease stand by")
return True
except Exception as e: except Exception as e:
trace_exception(e) trace_exception(e)
self.HideBlockingMessage("Adding file to vector store.\nPlease stand by") self.HideBlockingMessage("Adding file to vector store.\nPlease stand by")
@ -1950,10 +1953,10 @@ class APScript(StateMachine):
ASCIIColors.blue("*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*") ASCIIColors.blue("*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*-*")
def add_file(self, path, client:Client, callback=None): def add_file(self, path, client:Client, callback=None, process=True):
self.personality.add_file(path, client=client,callback=callback, process=process)
if callback is not None: if callback is not None:
callback("File added successfully",MSG_TYPE.MSG_TYPE_INFO) callback("File added successfully",MSG_TYPE.MSG_TYPE_INFO)
self.personality.add_file(path, client=client,callback=callback)
return True return True
def remove_file(self, path): def remove_file(self, path):

11
lollms/server/endpoints/lollms_skills_library.py
View File

@ -20,4 +20,15 @@ from typing import List
from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod from safe_store.text_vectorizer import TextVectorizer, VectorizationMethod, VisualizationMethod
import tqdm import tqdm
from pathlib import Path from pathlib import Path
# ----------------------- Defining router and main class ------------------------------
router = APIRouter()
lollmsElfServer:LOLLMSWebUI = LOLLMSWebUI.get_instance()
class DiscussionInfos(BaseModel):
client_id: str
@router.get("/add_discussion_to_skills_library")
def add_discussion_to_skills_library(discussionInfos:DiscussionInfos):
client = lollmsElfServer.session.get_client(discussionInfos.client_id)
lollmsElfServer.add_discussion_tto_skills_library(client)

110
lollms/services/motion_ctrl/lollms_motion_ctrl.py
View File

@ -31,7 +31,7 @@ from ascii_colors import ASCIIColors, trace_exception
from lollms.paths import LollmsPaths from lollms.paths import LollmsPaths
from lollms.utilities import git_pull from lollms.utilities import git_pull
import subprocess import subprocess
import shutil
@ -44,6 +44,8 @@ def verify_motion_ctrl(lollms_paths:LollmsPaths):
return motion_ctrl_folder.exists() return motion_ctrl_folder.exists()
def install_motion_ctrl(lollms_app:LollmsApplication): def install_motion_ctrl(lollms_app:LollmsApplication):
import conda.cli
root_dir = lollms_app.lollms_paths.personal_path root_dir = lollms_app.lollms_paths.personal_path
shared_folder = root_dir/"shared" shared_folder = root_dir/"shared"
motion_ctrl_folder = shared_folder / "auto_motion_ctrl" motion_ctrl_folder = shared_folder / "auto_motion_ctrl"
@ -51,9 +53,17 @@ def install_motion_ctrl(lollms_app:LollmsApplication):
if not show_yes_no_dialog("warning!","I have detected that there is a previous installation of motion ctrl.\nShould I remove it and continue installing?"): if not show_yes_no_dialog("warning!","I have detected that there is a previous installation of motion ctrl.\nShould I remove it and continue installing?"):
return return
else: else:
motion_ctrl_folder.unlink(True) try:
shutil.rmtree(motion_ctrl_folder)
except Exception as ex:
trace_exception(ex)
try:
conda.cli.main('conda', 'remove', '--name', env_name, '--all', '--yes')
except Exception as ex:
trace_exception(ex)
subprocess.run(["git", "clone", "https://github.com/ParisNeo/MotionCtrl.git", str(motion_ctrl_folder)]) subprocess.run(["git", "clone", "https://github.com/ParisNeo/MotionCtrl.git", str(motion_ctrl_folder)])
import conda.cli
env_name = "MotionCtrl" env_name = "MotionCtrl"
conda.cli.main('conda', 'create', '--name', env_name, 'python=3.10', '--yes') conda.cli.main('conda', 'create', '--name', env_name, 'python=3.10', '--yes')
@ -62,7 +72,7 @@ def install_motion_ctrl(lollms_app:LollmsApplication):
pip_install_command = "pip install -r " + str(motion_ctrl_folder) + "/requirements.txt" pip_install_command = "pip install -r " + str(motion_ctrl_folder) + "/requirements.txt"
# Run the combined command # Run the combined command
subprocess.run(activate_env_command + pip_install_command, shell=True, executable='/bin/bash') subprocess.run(activate_env_command + pip_install_command, shell=True)
#pip install -r requirements.txt #pip install -r requirements.txt
ASCIIColors.green("Motion ctrl installed successfully") ASCIIColors.green("Motion ctrl installed successfully")
@ -83,78 +93,68 @@ def get_motion_ctrl(lollms_paths:LollmsPaths):
return LollmsMotionCtrl return LollmsMotionCtrl
class LollmsMotionCtrl: class Service:
has_controlnet = False has_controlnet = False
def __init__( def __init__(
self, self,
app:LollmsApplication, app:LollmsApplication,
wm = "Artbot", base_url=None,
max_retries=50,
sampler="Euler a",
steps=20,
use_https=False,
username=None,
password=None,
auto_motion_ctrl_base_url=None,
share=False, share=False,
wait_for_service=True wait_for_service=True,
max_retries=5
): ):
if auto_motion_ctrl_base_url=="" or auto_motion_ctrl_base_url=="http://127.0.0.1:7860": if base_url=="" or base_url=="http://127.0.0.1:7861":
auto_motion_ctrl_base_url = None base_url = None
# Get the current directory # Get the current directory
lollms_paths = app.lollms_paths lollms_paths = app.lollms_paths
self.app = app self.app = app
root_dir = lollms_paths.personal_path root_dir = lollms_paths.personal_path
self.wm = wm
# Store the path to the script # Store the path to the script
if auto_motion_ctrl_base_url is None: if base_url is None:
self.auto_motion_ctrl_base_url = "http://127.0.0.1:7860" self.base_url = "http://127.0.0.1:7860"
if not verify_motion_ctrl(lollms_paths): if not verify_motion_ctrl(lollms_paths):
install_motion_ctrl(app.lollms_paths) install_motion_ctrl(app.lollms_paths)
else: else:
self.auto_motion_ctrl_base_url = auto_motion_ctrl_base_url self.base_url = base_url
self.auto_motion_ctrl_url = self.auto_motion_ctrl_base_url+"/sdapi/v1" self.auto_motion_ctrl_url = self.base_url+"/sdapi/v1"
shared_folder = root_dir/"shared" shared_folder = root_dir/"shared"
self.motion_ctrl_folder = shared_folder / "motion_ctrl" self.motion_ctrl_folder = shared_folder / "motion_ctrl"
self.output_dir = root_dir / "outputs/motion_ctrl" self.output_dir = root_dir / "outputs/motion_ctrl"
self.output_dir.mkdir(parents=True, exist_ok=True) self.output_dir.mkdir(parents=True, exist_ok=True)
ASCIIColors.red(" ") ASCIIColors.red(" ")
ASCIIColors.red(" __ _____ __ __ _____ _____ _____ ____ ") ASCIIColors.red("_ _ _ _ _ _ _ ")
ASCIIColors.red("| | | | | | | | | __| | __| \ ") ASCIIColors.red("| | | | | | | (_) | | | |")
ASCIIColors.red("| |__| | | |__| |__| | | |__ | |__ | | |") ASCIIColors.red("| | ___ | | |_ __ ___ ___ _ __ ___ ___ | |_ _ ___ _ __ ___| |_ _ __| |")
ASCIIColors.red("|_____|_____|_____|_____|_|_|_|_____|_____|_____|____/ ") ASCIIColors.red("| |/ _ \| | | '_ ` _ \/ __| | '_ ` _ \ / _ \| __| |/ _ \| '_ \ / __| __| '__| |")
ASCIIColors.red(" |_____| ") ASCIIColors.red("| | (_) | | | | | | | \__ \ | | | | | | (_) | |_| | (_) | | | || (__| |_| | | |")
ASCIIColors.red("|_|\___/|_|_|_| |_| |_|___/ |_| |_| |_|\___/ \__|_|\___/|_| |_| \___|\__|_| |_|")
ASCIIColors.red(" ______ ______ ")
ASCIIColors.red(" |______| |______| ")
ASCIIColors.red(" Forked from TencentARC's MotionCtrl api") ASCIIColors.red(" Forked from TencentARC's MotionCtrl api")
ASCIIColors.red(" Integration in lollms by ParisNeo") ASCIIColors.red(" Integration in lollms by ParisNeo")
motion_ctrl_folder = shared_folder / "auto_motion_ctrl" motion_ctrl_folder = shared_folder / "auto_motion_ctrl"
env_name = "MotionCtrl" env_name = "MotionCtrl"
if not self.wait_for_service(1,False) and auto_motion_ctrl_base_url is None: if not self.wait_for_service(1,False):
ASCIIColors.info("Loading lollms_motion_ctrl") ASCIIColors.info("Loading lollms_motion_ctrl")
os.environ['motion_ctrl_WEBUI_RESTARTING'] = '1' # To forbid sd webui from showing on the browser automatically os.environ['motion_ctrl_WEBUI_RESTARTING'] = '1' # To forbid sd webui from showing on the browser automatically
# Get the current operating system # Get the current operating system
os_name = platform.system() os_name = platform.system()
conda_path = get_conda_path() conda_path = get_conda_path()
import conda.cli
# Construct the command to activate the environment and start the server env_name = "MotionCtrl"
if os_name == "Windows": # Replace 'your_env_name' with the name of the environment you created
activate_env_command = f"call {conda_path}\\activate {env_name} && " activate_env_command = f"conda activate {env_name} && "
else: # Linux or macOS pip_install_command = "python -m app --share"
activate_env_command = f"source {conda_path}/activate {env_name} && "
start_server_command = f"python -m app --share"
# Run the combined command # Run the combined command
if os_name == "Windows": subprocess.run(activate_env_command + pip_install_command, shell=True)
subprocess.run(activate_env_command + start_server_command, shell=True, cwd=motion_ctrl_folder)
else: # Linux or macOS
subprocess.run(activate_env_command + start_server_command, shell=True, executable='/bin/bash', cwd=motion_ctrl_folder)
# Wait until the service is available at http://127.0.0.1:7860/ # Wait until the service is available at http://127.0.0.1:7860/
@ -163,12 +163,28 @@ class LollmsMotionCtrl:
else: else:
ASCIIColors.warning("We are not waiting for the MotionCtrl service to be up.\nThis means that you may need to wait a bit before you can use it.") ASCIIColors.warning("We are not waiting for the MotionCtrl service to be up.\nThis means that you may need to wait a bit before you can use it.")
self.default_sampler = sampler
self.default_steps = steps
self.session = requests.Session() self.session = requests.Session()
if username and password: def wait_for_service(self, max_retries = 50, show_warning=True):
self.set_auth(username, password) url = f"{self.base_url}"
else: # Adjust this value as needed
self.check_controlnet() retries = 0
while retries < max_retries or max_retries<0:
try:
response = requests.get(url)
if response.status_code == 200:
print("Service is available.")
if self.app is not None:
self.app.success("Motion ctrl is now available.")
return True
except requests.exceptions.RequestException:
pass
retries += 1
time.sleep(1)
if show_warning:
print("Service did not become available within the given time.")
if self.app is not None:
self.app.error("SD Service did not become available within the given time.")
return False