mirror of
https://github.com/ParisNeo/lollms.git
synced 2024-12-23 14:32:28 +00:00
added stuff
This commit is contained in:
parent
3a2bc02c1c
commit
8b833fb615
@ -1,5 +1,5 @@
|
|||||||
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
||||||
version: 106
|
version: 108
|
||||||
binding_name: null
|
binding_name: null
|
||||||
model_name: null
|
model_name: null
|
||||||
model_variant: null
|
model_variant: null
|
||||||
@ -31,7 +31,17 @@ app_custom_logo: ""
|
|||||||
discussion_prompt_separator: "!@>"
|
discussion_prompt_separator: "!@>"
|
||||||
start_header_id_template: "!@>"
|
start_header_id_template: "!@>"
|
||||||
end_header_id_template: ": "
|
end_header_id_template: ": "
|
||||||
|
|
||||||
separator_template: "\n"
|
separator_template: "\n"
|
||||||
|
|
||||||
|
start_user_header_id_template: "!@>"
|
||||||
|
end_user_header_id_template: ": "
|
||||||
|
end_user_message_id_template: ""
|
||||||
|
|
||||||
|
start_ai_header_id_template: "!@>"
|
||||||
|
end_ai_header_id_template: ": "
|
||||||
|
end_ai_message_id_template: ""
|
||||||
|
|
||||||
system_message_template: "system"
|
system_message_template: "system"
|
||||||
|
|
||||||
seed: -1
|
seed: -1
|
||||||
@ -156,6 +166,10 @@ openai_tts_voice: "alloy"
|
|||||||
enable_sd_service: false
|
enable_sd_service: false
|
||||||
sd_base_url: http://localhost:7860
|
sd_base_url: http://localhost:7860
|
||||||
|
|
||||||
|
# Image generation service
|
||||||
|
enable_fooocus_service: false
|
||||||
|
fooocus_base_url: http://localhost:7860
|
||||||
|
|
||||||
# diffuser
|
# diffuser
|
||||||
diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload
|
diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload
|
||||||
diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
|
diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
|
||||||
|
@ -419,29 +419,29 @@ class LollmsApplication(LoLLMsCom):
|
|||||||
trace_exception(ex)
|
trace_exception(ex)
|
||||||
self.warning(f"Couldn't load Motion control")
|
self.warning(f"Couldn't load Motion control")
|
||||||
|
|
||||||
if self.config.active_tti_service == "diffusers":
|
if self.config.active_tti_service == "diffusers" and type(self.tti)!=LollmsDiffusers:
|
||||||
from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers
|
from lollms.services.diffusers.lollms_diffusers import LollmsDiffusers
|
||||||
self.tti = LollmsDiffusers(self)
|
self.tti = LollmsDiffusers(self)
|
||||||
elif self.config.active_tti_service == "autosd":
|
elif self.config.active_tti_service == "autosd" and type(self.tti)!=LollmsSD:
|
||||||
from lollms.services.sd.lollms_sd import LollmsSD
|
from lollms.services.sd.lollms_sd import LollmsSD
|
||||||
self.tti = LollmsSD(self)
|
self.tti = LollmsSD(self)
|
||||||
elif self.config.active_tti_service == "dall-e":
|
elif self.config.active_tti_service == "dall-e" and type(self.tti)!=LollmsDalle:
|
||||||
from lollms.services.dalle.lollms_dalle import LollmsDalle
|
from lollms.services.dalle.lollms_dalle import LollmsDalle
|
||||||
self.tti = LollmsDalle(self, self.config.dall_e_key)
|
self.tti = LollmsDalle(self, self.config.dall_e_key)
|
||||||
elif self.config.active_tti_service == "midjourney":
|
elif self.config.active_tti_service == "midjourney" and type(self.tti)!=LollmsMidjourney:
|
||||||
from lollms.services.midjourney.lollms_midjourney import LollmsMidjourney
|
from lollms.services.midjourney.lollms_midjourney import LollmsMidjourney
|
||||||
self.tti = LollmsMidjourney(self, self.config.midjourney_key)
|
self.tti = LollmsMidjourney(self, self.config.midjourney_key)
|
||||||
|
|
||||||
if self.config.active_tts_service == "openai_tts":
|
if self.config.active_tts_service == "openai_tts" and type(self.tts)!=LollmsOpenAITTS:
|
||||||
from lollms.services.open_ai_tts.lollms_openai_tts import LollmsOpenAITTS
|
from lollms.services.open_ai_tts.lollms_openai_tts import LollmsOpenAITTS
|
||||||
self.tts = LollmsOpenAITTS(self, self.config.openai_tts_model, self.config.openai_tts_voice, self.config.openai_tts_key)
|
self.tts = LollmsOpenAITTS(self, self.config.openai_tts_model, self.config.openai_tts_voice, self.config.openai_tts_key)
|
||||||
elif self.config.active_tts_service == "xtts" and self.xtts:
|
elif self.config.active_tts_service == "xtts" and self.xtts:
|
||||||
self.tts = self.xtts
|
self.tts = self.xtts
|
||||||
|
|
||||||
if self.config.active_stt_service == "openai_whisper":
|
if self.config.active_stt_service == "openai_whisper" and type(self.stt)!=LollmsOpenAIWhisper:
|
||||||
from lollms.services.openai_whisper.lollms_openai_whisper import LollmsOpenAIWhisper
|
from lollms.services.openai_whisper.lollms_openai_whisper import LollmsOpenAIWhisper
|
||||||
self.stt = LollmsOpenAIWhisper(self, self.config.openai_whisper_model, self.config.openai_whisper_key)
|
self.stt = LollmsOpenAIWhisper(self, self.config.openai_whisper_model, self.config.openai_whisper_key)
|
||||||
elif self.config.active_stt_service == "whisper":
|
elif self.config.active_stt_service == "whisper" and type(self.stt)!=LollmsWhisper :
|
||||||
from lollms.services.whisper.lollms_whisper import LollmsWhisper
|
from lollms.services.whisper.lollms_whisper import LollmsWhisper
|
||||||
self.stt = LollmsWhisper(self, self.config.whisper_model)
|
self.stt = LollmsWhisper(self, self.config.whisper_model)
|
||||||
|
|
||||||
@ -771,11 +771,19 @@ class LollmsApplication(LoLLMsCom):
|
|||||||
"""
|
"""
|
||||||
start_header_id_template = self.config.start_header_id_template
|
start_header_id_template = self.config.start_header_id_template
|
||||||
end_header_id_template = self.config.end_header_id_template
|
end_header_id_template = self.config.end_header_id_template
|
||||||
|
|
||||||
separator_template = self.config.separator_template
|
separator_template = self.config.separator_template
|
||||||
|
|
||||||
|
start_user_header_id_template = self.config.start_user_header_id_template
|
||||||
|
end_user_header_id_template = self.config.end_user_header_id_template
|
||||||
|
end_user_message_id_template = self.config.end_user_message_id_template
|
||||||
|
|
||||||
|
start_ai_header_id_template = self.config.start_ai_header_id_template
|
||||||
|
end_ai_header_id_template = self.config.end_ai_header_id_template
|
||||||
|
end_ai_message_id_template = self.config.end_ai_message_id_template
|
||||||
|
|
||||||
system_message_template = self.config.system_message_template
|
system_message_template = self.config.system_message_template
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
if self.personality.callback is None:
|
if self.personality.callback is None:
|
||||||
self.personality.callback = partial(self.process_chunk, client_id=client_id)
|
self.personality.callback = partial(self.process_chunk, client_id=client_id)
|
||||||
# Get the list of messages
|
# Get the list of messages
|
||||||
@ -1069,13 +1077,13 @@ class LollmsApplication(LoLLMsCom):
|
|||||||
# Tokenize the message content
|
# Tokenize the message content
|
||||||
if self.config.use_model_name_in_discussions:
|
if self.config.use_model_name_in_discussions:
|
||||||
if message.model:
|
if message.model:
|
||||||
msg = f"{separator_template}{start_header_id_template}{message.sender}({message.model}){end_header_id_template}" + message.content.strip()
|
msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}({message.model}){end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip()
|
||||||
else:
|
else:
|
||||||
msg = f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip()
|
msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip()
|
||||||
message_tokenized = self.model.tokenize(msg)
|
message_tokenized = self.model.tokenize(msg)
|
||||||
else:
|
else:
|
||||||
message_tokenized = self.model.tokenize(
|
message_tokenized = self.model.tokenize(
|
||||||
f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip()
|
f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip()
|
||||||
)
|
)
|
||||||
# Check if adding the message will exceed the available space
|
# Check if adding the message will exceed the available space
|
||||||
if tokens_accumulated + len(message_tokenized) > available_space-n_tokens:
|
if tokens_accumulated + len(message_tokenized) > available_space-n_tokens:
|
||||||
@ -1099,13 +1107,13 @@ class LollmsApplication(LoLLMsCom):
|
|||||||
|
|
||||||
if self.config.use_model_name_in_discussions:
|
if self.config.use_model_name_in_discussions:
|
||||||
if message.model:
|
if message.model:
|
||||||
msg = f"{separator_template}{start_header_id_template}{message.sender}({message.model}){end_header_id_template}" + message.content.strip()
|
msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}({message.model}){end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip()
|
||||||
else:
|
else:
|
||||||
msg = f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip()
|
msg = f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip()
|
||||||
message_tokenized = self.model.tokenize(msg)
|
message_tokenized = self.model.tokenize(msg)
|
||||||
else:
|
else:
|
||||||
message_tokenized = self.model.tokenize(
|
message_tokenized = self.model.tokenize(
|
||||||
f"{separator_template}{start_header_id_template}{message.sender}{end_header_id_template}" + message.content.strip()
|
f"{separator_template}{start_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else start_user_header_id_template}{message.sender}{end_ai_header_id_template if message.sender_type == SENDER_TYPES.SENDER_TYPES_AI else end_user_header_id_template}" + message.content.strip()
|
||||||
)
|
)
|
||||||
|
|
||||||
# Add the tokenized message to the full_message_list
|
# Add the tokenized message to the full_message_list
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
|
||||||
version: 106
|
version: 108
|
||||||
binding_name: null
|
binding_name: null
|
||||||
model_name: null
|
model_name: null
|
||||||
model_variant: null
|
model_variant: null
|
||||||
@ -31,7 +31,17 @@ app_custom_logo: ""
|
|||||||
discussion_prompt_separator: "!@>"
|
discussion_prompt_separator: "!@>"
|
||||||
start_header_id_template: "!@>"
|
start_header_id_template: "!@>"
|
||||||
end_header_id_template: ": "
|
end_header_id_template: ": "
|
||||||
|
|
||||||
separator_template: "\n"
|
separator_template: "\n"
|
||||||
|
|
||||||
|
start_user_header_id_template: "!@>"
|
||||||
|
end_user_header_id_template: ": "
|
||||||
|
end_user_message_id_template: ""
|
||||||
|
|
||||||
|
start_ai_header_id_template: "!@>"
|
||||||
|
end_ai_header_id_template: ": "
|
||||||
|
end_ai_message_id_template: ""
|
||||||
|
|
||||||
system_message_template: "system"
|
system_message_template: "system"
|
||||||
|
|
||||||
seed: -1
|
seed: -1
|
||||||
@ -156,6 +166,10 @@ openai_tts_voice: "alloy"
|
|||||||
enable_sd_service: false
|
enable_sd_service: false
|
||||||
sd_base_url: http://localhost:7860
|
sd_base_url: http://localhost:7860
|
||||||
|
|
||||||
|
# Image generation service
|
||||||
|
enable_fooocus_service: false
|
||||||
|
fooocus_base_url: http://localhost:7860
|
||||||
|
|
||||||
# diffuser
|
# diffuser
|
||||||
diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload
|
diffusers_offloading_mode: sequential_cpu_offload # sequential_cpu_offload
|
||||||
diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
|
diffusers_model: PixArt-alpha/PixArt-Sigma-XL-2-1024-MS
|
||||||
|
@ -206,3 +206,153 @@ if __name__ == "__main__":
|
|||||||
|
|
||||||
# Print the HTML output
|
# Print the HTML output
|
||||||
print(html_output)
|
print(html_output)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
# Core function to search for PDFs on arXiv and download them to a specified directory
|
||||||
|
def search_and_rank(llm, search_subject:str, score_threshold:float, query: str, max_results: Optional[int] = 5, sort_by: Optional[str] = 'relevance', start_date: Optional[str] = None, end_date: Optional[str] = None, author: Optional[str] = None, client: Optional[Any] = None) -> (str, Dict[str, Any]):
|
||||||
|
try:
|
||||||
|
|
||||||
|
if client is None:
|
||||||
|
download_to = Path("./pdf_search")
|
||||||
|
else:
|
||||||
|
download_to = client.discussion.discussion_folder / "pdf_search"
|
||||||
|
|
||||||
|
# Construct the search URL with additional parameters
|
||||||
|
url = f'http://export.arxiv.org/api/query?search_query={query}&start=0&max_results={max_results}&sortBy={sort_by}'
|
||||||
|
if start_date:
|
||||||
|
url += f'&startDate={start_date}'
|
||||||
|
if end_date:
|
||||||
|
url += f'&endDate={end_date}'
|
||||||
|
if author:
|
||||||
|
url += f'&author={author}'
|
||||||
|
|
||||||
|
response = requests.get(url)
|
||||||
|
response.raise_for_status()
|
||||||
|
|
||||||
|
# Parse the response
|
||||||
|
soup = BeautifulSoup(response.content, 'xml')
|
||||||
|
entries = soup.find_all('entry')
|
||||||
|
|
||||||
|
# Create the directory if it doesn't exist
|
||||||
|
download_to.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# Extract PDF URLs and additional information
|
||||||
|
html_output = "<html><body>"
|
||||||
|
report_content = ""
|
||||||
|
pdf_info = {}
|
||||||
|
for entry in entries:
|
||||||
|
pdf_url = entry.id.text.replace('abs', 'pdf') + '.pdf'
|
||||||
|
pdf_name = pdf_url.split('/')[-1]
|
||||||
|
pdf_path = download_to / pdf_name
|
||||||
|
|
||||||
|
if client is None:
|
||||||
|
local_url = f'/discussions/pdf_search/{pdf_name}'
|
||||||
|
else:
|
||||||
|
local_url = discussion_path_to_url(pdf_path)
|
||||||
|
|
||||||
|
# Extract additional information
|
||||||
|
title = entry.title.text
|
||||||
|
authors = ', '.join(author.find('name').text for author in entry.find_all('author'))
|
||||||
|
affiliations = ', '.join(affiliation.text for affiliation in entry.find_all('affiliation'))
|
||||||
|
abstract = entry.summary.text
|
||||||
|
published_date = entry.published.text
|
||||||
|
journal_ref = entry.find('journal_ref').text if entry.find('journal_ref') else 'N/A'
|
||||||
|
|
||||||
|
# Use the LLM to rate the relevance of the text example compared to the search prompt
|
||||||
|
relevance_score = llm.fast_gen(f"Rate the relevance of the following text compared to the prompt on a scale of 0 to 1. Only respond with a number between 0 and 1:\nPrompt: {search_subject}\nText: {abstract}\nRelevance score:")
|
||||||
|
|
||||||
|
# Extract the relevance score from the generated text
|
||||||
|
try:
|
||||||
|
score = float(relevance_score.strip())
|
||||||
|
except ValueError:
|
||||||
|
score = 0.0 # Default score in case of parsing issues
|
||||||
|
|
||||||
|
if score<score_threshold: # drop irrelevant scores
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Create the output tuple
|
||||||
|
result = (f"Relevance Score: {score}", {"score": score})
|
||||||
|
|
||||||
|
# Write abstract and additional information to text file
|
||||||
|
abstract_path = download_to / f"{pdf_name}_abstract.txt"
|
||||||
|
with abstract_path.open('w', encoding='utf-8') as abstract_file:
|
||||||
|
abstract_file.write(f"Title: {title}\n")
|
||||||
|
abstract_file.write(f"Authors: {authors}\n")
|
||||||
|
abstract_file.write(f"Affiliations: {affiliations}\n")
|
||||||
|
abstract_file.write(f"Abstract: {abstract}\n")
|
||||||
|
abstract_file.write(f"Published Date: {published_date}\n")
|
||||||
|
abstract_file.write(f"Journal/Conference: {journal_ref}\n")
|
||||||
|
abstract_file.write(f"Relevance score: {score}\n")
|
||||||
|
|
||||||
|
# Download PDF
|
||||||
|
pdf_response = requests.get(pdf_url)
|
||||||
|
with pdf_path.open('wb') as pdf_file:
|
||||||
|
pdf_file.write(pdf_response.content)
|
||||||
|
|
||||||
|
# Append to HTML output
|
||||||
|
html_output += f"""
|
||||||
|
<div>
|
||||||
|
<h2>{title}</h2>
|
||||||
|
<p><strong>Authors:</strong> {authors}</p>
|
||||||
|
<p><strong>Affiliations:</strong> {affiliations}</p>
|
||||||
|
<p><strong>Abstract:</strong> {abstract}</p>
|
||||||
|
<p><strong>Published Date:</strong> {published_date}</p>
|
||||||
|
<p><strong>Journal/Conference:</strong> {journal_ref}</p>
|
||||||
|
<p><strong>Relevance score:</strong> {score}</p>
|
||||||
|
<p><a href="{pdf_url}" target="_blank">PDF Link</a></p>
|
||||||
|
<p><a href="{local_url}" target="_blank">Local PDF</a></p>
|
||||||
|
</div>
|
||||||
|
"""
|
||||||
|
# Append to report content
|
||||||
|
report_content += f"""
|
||||||
|
Title: {title}
|
||||||
|
Authors: {authors}
|
||||||
|
Affiliations: {affiliations}
|
||||||
|
Abstract: {abstract}
|
||||||
|
Published Date: {published_date}
|
||||||
|
Journal/Conference: {journal_ref}
|
||||||
|
Relevance score: {score}
|
||||||
|
PDF Link: {pdf_url}
|
||||||
|
Local PDF: {local_url}
|
||||||
|
------------------------
|
||||||
|
"""
|
||||||
|
# Append to pdf_info dict
|
||||||
|
pdf_info[pdf_name] = {
|
||||||
|
"title": title,
|
||||||
|
"authors": authors,
|
||||||
|
"affiliations": affiliations,
|
||||||
|
"abstract": abstract,
|
||||||
|
"published_date": published_date,
|
||||||
|
"journal_ref": journal_ref,
|
||||||
|
"pdf_url": pdf_url,
|
||||||
|
"local_url": local_url
|
||||||
|
}
|
||||||
|
|
||||||
|
# Save the report to a text file
|
||||||
|
report_path = download_to / "pdf_search_report.txt"
|
||||||
|
with report_path.open('w', encoding='utf-8') as report_file:
|
||||||
|
report_file.write(report_content)
|
||||||
|
|
||||||
|
html_output += "</body></html>"
|
||||||
|
return html_output, pdf_info
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
return trace_exception(e), {}
|
||||||
|
|
||||||
|
# Metadata function
|
||||||
|
def search_and_rank_function(llm, score_threshold:float, client: Optional[Any] = None):
|
||||||
|
return {
|
||||||
|
"function_name": "arxiv_pdf_search", # The function name in string
|
||||||
|
"function": partial(arxiv_pdf_search, llm=llm, client=client, score_threshold= score_threshold), # The function to be called with partial to preset client
|
||||||
|
"function_description": "Searches for PDFs on arXiv based on a query, downloads them to a specified directory, rates them, sort them by rating and returns a HTML string containing article details and links, along with a dictionary containing detailed information about each PDF.", # Description of the function
|
||||||
|
"function_parameters": [ # The set of parameters
|
||||||
|
{"name": "search_subject", "type": "str", "description": "The search subject."},
|
||||||
|
{"name": "query", "type": "str", "description": "The search query for arXiv."},
|
||||||
|
{"name": "max_results", "type": "int", "description": "The maximum number of results to return. (Optional)", "optional": True, "default": 5},
|
||||||
|
{"name": "sort_by", "type": "str", "description": "The sorting criteria for the search results (e.g., relevance, lastUpdatedDate). (Optional)", "optional": True, "default": "relevance"},
|
||||||
|
{"name": "start_date", "type": "str", "description": "The start date for the search results in the format YYYY-MM-DD. (Optional)", "optional": True},
|
||||||
|
{"name": "end_date", "type": "str", "description": "The end date for the search results in the format YYYY-MM-DD. (Optional)", "optional": True},
|
||||||
|
{"name": "author", "type": "str", "description": "The author name for the search results. (Optional)", "optional": True},
|
||||||
|
]
|
||||||
|
}
|
@ -3401,8 +3401,9 @@ The AI should respond in this format using data from actions_list:
|
|||||||
start_header_id_template = self.config.start_header_id_template
|
start_header_id_template = self.config.start_header_id_template
|
||||||
end_header_id_template = self.config.end_header_id_template
|
end_header_id_template = self.config.end_header_id_template
|
||||||
system_message_template = self.config.system_message_template
|
system_message_template = self.config.system_message_template
|
||||||
|
separator_template = self.config.separator_template
|
||||||
|
|
||||||
function_descriptions = [f"{start_header_id_template}{system_message_template}{end_header_id_template}If you need to call a function to fulfull the user request, use a function markdown tag with the function call as the following json format:",
|
function_descriptions = [f"{start_header_id_template}{system_message_template}{end_header_id_template}If you need to call a function to fulfill the user request, use a function markdown tag with the function call as the following json format:",
|
||||||
"```function",
|
"```function",
|
||||||
"{",
|
"{",
|
||||||
'"function_name":the name of the function to be called,',
|
'"function_name":the name of the function to be called,',
|
||||||
@ -3412,8 +3413,9 @@ The AI should respond in this format using data from actions_list:
|
|||||||
"Only use available functions.",
|
"Only use available functions.",
|
||||||
"You can call multiple functions in one generation.",
|
"You can call multiple functions in one generation.",
|
||||||
"Each function call needs to be in a separate function markdown tag.",
|
"Each function call needs to be in a separate function markdown tag.",
|
||||||
"Do not add status of the execution as it will be added automatically by the system.",
|
f"Do not add status of the execution as it will be added automatically by the system.{separator_template}"
|
||||||
f"{start_header_id_template}Available functions{end_header_id_template}\n"]
|
f"{start_header_id_template}Available functions{end_header_id_template}\n"]
|
||||||
|
|
||||||
for function in functions:
|
for function in functions:
|
||||||
description = f"{function['function_name']}: {function['function_description']}\nparameters:{function['function_parameters']}"
|
description = f"{function['function_name']}: {function['function_description']}\nparameters:{function['function_parameters']}"
|
||||||
function_descriptions.append(description)
|
function_descriptions.append(description)
|
||||||
|
179
lollms/services/fooocus/lollms_fooocus.py
Normal file
179
lollms/services/fooocus/lollms_fooocus.py
Normal file
@ -0,0 +1,179 @@
|
|||||||
|
# Title LollmsFooocus
|
||||||
|
# Licence: MIT
|
||||||
|
# Author : Paris Neo
|
||||||
|
# All rights are reserved
|
||||||
|
|
||||||
|
from pathlib import Path
|
||||||
|
import sys
|
||||||
|
from lollms.app import LollmsApplication
|
||||||
|
from lollms.paths import LollmsPaths
|
||||||
|
from lollms.config import TypedConfig, ConfigTemplate, BaseConfig
|
||||||
|
from lollms.utilities import PackageManager, check_and_install_torch, find_next_available_filename
|
||||||
|
import time
|
||||||
|
import io
|
||||||
|
import sys
|
||||||
|
import requests
|
||||||
|
import os
|
||||||
|
import base64
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
import platform
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from PIL import Image, PngImagePlugin
|
||||||
|
from enum import Enum
|
||||||
|
from typing import List, Dict, Any
|
||||||
|
|
||||||
|
from ascii_colors import ASCIIColors, trace_exception
|
||||||
|
from lollms.paths import LollmsPaths
|
||||||
|
from lollms.tti import LollmsTTI
|
||||||
|
from lollms.utilities import git_pull, show_yes_no_dialog, run_script_in_env, create_conda_env
|
||||||
|
import subprocess
|
||||||
|
import shutil
|
||||||
|
from tqdm import tqdm
|
||||||
|
import threading
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def download_file(url, folder_path, local_filename):
|
||||||
|
# Make sure 'folder_path' exists
|
||||||
|
folder_path.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
with requests.get(url, stream=True) as r:
|
||||||
|
r.raise_for_status()
|
||||||
|
total_size = int(r.headers.get('content-length', 0))
|
||||||
|
progress_bar = tqdm(total=total_size, unit='B', unit_scale=True)
|
||||||
|
with open(folder_path / local_filename, 'wb') as f:
|
||||||
|
for chunk in r.iter_content(chunk_size=8192):
|
||||||
|
f.write(chunk)
|
||||||
|
progress_bar.update(len(chunk))
|
||||||
|
progress_bar.close()
|
||||||
|
|
||||||
|
return local_filename
|
||||||
|
|
||||||
|
def install_model(lollms_app:LollmsApplication, model_url):
|
||||||
|
root_dir = lollms_app.lollms_paths.personal_path
|
||||||
|
shared_folder = root_dir/"shared"
|
||||||
|
diffusers_folder = shared_folder / "diffusers"
|
||||||
|
if not PackageManager.check_package_installed("diffusers"):
|
||||||
|
PackageManager.install_or_update("diffusers")
|
||||||
|
if not PackageManager.check_package_installed("torch"):
|
||||||
|
check_and_install_torch(True)
|
||||||
|
|
||||||
|
import torch
|
||||||
|
from diffusers import PixArtSigmaPipeline
|
||||||
|
|
||||||
|
# You can replace the checkpoint id with "PixArt-alpha/PixArt-Sigma-XL-2-512-MS" too.
|
||||||
|
pipe = PixArtSigmaPipeline.from_pretrained(
|
||||||
|
"PixArt-alpha/PixArt-Sigma-XL-2-1024-MS", torch_dtype=torch.float16
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def install_diffusers(lollms_app:LollmsApplication):
|
||||||
|
root_dir = lollms_app.lollms_paths.personal_path
|
||||||
|
shared_folder = root_dir/"shared"
|
||||||
|
diffusers_folder = shared_folder / "diffusers"
|
||||||
|
diffusers_folder.mkdir(exist_ok=True, parents=True)
|
||||||
|
if not PackageManager.check_package_installed("diffusers"):
|
||||||
|
PackageManager.install_or_update("diffusers")
|
||||||
|
PackageManager.install_or_update("xformers")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade_diffusers(lollms_app:LollmsApplication):
|
||||||
|
PackageManager.install_or_update("diffusers")
|
||||||
|
PackageManager.install_or_update("xformers")
|
||||||
|
|
||||||
|
|
||||||
|
class LollmsFooocus(LollmsTTI):
|
||||||
|
has_controlnet = False
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
app:LollmsApplication,
|
||||||
|
wm = "Artbot",
|
||||||
|
):
|
||||||
|
super().__init__(app)
|
||||||
|
self.ready = False
|
||||||
|
# Get the current directory
|
||||||
|
lollms_paths = app.lollms_paths
|
||||||
|
root_dir = lollms_paths.personal_path
|
||||||
|
|
||||||
|
self.wm = wm
|
||||||
|
|
||||||
|
shared_folder = root_dir/"shared"
|
||||||
|
self.diffusers_folder = shared_folder / "diffusers"
|
||||||
|
self.output_dir = root_dir / "outputs/diffusers"
|
||||||
|
self.models_dir = self.diffusers_folder / "models"
|
||||||
|
self.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
self.models_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
ASCIIColors.red(" _ _ _ ______ ")
|
||||||
|
ASCIIColors.red("| | | | | | ___| ")
|
||||||
|
ASCIIColors.red("| | ___ | | |_ __ ___ ___ | |_ ___ ___ ___ ___ _ _ ___ ")
|
||||||
|
ASCIIColors.red("| | / _ \| | | '_ ` _ \/ __| | _/ _ \ / _ \ / _ \ / __| | | / __|")
|
||||||
|
ASCIIColors.red("| |___| (_) | | | | | | | \__ \ | || (_) | (_) | (_) | (__| |_| \__ \ ")
|
||||||
|
ASCIIColors.red("\_____/\___/|_|_|_| |_| |_|___/ \_| \___/ \___/ \___/ \___|\__,_|___/")
|
||||||
|
ASCIIColors.red(" ______ ")
|
||||||
|
ASCIIColors.red(" |______| ")
|
||||||
|
import torch
|
||||||
|
from diffusers import PixArtSigmaPipeline
|
||||||
|
self.model = PixArtSigmaPipeline.from_pretrained(
|
||||||
|
app.config.diffusers_model, torch_dtype=torch.float16, cache_dir=self.models_dir,
|
||||||
|
use_safetensors=True,
|
||||||
|
)
|
||||||
|
# Enable memory optimizations.
|
||||||
|
if app.config.diffusers_offloading_mode=="sequential_cpu_offload":
|
||||||
|
self.model.enable_sequential_cpu_offload()
|
||||||
|
elif app.coinfig.diffusers_offloading_mode=="model_cpu_offload":
|
||||||
|
self.model.enable_model_cpu_offload()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def verify(app:LollmsApplication):
|
||||||
|
# Clone repository
|
||||||
|
root_dir = app.lollms_paths.personal_path
|
||||||
|
shared_folder = root_dir/"shared"
|
||||||
|
diffusers_folder = shared_folder / "diffusers"
|
||||||
|
return diffusers_folder.exists()
|
||||||
|
|
||||||
|
def get(app:LollmsApplication):
|
||||||
|
root_dir = app.lollms_paths.personal_path
|
||||||
|
shared_folder = root_dir/"shared"
|
||||||
|
diffusers_folder = shared_folder / "diffusers"
|
||||||
|
diffusers_script_path = diffusers_folder / "lollms_diffusers.py"
|
||||||
|
git_pull(diffusers_folder)
|
||||||
|
|
||||||
|
if diffusers_script_path.exists():
|
||||||
|
ASCIIColors.success("lollms_diffusers found.")
|
||||||
|
ASCIIColors.success("Loading source file...",end="")
|
||||||
|
# use importlib to load the module from the file path
|
||||||
|
from lollms.services.diffusers.lollms_diffusers import LollmsFooocus
|
||||||
|
ASCIIColors.success("ok")
|
||||||
|
return LollmsFooocus
|
||||||
|
|
||||||
|
|
||||||
|
def paint(
|
||||||
|
self,
|
||||||
|
positive_prompt,
|
||||||
|
negative_prompt,
|
||||||
|
files=[],
|
||||||
|
sampler_name="Euler",
|
||||||
|
seed=-1,
|
||||||
|
scale=7.5,
|
||||||
|
steps=20,
|
||||||
|
img2img_denoising_strength=0.9,
|
||||||
|
width=512,
|
||||||
|
height=512,
|
||||||
|
restore_faces=True,
|
||||||
|
output_path=None
|
||||||
|
):
|
||||||
|
if output_path is None:
|
||||||
|
output_path = self.output_dir
|
||||||
|
from diffusers.utils.pil_utils import pt_to_pil
|
||||||
|
image = self.model(positive_prompt, negative_prompt=negative_prompt, guidance_scale=scale, num_inference_steps=steps,).images[0]
|
||||||
|
output_path = Path(output_path)
|
||||||
|
fn = find_next_available_filename(output_path,"diff_img_")
|
||||||
|
# Save the image
|
||||||
|
image.save(fn)
|
||||||
|
return fn, {"prompt":positive_prompt, "negative_prompt":negative_prompt}
|
||||||
|
|
@ -767,7 +767,7 @@ class TasksLibrary:
|
|||||||
end_header_id_template = self.config.end_header_id_template
|
end_header_id_template = self.config.end_header_id_template
|
||||||
system_message_template = self.config.system_message_template
|
system_message_template = self.config.system_message_template
|
||||||
separator_template = self.config.separator_template
|
separator_template = self.config.separator_template
|
||||||
function_descriptions = [f"{start_header_id_template}information{end_header_id_template}If you need to call a function to fulfull the user request, use a function markdown tag with the function call as the following json format:",
|
function_descriptions = [f"{start_header_id_template}information{end_header_id_template}If you need to call a function to fulfill the user request, use a function markdown tag with the function call as the following json format:",
|
||||||
"```function",
|
"```function",
|
||||||
"{",
|
"{",
|
||||||
'"function_name":the name of the function to be called,',
|
'"function_name":the name of the function to be called,',
|
||||||
|
Loading…
Reference in New Issue
Block a user