ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2025-03-10 22:43:55 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

removed useless junk

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-01-21 05:35:18 +01:00
parent f1b081dc3a
commit d100194b70
31 changed files with 129 additions and 6422 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
lollms/apps/__init__.py
View File

319
lollms/apps/console/__init__.py
View File

@ -1,319 +0,0 @@
from lollms.config import InstallOption
from lollms.binding import BindingBuilder, ModelBuilder
from lollms.personality import MSG_TYPE, PersonalityBuilder
from lollms.main_config import LOLLMSConfig
from ascii_colors import ASCIIColors
from lollms.paths import LollmsPaths
from lollms.app import LollmsApplication
from lollms.terminal import MainMenu
from typing import Callable
from pathlib import Path
import argparse
import yaml
import time
import sys
class Conversation(LollmsApplication):
def __init__(
self,
configuration_path:str|Path=None,
show_logo:bool=True,
show_commands_list:bool=False,
show_personality_infos:bool=True,
show_model_infos:bool=True,
show_welcome_message:bool=True,
show_time_elapsed:bool = False
):
# Fore it to be a path
self.configuration_path = configuration_path
self.is_logging = False
self.log_file_path = ""
self.show_time_elapsed = show_time_elapsed
self.bot_says = ""
# get paths
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_")
# Configuration loading part
config = LOLLMSConfig.autoload(lollms_paths, configuration_path)
super().__init__("lollms-console",config, lollms_paths)
if show_logo:
self.menu.show_logo()
if show_commands_list:
self.menu.show_commands_list()
if show_personality_infos:
print()
print(f"{ASCIIColors.color_green}-----------------------------------------------------------------")
print(f"{ASCIIColors.color_green}Current personality : {ASCIIColors.color_reset}{self.personality}")
print(f"{ASCIIColors.color_green}Version : {ASCIIColors.color_reset}{self.personality.version}")
print(f"{ASCIIColors.color_green}Author : {ASCIIColors.color_reset}{self.personality.author}")
print(f"{ASCIIColors.color_green}Description : {ASCIIColors.color_reset}{self.personality.personality_description}")
print(f"{ASCIIColors.color_green}-----------------------------------------------------------------")
print()
if show_model_infos:
print()
print(f"{ASCIIColors.color_green}-----------------------------------------------------------------")
print(f"{ASCIIColors.color_green}Current binding : {ASCIIColors.color_reset}{self.config['binding_name']}")
print(f"{ASCIIColors.color_green}Current model : {ASCIIColors.color_reset}{self.config['model_name']}")
print(f"{ASCIIColors.color_green}Personal data path : {ASCIIColors.color_reset}{self.lollms_paths.personal_path}")
print(f"{ASCIIColors.color_green}-----------------------------------------------------------------")
print()
# If there is a disclaimer, show it
if self.personality.disclaimer != "":
print(f"\n{ASCIIColors.color_red}Disclaimer")
print(self.personality.disclaimer)
print(f"{ASCIIColors.color_reset}")
if show_welcome_message and self.personality.welcome_message:
ASCIIColors.red(self.personality.name+": ", end="")
print(self.personality.welcome_message)
def ask_override_file(self):
user_input = input("Would you like to override the existing file? (Y/N): ")
user_input = user_input.lower()
if user_input == "y" or user_input == "yes":
print("File will be overridden.")
return True
elif user_input == "n" or user_input == "no":
print("File will not be overridden.")
return False
else:
print("Invalid input. Please enter 'Y' or 'N'.")
# Call the function again recursively to prompt the user for valid input
return self.ask_override_file()
def start_log(self, file_name):
if Path(file_name).is_absolute():
self.log_file_path = Path(file_name)
else:
home_dir = self.lollms_paths.personal_log_path
home_dir.mkdir(parents=True, exist_ok=True)
self.log_file_path = home_dir/file_name
if self.log_file_path.exists():
if not self.ask_override_file():
print("Cancelled")
return
try:
with(open(self.log_file_path, "w") as f):
self.header = f"""------------------------
Log file for lollms discussion
Participating personalities:
{self.config['personalities']}
------------------------
"""
f.write(self.header)
self.is_logging = True
return True
except:
return False
def log(self, text, append=False):
try:
with(open(self.log_file_path, "a" if append else "w") as f):
f.write(text) if append else f.write(self.header+self.personality.personality_conditioning+text)
return True
except:
return False
def stop_log(self):
self.is_logging = False
def reset_context(self):
if self.personality.include_welcome_message_in_disucssion:
full_discussion = (
self.personality.ai_message_prefix +
self.personality.welcome_message +
self.personality.link_text
)
else:
full_discussion = ""
return full_discussion
def remove_text_from_string(self, string, text_to_find):
"""
Removes everything from the first occurrence of the specified text in the string (case-insensitive).
Parameters:
string (str): The original string.
text_to_find (str): The text to find in the string.
Returns:
str: The updated string.
"""
index = string.lower().find(text_to_find.lower())
if index != -1:
string = string[:index]
return string
def start_conversation(self):
full_discussion = self.reset_context()
while True:
try:
ump = self.config.discussion_prompt_separator +self.config.user_name+": " if self.config.use_user_name_in_discussions else self.personality.user_message_prefix
if self.config.use_user_name_in_discussions:
prompt = input(f"{ASCIIColors.color_green}{self.config.user_name}: {ASCIIColors.color_reset}")
else:
prompt = input(f"{ASCIIColors.color_green}You: {ASCIIColors.color_reset}")
if self.show_time_elapsed:
t0 = time.time() #Time at start of request
if prompt == "exit":
return
if prompt == "menu":
self.menu.main_menu()
continue
if prompt == "reset":
self.reset_context()
print(f"{ASCIIColors.color_red}Context reset issued{ASCIIColors.color_reset}")
continue
if prompt == "start_log":
fp = input("Please enter a log file path (ex: log.txt): ")
self.start_log(fp)
print(f"{ASCIIColors.color_red}Started logging to : {self.log_file_path}{ASCIIColors.color_reset}")
continue
if prompt == "stop_log":
self.stop_log()
print(f"{ASCIIColors.color_red}Log stopped{ASCIIColors.color_reset}")
continue
if prompt == "send_file":
if self.personality.processor is None:
print(f"{ASCIIColors.color_red}This personality doesn't support file reception{ASCIIColors.color_reset}")
continue
fp = input("Please enter a file path: ")
# Remove double quotes using string slicing
if fp.startswith('"') and fp.endswith('"'):
fp = fp[1:-1]
if self.personality.processor.add_file(fp):
print(f"{ASCIIColors.color_green}File imported{ASCIIColors.color_reset}")
else:
print(f"{ASCIIColors.color_red}Couldn't load file{ASCIIColors.color_reset}")
continue
if prompt == "context_infos":
tokens = self.personality.model.tokenize(full_discussion)
print(f"{ASCIIColors.color_green}Current context has {len(tokens)} tokens/ {self.config.ctx_size}{ASCIIColors.color_reset}")
continue
if prompt != '':
if self.personality.processor is not None and self.personality.processor_cfg["process_model_input"]:
preprocessed_prompt = self.personality.processor.process_model_input(prompt)
else:
preprocessed_prompt = prompt
if self.personality.processor is not None and self.personality.processor_cfg["custom_workflow"]:
full_discussion += (
ump +
preprocessed_prompt
)
else:
full_discussion += (
ump +
preprocessed_prompt +
self.personality.link_text +
self.personality.ai_message_prefix
)
def callback(text, type:MSG_TYPE=None, metadata:dict={}):
if type == MSG_TYPE.MSG_TYPE_CHUNK:
# Replace stdout with the default stdout
sys.stdout = sys.__stdout__
print(text, end="", flush=True)
bot_says = self.bot_says + text
antiprompt = self.personality.detect_antiprompt(bot_says)
if antiprompt:
self.bot_says = self.remove_text_from_string(bot_says,antiprompt)
ASCIIColors.warning(f"Detected hallucination with antiprompt {antiprompt}")
return False
else:
self.bot_says = bot_says
return True
tk = self.personality.model.tokenize(full_discussion)
n_tokens = len(tk)
fd = self.personality.model.detokenize(tk[-min(self.config.ctx_size-self.n_cond_tk,n_tokens):])
print(f"{ASCIIColors.color_red}{self.personality.name}:{ASCIIColors.color_reset}", end='', flush=True)
if self.personality.processor is not None and self.personality.processor_cfg["custom_workflow"]:
output = self.personality.processor.run_workflow(prompt, previous_discussion_text=self.personality.personality_conditioning+fd, callback=callback)
print(output)
else:
output = self.personality.model.generate(self.personality.personality_conditioning+fd, n_predict=self.personality.model_n_predicts, callback=callback)
full_discussion += output.strip()
print()
if self.personality.processor is not None and self.personality.processor_cfg["process_model_output"]:
output = self.personality.processor.process_model_output(output)
self.log(full_discussion)
if self.show_time_elapsed:
t1 = time.time() # Time at end of response
print(f"{ASCIIColors.color_cyan}Time Elapsed: {ASCIIColors.color_reset}",str(int((t1-t0)*1000)),"ms\n") # Total time elapsed since t0 in ms
except KeyboardInterrupt:
print("Keyboard interrupt detected.\nBye")
break
print("Done")
print(f"{self.personality}")
def main():
# Create the argument parser
parser = argparse.ArgumentParser(description='App Description')
# Add the configuration path argument
parser.add_argument('--configuration_path', default=None,
help='Path to the configuration file')
parser.add_argument('--reset_personal_path', action='store_true', help='Reset the personal path')
parser.add_argument('--reset_config', action='store_true', help='Reset the configurations')
parser.add_argument('--reset_installs', action='store_true', help='Reset all installation status')
parser.add_argument('--show_time_elapsed', action='store_true', help='Enables response time')
# Parse the command-line arguments
args = parser.parse_args()
if args.reset_installs:
LollmsApplication.reset_all_installs()
if args.reset_personal_path:
LollmsPaths.reset_configs()
if args.reset_config:
lollms_paths = LollmsPaths.find_paths(tool_prefix="lollms_server_")
cfg_path = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml"
try:
cfg_path.unlink()
ASCIIColors.success("LOLLMS configuration reset successfully")
except:
ASCIIColors.success("Couldn't reset LOLLMS configuration")
if args.show_time_elapsed:
show_time_elapsed = True
else:
show_time_elapsed = False
# Parse the command-line arguments
args = parser.parse_args()
configuration_path = args.configuration_path
lollms_app = Conversation(configuration_path=configuration_path, show_commands_list=True, show_time_elapsed=show_time_elapsed)
lollms_app.start_conversation()
if __name__ == "__main__":
main()

3
lollms/apps/create/.vite/deps_temp_0b6cd151/package.json
View File

@ -1,3 +0,0 @@
{
"type": "module"
}

3
lollms/apps/create/.vite/deps_temp_889a01ff/package.json
View File

@ -1,3 +0,0 @@
{
"type": "module"
}

2
lollms/apps/discord_bot/.gitignore vendored
View File

@ -1,2 +0,0 @@
discord_config.yaml
data

31
lollms/apps/discord_bot/README.md
View File

@ -1,31 +0,0 @@
# Project Name
## Description
This project is a Discord bot that utilizes the lollms library to provide assistance and generate responses based on user input.
## Installation
1. Clone the repository:
```shell
git clone https://github.com/your-username/your-repository.git
```
2. Install the required dependencies:
```shell
pip install -r requirements.txt
```
3. Create a `discord_config.yaml` file in the project directory and add your bot token to it.
## Usage
1. Run the bot:
```shell
python main.py
```
2. The bot will join your Discord server and send a welcome message to new members.
3. Use the following commands to interact with the bot:
- `!lollms <prompt>`: Chat with the bot and receive generated responses based on the input prompt.
- `!install`: Get instructions on how to install the lollms library.
## Contributing
Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
## License
[Apache 2.0](https://choosealicense.com/licenses/apache-2.0/)

267
lollms/apps/discord_bot/__init__.py
View File

@ -1,267 +0,0 @@
import discord
from discord.ext import commands
from lollms.app import LollmsApplication
from lollms.paths import LollmsPaths
from lollms.main_config import LOLLMSConfig
import yaml
from pathlib import Path
from ascii_colors import ASCIIColors
from safe_store import TextVectorizer, GenericDataLoader, VisualizationMethod, VectorizationMethod
from typing import Tuple, List
context={
"discussion":""
}
client = commands.Bot(command_prefix='!', intents=discord.Intents.all())
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_discord_")
config = LOLLMSConfig.autoload(lollms_paths)
lollms_app = LollmsApplication("lollms_bot", config=config, lollms_paths=lollms_paths)
current_path = Path(__file__).resolve().parent
root_config_path = lollms_app.lollms_paths.personal_configuration_path / "discord_bot"
root_config_path.mkdir(parents=True, exist_ok=True)
root_data_path = lollms_app.lollms_paths.personal_data_path / "discord_bot"
root_data_path.mkdir(parents=True, exist_ok=True)
config_path = root_config_path / 'config.yaml'
text_vectorzer = TextVectorizer(
database_path=root_data_path / "db.json",
vectorization_method=VectorizationMethod.TFIDF_VECTORIZER,
save_db=True
)
files = [f for f in root_data_path.iterdir() if f.suffix in [".txt", ".md", ".pdf"]]
for f in files:
txt = GenericDataLoader.read_file(f)
text_vectorzer.add_document(f, txt, 512, 128, False)
text_vectorzer.index()
if not config_path.exists():
bot_token = input("Please enter your bot token: ")
config = {'bot_token': bot_token, "summoning_word":"!lollms"}
with open(config_path, 'w') as config_file:
yaml.safe_dump(config, config_file)
else:
with open(config_path, 'r') as config_file:
config = yaml.safe_load(config_file)
bot_token = config['bot_token']
@client.event
async def on_ready():
print(f'Logged in as {client.user}')
print('------')
@client.event
async def on_member_join(member):
channel = member.guild.system_channel
if channel is not None:
await channel.send(f'Welcome {member.mention} to the server! How can I assist you today?')
def prepare_query(discussion, prompt, n_tokens: int = 0) -> Tuple[str, str, List[str]]:
"""
Prepares the query for the model.
Args:
client_id (str): The client ID.
message_id (int): The message ID. Default is -1.
is_continue (bool): Whether the query is a continuation. Default is False.
n_tokens (int): The number of tokens. Default is 0.
Returns:
Tuple[str, str, List[str]]: The prepared query, original message content, and tokenized query.
"""
# Define current message
current_message = prompt
# Build the conditionning text block
conditionning = lollms_app.personality.personality_conditioning
# Check if there are document files to add to the prompt
documentation = ""
if lollms_app.personality.persona_data_vectorizer:
if documentation=="":
documentation="!@>Documentation:\n"
docs, sorted_similarities = lollms_app.personality.persona_data_vectorizer.recover_text(current_message, top_k=lollms_app.config.data_vectorization_nb_chunks)
for doc, infos in zip(docs, sorted_similarities):
documentation += f"document chunk:\n{doc}"
if len(lollms_app.personality.text_files) > 0 and lollms_app.personality.vectorizer:
if documentation=="":
documentation="!@>Documentation:\n"
docs, sorted_similarities = lollms_app.personality.vectorizer.recover_text(current_message.content, top_k=lollms_app.config.data_vectorization_nb_chunks)
for doc, infos in zip(docs, sorted_similarities):
documentation += f"document chunk:\nchunk path: {infos[0]}\nchunk content:{doc}"
# Check if there is discussion history to add to the prompt
history = ""
if lollms_app.config.use_discussions_history and lollms_app.long_term_memory is not None:
if history=="":
documentation="!@>History:\n"
docs, sorted_similarities = lollms_app.long_term_memory.recover_text(current_message.content, top_k=lollms_app.config.data_vectorization_nb_chunks)
for doc, infos in zip(docs, sorted_similarities):
history += f"discussion chunk:\ndiscussion title: {infos[0]}\nchunk content:{doc}"
# Add information about the user
user_description=""
if lollms_app.config.use_user_name_in_discussions:
user_description="!@>User description:\n"+lollms_app.config.user_description
# Tokenize the conditionning text and calculate its number of tokens
tokens_conditionning = lollms_app.model.tokenize(conditionning)
n_cond_tk = len(tokens_conditionning)
# Tokenize the documentation text and calculate its number of tokens
if len(documentation)>0:
tokens_documentation = lollms_app.model.tokenize(documentation)
n_doc_tk = len(tokens_documentation)
else:
tokens_documentation = []
n_doc_tk = 0
# Tokenize the history text and calculate its number of tokens
if len(history)>0:
tokens_history = lollms_app.model.tokenize(history)
n_history_tk = len(tokens_history)
else:
tokens_history = []
n_history_tk = 0
# Tokenize user description
if len(user_description)>0:
tokens_user_description = lollms_app.model.tokenize(user_description)
n_user_description_tk = len(tokens_user_description)
else:
tokens_user_description = []
n_user_description_tk = 0
# Calculate the total number of tokens between conditionning, documentation, and history
total_tokens = n_cond_tk + n_doc_tk + n_history_tk + n_user_description_tk
# Calculate the available space for the messages
available_space = lollms_app.config.ctx_size - n_tokens - total_tokens
# Raise an error if the available space is 0 or less
if available_space<1:
raise Exception("Not enough space in context!!")
# Accumulate messages until the cumulative number of tokens exceeds available_space
tokens_accumulated = 0
# Initialize a list to store the full messages
full_message_list = []
# If this is not a continue request, we add the AI prompt
message_tokenized = lollms_app.model.tokenize(
"\n" +lollms_app.personality.ai_message_prefix.strip()
)
full_message_list.append(message_tokenized)
# Update the cumulative number of tokens
tokens_accumulated += len(message_tokenized)
message_tokenized = lollms_app.model.tokenize(discussion)
if len(message_tokenized)>lollms_app.config.ctx_size-1024:
pos = message_tokenized[-(lollms_app.config.ctx_size-1024)]
detokenized = lollms_app.model.detokenize(message_tokenized[pos:pos+20])
position = discussion.find(detokenized)
if position!=-1 and position>0:
discussion_messages = discussion[-position:]
else:
discussion_messages = discussion[-(lollms_app.config.ctx_size-1024):]
else:
discussion_messages = discussion
# Build the final prompt by concatenating the conditionning and discussion messages
prompt_data = conditionning + documentation + history + user_description + discussion_messages
# Tokenize the prompt data
tokens = lollms_app.model.tokenize(prompt_data)
# if this is a debug then show prompt construction details
if lollms_app.config["debug"]:
ASCIIColors.bold("CONDITIONNING")
ASCIIColors.yellow(conditionning)
ASCIIColors.bold("DOC")
ASCIIColors.yellow(documentation)
ASCIIColors.bold("HISTORY")
ASCIIColors.yellow(history)
ASCIIColors.bold("DISCUSSION")
ASCIIColors.hilight(discussion_messages,"!@>",ASCIIColors.color_yellow,ASCIIColors.color_bright_red,False)
ASCIIColors.bold("Final prompt")
ASCIIColors.hilight(prompt_data,"!@>",ASCIIColors.color_yellow,ASCIIColors.color_bright_red,False)
ASCIIColors.info(f"prompt size:{len(tokens)} tokens")
ASCIIColors.info(f"available space after doc and history:{available_space} tokens")
# Return the prepared query, original message content, and tokenized query
return prompt_data
@client.event
async def on_message(message):
if message.author == client.user:
return
if message.content.startswith(config["summoning_word"]):
prompt = message.content[len(config["summoning_word"])+1:]
context['discussion'] = prepare_query(context['discussion'], prompt, 1024)
context['discussion']+= "\n!@>" + message.author.name +": "+ prompt + "\n" + f"{lollms_app.personality.ai_message_prefix}"
context['current_response']=""
def callback(text, type=None):
antiprompt = lollms_app.personality.detect_antiprompt(context['current_response'])
if antiprompt:
ASCIIColors.warning(f"\nDetected hallucination with antiprompt: {antiprompt}")
context['current_response'] = lollms_app.remove_text_from_string(context['current_response'],antiprompt)
return False
context['current_response'] += text
print(text,end="")
return True
ASCIIColors.green("Warming up ...")
lollms_app.safe_generate(context['discussion'], n_predict=1024, callback=callback, placeholder={"discussion":context['discussion']},place_holders_to_sacrifice=["discussion"], debug=True)
context['discussion'] += context['current_response']
await message.channel.send(context['current_response'])
elif message.content.startswith('!mount'):
personality_name = message.content[len('!mount')+1:]
lollms_app.config.personalities.append(personality_name)
lollms_app.personality = lollms_app.mount_personality(len(lollms_app.config.personalities)-1)
lollms_app.config.active_personality_id = len(lollms_app.config.personalities)-1
await message.channel.send(f"Personality {personality_name} mounted successfuly")
elif message.content.startswith('!list'):
await message.channel.send(f"Mounted personalities:\n{[p.name for p in lollms_app.mounted_personalities]}")
elif message.content.startswith('!select'):
personality_name = message.content[len('!select')+1:]
index = 0
for i in range(len(lollms_app.mounted_personalities)):
if lollms_app.mounted_personalities[i].name.lower().strip()==personality_name.lower():
index = i
lollms_app.config.active_personality_id = index
await message.channel.send(f"Activated personality:\n{personality_name}")
elif message.content.startswith('!reset'):
context["discussion"]=""
await message.channel.send(f"Content reset")
elif message.content.startswith('!install'):
response = "To install lollms, make sure you have installed the currently supported python version (consult the repository to verify what version is currently supported, but as of 10/22/2023, the version is 3.10).\nThen you can follow these steps:\n1. Open your command line interface.\n2. Navigate to the directory where you want to install lollms.\n3. Run the following command to clone the lollms repository: `git clone https://github.com/lollms/lollms.git`.\n4. Once the repository is cloned, navigate into the lollms directory.\n5. Run `pip install -r requirements.txt` to install the required dependencies.\n6. You can now use lollms by importing it in your Python code."
await message.channel.send(response)
@client.event
async def on_ready():
print(f'Logged in as {client.user}')
print('------')
@client.event
async def on_member_join(member):
channel = member.guild.system_channel
if channel is not None:
await channel.send(f'Welcome {member.mention} to the server! How can I assist you today?')
client.run(bot_token)

533
lollms/apps/elf/__init__.py
View File

@ -1,533 +0,0 @@
from lollms.app import LollmsApplication
from lollms.paths import LollmsPaths
from lollms.main_config import LOLLMSConfig
import sys
import time
maxtry=10
from pathlib import Path
import json
import re
import random
from flask import Flask, make_response, request, abort
from flask.json import jsonify
from typing import Callable
import string
import argparse
from ascii_colors import ASCIIColors
BUNDLES=4
MAXWORDS=1048
DEBUG=True
class Elf(LollmsApplication):
def __init__(self):
pass
def init(self, custom_default_cfg_path):
lollms_paths = LollmsPaths.find_paths(custom_global_paths_cfg_path=custom_default_cfg_path, tool_prefix="lollms_elf_")
config = LOLLMSConfig.autoload(lollms_paths, None)
super().__init__("Elf", config, lollms_paths)
def split_fibers(self,fibers, max_words=MAXWORDS):
# Split each fiber into chunks of up to max_words words
sfibers = []
for fiber in fibers:
words = fiber.split()
for i in range(0, len(words), max_words):
chunk = " ".join(words[i : i + max_words])
sfibers.append(chunk)
return sfibers
def refactor_into_fiber_bundles(self, lines, bundle_size):
bundles = []
temp = []
for line in lines:
# Split the line into fibers
# fibers = line.split('.')
fibers = re.split(r"[\.\n]", line)
# Filter out empty lines or lines with only whitespace
fibers = [fiber.strip() for fiber in fibers if re.search(r"\S", fiber)]
# Add filtered fibers to the current bundle
temp.extend(self.split_fibers(fibers))
# now lete
current_bundle = []
# print(temp)
for line in temp:
current_bundle.append(line)
# Check if the current bundle size exceeds the desired bundle size
if len(current_bundle) >= bundle_size:
# Add the current bundle to the list of bundles
bundles.append(current_bundle)
# Start a new bundle
current_bundle = []
# Add the last bundle if it's not empty
if current_bundle:
bundles.append(current_bundle)
return bundles
def read_input_file(self, file_path):
with open(file_path, "r") as file:
lines = file.readlines()
lines = self.refactor_into_fiber_bundles(lines, BUNDLES)
with open("debug.txt", "w") as fo:
for line in lines:
fo.write("|\n".join(line))
for line in lines:
self.text_ring_buffer.append(
self.personality.user_message_prefix + "\n".join(line)
)
print("start COUNT",len(self.text_ring_buffer))
def safe_generate(
self,
full_discussion:str,
n_predict=None,
callback: Callable[[str, int, dict], bool]=None,
temperature=0.1,
top_k=50,
top_p=0.9,
repeat_penalty=1.3,
last_n_tokens=60,
seed=-1,
n_threads=8,
batch_size=1,
stream=None):
"""safe_generate
Args:
full_discussion (string): A prompt or a long discussion to use for generation
callback (_type_, optional): A callback to call for each received token. Defaults to None.
Returns:
str: Model output
"""
if n_predict == None:
n_predict =self.config.n_predict
tk = self.personality.model.tokenize(full_discussion)
n_tokens = len(tk)
fd = self.personality.model.detokenize(tk[-min(self.config.ctx_size-self.n_cond_tk,n_tokens):])
self.bot_says = ""
output = self.personality.model.generate(
fd,
n_predict=n_predict,
temperature=temperature,
top_k=top_k,
top_p=top_p,
repeat_penalty=repeat_penalty,
last_n_tokens=last_n_tokens,
seed=seed,
n_threads=n_threads,
batch_size=batch_size,
callback=callback)
return output
def gen_rewrite(self):
topic = "Snippet"
target= "Protobuf Server"
return random.choice(
[
f"Transform this {topic} into a Python code representation of a {target}.",
f"Generate a Python code snippet for a {target} that implements this {topic}.",
f"Craft a Python implementation of a {target} that embodies the essence of this {topic}.",
]
)
def callback(self, text, type=None, metadata: dict = {}):
if DEBUG:
print("DBG:" + text, end="")
sys.stdout.flush()
return True
# set up the Flask application
app = Flask(__name__)
cv = Elf()
# input_file_path = "user_input.txt"
# try:
# cv.read_input_file(input_file_path)
# cv.start_conversation2()
# except Exception as e:
# print(e)
# raise e
models = {}
@app.route("/v1/models", methods=['GET'])
def models():
data = [
{
"id": model,
"object": "model",
"owned_by": "",
"tokens": 99999,
"fallbacks": None,
"endpoints": [
"/v1/chat/completions"
],
"limits": None,
"permission": []
}
for model in cv.binding.list_models()
]
return {'data': data, 'object': 'list'}
@app.route("/v1/completions", methods=['POST'])
def completions():
request_data = request.get_json()
model = request_data.get('model', None).replace("neuro-", "")
prompt = request_data.get('prompt')
temperature = request_data.get('temperature')
max_tokens = request_data.get('max_tokens', 1024)
prompt_tokens = cv.model.tokenize(prompt)
n_prompt_tokens = len(prompt_tokens)
if model is not None:
# TODO add model selection
pass
completion_id = "".join(random.choices(string.ascii_letters + string.digits, k=28))
completion_timestamp = int(time.time())
response = cv.safe_generate(full_discussion=prompt, temperature=temperature, n_predict=max_tokens)
completion_tokens = cv.model.tokenize(response)
n_completion_tokens = len(completion_tokens)
completion_timestamp = int(time.time())
completion_id = ''.join(random.choices(
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', k=28))
system_fingerprint = ''.join(random.choices(
'abcdefghijklmnopqrstuvwxyz0123456789', k=10))
return {
"id": f"cmpl-{completion_id}",
"object": "text_completion",
"created": completion_timestamp,
"model": model,
"system_fingerprint": "fp_"+system_fingerprint,
"choices": [
{
"index": 0,
"text": response,
"logprobs": None,
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": n_prompt_tokens,
"completion_tokens": n_completion_tokens,
"total_tokens": n_prompt_tokens + n_completion_tokens,
},
}
@app.route("/chat/completions", methods=['POST'])
@app.route("/v1/chat/completions", methods=['POST'])
@app.route("/", methods=['POST'])
def chat_completions():
request_data = request.get_json()
model = request_data.get('model', None).replace("neuro-", "")
messages = request_data.get('messages')
stream = request_data.get('stream', False)
streaming_ = request_data.get('stream', False)
temperature = request_data.get('temperature', 1.0)
top_p = request_data.get('top_p', 1.0)
max_tokens = request_data.get('max_tokens', 1024)
if model is not None:
# TODO add model selection
pass
full_discussion = ""
for message in messages:
if message["role"]:
full_discussion += f'{message["role"]}: {message["content"]}\n'
else:
full_discussion += f'{message["content"]}\n'
completion_id = "".join(random.choices(string.ascii_letters + string.digits, k=28))
completion_timestamp = int(time.time())
prompt_tokens = cv.model.tokenize(full_discussion)
n_prompt_tokens = len(prompt_tokens)
if not streaming_:
response = cv.safe_generate(full_discussion=full_discussion, temperature=temperature, top_p=top_p, n_predict=max_tokens)
completion_tokens = cv.model.tokenize(response)
n_completion_tokens = len(completion_tokens)
completion_timestamp = int(time.time())
completion_id = ''.join(random.choices(
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', k=28))
return {
"id": f"chatcmpl-{completion_id}",
"object": "chat.completion",
"created": completion_timestamp,
"model": model,
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": response,
},
"finish_reason": "stop",
}
],
"usage": {
"prompt_tokens": n_prompt_tokens,
"completion_tokens": n_completion_tokens,
"total_tokens": n_prompt_tokens + n_completion_tokens,
},
}
else:
print('Streaming ...')
if True:
def callback(token, data_type):
print(token,end="",flush=True)
return True
response = cv.safe_generate(
full_discussion=full_discussion,
temperature=temperature,
top_p=top_p,
n_predict=max_tokens,
callback=callback
)
def stream():
nonlocal response
for token in response.split(" "):
completion_timestamp = int(time.time())
completion_id = ''.join(random.choices(
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', k=28))
completion_data = {
'id': f'chatcmpl-{completion_id}',
'object': 'chat.completion.chunk',
'created': completion_timestamp,
'choices': [
{
'delta': {
'content': token+" "
},
'index': 0,
'finish_reason': None
}
]
}
yield 'data: %s\n\n' % json.dumps(completion_data, separators=(',' ':'))
time.sleep(0.02)
return app.response_class(
stream(),
mimetype='text/event-stream'
)
else:
def stream_callback(token, message_type=None):
print(token)
completion_timestamp = int(time.time())
completion_id = ''.join(random.choices(
'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789', k=28))
completion_data = {
'id': f'chatcmpl-{completion_id}',
'object': 'chat.completion.chunk',
'created': completion_timestamp,
'choices': [
{
'delta': {
'content': token
},
'index': 0,
'finish_reason': None
}
]
}
#yield 'data: %s\n\n' % json.dumps(completion_data, separators=(',' ':'))
time.sleep(0.02)
return True
return app.response_class(
cv.safe_generate(
full_discussion=full_discussion,
temperature=temperature,
top_p=top_p,
n_predict=max_tokens,
callback=lambda x,y:stream_callback(x,y)
),
mimetype='text/event-stream'
)
# define the engines endpoint
@app.route('/v1/engines')
@app.route('/v1/models')
@app.route("/models", methods=['GET'])
def v1_engines():
return make_response(jsonify({
'data': [{
'object': 'engine',
'id': id,
'ready': True,
'owner': 'huggingface',
'permissions': None,
'created': None
} for id in models.keys()]
}))
@app.route('/v1/embeddings', methods=['POST'])
@app.route('/embeddings', methods=['POST'])
def create_embedding():
j_input = request.get_json()
#model = embedding_processing()
embedding = cv.model.embedding(text_list=j_input['input'])
return jsonify(
embedding
)
@app.route("/v1/dashboard/billing/subscription", methods=['GET'])
@app.route("/dashboard/billing/subscription", methods=['GET'])
def billing_subscription():
return jsonify({
"object": "billing_subscription",
"has_payment_method": True,
"canceled": False,
"canceled_at": None,
"delinquent": None,
"access_until": 2556028800,
"soft_limit": 6944500,
"hard_limit": 166666666,
"system_hard_limit": 166666666,
"soft_limit_usd": 416.67,
"hard_limit_usd": 9999.99996,
"system_hard_limit_usd": 9999.99996,
"plan": {
"title": "Pay-as-you-go",
"id": "payg"
},
"primary": True,
"account_name": "OpenAI",
"po_number": None,
"billing_email": None,
"tax_ids": None,
"billing_address": {
"city": "New York",
"line1": "OpenAI",
"country": "US",
"postal_code": "NY10031"
},
"business_address": None
}
)
@app.route("/v1/dashboard/billing/usage", methods=['GET'])
@app.route("/dashboard/billing/usage", methods=['GET'])
def billing_usage():
return jsonify({
"object": "list",
"daily_costs": [
{
"timestamp": time.time(),
"line_items": [
{
"name": "GPT-4",
"cost": 0.0
},
{
"name": "Chat models",
"cost": 1.01
},
{
"name": "InstructGPT",
"cost": 0.0
},
{
"name": "Fine-tuning models",
"cost": 0.0
},
{
"name": "Embedding models",
"cost": 0.0
},
{
"name": "Image models",
"cost": 16.0
},
{
"name": "Audio models",
"cost": 0.0
}
]
}
],
"total_usage": 1.01
}
)
@app.route("/v1/providers", methods=['GET'])
@app.route("/providers", methods=['GET'])
def providers():
#todo : implement
return jsonify({})
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--host', '-hst', default=None, help='Host name')
parser.add_argument('--port', '-prt', default=None, help='Port number')
parser.add_argument('--reset_personal_path', action='store_true', help='Reset the personal path')
parser.add_argument('--reset_config', action='store_true', help='Reset the configurations')
parser.add_argument('--reset_installs', action='store_true', help='Reset all installation status')
parser.add_argument('--default_cfg_path', type=str, default=None, help='Reset all installation status')
ASCIIColors.yellow(" _ _ _ _ _ _ _ ")
ASCIIColors.yellow(" _\ \ /\ \ _\ \ _\ \ /\_\/\_\ _ / /\ ")
ASCIIColors.yellow(" /\__ \ / \ \ /\__ \ /\__ \ / / / / //\_\ / / \ ")
ASCIIColors.yellow(" / /_ \_\ / /\ \ \ / /_ \_\ / /_ \_\ /\ \/ \ \/ / // / /\ \__ ")
ASCIIColors.yellow(" / / /\/_/ / / /\ \ \ / / /\/_/ / / /\/_/ / \____\__/ // / /\ \___\ ")
ASCIIColors.yellow(" / / / / / / \ \_\ / / / / / / / /\/________/ \ \ \ \/___/ ")
ASCIIColors.yellow(" / / / / / / / / // / / / / / / / /\/_// / / \ \ \ ")
ASCIIColors.yellow(" / / / ____ / / / / / // / / ____ / / / ____ / / / / / /_ \ \ \ ")
ASCIIColors.yellow(" / /_/_/ ___/\ / / /___/ / // /_/_/ ___/\ / /_/_/ ___/\/ / / / / //_/\__/ / / ")
ASCIIColors.yellow("/_______/\__\// / /____\/ //_______/\__\//_______/\__\/\/_/ / / / \ \/___/ / ")
ASCIIColors.yellow("\_______\/ \/_________/ \_______\/ \_______\/ \/_/ \_____\/ ")
ASCIIColors.yellow(" _ _ _ ")
ASCIIColors.yellow(" /\ \ _\ \ /\ \ ")
ASCIIColors.yellow(" / \ \ /\__ \ / \ \ ")
ASCIIColors.yellow(" / /\ \ \ / /_ \_\ / /\ \ \ ")
ASCIIColors.yellow(" / / /\ \_\ / / /\/_/ / / /\ \_\ ")
ASCIIColors.yellow(" / /_/_ \/_/ / / / / /_/_ \/_/ ")
ASCIIColors.yellow(" / /____/\ / / / / /____/\ ")
ASCIIColors.yellow(" / /\____\/ / / / ____ / /\____\/ ")
ASCIIColors.yellow(" / / /______ / /_/_/ ___/\ / / / ")
ASCIIColors.yellow("/ / /_______\/_______/\__\// / / ")
ASCIIColors.yellow("\/__________/\_______\/ \/_/ ")
args = parser.parse_args()
if args.reset_personal_path:
LollmsPaths.reset_configs()
cv.init(args.default_cfg_path)
app.run(host=args.host, port=args.port)
if __name__ == "__main__":
main()

4
lollms/apps/playground/.env.development
View File

@ -1,4 +0,0 @@
VITE_LOLLMS_API = http://127.0.0.1:9601 # http://localhost:9600
VITE_LOLLMS_API_CHANGE_ORIGIN = 1 # FALSE
VITE_LOLLMS_API_SECURE = 0 # FALSE
VITE_LOLLMS_API_BASEURL = /api/ # FALSE

14
lollms/apps/playground/.eslintrc.cjs
View File

@ -1,14 +0,0 @@
/* eslint-env node */
require('@rushstack/eslint-patch/modern-module-resolution')
module.exports = {
root: true,
'extends': [
'plugin:vue/vue3-essential',
'eslint:recommended',
'@vue/eslint-config-prettier/skip-formatting'
],
parserOptions: {
ecmaVersion: 'latest'
}
}

30
lollms/apps/playground/.gitignore vendored
View File

@ -1,30 +0,0 @@
# Logs
logs
*.log
npm-debug.log*
yarn-debug.log*
yarn-error.log*
pnpm-debug.log*
lerna-debug.log*
node_modules
.DS_Store
dist-ssr
coverage
*.local
/cypress/videos/
/cypress/screenshots/
# Editor directories and files
.vscode
!.vscode/extensions.json
.idea
*.suo
*.ntvs*
*.njsproj
*.sln
*.sw?
# REST Client files (VSCODE extension for making GET POST requests easy and fst from text files)
*.http

8
lollms/apps/playground/.prettierrc.json
View File

@ -1,8 +0,0 @@
{
"$schema": "https://json.schemastore.org/prettierrc",
"semi": false,
"tabWidth": 2,
"singleQuote": true,
"printWidth": 100,
"trailingComma": "none"
}

201
lollms/apps/playground/LICENSE
View File

@ -1,201 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2023 Saifeddine ALOUI (aka parisneo)
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

160
lollms/apps/playground/__init__.py
View File

@ -1,160 +0,0 @@
from flask import Flask, send_from_directory, request, jsonify
from lollms.helpers import get_trace_exception
from lollms.paths import LollmsPaths
from lollms.main_config import LOLLMSConfig
from pathlib import Path
import requests
import json
import shutil
import yaml
import os
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_")
config = LOLLMSConfig.autoload(lollms_paths)
app = Flask(__name__, static_folder='dist/')
@app.route('/')
def index():
return app.send_static_file('index.html')
@app.route('/<path:filename>')
def serve_file(filename):
root_dir = "dist/"
return send_from_directory(root_dir, filename)
@app.route("/execute_python_code", methods=["POST"])
def execute_python_code():
"""Executes Python code and returns the output."""
data = request.get_json()
code = data["code"]
# Import the necessary modules.
import io
import sys
import time
# Create a Python interpreter.
interpreter = io.StringIO()
sys.stdout = interpreter
# Execute the code.
start_time = time.time()
try:
exec(code)
# Get the output.
output = interpreter.getvalue()
except Exception as ex:
output = str(ex)+"\n"+get_trace_exception(ex)
end_time = time.time()
return jsonify({"output":output,"execution_time":end_time - start_time})
def copy_files(src, dest):
for item in os.listdir(src):
src_file = os.path.join(src, item)
dest_file = os.path.join(dest, item)
if os.path.isfile(src_file):
shutil.copy2(src_file, dest_file)
@app.route("/get_presets", methods=["GET"])
def get_presets():
presets_folder = lollms_paths.personal_databases_path/"lollms_playground_presets"
if not presets_folder.exists():
presets_folder.mkdir(exist_ok=True, parents=True)
path = Path(__file__).parent/"presets"
copy_files(str(path),presets_folder)
presets = []
for filename in presets_folder.glob('*.yaml'):
print(filename)
with open(filename, 'r', encoding='utf-8') as file:
preset = yaml.safe_load(file)
if preset is not None:
presets.append(preset)
return jsonify(presets)
@app.route("/add_preset", methods=["POST"])
def add_preset():
# Get the JSON data from the POST request.
preset_data = request.get_json()
presets_folder = lollms_paths.personal_databases_path/"lollms_playground_presets"
if not presets_folder.exists():
presets_folder.mkdir(exist_ok=True, parents=True)
copy_files("presets",presets_folder)
fn = preset_data.name.lower().replace(" ","_")
filename = presets_folder/f"{fn}.yaml"
with open(filename, 'w', encoding='utf-8') as file:
yaml.dump(preset_data)
return jsonify({"status": True})
@app.route("/del_preset", methods=["POST"])
def del_preset():
presets_folder = lollms_paths.personal_databases_path/"lollms_playground_presets"
if not presets_folder.exists():
presets_folder.mkdir(exist_ok=True, parents=True)
copy_files("presets",presets_folder)
presets = []
for filename in presets_folder.glob('*.yaml'):
print(filename)
with open(filename, 'r') as file:
preset = yaml.safe_load(file)
if preset is not None:
presets.append(preset)
return jsonify(presets)
@app.route("/save_preset", methods=["POST"])
def save_presets():
"""Saves a preset to a file.
Args:
None.
Returns:
None.
"""
# Get the JSON data from the POST request.
preset_data = request.get_json()
presets_file = lollms_paths.personal_databases_path/"presets.json"
# Save the JSON data to a file.
with open(presets_file, "w") as f:
json.dump(preset_data, f, indent=4)
return jsonify({"status":True,"message":"Preset saved successfully!"})
@app.route("/list_models", methods=["GET"])
def list_models():
host = f"http://{config.host}:{config.port}"
response = requests.get(f"{host}/list_models")
data = response.json()
return jsonify(data)
@app.route("/get_active_model", methods=["GET"])
def get_active_model():
host = f"http://{config.host}:{config.port}"
response = requests.get(f"{host}/get_active_model")
data = response.json()
return jsonify(data)
@app.route("/update_setting", methods=["POST"])
def update_setting():
host = f"http://{config.host}:{config.port}"
response = requests.post(f"{host}/update_setting", headers={'Content-Type': 'application/json'}, data=request.data)
data = response.json()
return jsonify(data)
def main():
app.run()
if __name__ == '__main__':
main()

14
lollms/apps/playground/index.html
View File

@ -1,14 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<link rel="icon" href="/favicon.ico">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LoLLMS Playground WebUI - Welcome</title>
</head>
<body>
<div id="app"></div>
<script type="module" src="/src/main.js"></script>
</body>
</html>

3363
lollms/apps/playground/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

44
lollms/apps/playground/package.json
View File

@ -1,44 +0,0 @@
{
"name": "lollms-playground-vue",
"version": "0.0.0",
"private": true,
"scripts": {
"dev": "vite",
"build": "vite build",
"preview": "vite preview",
"lint": "eslint . --ext .vue,.js,.jsx,.cjs,.mjs --fix --ignore-path .gitignore",
"format": "prettier --write src/"
},
"dependencies": {
"axios": "^1.3.6",
"feather-icons": "^4.29.0",
"flowbite": "^1.6.5",
"flowbite-vue": "^0.0.10",
"highlight.js": "^11.8.0",
"markdown-it": "^13.0.1",
"markdown-it-anchor": "^8.6.7",
"markdown-it-attrs": "^4.1.6",
"markdown-it-emoji": "^2.0.2",
"markdown-it-implicit-figures": "^0.11.0",
"markdown-it-multimd-table": "^4.2.2",
"papaparse": "^5.4.1",
"prismjs": "^1.29.0",
"socket.io-client": "^4.6.1",
"vue": "^3.2.47",
"vue-router": "^4.1.6",
"vuex": "^4.0.2"
},
"devDependencies": {
"@rushstack/eslint-patch": "^1.2.0",
"@vitejs/plugin-vue": "^4.0.0",
"@vue/eslint-config-prettier": "^7.1.0",
"autoprefixer": "^10.4.14",
"eslint": "^8.34.0",
"eslint-plugin-vue": "^9.9.0",
"postcss": "^8.4.23",
"prettier": "^2.8.4",
"tailwind-scrollbar": "^3.0.1",
"tailwindcss": "^3.3.1",
"vite": "^4.1.4"
}
}

6
lollms/apps/playground/postcss.config.js
View File

@ -1,6 +0,0 @@
module.exports = {
plugins: {
tailwindcss: {},
autoprefixer: {},
},
}

5
lollms/apps/playground/presets.json
View File

@ -1,5 +0,0 @@
{
"preset1" : "Once apon a time",
"preset2" : "1+1="
}

BIN
lollms/apps/playground/public/chime_aud.wav
View File

Binary file not shown.

BIN
lollms/apps/playground/public/favicon.ico
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 66 KiB

39
lollms/apps/playground/tailwind.config.js
View File

@ -1,39 +0,0 @@
/** @type {import('tailwindcss').Config} */
module.exports = {
content: [
'./index.html',
'./src/**/*.{vue,js,ts,jsx,tsx}',
'node_modules/flowbite-vue/**/*.{js,jsx,ts,tsx}'
],
darkMode: 'class',
theme: {
extend: {
colors: {
primary: '#0e8ef0',
'primary-light': '#3dabff',
secondary: '#0fd974',
accent: '#f0700e',
'bg-dark': '#132e59',
'bg-dark-tone': '#25477d',
'bg-dark-tone-panel': '#4367a3',
'bg-dark-code-block': '#2254a7',
'bg-light': '#e2edff',
'bg-light-tone': '#b9d2f7',
'bg-light-code-block': '#cad7ed',
'bg-light-tone-panel': '#8fb5ef',
'bg-dark-discussion': '#435E8A',
'bg-dark-discussion-odd': '#284471',
'bg-light-discussion': '#c5d8f8',
'bg-light-discussion-odd': '#d6e7ff'
},
fontFamily: {
sans: ['PTSans', 'Roboto', 'sans-serif']
},
container: {
padding: '2rem',
center: true
}
}
},
plugins: [require('flowbite/plugin'), require('tailwind-scrollbar')]
}

42
lollms/apps/playground/vite.config.mjs
View File

@ -1,42 +0,0 @@
import { fileURLToPath, URL } from 'node:url'
import { defineConfig,loadEnv } from 'vite'
import vue from '@vitejs/plugin-vue'
// https://vitejs.dev/config/
export default ({ mode }) => {
// Load app-level env vars to node-level env vars.
process.env = {...process.env, ...loadEnv(mode, process.cwd())};
return defineConfig({
plugins: [
vue()
],
resolve: {
alias: {
'@': fileURLToPath(new URL('./src', import.meta.url))
}
},
server: {
serveStatic: {
// This will force Vite to serve the YAML file.
paths: ['./presets.json'],
},
proxy: {
"/api/": {
target: process.env.VITE_LOLLMS_API,
changeOrigin: process.env.VITE_LOLLMS_API_CHANGE_ORIGIN,
secure: process.env.VITE_LOLLMS_API_SECURE,
rewrite: (path) => path.replace(/^\/api/, ""),
},
// "/": {
// target: process.env.VITE_LOLLMS_API,
// changeOrigin: process.env.VITE_LOLLMS_API_CHANGE_ORIGIN,
// secure: process.env.VITE_LOLLMS_API_SECURE,
// },
},
}
})}

823
lollms/apps/server/__init__.py
View File

@ -1,823 +0,0 @@
from lollms.config import InstallOption
from flask import Flask, request, jsonify
from flask_socketio import SocketIO, emit
from flask_cors import CORS
from lollms.types import MSG_TYPE
from lollms.personality import AIPersonality
from lollms.main_config import LOLLMSConfig
from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder
from lollms.personality import PersonalityBuilder
from lollms.apps.console import MainMenu
from lollms.paths import LollmsPaths
from lollms.apps.console import MainMenu
from lollms.app import LollmsApplication
from safe_store import TextVectorizer
from ascii_colors import ASCIIColors, trace_exception
from typing import List, Tuple
from typing import Callable
import importlib
from pathlib import Path
import argparse
import logging
import yaml
import copy
import gc
import json
import traceback
import shutil
import psutil
import subprocess
import pkg_resources
def reset_all_installs(lollms_paths:LollmsPaths):
ASCIIColors.info("Removeing all configuration files to force reinstall")
ASCIIColors.info(f"Searching files from {lollms_paths.personal_configuration_path}")
for file_path in lollms_paths.personal_configuration_path.iterdir():
if file_path.name!=f"{lollms_paths.tool_prefix}local_config.yaml" and file_path.suffix.lower()==".yaml":
file_path.unlink()
ASCIIColors.info(f"Deleted file: {file_path}")
class LoLLMsServer(LollmsApplication):
def __init__(self):
self.clients = {}
self.current_binding = None
self.model = None
self.personalities = []
self.answer = ['']
self.busy = False
parser = argparse.ArgumentParser()
parser.add_argument('--host', '-hst', default=None, help='Host name')
parser.add_argument('--port', '-prt', default=None, help='Port number')
parser.add_argument('--reset_personal_path', action='store_true', help='Reset the personal path')
parser.add_argument('--reset_config', action='store_true', help='Reset the configurations')
parser.add_argument('--reset_installs', action='store_true', help='Reset all installation status')
parser.add_argument('--default_cfg_path', type=str, default=None, help='Reset all installation status')
args = parser.parse_args()
if args.reset_installs:
self.reset_all_installs()
if args.reset_personal_path:
LollmsPaths.reset_configs()
if args.reset_config:
lollms_paths = LollmsPaths.find_paths(custom_default_cfg_path=args.default_cfg_path, tool_prefix="lollms_server_")
cfg_path = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml"
try:
cfg_path.unlink()
ASCIIColors.success("LOLLMS configuration reset successfully")
except:
ASCIIColors.success("Couldn't reset LOLLMS configuration")
else:
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_")
# Configuration loading part
config = LOLLMSConfig.autoload(lollms_paths, None)
if args.host is not None:
config.host = args.host
if args.port is not None:
config.port = args.port
super().__init__("lollms-server", config, lollms_paths)
self.menu.show_logo()
self.app = Flask("LoLLMsServer")
#self.app.config['SECRET_KEY'] = 'lollmssecret'
CORS(self.app) # Enable CORS for all routes
def get_config():
ASCIIColors.yellow("Requested configuration")
return jsonify(self.config.to_dict())
self.app.add_url_rule(
"/get_config", "get_config", get_config, methods=["GET"]
)
def list_models():
if self.binding is not None:
ASCIIColors.yellow("Listing models")
models = self.binding.list_models()
ASCIIColors.green("ok")
return jsonify(models)
else:
return jsonify([])
self.app.add_url_rule(
"/list_models", "list_models", list_models, methods=["GET"]
)
def get_active_model():
if self.binding is not None:
models = self.binding.list_models()
index = models.index(self.config.model_name)
ASCIIColors.yellow(f"Recovering active model: {models[index]}")
return jsonify({"model":models[index],"index":index})
else:
return jsonify(None)
self.app.add_url_rule(
"/get_active_model", "get_active_model", get_active_model, methods=["GET"]
)
def get_server_ifos(self):
"""
Returns information about the server.
"""
server_infos = {}
if self.binding is not None:
models = self.binding.list_models()
index = models.index(self.config.model_name)
ASCIIColors.yellow(f"Recovering active model: {models[index]}")
server_infos["binding"]=self.binding.name
server_infos["models_list"]=models
server_infos["model"]=models[index]
server_infos["model_index"]=index
else:
server_infos["models_list"]=[]
server_infos["model"]=""
server_infos["model_index"]=-1
ram = psutil.virtual_memory()
server_infos["lollms_version"]= pkg_resources.get_distribution('lollms').version
server_infos["total_space"]=ram.total
server_infos["available_space"]=ram.free
server_infos["percent_usage"]=ram.percent
server_infos["ram_usage"]=ram.used
try:
output = subprocess.check_output(['nvidia-smi', '--query-gpu=memory.total,memory.used,gpu_name', '--format=csv,nounits,noheader'])
lines = output.decode().strip().split('\n')
vram_info = [line.split(',') for line in lines]
server_infos["nb_gpus"]= len(vram_info)
if vram_info is not None:
for i, gpu in enumerate(vram_info):
server_infos[f"gpu_{i}_total_vram"] = int(gpu[0])*1024*1024
server_infos[f"gpu_{i}_used_vram"] = int(gpu[1])*1024*1024
server_infos[f"gpu_{i}_model"] = gpu[2].strip()
else:
# Set all VRAM-related entries to None
server_infos["gpu_0_total_vram"] = None
server_infos["gpu_0_used_vram"] = None
server_infos["gpu_0_model"] = None
except (subprocess.CalledProcessError, FileNotFoundError):
server_infos["nb_gpus"]= 0
server_infos["gpu_0_total_vram"] = None
server_infos["gpu_0_used_vram"] = None
server_infos["gpu_0_model"] = None
current_drive = Path.cwd().anchor
drive_disk_usage = psutil.disk_usage(current_drive)
server_infos["system_disk_total_space"]=drive_disk_usage.total
server_infos["system_disk_available_space"]=drive_disk_usage.free
try:
models_folder_disk_usage = psutil.disk_usage(str(self.lollms_paths.binding_models_paths[0]))
server_infos["binding_disk_total_space"]=models_folder_disk_usage.total
server_infos["binding_disk_available_space"]=models_folder_disk_usage.free
except Exception as ex:
server_infos["binding_disk_total_space"]=None
server_infos["binding_disk_available_space"]=None
return jsonify(server_infos)
self.app.add_url_rule(
"/get_server_ifos", "get_server_ifos", get_server_ifos, methods=["GET"]
)
def update_setting():
data = request.get_json()
setting_name = data['setting_name']
if setting_name== "model_name":
self.config["model_name"]=data['setting_value']
if self.config["model_name"] is not None:
try:
self.model = None
if self.binding:
del self.binding
self.binding = None
for per in self.mounted_personalities:
if per is not None:
per.model = None
gc.collect()
self.binding = BindingBuilder().build_binding(self.config, self.lollms_paths, lollmsCom=self)
self.model = self.binding.build_model()
for per in self.mounted_personalities:
if per is not None:
per.model = self.model
except Exception as ex:
# Catch the exception and get the traceback as a list of strings
traceback_lines = traceback.format_exception(type(ex), ex, ex.__traceback__)
# Join the traceback lines into a single string
traceback_text = ''.join(traceback_lines)
ASCIIColors.error(f"Couldn't load model: [{ex}]")
ASCIIColors.error(traceback_text)
return jsonify({ "status":False, 'error':str(ex)})
else:
ASCIIColors.warning("Trying to set a None model. Please select a model for the binding")
print("update_settings : New model selected")
return jsonify({'setting_name': data['setting_name'], "status":True})
self.app.add_url_rule(
"/update_setting", "update_setting", update_setting, methods=["POST"]
)
def list_mounted_personalities():
if self.binding is not None:
ASCIIColors.yellow("Listing mounted personalities")
return jsonify(self.config.personalities)
else:
return jsonify([])
self.app.add_url_rule(
"/list_mounted_personalities", "list_mounted_personalities", list_mounted_personalities, methods=["GET"]
)
self.sio = SocketIO(self.app, cors_allowed_origins='*', ping_timeout=1200, ping_interval=4000)
# Set log level to warning
self.app.logger.setLevel(logging.WARNING)
# Configure a custom logger for Flask-SocketIO
self.sio_log = logging.getLogger('socketio')
self.sio_log.setLevel(logging.WARNING)
self.sio_log.addHandler(logging.StreamHandler())
self.initialize_routes()
self.run(self.config.host, self.config.port)
def initialize_routes(self):
@self.sio.on('connect')
def handle_connect():
client_id = request.sid
self.clients[client_id] = {
"namespace": request.namespace,
"full_discussion_blocks": [],
"is_generating":False,
"requested_stop":False
}
ASCIIColors.success(f'Client connected with session ID: {client_id}')
@self.sio.on('disconnect')
def handle_disconnect():
client_id = request.sid
if client_id in self.clients:
del self.clients[client_id]
print(f'Client disconnected with session ID: {client_id}')
@self.sio.on('list_available_bindings')
def handle_list_bindings():
binding_infs = []
for p in self.bindings_path.iterdir():
if p.is_dir():
with open(p/"binding_card.yaml", "r") as f:
card = yaml.safe_load(f)
with open(p/"models.yaml", "r") as f:
models = yaml.safe_load(f)
entry={
"name":p.name,
"card":card,
"models":models
}
binding_infs.append(entry)
emit('bindings_list', {'success':True, 'bindings': binding_infs}, room=request.sid)
@self.sio.on('list_available_personalities')
def handle_list_available_personalities():
personalities_folder = self.personalities_path
personalities = {}
for language_folder in personalities_folder.iterdir():
if language_folder.is_dir():
personalities[language_folder.name] = {}
for category_folder in language_folder.iterdir():
if category_folder.is_dir():
personalities[language_folder.name][category_folder.name] = []
for personality_folder in category_folder.iterdir():
if personality_folder.is_dir():
try:
personality_info = {"folder":personality_folder.stem}
config_path = personality_folder / 'config.yaml'
with open(config_path) as config_file:
config_data = yaml.load(config_file, Loader=yaml.FullLoader)
personality_info['name'] = config_data.get('name',"No Name")
personality_info['description'] = config_data.get('personality_description',"")
personality_info['author'] = config_data.get('author', 'ParisNeo')
personality_info['version'] = config_data.get('version', '1.0.0')
scripts_path = personality_folder / 'scripts'
personality_info['has_scripts'] = scripts_path.is_dir()
assets_path = personality_folder / 'assets'
gif_logo_path = assets_path / 'logo.gif'
webp_logo_path = assets_path / 'logo.webp'
png_logo_path = assets_path / 'logo.png'
jpg_logo_path = assets_path / 'logo.jpg'
jpeg_logo_path = assets_path / 'logo.jpeg'
bmp_logo_path = assets_path / 'logo.bmp'
personality_info['has_logo'] = png_logo_path.is_file() or gif_logo_path.is_file()
if gif_logo_path.exists():
personality_info['avatar'] = str(gif_logo_path).replace("\\","/")
elif webp_logo_path.exists():
personality_info['avatar'] = str(webp_logo_path).replace("\\","/")
elif png_logo_path.exists():
personality_info['avatar'] = str(png_logo_path).replace("\\","/")
elif jpg_logo_path.exists():
personality_info['avatar'] = str(jpg_logo_path).replace("\\","/")
elif jpeg_logo_path.exists():
personality_info['avatar'] = str(jpeg_logo_path).replace("\\","/")
elif bmp_logo_path.exists():
personality_info['avatar'] = str(bmp_logo_path).replace("\\","/")
else:
personality_info['avatar'] = ""
personalities[language_folder.name][category_folder.name].append(personality_info)
except Exception as ex:
print(f"Couldn't load personality from {personality_folder} [{ex}]")
emit('personalities_list', {'personalities': personalities}, room=request.sid)
@self.sio.on('list_available_models')
def handle_list_available_models():
"""List the available models
Returns:
_type_: _description_
"""
if self.binding is None:
emit('available_models_list', {'success':False, 'error': "No binding selected"}, room=request.sid)
model_list = self.binding.get_available_models(self)
models = []
for model in model_list:
try:
filename = model.get('filename',"")
server = model.get('server',"")
image_url = model.get("icon", '/images/default_model.png')
license = model.get("license", 'unknown')
owner = model.get("owner", 'unknown')
owner_link = model.get("owner_link", 'https://github.com/ParisNeo')
filesize = int(model.get('filesize',0))
description = model.get('description',"")
model_type = model.get("model_type","")
if server.endswith("/"):
path = f'{server}{filename}'
else:
path = f'{server}/{filename}'
local_path = self.models_path/f'{self.config["binding_name"]}/{filename}'
is_installed = local_path.exists() or model_type.lower()=="api"
models.append({
'title': filename,
'icon': image_url, # Replace with the path to the model icon
'license': license,
'owner': owner,
'owner_link': owner_link,
'description': description,
'isInstalled': is_installed,
'path': path,
'filesize': filesize,
'model_type': model_type
})
except Exception as ex:
print("#################################")
print(ex)
print("#################################")
print(f"Problem with model : {model}")
emit('available_models_list', {'success':True, 'available_models': models}, room=request.sid)
@self.sio.on('list_available_personalities_categories')
def handle_list_available_personalities_categories(data):
try:
categories = [l for l in (self.personalities_path).iterdir()]
emit('available_personalities_categories_list', {'success': True, 'available_personalities_categories': categories})
except Exception as ex:
emit('available_personalities_categories_list', {'success': False, 'error':str(ex)})
@self.sio.on('list_available_personalities_names')
def handle_list_available_personalities_names(data):
try:
category = data["category"]
personalities = [l for l in (self.personalities_path/category).iterdir()]
emit('list_available_personalities_names_list', {'success': True, 'list_available_personalities_names': personalities})
except Exception as ex:
emit('list_available_personalities_names_list', {'success': False, 'error':str(ex)})
@self.sio.on('select_binding')
def handle_select_binding(data):
self.cp_config = copy.deepcopy(self.config)
self.cp_config["binding_name"] = data['binding_name']
try:
del self.model
del self.binding
self.model = None
self.binding = None
gc.collect()
for personality in self.mount_personalities:
personality.model = None
self.binding = self.build_binding(self.bindings_path, self.cp_config)
self.config = self.cp_config
self.mount_personalities()
gc.collect()
emit('select_binding', {'success':True, 'binding_name': self.cp_config["binding_name"]}, room=request.sid)
except Exception as ex:
print(ex)
emit('select_binding', {'success':False, 'binding_name': self.cp_config["binding_name"], 'error':f"Couldn't load binding:\n{ex}"}, room=request.sid)
@self.sio.on('select_model')
def handle_select_model(data):
model_name = data['model_name']
if self.binding is None:
emit('select_model', {'success':False, 'model_name': model_name, 'error':f"Please select a binding first"}, room=request.sid)
return
self.cp_config = copy.deepcopy(self.config)
self.cp_config["model_name"] = data['model_name']
try:
del self.model
self.model = None
gc.collect()
for personality in self.mount_personalities:
personality.model = None
self.model = self.binding.build_model()
self.mount_personalities()
gc.collect()
emit('select_model', {'success':True, 'model_name': model_name}, room=request.sid)
except Exception as ex:
print(ex)
emit('select_model', {'success':False, 'model_name': model_name, 'error':f"Please select a binding first"}, room=request.sid)
@self.sio.on('add_personality')
def handle_add_personality(data):
personality_path = data['path']
try:
personality = AIPersonality(
personality_path,
self.lollms_paths,
self.config,
self.model,
self
)
self.personalities.append(personality)
self.config["personalities"].append(personality_path)
emit('personality_added', {'success':True, 'name': personality.name, 'id':len(self.personalities)-1}, room=request.sid)
self.config.save_config()
except Exception as e:
error_message = str(e)
emit('personality_add_failed', {'success':False, 'error': error_message}, room=request.sid)
@self.sio.on('vectorize_text')
def vectorize_text(parameters:dict):
"""Vectorizes text
Args:
parameters (dict): contains
'chunk_size': the maximum size of a text chunk (512 by default)
'vectorization_method': can be either "model_embedding" or "ftidf_vectorizer" (default is "ftidf_vectorizer")
'payloads': a list of dicts. each entry has the following format
{
"path": the path to the document
"text": the text of the document
},
'return_database': If true the vectorized database will be sent to the client (default is True)
'database_path': the path to store the database (default is none)
returns a dict
status: True if success and false if not
if you asked for the database to be sent back you will ahve those fields too:
embeddings: a dictionary containing the text chunks with their ids and embeddings
"texts": a dictionary of text chunks for each embedding (use index for correspondance)
"infos": extra information
"vectorizer": The vectorize if this is using tfidf or none if it uses model
"""
vectorization_method = parameters.get('vectorization_method',"ftidf_vectorizer")
chunk_size = parameters.get("chunk_size",512)
payloads = parameters["payloads"]
database_path = parameters.get("database_path",None)
return_database = parameters.get("return_database",True)
if database_path is None and return_database is None:
ASCIIColors.warning("Vectorization should either ask to save the database or to recover it. You didn't ask for any one!")
emit('vectorized_db',{"status":False, "error":"Vectorization should either ask to save the database or to recover it. You didn't ask for any one!"})
return
tv = TextVectorizer(vectorization_method, self.model)
for payload in payloads:
tv.add_document(payload["path"],payload["text"],chunk_size=chunk_size)
json_db = tv.toJson()
if return_database:
emit('vectorized_db',{**{"status":True}, **json_db})
else:
emit('vectorized_db',{"status":True})
with open(database_path, "w") as file:
json.dump(json_db, file, indent=4)
@self.sio.on('query_database')
def query_database(parameters:dict):
"""queries a database
Args:
parameters (dict): contains
'vectorization_method': can be either "model_embedding" or "ftidf_vectorizer"
'database': a list of dicts. each entry has the following format
{
embeddings: a dictionary containing the text chunks with their ids and embeddings
"texts": a dictionary of text chunks for each embedding (use index for correspondance)
"infos": extra information
"vectorizer": The vectorize if this is using tfidf or none if it uses model
}
'database_path': If supplied, the database is loaded from a path
'query': a query to search in the database
"""
vectorization_method = parameters['vectorization_method']
database = parameters.get("database",None)
query = parameters.get("query",None)
if query is None:
ASCIIColors.error("No query given!")
emit('vector_db_query',{"status":False, "error":"Please supply a query"})
return
if database is None:
database_path = parameters.get("database_path",None)
if database_path is None:
ASCIIColors.error("No database given!")
emit('vector_db_query',{"status":False, "error":"You did not supply a database file nor a database content"})
return
else:
with open(database_path, "r") as file:
database = json.load(file)
tv = TextVectorizer(vectorization_method, self.model, database_dict=database)
docs, sorted_similarities = tv.recover_text(tv.embed_query(query))
emit('vectorized_db',{
"chunks":docs,
"refs":sorted_similarities
})
@self.sio.on('list_active_personalities')
def handle_list_active_personalities():
personality_names = [p.name for p in self.personalities]
emit('active_personalities_list', {'success':True, 'personalities': personality_names}, room=request.sid)
@self.sio.on('activate_personality')
def handle_activate_personality(data):
personality_id = data['id']
if personality_id<len(self.personalities):
self.active_personality=self.personalities[personality_id]
emit('activate_personality', {'success':True, 'name': self.active_personality, 'id':len(self.personalities)-1}, room=request.sid)
self.config["active_personality_id"]=personality_id
self.config.save_config()
else:
emit('personality_add_failed', {'success':False, 'error': "Personality ID not valid"}, room=request.sid)
@self.sio.on('tokenize')
def tokenize(data):
client_id = request.sid
prompt = data['prompt']
tk = self.model.tokenize(prompt)
emit("tokenized", {"tokens":tk}, room=client_id)
@self.sio.on('detokenize')
def detokenize(data):
client_id = request.sid
prompt = data['prompt']
txt = self.model.detokenize(prompt)
emit("detokenized", {"text":txt}, room=client_id)
@self.sio.on('embed')
def detokenize(data):
client_id = request.sid
prompt = data['prompt']
txt = self.model.embed(prompt)
self.sio.emit("embeded", {"text":txt}, room=client_id)
@self.sio.on('cancel_text_generation')
def cancel_text_generation(data):
client_id = request.sid
self.clients[client_id]["requested_stop"]=True
print(f"Client {client_id} requested canceling generation")
self.sio.emit("generation_canceled", {"message":"Generation is canceled."}, room=client_id)
self.sio.sleep(0)
self.busy = False
# A copy of the original lollms-server generation code needed for playground
@self.sio.on('generate_text')
def handle_generate_text(data):
client_id = request.sid
ASCIIColors.info(f"Text generation requested by client: {client_id}")
if self.busy:
self.sio.emit("busy", {"message":"I am busy. Come back later."}, room=client_id)
self.sio.sleep(0)
ASCIIColors.warning(f"OOps request {client_id} refused!! Server busy")
return
def generate_text():
self.busy = True
try:
model = self.model
self.clients[client_id]["is_generating"]=True
self.clients[client_id]["requested_stop"]=False
prompt = data['prompt']
tokenized = model.tokenize(prompt)
personality_id = int(data.get('personality', -1))
n_crop = int(data.get('n_crop', len(tokenized)))
if n_crop!=-1:
prompt = model.detokenize(tokenized[-n_crop:])
n_predicts = int(data.get("n_predicts",1024))
parameters = data.get("parameters",{
"temperature":data.get("temperature",self.config["temperature"]),
"top_k":data.get("top_k",self.config["top_k"]),
"top_p":data.get("top_p",self.config["top_p"]),
"repeat_penalty":data.get("repeat_penalty",self.config["repeat_penalty"]),
"repeat_last_n":data.get("repeat_last_n",self.config["repeat_last_n"]),
"seed":data.get("seed",self.config["seed"])
})
if personality_id==-1:
# Raw text generation
self.answer = {"full_text":""}
def callback(text, message_type: MSG_TYPE, metadata:dict={}):
if message_type == MSG_TYPE.MSG_TYPE_CHUNK:
ASCIIColors.success(f"generated:{len(self.answer['full_text'].split())} words", end='\r')
self.answer["full_text"] = self.answer["full_text"] + text
self.sio.emit('text_chunk', {'chunk': text, 'type':MSG_TYPE.MSG_TYPE_CHUNK.value}, room=client_id)
self.sio.sleep(0)
if client_id in self.clients:# Client disconnected
if self.clients[client_id]["requested_stop"]:
return False
else:
return True
else:
return False
tk = model.tokenize(prompt)
n_tokens = len(tk)
fd = model.detokenize(tk[-min(self.config.ctx_size-n_predicts,n_tokens):])
try:
ASCIIColors.print("warming up", ASCIIColors.color_bright_cyan)
generated_text = model.generate(fd,
n_predict=n_predicts,
callback=callback,
temperature = parameters["temperature"],
top_k = parameters["top_k"],
top_p = parameters["top_p"],
repeat_penalty = parameters["repeat_penalty"],
repeat_last_n = parameters["repeat_last_n"],
seed = parameters["seed"]
)
ASCIIColors.success(f"\ndone")
if client_id in self.clients:
if not self.clients[client_id]["requested_stop"]:
# Emit the generated text to the client
self.sio.emit('text_generated', {'text': generated_text}, room=client_id)
self.sio.sleep(0)
except Exception as ex:
self.sio.emit('generation_error', {'error': str(ex)}, room=client_id)
ASCIIColors.error(f"\ndone")
self.busy = False
else:
try:
personality: AIPersonality = self.personalities[personality_id]
ump = self.config.discussion_prompt_separator +self.config.user_name+": " if self.config.use_user_name_in_discussions else self.personality.user_message_prefix
personality.model = model
cond_tk = personality.model.tokenize(personality.personality_conditioning)
n_cond_tk = len(cond_tk)
# Placeholder code for text generation
# Replace this with your actual text generation logic
print(f"Text generation requested by client: {client_id}")
self.answer["full_text"] = ''
full_discussion_blocks = self.clients[client_id]["full_discussion_blocks"]
if prompt != '':
if personality.processor is not None and personality.processor_cfg["process_model_input"]:
preprocessed_prompt = personality.processor.process_model_input(prompt)
else:
preprocessed_prompt = prompt
if personality.processor is not None and personality.processor_cfg["custom_workflow"]:
full_discussion_blocks.append(ump)
full_discussion_blocks.append(preprocessed_prompt)
else:
full_discussion_blocks.append(ump)
full_discussion_blocks.append(preprocessed_prompt)
full_discussion_blocks.append(personality.link_text)
full_discussion_blocks.append(personality.ai_message_prefix)
full_discussion = personality.personality_conditioning + ''.join(full_discussion_blocks)
def callback(text, message_type: MSG_TYPE, metadata:dict={}):
if message_type == MSG_TYPE.MSG_TYPE_CHUNK:
self.answer["full_text"] = self.answer["full_text"] + text
self.sio.emit('text_chunk', {'chunk': text}, room=client_id)
self.sio.sleep(0)
try:
if self.clients[client_id]["requested_stop"]:
return False
else:
return True
except: # If the client is disconnected then we stop talking to it
return False
tk = personality.model.tokenize(full_discussion)
n_tokens = len(tk)
fd = personality.model.detokenize(tk[-min(self.config.ctx_size-n_cond_tk-personality.model_n_predicts,n_tokens):])
if personality.processor is not None and personality.processor_cfg["custom_workflow"]:
ASCIIColors.info("processing...")
generated_text = personality.processor.run_workflow(prompt, previous_discussion_text=personality.personality_conditioning+fd, callback=callback)
else:
ASCIIColors.info("generating...")
generated_text = personality.model.generate(
personality.personality_conditioning+fd,
n_predict=personality.model_n_predicts,
callback=callback)
if personality.processor is not None and personality.processor_cfg["process_model_output"]:
generated_text = personality.processor.process_model_output(generated_text)
full_discussion_blocks.append(generated_text.strip())
ASCIIColors.success("\ndone")
# Emit the generated text to the client
self.sio.emit('text_generated', {'text': generated_text}, room=client_id)
self.sio.sleep(0)
except Exception as ex:
self.sio.emit('generation_error', {'error': str(ex)}, room=client_id)
ASCIIColors.error(f"\ndone")
self.busy = False
except Exception as ex:
trace_exception(ex)
self.sio.emit('generation_error', {'error': str(ex)}, room=client_id)
self.busy = False
# Start the text generation task in a separate thread
task = self.sio.start_background_task(target=generate_text)
def build_binding(self, bindings_path: Path, cfg: LOLLMSConfig)->LLMBinding:
binding_path = Path(bindings_path) / cfg["binding_name"]
# define the full absolute path to the module
absolute_path = binding_path.resolve()
# infer the module name from the file path
module_name = binding_path.stem
# use importlib to load the module from the file path
loader = importlib.machinery.SourceFileLoader(module_name, str(absolute_path / "__init__.py"))
binding_module = loader.load_module()
binding = getattr(binding_module, binding_module.binding_name)
return binding
def run(self, host="localhost", port="9601"):
if self.binding is None:
ASCIIColors.warning("No binding selected. Please select one")
self.menu.select_binding()
print(f"{ASCIIColors.color_red}Current binding (model) : {ASCIIColors.color_reset}{self.binding}")
print(f"{ASCIIColors.color_red}Mounted personalities : {ASCIIColors.color_reset}{self.config.personalities}")
if len(self.config.personalities)==0:
ASCIIColors.warning("No personality selected. Selecting lollms. You can mount other personalities using lollms-settings application")
self.config.personalities = ["generic/lollms"]
self.config.save_config()
if self.config.active_personality_id>=len(self.config.personalities):
self.config.active_personality_id = 0
print(f"{ASCIIColors.color_red}Current personality : {ASCIIColors.color_reset}{self.config.personalities[self.config.active_personality_id]}")
ASCIIColors.info(f"Serving on address: http://{host}:{port}")
self.sio.run(self.app, host=host, port=port)
def main():
LoLLMsServer()
if __name__ == '__main__':
main()

73
lollms/apps/settings/README.md
View File

@ -1,73 +0,0 @@
# lollms-settings
## Description
The `lollms-settings` tool is used to configure multiple aspects of the lollms project. Lollms is a multi-bindings LLM service that serves multiple LLM models that can generate text out of a prompt.
## Usage
To use the `lollms-settings` tool, you can run the following command:
```
python lollms_settings.py [--configuration_path CONFIGURATION_PATH] [--reset_personal_path] [--reset_config] [--reset_installs] [--default_cfg_path DEFAULT_CFG_PATH] [--tool_prefix TOOL_PREFIX] [--set_personal_folder_path SET_PERSONAL_FOLDER_PATH] [--install_binding INSTALL_BINDING] [--install_model INSTALL_MODEL] [--select_model SELECT_MODEL] [--mount_personalities MOUNT_PERSONALITIES] [--set_personal_foldrer SET_PERSONAL_FOLDRE] [--silent]
```
### Arguments
- `--configuration_path`: Path to the configuration file.
- `--reset_personal_path`: Reset the personal path.
- `--reset_config`: Reset the configurations.
- `--reset_installs`: Reset all installation status.
- `--default_cfg_path`: Reset all installation status.
- `--tool_prefix`: A prefix to define what tool is being used (default `lollms_server_`).
- lollms applications prefixes:
- lollms server: `lollms_server_`
- lollms-elf: `lollms_elf_`
- lollms-webui: `""`
- lollms-discord-bot: `lollms_discord_`
- `--set_personal_folder_path`: Forces installing and selecting a specific binding.
- `--install_binding`: Forces installing and selecting a specific binding.
- `--install_model`: Forces installing and selecting a specific model.
- `--select_model`: Forces selecting a specific model.
- `--mount_personalities`: Forces mounting a list of personas.
- `--set_personal_foldrer`: Forces setting personal folder to a specific value.
- `--silent`: This will operate in silent mode, no menu will be shown.
### Examples
Here are some examples of how to use the `lollms-settings` tool:
1. Reset the configurations:
```
python lollms_settings.py --reset_config
```
2. Install and select a specific binding:
```
python lollms_settings.py --install_binding <binding_name>
```
3. Install and select a specific model:
```
python lollms_settings.py --install_model <model_path>
```
4. Select a specific model:
```
python lollms_settings.py --select_model <model_name>
```
5. Mount a list of personas:
```
python lollms_settings.py --mount_personalities <persona1> <persona2> ...
```
6. Set personal folder to a specific value:
```
python lollms_settings.py --set_personal_foldrer <folder_path>
```
7. Run in silent mode:
```
python lollms_settings.py --silent
```
## License
This project is licensed under the Apache 2.0 License.

272
lollms/apps/settings/__init__.py
View File

@ -1,272 +0,0 @@
from lollms.main_config import LOLLMSConfig
from ascii_colors import ASCIIColors
from lollms.paths import LollmsPaths
from pathlib import Path
import argparse
from lollms.app import LollmsApplication
from typing import Callable
from lollms.config import BaseConfig
from lollms.binding import BindingBuilder, InstallOption
from tqdm import tqdm
class Settings(LollmsApplication):
def __init__(
self,
lollms_paths:LollmsPaths=None,
configuration_path:str|Path=None,
show_logo:bool=True,
show_commands_list:bool=False,
show_personality_infos:bool=True,
show_model_infos:bool=True,
show_welcome_message:bool=True
):
# get paths
if lollms_paths is None:
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_")
# Load maoin configuration
config = LOLLMSConfig.autoload(lollms_paths)
super().__init__("lollms-settings", config, lollms_paths, load_model=False)
if show_logo:
self.menu.show_logo()
if show_personality_infos:
try:
print()
print(f"{ASCIIColors.color_green}Current personality : {ASCIIColors.color_reset}{self.personality}")
print(f"{ASCIIColors.color_green}Version : {ASCIIColors.color_reset}{self.personality.version}")
print(f"{ASCIIColors.color_green}Author : {ASCIIColors.color_reset}{self.personality.author}")
print(f"{ASCIIColors.color_green}Description : {ASCIIColors.color_reset}{self.personality.personality_description}")
print()
except:
pass
if show_model_infos:
try:
print()
print(f"{ASCIIColors.color_green}Current binding : {ASCIIColors.color_reset}{self.config['binding_name']}")
print(f"{ASCIIColors.color_green}Current model : {ASCIIColors.color_reset}{self.config['model_name']}")
print()
except:
pass
def start(self):
self.menu.main_menu()
def ask_override_file(self):
user_input = input("Would you like to override the existing file? (Y/N): ")
user_input = user_input.lower()
if user_input == "y" or user_input == "yes":
print("File will be overridden.")
return True
elif user_input == "n" or user_input == "no":
print("File will not be overridden.")
return False
else:
print("Invalid input. Please enter 'Y' or 'N'.")
# Call the function again recursively to prompt the user for valid input
return self.ask_override_file()
def start_log(self, file_name):
if Path(file_name).is_absolute():
self.log_file_path = Path(file_name)
else:
home_dir = self.lollms_paths.personal_path/"logs"
home_dir.mkdir(parents=True, exist_ok=True)
self.log_file_path = home_dir/file_name
if self.log_file_path.exists():
if not self.ask_override_file():
print("Canceled")
return
try:
with(open(self.log_file_path, "w") as f):
self.header = f"""------------------------
Log file for lollms discussion
Participating personalities:
{self.config['personalities']}
------------------------
"""
f.write(self.header)
self.is_logging = True
return True
except:
return False
def log(self, text, append=False):
try:
with(open(self.log_file_path, "a" if append else "w") as f):
f.write(text) if append else f.write(self.header+self.personality.personality_conditioning+text)
return True
except:
return False
def stop_log(self):
self.is_logging = False
def reset_context(self):
if self.personality.include_welcome_message_in_disucssion:
full_discussion = (
self.personality.ai_message_prefix +
self.personality.welcome_message +
self.personality.link_text
)
else:
full_discussion = ""
return full_discussion
def safe_generate(self, full_discussion:str, n_predict=None, callback: Callable[[str, int, dict], bool]=None):
"""safe_generate
Args:
full_discussion (string): A prompt or a long discussion to use for generation
callback (_type_, optional): A callback to call for each received token. Defaults to None.
Returns:
str: Model output
"""
if n_predict == None:
n_predict =self.personality.model_n_predicts
tk = self.personality.model.tokenize(full_discussion)
n_tokens = len(tk)
fd = self.personality.model.detokenize(tk[-min(self.config.ctx_size-self.n_cond_tk,n_tokens):])
self.bot_says = ""
output = self.personality.model.generate(self.personality.personality_conditioning+fd, n_predict=n_predict, callback=callback)
return output
def remove_text_from_string(self, string, text_to_find):
"""
Removes everything from the first occurrence of the specified text in the string (case-insensitive).
Parameters:
string (str): The original string.
text_to_find (str): The text to find in the string.
Returns:
str: The updated string.
"""
index = string.lower().find(text_to_find.lower())
if index != -1:
string = string[:index]
return string
def main():
# Create the argument parser
parser = argparse.ArgumentParser(description='The lollms-settings app is used to configure multiple aspects of the lollms project. Lollms is a multi bindings LLM service that serves multiple LLM models that can generate text out of a prompt.')
# Add the configuration path argument
parser.add_argument('--configuration_path', default=None,
help='Path to the configuration file')
parser.add_argument('--reset_personal_path', action='store_true', help='Reset the personal path')
parser.add_argument('--reset_config', action='store_true', help='Reset the configurations')
parser.add_argument('--reset_installs', action='store_true', help='Reset all installation status')
parser.add_argument('--default_cfg_path', type=str, default=None, help='Reset all installation status')
parser.add_argument('--tool_prefix', type=str, default="lollms_server_", help='A prefix to define what tool is being used (default lollms_server_)\nlollms applications prefixes:\n lollms server: lollms_server_\nlollms-elf: lollms_elf_\nlollms-webui: ""\nlollms-discord-bot: lollms_discord_')
parser.add_argument('--set_personal_folder_path', type=str, default=None, help='Forces installing and selecting a specific binding')
parser.add_argument('--install_binding', type=str, default=None, help='Forces installing and selecting a specific binding')
parser.add_argument('--install_model', type=str, default=None, help='Forces installing and selecting a specific model')
parser.add_argument('--select_model', type=str, default=None, help='Forces selecting a specific model')
parser.add_argument('--mount_personalities', nargs='+', type=str, default=None, help='Forces mounting a list of personas')
parser.add_argument('--set_personal_foldrer', type=str, default=None, help='Forces setting personal folder to a specific value')
parser.add_argument('--silent', action='store_true', help='This will operate in scilent mode, no menu will be shown')
# Parse the command-line arguments
args = parser.parse_args()
tool_prefix = args.tool_prefix
if args.reset_installs:
LollmsApplication.reset_all_installs()
if args.reset_personal_path:
LollmsPaths.reset_configs()
if args.reset_config:
lollms_paths = LollmsPaths.find_paths(custom_default_cfg_path=args.default_cfg_path, tool_prefix=tool_prefix, force_personal_path=args.set_personal_folder_path)
cfg_path = lollms_paths.personal_configuration_path / f"{lollms_paths.tool_prefix}local_config.yaml"
try:
cfg_path.unlink()
ASCIIColors.success("LOLLMS configuration reset successfully")
except:
ASCIIColors.success("Couldn't reset LOLLMS configuration")
else:
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix=tool_prefix, force_personal_path=args.set_personal_folder_path)
configuration_path = args.configuration_path
if args.set_personal_folder_path:
cfg = BaseConfig(config={
"lollms_path":str(Path(__file__).parent),
"lollms_personal_path":args.set_personal_folder_path
})
ASCIIColors.green(f"Selected personal path: {cfg.lollms_personal_path}")
pp= Path(cfg.lollms_personal_path)
if not pp.exists():
try:
pp.mkdir(parents=True)
ASCIIColors.green(f"Personal path set to {pp}")
except:
print(f"{ASCIIColors.color_red}It seams there is an error in the path you rovided{ASCIIColors.color_reset}")
global_paths_cfg_path = Path.home()/f"{tool_prefix}global_paths_cfg.yaml"
lollms_paths = LollmsPaths.find_paths(force_local=False, custom_global_paths_cfg_path=global_paths_cfg_path, tool_prefix=tool_prefix)
cfg.save_config(global_paths_cfg_path)
settings_app = Settings(configuration_path=configuration_path, lollms_paths=lollms_paths, show_commands_list=True)
if args.install_binding:
settings_app.config.binding_name= args.install_binding
settings_app.binding = BindingBuilder().build_binding(settings_app.config, settings_app.lollms_paths,InstallOption.FORCE_INSTALL, lollmsCom=settings_app)
settings_app.config.save_config()
if args.install_model:
if not settings_app.binding:
settings_app.binding = BindingBuilder().build_binding(settings_app.config, settings_app.lollms_paths,InstallOption.FORCE_INSTALL, lollmsCom=settings_app)
def progress_callback(blocks, block_size, total_size):
tqdm_bar.total=total_size
tqdm_bar.update(block_size)
# Usage example
with tqdm(total=100, unit="%", desc="Download Progress", ncols=80) as tqdm_bar:
settings_app.config.download_model(args.install_model,settings_app.binding, progress_callback)
settings_app.config.model_name = args.install_model.split("/")[-1]
settings_app.model = settings_app.binding.build_model()
settings_app.config.save_config()
if args.select_model:
settings_app.config.model_name = args.select_model
if not settings_app.binding:
settings_app.binding = BindingBuilder().build_binding(settings_app.config, settings_app.lollms_paths,InstallOption.FORCE_INSTALL, lollmsCom=settings_app)
settings_app.model = settings_app.binding.build_model()
settings_app.config.save_config()
if args.mount_personalities:
for entry in args.mount_personalities:
try:
settings_app.config.personalities.append(entry)
settings_app.mount_personality(-1)
ASCIIColors.green(f"Personality : {entry} mounted")
except:
settings_app.config.personalities.pop()
ASCIIColors.red(f"Personality : {entry} couldn't be mounted")
if not args.silent:
settings_app.start()
if __name__ == "__main__":
main()

68
lollms/apps/train/__init__.py
View File

@ -1,68 +0,0 @@
from lollms.config import InstallOption
from lollms.binding import BindingBuilder, ModelBuilder
from lollms.personality import MSG_TYPE, PersonalityBuilder
from lollms.main_config import LOLLMSConfig
from lollms.paths import LollmsPaths
from lollms.app import LollmsApplication
from lollms.terminal import MainMenu
from ascii_colors import ASCIIColors
from typing import Callable
from pathlib import Path
import argparse
import yaml
import time
import sys
class Trainer(LollmsApplication):
def __init__(
self,
configuration_path:str|Path=None,
show_logo:bool=True,
show_time_elapsed:bool = False
):
# Fore it to be a path
self.configuration_path = configuration_path
self.is_logging = False
self.log_file_path = ""
self.show_time_elapsed = show_time_elapsed
self.bot_says = ""
# get paths
lollms_paths = LollmsPaths.find_paths(force_local=False, tool_prefix="lollms_server_")
# Configuration loading part
config = LOLLMSConfig.autoload(lollms_paths, configuration_path)
super().__init__("lollms-console",config, lollms_paths)
if show_logo:
self.menu.show_logo()
def start_training(self):
print(self.model)
def main():
# Create the argument parser
parser = argparse.ArgumentParser(description='App Description')
# Add the configuration path argument
parser.add_argument('--configuration_path', default=None,
help='Path to the configuration file')
# Parse the command-line arguments
args = parser.parse_args()
# Parse the command-line arguments
args = parser.parse_args()
configuration_path = args.configuration_path
lollms_app = Trainer(configuration_path=configuration_path)
lollms_app.start_training()
if __name__ == "__main__":
main()

50
lollms/generation.py Normal file
View File

@ -0,0 +1,50 @@
from enum import Enum
from ascii_colors import ASCIIColors
class ROLE_CHANGE_DECISION(Enum):
"""Roles change detection."""
MOVE_ON = 0
"""The received chunk is a normal chunk so move on."""
PROGRESSING = 1
"""Started receiving Role change."""
ROLE_CHANGED = 2
"""Role change detected."""
FALSE_ALERT = 3
"""False alert (didn't detect the full role change)."""
class ROLE_CHANGE_OURTPUT:
status:ROLE_CHANGE_DECISION
value:str=""
def __init__(self, status, value='') -> None:
self.status = status
self.value = value
class RECPTION_MANAGER:
done:bool=False
chunk:str=""
new_role:str=""
reception_buffer:str=""
def new_chunk(self, chunk):
self.chunk = chunk
if chunk=="!" and self.new_role == "":
self.new_role+=chunk
return ROLE_CHANGE_OURTPUT(ROLE_CHANGE_DECISION.PROGRESSING)
elif self.new_role != "":
if self.new_role=="!" and chunk=="@":
self.new_role+=chunk
return ROLE_CHANGE_OURTPUT(ROLE_CHANGE_DECISION.PROGRESSING)
elif self.new_role=="!@" and chunk==">":
self.new_role=""
self.done=True
ASCIIColors.yellow("Delected end of sentence")
return ROLE_CHANGE_OURTPUT(ROLE_CHANGE_DECISION.ROLE_CHANGED, self.reception_buffer)
else:
rc = ROLE_CHANGE_OURTPUT(ROLE_CHANGE_DECISION.FALSE_ALERT, self.reception_buffer)
self.reception_buffer += self.new_role
return rc
self.reception_buffer += chunk
return ROLE_CHANGE_OURTPUT(ROLE_CHANGE_DECISION.MOVE_ON)

159
lollms/server/endpoints/lollms_generator.py
View File

@ -8,12 +8,13 @@ description:
""" """
from fastapi import APIRouter, Request, Body from fastapi import APIRouter, Request, Body, Response
from lollms.server.elf_server import LOLLMSElfServer from lollms.server.elf_server import LOLLMSElfServer
from pydantic import BaseModel from pydantic import BaseModel
from starlette.responses import StreamingResponse from starlette.responses import StreamingResponse
from lollms.types import MSG_TYPE from lollms.types import MSG_TYPE
from lollms.utilities import detect_antiprompt, remove_text_from_string, trace_exception from lollms.utilities import detect_antiprompt, remove_text_from_string, trace_exception
from lollms.generation import RECPTION_MANAGER, ROLE_CHANGE_DECISION, ROLE_CHANGE_OURTPUT
from ascii_colors import ASCIIColors from ascii_colors import ASCIIColors
import time import time
import threading import threading
@ -21,60 +22,15 @@ from typing import List, Optional, Union
import random import random
import string import string
import json import json
from enum import Enum
import asyncio
def _generate_id(length=10): def _generate_id(length=10):
letters_and_digits = string.ascii_letters + string.digits letters_and_digits = string.ascii_letters + string.digits
random_id = ''.join(random.choice(letters_and_digits) for _ in range(length)) random_id = ''.join(random.choice(letters_and_digits) for _ in range(length))
return random_id return random_id
class GenerateRequest(BaseModel):
text: str
n_predict: int = 1024
stream: bool = False
temperature: float = 0.4
top_k: int = 50
top_p: float = 0.6
repeat_penalty: float = 1.3
repeat_last_n: int = 40
seed: int = -1
n_threads: int = 1
class V1ChatGenerateRequest(BaseModel):
"""
Data model for the V1 Chat Generate Request.
Attributes:
- model: str representing the model to be used for text generation.
- messages: list of messages to be used as prompts for text generation.
- stream: bool indicating whether to stream the generated text or not.
- temperature: float representing the temperature parameter for text generation.
- max_tokens: float representing the maximum number of tokens to generate.
"""
model: str
messages: list
stream: bool
temperature: float
max_tokens: float
class V1InstructGenerateRequest(BaseModel):
"""
Data model for the V1 Chat Generate Request.
Attributes:
- model: str representing the model to be used for text generation.
- messages: list of messages to be used as prompts for text generation.
- stream: bool indicating whether to stream the generated text or not.
- temperature: float representing the temperature parameter for text generation.
- max_tokens: float representing the maximum number of tokens to generate.
"""
model: str
prompt: str
stream: bool
temperature: float
max_tokens: float
# ----------------------- Defining router and main class ------------------------------ # ----------------------- Defining router and main class ------------------------------
router = APIRouter() router = APIRouter()
@ -240,7 +196,7 @@ class Delta(BaseModel):
class Choices(BaseModel): class Choices(BaseModel):
finish_reason: Optional[str] = None, finish_reason: Optional[str] = None,
index: Optional[int] = 0, index: Optional[int] = 0,
message: Optional[str] = "", message: Optional[Message] = None,
logprobs: Optional[float] = None logprobs: Optional[float] = None
@ -283,11 +239,6 @@ class StreamingModelResponse(BaseModel):
usage: Optional[Usage] = None usage: Optional[Usage] = None
"""Usage statistics for the completion request.""" """Usage statistics for the completion request."""
_hidden_params: dict = {}
def encode(self, charset):
encoded = json.dumps(self.dict()).encode(charset)
return encoded
class ModelResponse(BaseModel): class ModelResponse(BaseModel):
id: str id: str
"""A unique identifier for the completion.""" """A unique identifier for the completion."""
@ -314,105 +265,121 @@ class ModelResponse(BaseModel):
usage: Optional[Usage] = None usage: Optional[Usage] = None
"""Usage statistics for the completion request.""" """Usage statistics for the completion request."""
_hidden_params: dict = {}
class GenerationRequest(BaseModel): class GenerationRequest(BaseModel):
model: str = ""
messages: List[Message] messages: List[Message]
max_tokens: Optional[int] = 1024 max_tokens: Optional[int] = 1024
stream: Optional[bool] = False stream: Optional[bool] = False
temperature: Optional[float] = 0.1 temperature: Optional[float] = 0.1
@router.post("/v1/chat/completions") @router.post("/v1/chat/completions", response_class=StreamingModelResponse|ModelResponse)
async def v1_chat_completions(request: GenerationRequest): async def v1_chat_completions(request: GenerationRequest):
try: try:
reception_manager=RECPTION_MANAGER()
messages = request.messages messages = request.messages
text = "" prompt = ""
roles= False
for message in messages: for message in messages:
text += f"{message.role}: {message.content}\n" if message.role!="":
prompt += f"!@>{message.role}: {message.content}\n"
roles = True
else:
prompt += f"{message.content}\n"
if roles:
prompt += "!@>assistant:"
n_predict = request.max_tokens if request.max_tokens>0 else 1024 n_predict = request.max_tokens if request.max_tokens>0 else 1024
stream = request.stream stream = request.stream
prompt_tokens = len(elf_server.binding.tokenize(prompt))
if elf_server.binding is not None: if elf_server.binding is not None:
if stream: if stream:
output = {"text":"","waiting":True,"new":[]} new_output={"new_values":[]}
def generate_chunks(): async def generate_chunks():
lk = threading.Lock() lk = threading.Lock()
def callback(chunk, chunk_type:MSG_TYPE=MSG_TYPE.MSG_TYPE_CHUNK): def callback(chunk, chunk_type:MSG_TYPE=MSG_TYPE.MSG_TYPE_CHUNK):
if elf_server.cancel_gen: if elf_server.cancel_gen:
return False return False
if chunk is None: if chunk is None:
return return
output["text"] += chunk
rx = reception_manager.new_chunk(chunk)
if rx.status!=ROLE_CHANGE_DECISION.MOVE_ON:
if rx.status==ROLE_CHANGE_DECISION.PROGRESSING:
return True
elif rx.status==ROLE_CHANGE_DECISION.ROLE_CHANGED:
return False
else:
chunk = chunk + rx.value
# Yield each chunk of data # Yield each chunk of data
lk.acquire() lk.acquire()
try: try:
antiprompt = detect_antiprompt(output["text"]) new_output["new_values"].append(reception_manager.chunk)
if antiprompt: lk.release()
ASCIIColors.warning(f"\nDetected hallucination with antiprompt: {antiprompt}") return True
output["text"] = remove_text_from_string(output["text"],antiprompt)
lk.release()
return False
else:
output["new"].append(chunk)
lk.release()
return True
except Exception as ex: except Exception as ex:
trace_exception(ex) trace_exception(ex)
lk.release() lk.release()
return True return False
def chunks_builder(): def chunks_builder():
elf_server.binding.generate( elf_server.binding.generate(
text, prompt,
n_predict, n_predict,
callback=callback, callback=callback,
temperature=request.temperature or elf_server.config.temperature temperature=request.temperature or elf_server.config.temperature
) )
output["waiting"] = False reception_manager.done = True
thread = threading.Thread(target=chunks_builder) thread = threading.Thread(target=chunks_builder)
thread.start() thread.start()
current_index = 0 current_index = 0
while (output["waiting"] and elf_server.cancel_gen == False): while (not reception_manager.done and elf_server.cancel_gen == False):
while (output["waiting"] and len(output["new"])==0): while (not reception_manager.done and len(new_output["new_values"])==0):
time.sleep(0.001) time.sleep(0.001)
lk.acquire() lk.acquire()
for i in range(len(output["new"])): for i in range(len(new_output["new_values"])):
output_val = StreamingModelResponse( output_val = StreamingModelResponse(
id = _generate_id(), id = _generate_id(),
choices = [StreamingChoices(index= current_index, delta=Delta(content=output["new"][i]))], choices = [StreamingChoices(index= current_index, delta=Delta(content=new_output["new_values"][i]))],
created=int(time.time()), created=int(time.time()),
model=elf_server.config.model_name, model=elf_server.config.model_name,
usage=Usage(prompt_tokens= 0, completion_tokens= 10) object="chat.completion.chunk",
usage=Usage(prompt_tokens= prompt_tokens, completion_tokens= 1)
) )
current_index += 1 current_index += 1
yield output_val yield (output_val.json() + '\n')
output["new"]=[] new_output["new_values"]=[]
lk.release() lk.release()
elf_server.cancel_gen = False elf_server.cancel_gen = False
return StreamingResponse(generate_chunks(), media_type="application/json")
return StreamingResponse(iter(generate_chunks()))
else: else:
output = {"text":""}
def callback(chunk, chunk_type:MSG_TYPE=MSG_TYPE.MSG_TYPE_CHUNK): def callback(chunk, chunk_type:MSG_TYPE=MSG_TYPE.MSG_TYPE_CHUNK):
# Yield each chunk of data # Yield each chunk of data
if chunk is None: if chunk is None:
return
output["text"] += chunk
antiprompt = detect_antiprompt(output["text"])
if antiprompt:
ASCIIColors.warning(f"\nDetected hallucination with antiprompt: {antiprompt}")
output["text"] = remove_text_from_string(output["text"],antiprompt)
return False
else:
return True return True
rx = reception_manager.new_chunk(chunk)
if rx.status!=ROLE_CHANGE_DECISION.MOVE_ON:
if rx.status==ROLE_CHANGE_DECISION.PROGRESSING:
return True
elif rx.status==ROLE_CHANGE_DECISION.ROLE_CHANGED:
return False
else:
chunk = chunk + rx.value
return True
elf_server.binding.generate( elf_server.binding.generate(
text, prompt,
n_predict, n_predict,
callback=callback, callback=callback,
temperature=request.temperature or elf_server.config.temperature temperature=request.temperature or elf_server.config.temperature
) )
return ModelResponse(id = _generate_id(), choices = [Choices(message=output["text"])], created=int(time.time())) completion_tokens = len(elf_server.binding.tokenize(reception_manager.reception_buffer))
return ModelResponse(id = _generate_id(), choices = [Choices(message=Message(role="assistant", content=reception_manager.reception_buffer), finish_reason="stop", index=0)], created=int(time.time()), model=request.model,usage=Usage(prompt_tokens=prompt_tokens, completion_tokens=completion_tokens))
else: else:
return None return None
except Exception as ex: except Exception as ex:

7
requirements.txt
View File

@ -14,3 +14,10 @@ safe_store
ascii_colors>=0.1.3 ascii_colors>=0.1.3
autopep8 autopep8
fastapi
uvicorn
python-multipart
python-socketio
python-socketio[client]
python-socketio[asyncio_client]

7
requirements_dev.txt
View File

@ -11,3 +11,10 @@ requests
safe_store safe_store
autopep8 autopep8
fastapi
uvicorn
python-multipart
python-socketio
python-socketio[client]
python-socketio[asyncio_client]