From c6fcfa36868416472c859476634a010d8f83bdcb Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Mon, 11 Mar 2024 00:56:55 +0100 Subject: [PATCH] skills library is now working --- lollms/app.py | 69 +++++++++++++++- lollms/binding.py | 5 ++ lollms/databases/discussions_database.py | 5 +- lollms/databases/skills_database.py | 82 ++++++++++++------- .../server/endpoints/lollms_skills_library.py | 13 ++- 5 files changed, 140 insertions(+), 34 deletions(-) diff --git a/lollms/app.py b/lollms/app.py index cca47cc..2613e15 100644 --- a/lollms/app.py +++ b/lollms/app.py @@ -2,6 +2,7 @@ from lollms.main_config import LOLLMSConfig from lollms.paths import LollmsPaths from lollms.personality import PersonalityBuilder, AIPersonality from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder +from lollms.databases.discussions_database import Message from lollms.extension import LOLLMSExtension, ExtensionBuilder from lollms.config import InstallOption from lollms.helpers import ASCIIColors, trace_exception @@ -128,8 +129,72 @@ class LollmsApplication(LoLLMsCom): self.mount_personalities() self.mount_extensions() - def add_discussion_tto_skills_library(self, client:Client): - pass + def add_discussion_to_skills_library(self, client: Client): + messages = client.discussion.get_messages() + db = client.discussion.skills_db + + # Extract relevant information from messages + content = self._extract_content(messages) + + # Generate title + title_prompt = f"Generate a concise title for the following content without any additional explanations or context:\n\n{content}\n\nTitle:" + title = self._generate_text(title_prompt) + + # Determine category + category_prompt = f"Determine the most appropriate category for the following title and content, and provide only the category name:\n\nTitle: {title}\nContent: {content}\n\nCategory:" + category = self._generate_text(category_prompt) + + # Add entry to skills library + db.add_entry(1, category, title, content) + return category, title, content + + def _extract_content(self, messages:List[Message]): + ranked_messages = sorted(messages, key=lambda m: m.rank, reverse=True) + + max_chunk_size = int(self.config.ctx_size * 0.75) + + chunks = [] + current_chunk = "" + current_chunk_tokens = 0 + + for message in ranked_messages: + rank = message.rank + sender = message.sender + text = message.content + message_content = f"Rank {rank} - {sender}: {text}\n" + + message_tokens = self.model.get_nb_tokens(message_content) + + if current_chunk_tokens + message_tokens <= max_chunk_size: + current_chunk += message_content + current_chunk_tokens += message_tokens + else: + chunks.append(current_chunk) + current_chunk = message_content + current_chunk_tokens = message_tokens + + if current_chunk: + chunks.append(current_chunk) + + summarized_chunks = [] + for chunk in chunks: + prompt = f"Summarize the following discussion chunk, keeping only important information that can be used as skills. Use bullet points:\n\n{chunk}" + max_tokens = self.config.ctx_size - self.model.get_nb_tokens(prompt) + + summarized_chunk = self.model.generate(prompt, max_tokens) + summarized_chunks.append(summarized_chunk.strip()) + + summarized_content = "\n".join(summarized_chunks) + return summarized_content + + + def _generate_text(self, prompt): + max_tokens = self.config.ctx_size - self.model.get_nb_tokens(prompt) + generated_text = self.model.generate(prompt, max_tokens) + return generated_text.strip() + + + def get_uploads_path(self, client_id): return self.lollms_paths.personal_uploads_path diff --git a/lollms/binding.py b/lollms/binding.py index 5cc2d99..a5c5e44 100644 --- a/lollms/binding.py +++ b/lollms/binding.py @@ -111,6 +111,11 @@ class LLMBinding: models_folder.mkdir(parents=True, exist_ok=True) + def get_nb_tokens(self, prompt): + """ + Counts the number of tokens in a prtompt + """ + return len(self.tokenize(prompt)) def searchModelFolder(self, model_name:str): for mn in self.models_folders: diff --git a/lollms/databases/discussions_database.py b/lollms/databases/discussions_database.py index d59423f..c8f4009 100644 --- a/lollms/databases/discussions_database.py +++ b/lollms/databases/discussions_database.py @@ -4,6 +4,7 @@ from pathlib import Path from datetime import datetime from lollms.helpers import ASCIIColors from lollms.paths import LollmsPaths +from lollms.databases.skills_database import SkillsLibrary import json __author__ = "parisneo" @@ -598,8 +599,8 @@ class Discussion: self.discussion_skills_folder.mkdir(exist_ok=True) self.discussion_rag_folder.mkdir(exist_ok=True) self.messages = self.get_messages() - self.skills_db = self.discussion_skills_folder/"skills_db.db" - + self.skills_db_path = self.discussion_skills_folder/"skills_db.db" + self.skills_db = SkillsLibrary(self.skills_db_path) if len(self.messages)>0: self.current_message = self.messages[-1] diff --git a/lollms/databases/skills_database.py b/lollms/databases/skills_database.py index b39ffe4..9546edf 100644 --- a/lollms/databases/skills_database.py +++ b/lollms/databases/skills_database.py @@ -1,19 +1,15 @@ import sqlite3 class SkillsLibrary: - def __init__(self, db_path): - self.conn = sqlite3.connect(db_path) - self.cursor = self.conn.cursor() - self._create_table() - def __init__(self, db_path): - self.conn = sqlite3.connect(db_path) - self.cursor = self.conn.cursor() + self.db_path =db_path self._initialize_db() def _initialize_db(self): - self.cursor.execute(""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute(""" CREATE TABLE IF NOT EXISTS skills_library ( id INTEGER PRIMARY KEY, version INTEGER, @@ -22,29 +18,39 @@ class SkillsLibrary: content TEXT ) """) - self.cursor.execute(""" + cursor.execute(""" CREATE TABLE IF NOT EXISTS db_info ( version INTEGER ) """) - self.cursor.execute("SELECT version FROM db_info") - version = self.cursor.fetchone() + cursor.execute("SELECT version FROM db_info") + version = cursor.fetchone() if version is None: - self.cursor.execute("INSERT INTO db_info (version) VALUES (1)") - self.conn.commit() + cursor.execute("INSERT INTO db_info (version) VALUES (1)") + conn.commit() + cursor.close() + conn.close() else: + cursor.close() + conn.close() self._migrate_db(version[0]) def _migrate_db(self, version): + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() # Perform migrations based on the current version # For example, if the current version is 1 and the latest version is 2: if version < 2: - self.cursor.execute("ALTER TABLE skills_library ADD COLUMN new_column TEXT") - self.cursor.execute("UPDATE db_info SET version = 2") - self.conn.commit() + cursor.execute("ALTER TABLE skills_library ADD COLUMN new_column TEXT") + cursor.execute("UPDATE db_info SET version = 2") + conn.commit() + cursor.close() + conn.close() def _create_table(self): - self.cursor.execute(""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute(""" CREATE TABLE IF NOT EXISTS skills_library ( id INTEGER PRIMARY KEY, version INTEGER, @@ -53,29 +59,49 @@ class SkillsLibrary: content TEXT ) """) - self.conn.commit() + conn.commit() + cursor.close() + conn.close() def add_entry(self, version, category, title, content): - self.cursor.execute(""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute(""" INSERT INTO skills_library (version, category, title, content) VALUES (?, ?, ?, ?) """, (version, category, title, content)) - self.conn.commit() + conn.commit() + cursor.close() + conn.close() def list_entries(self): - self.cursor.execute("SELECT * FROM skills_library") - return self.cursor.fetchall() - + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute("SELECT * FROM skills_library") + res = cursor.fetchall() + cursor.close() + conn.close() + return res + def query_entry(self, text): - self.cursor.execute(""" + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute(""" SELECT * FROM skills_library WHERE category LIKE ? OR title LIKE ? OR content LIKE ? """, (f'%{text}%', f'%{text}%', f'%{text}%')) - return self.cursor.fetchall() - + res= cursor.fetchall() + cursor.close() + conn.close() + return res + def remove_entry(self, id): - self.cursor.execute("DELETE FROM skills_library WHERE id = ?", (id,)) - self.conn.commit() + conn = sqlite3.connect(self.db_path) + cursor = conn.cursor() + cursor.execute("DELETE FROM skills_library WHERE id = ?", (id,)) + conn.commit() + cursor.close() + conn.close() def export_entries(self, file_path): with open(file_path, 'w') as f: diff --git a/lollms/server/endpoints/lollms_skills_library.py b/lollms/server/endpoints/lollms_skills_library.py index 619d4a2..68c7fac 100644 --- a/lollms/server/endpoints/lollms_skills_library.py +++ b/lollms/server/endpoints/lollms_skills_library.py @@ -30,5 +30,14 @@ class DiscussionInfos(BaseModel): @router.post("/add_discussion_to_skills_library") def add_discussion_to_skills_library(discussionInfos:DiscussionInfos): - client = lollmsElfServer.session.get_client(discussionInfos.client_id) - lollmsElfServer.add_discussion_tto_skills_library(client) + lollmsElfServer.ShowBlockingMessage("Learning...") + try: + client = lollmsElfServer.session.get_client(discussionInfos.client_id) + category, title, content = lollmsElfServer.add_discussion_to_skills_library(client) + lollmsElfServer.InfoMessage(f"Discussion skill added to skills library:\ntitle:{title}\ncategory:{category}") + except Exception as ex: + trace_exception(ex) + ASCIIColors.error(ex) + lollmsElfServer.InfoMessage(f"Failed to learn from this discussion because of the follwoing error:\n{ex}") + return {"status":False,"error":f"{ex}"} + return {"status":True}