skills library is now working

lollms/app.py

@@ -2,6 +2,7 @@ from lollms.main_config import LOLLMSConfig
 from lollms.paths import LollmsPaths
 from lollms.personality import PersonalityBuilder, AIPersonality
 from lollms.binding import LLMBinding, BindingBuilder, ModelBuilder
+from lollms.databases.discussions_database import Message
 from lollms.extension import LOLLMSExtension, ExtensionBuilder
 from lollms.config import InstallOption
 from lollms.helpers import ASCIIColors, trace_exception
@@ -128,8 +129,72 @@ class LollmsApplication(LoLLMsCom):
             self.mount_personalities()
             self.mount_extensions()
 
-    def add_discussion_tto_skills_library(self, client:Client):
+    def add_discussion_to_skills_library(self, client: Client):
-        pass
+        messages = client.discussion.get_messages()
+        db = client.discussion.skills_db
+
+        # Extract relevant information from messages
+        content = self._extract_content(messages)
+
+        # Generate title
+        title_prompt =  f"Generate a concise title for the following content without any additional explanations or context:\n\n{content}\n\nTitle:"
+        title = self._generate_text(title_prompt)
+
+        # Determine category
+        category_prompt = f"Determine the most appropriate category for the following title and content, and provide only the category name:\n\nTitle: {title}\nContent: {content}\n\nCategory:"
+        category = self._generate_text(category_prompt)
+
+        # Add entry to skills library
+        db.add_entry(1, category, title, content)
+        return category, title, content
+
+    def _extract_content(self, messages:List[Message]):
+        ranked_messages = sorted(messages, key=lambda m: m.rank, reverse=True)
+        
+        max_chunk_size = int(self.config.ctx_size * 0.75)
+        
+        chunks = []
+        current_chunk = ""
+        current_chunk_tokens = 0
+        
+        for message in ranked_messages:
+            rank = message.rank
+            sender = message.sender
+            text = message.content
+            message_content = f"Rank {rank} - {sender}: {text}\n"
+            
+            message_tokens = self.model.get_nb_tokens(message_content)
+            
+            if current_chunk_tokens + message_tokens <= max_chunk_size:
+                current_chunk += message_content
+                current_chunk_tokens += message_tokens
+            else:
+                chunks.append(current_chunk)
+                current_chunk = message_content
+                current_chunk_tokens = message_tokens
+        
+        if current_chunk:
+            chunks.append(current_chunk)
+        
+        summarized_chunks = []
+        for chunk in chunks:
+            prompt = f"Summarize the following discussion chunk, keeping only important information that can be used as skills. Use bullet points:\n\n{chunk}"
+            max_tokens = self.config.ctx_size - self.model.get_nb_tokens(prompt)
+            
+            summarized_chunk = self.model.generate(prompt, max_tokens)
+            summarized_chunks.append(summarized_chunk.strip())
+        
+        summarized_content = "\n".join(summarized_chunks)
+        return summarized_content
+
+
+    def _generate_text(self, prompt):
+        max_tokens = self.config.ctx_size - self.model.get_nb_tokens(prompt)
+        generated_text = self.model.generate(prompt, max_tokens)
+        return generated_text.strip()
+
+
+
 
     def get_uploads_path(self, client_id):
         return self.lollms_paths.personal_uploads_path

lollms/binding.py

@@ -111,6 +111,11 @@ class LLMBinding:
             models_folder.mkdir(parents=True, exist_ok=True)
 
 
+    def get_nb_tokens(self, prompt):
+        """
+        Counts the number of tokens in a prtompt
+        """
+        return len(self.tokenize(prompt))
 
     def searchModelFolder(self, model_name:str):
         for mn in self.models_folders:

lollms/databases/discussions_database.py

@@ -4,6 +4,7 @@ from pathlib import Path
 from datetime import datetime
 from lollms.helpers import ASCIIColors
 from lollms.paths import LollmsPaths
+from lollms.databases.skills_database import SkillsLibrary
 import json
 
 __author__ = "parisneo"
@@ -598,8 +599,8 @@ class Discussion:
         self.discussion_skills_folder.mkdir(exist_ok=True)
         self.discussion_rag_folder.mkdir(exist_ok=True)
         self.messages = self.get_messages()
-        self.skills_db = self.discussion_skills_folder/"skills_db.db"
+        self.skills_db_path = self.discussion_skills_folder/"skills_db.db"
-
+        self.skills_db = SkillsLibrary(self.skills_db_path)
         if len(self.messages)>0:
             self.current_message = self.messages[-1]
 

lollms/databases/skills_database.py

@@ -1,19 +1,15 @@
 import sqlite3
 
 class SkillsLibrary:
-    def __init__(self, db_path):
-        self.conn = sqlite3.connect(db_path)
-        self.cursor = self.conn.cursor()
-        self._create_table()
-
         
     def __init__(self, db_path):
-        self.conn = sqlite3.connect(db_path)
+        self.db_path =db_path
-        self.cursor = self.conn.cursor()
         self._initialize_db()
 
     def _initialize_db(self):
-        self.cursor.execute("""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()        
+        cursor.execute("""
             CREATE TABLE IF NOT EXISTS skills_library (
                 id INTEGER PRIMARY KEY,
                 version INTEGER,
@@ -22,29 +18,39 @@ class SkillsLibrary:
                 content TEXT
             )
         """)
-        self.cursor.execute("""
+        cursor.execute("""
             CREATE TABLE IF NOT EXISTS db_info (
                 version INTEGER
             )
         """)
-        self.cursor.execute("SELECT version FROM db_info")
+        cursor.execute("SELECT version FROM db_info")
-        version = self.cursor.fetchone()
+        version = cursor.fetchone()
         if version is None:
-            self.cursor.execute("INSERT INTO db_info (version) VALUES (1)")
+            cursor.execute("INSERT INTO db_info (version) VALUES (1)")
-            self.conn.commit()
+            conn.commit()
+            cursor.close()
+            conn.close()
         else:
+            cursor.close()
+            conn.close()
             self._migrate_db(version[0])
 
     def _migrate_db(self, version):
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()        
         # Perform migrations based on the current version
         # For example, if the current version is 1 and the latest version is 2:
         if version < 2:
-            self.cursor.execute("ALTER TABLE skills_library ADD COLUMN new_column TEXT")
+            cursor.execute("ALTER TABLE skills_library ADD COLUMN new_column TEXT")
-            self.cursor.execute("UPDATE db_info SET version = 2")
+            cursor.execute("UPDATE db_info SET version = 2")
-            self.conn.commit()
+            conn.commit()
+        cursor.close()
+        conn.close()
 
     def _create_table(self):
-        self.cursor.execute("""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()        
+        cursor.execute("""
             CREATE TABLE IF NOT EXISTS skills_library (
                 id INTEGER PRIMARY KEY,
                 version INTEGER,
@@ -53,29 +59,49 @@ class SkillsLibrary:
                 content TEXT
             )
         """)
-        self.conn.commit()
+        conn.commit()
+        cursor.close()
+        conn.close()
 
     def add_entry(self, version, category, title, content):
-        self.cursor.execute("""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()        
+        cursor.execute("""
             INSERT INTO skills_library (version, category, title, content) 
             VALUES (?, ?, ?, ?)
         """, (version, category, title, content))
-        self.conn.commit()
+        conn.commit()
+        cursor.close()
+        conn.close()
 
     def list_entries(self):
-        self.cursor.execute("SELECT * FROM skills_library")
+        conn = sqlite3.connect(self.db_path)
-        return self.cursor.fetchall()
+        cursor = conn.cursor()        
-
+        cursor.execute("SELECT * FROM skills_library")
+        res = cursor.fetchall()
+        cursor.close()
+        conn.close()
+        return res
+    
     def query_entry(self, text):
-        self.cursor.execute("""
+        conn = sqlite3.connect(self.db_path)
+        cursor = conn.cursor()        
+        cursor.execute("""
             SELECT * FROM skills_library 
             WHERE category LIKE ? OR title LIKE ? OR content LIKE ?
         """, (f'%{text}%', f'%{text}%', f'%{text}%'))
-        return self.cursor.fetchall()
+        res= cursor.fetchall()
-
+        cursor.close()
+        conn.close()
+        return res
+    
     def remove_entry(self, id):
-        self.cursor.execute("DELETE FROM skills_library WHERE id = ?", (id,))
+        conn = sqlite3.connect(self.db_path)
-        self.conn.commit()
+        cursor = conn.cursor()        
+        cursor.execute("DELETE FROM skills_library WHERE id = ?", (id,))
+        conn.commit()
+        cursor.close()
+        conn.close()
 
     def export_entries(self, file_path):
         with open(file_path, 'w') as f:

lollms/server/endpoints/lollms_skills_library.py

@@ -30,5 +30,14 @@ class DiscussionInfos(BaseModel):
 
 @router.post("/add_discussion_to_skills_library")
 def add_discussion_to_skills_library(discussionInfos:DiscussionInfos):
-    client = lollmsElfServer.session.get_client(discussionInfos.client_id)
+    lollmsElfServer.ShowBlockingMessage("Learning...")
-    lollmsElfServer.add_discussion_tto_skills_library(client)    
+    try:
+        client = lollmsElfServer.session.get_client(discussionInfos.client_id)
+        category, title, content = lollmsElfServer.add_discussion_to_skills_library(client)    
+        lollmsElfServer.InfoMessage(f"Discussion skill added to skills library:\ntitle:{title}\ncategory:{category}")
+    except Exception as ex:
+        trace_exception(ex)
+        ASCIIColors.error(ex)
+        lollmsElfServer.InfoMessage(f"Failed to learn from this discussion because of the follwoing error:\n{ex}")
+        return {"status":False,"error":f"{ex}"}
+    return {"status":True}