new search

endpoints/chat_bar.py

@@ -8,6 +8,7 @@ description:
 
 """
 from fastapi import APIRouter, Request
+from fastapi import HTTPException
 from pydantic import BaseModel, Field
 from lollms_webui import LOLLMSWebUI
 from pydantic import BaseModel
@@ -32,6 +33,8 @@ from utilities.execution_engines.python_execution_engine import execute_python
 from utilities.execution_engines.latex_execution_engine import execute_latex
 from utilities.execution_engines.shell_execution_engine import execute_bash
 
+from lollms.internet import scrape_and_save
+import threading
 # ----------------------- Defining router and main class ------------------------------
 
 router = APIRouter()
@@ -46,6 +49,9 @@ class CmdExecutionRequest(BaseModel):
     command: str = Field(..., description="Url to be used")
     parameters: List
 
+
+
+"""
 @router.post("/execute_personality_command")
 async def execute_personality_command(request: CmdExecutionRequest):
     client_id = request.client_id
@@ -100,26 +106,38 @@ async def execute_personality_command(request: CmdExecutionRequest):
 
     lollmsElfServer.busy=False
     return {'status':True,}
+"""
+
 
 @router.post("/add_webpage")
 async def add_webpage(request: AddWebPageRequest):
-    lollmsElfServer.ShowBlockingMessage("Scraping web page\nPlease wait...")
-    ASCIIColors.yellow("Scaping web page")
     client = lollmsElfServer.session.get_client(request.client_id)
-    url = request.url
-    index =  find_first_available_file_index(lollmsElfServer.lollms_paths.personal_uploads_path,"web_",".txt")
-    file_path=lollmsElfServer.lollms_paths.personal_uploads_path/f"web_{index}.txt"
-    lollmsElfServer.scrape_and_save(url=url, file_path=file_path)
-    try:
-        if not lollmsElfServer.personality.processor is None:
-            lollmsElfServer.personality.processor.add_file(file_path, client, partial(lollmsElfServer.process_chunk, client_id = request.client_id))
-            # File saved successfully
-        else:
-            lollmsElfServer.personality.add_file(file_path, client, partial(lollmsElfServer.process_chunk, client_id = request.client_id))
-            # File saved successfully
-        lollmsElfServer.HideBlockingMessage()
-        return {'status':True,}
-    except Exception as e:
-        # Error occurred while saving the file
-        lollmsElfServer.HideBlockingMessage()
-        return {'status':False,"error":str(e)}
+    if client is None:
+        raise HTTPException(status_code=400, detail="Unknown client. This service only accepts lollms webui requests")
+        
+    def do_scraping():
+        lollmsElfServer.ShowBlockingMessage("Scraping web page\nPlease wait...")
+        ASCIIColors.yellow("Scaping web page")
+        client = lollmsElfServer.session.get_client(request.client_id)
+        url = request.url
+        index =  find_first_available_file_index(lollmsElfServer.lollms_paths.personal_uploads_path,"web_",".txt")
+        file_path=lollmsElfServer.lollms_paths.personal_uploads_path/f"web_{index}.txt"
+        scrape_and_save(url=url, file_path=file_path)
+        try:
+            if not lollmsElfServer.personality.processor is None:
+                lollmsElfServer.personality.processor.add_file(file_path, client, partial(lollmsElfServer.process_chunk, client_id = request.client_id))
+                # File saved successfully
+            else:
+                lollmsElfServer.personality.add_file(file_path, client, partial(lollmsElfServer.process_chunk, client_id = request.client_id))
+                # File saved successfully
+            lollmsElfServer.HideBlockingMessage()
+            lollmsElfServer.refresh_files()
+        except Exception as e:
+            # Error occurred while saving the file
+            lollmsElfServer.HideBlockingMessage()
+            lollmsElfServer.refresh_files()
+            return {'status':False,"error":str(e)}
+    client.generation_thread = threading.Thread(target=do_scraping)
+    client.generation_thread.start()
+        
+    return {'status':True}

events/lollms_chatbox_events.py

@@ -28,6 +28,7 @@ import threading
 import os
 import time
 
+from lollms.internet import scrape_and_save
 from lollms.databases.discussions_database import Discussion
 from datetime import datetime
 
@@ -73,7 +74,7 @@ def add_events(sio:socketio):
         url = data['url']
         index =  find_first_available_file_index(lollmsElfServer.lollms_paths.personal_uploads_path,"web_",".txt")
         file_path=lollmsElfServer.lollms_paths.personal_uploads_path/f"web_{index}.txt"
-        lollmsElfServer.scrape_and_save(url=url, file_path=file_path)
+        scrape_and_save(url=url, file_path=file_path)
         try:
             if not lollmsElfServer.personality.processor is None:
                 lollmsElfServer.personality.processor.add_file(file_path, client, partial(lollmsElfServer.process_chunk, client_id = sid))

lollms_core

@@ -1 +1 @@
-Subproject commit ddba6ed36413048fed0c5363926aa1520987c064
+Subproject commit 46a694fa0b0423b0b36a434a00c4c50cb01a976a

lollms_webui.py

@@ -51,8 +51,8 @@ if not PackageManager.check_package_installed("requests"):
 if not PackageManager.check_package_installed("bs4"):
     PackageManager.install_package("beautifulsoup4")
 import requests
-from bs4 import BeautifulSoup
 
+from lollms.internet import scrape_and_save
 
 
 def terminate_thread(thread):
@@ -374,24 +374,28 @@ class LOLLMSWebUI(LOLLMSElfServer):
             if "lollms" in text.lower():
                 self.summoned = True
 
-    def scrape_and_save(self, url, file_path):
-        # Send a GET request to the URL
-        response = requests.get(url)
+    # def scrape_and_save(self, url, file_path):
+    #     # Send a GET request to the URL
+    #     response = requests.get(url)
         
-        # Parse the HTML content using BeautifulSoup
-        soup = BeautifulSoup(response.content, 'html.parser')
+    #     # Parse the HTML content using BeautifulSoup
+    #     soup = BeautifulSoup(response.content, 'html.parser')
         
-        # Find all the text content in the webpage
-        text_content = soup.get_text()
+    #     # Find all the text content in the webpage
+    #     text_content = soup.get_text()
         
-        # Remove extra returns and spaces
-        text_content = ' '.join(text_content.split())
+    #     # Remove extra returns and spaces
+    #     text_content = ' '.join(text_content.split())
         
-        # Save the text content as a text file
-        with open(file_path, 'w', encoding="utf-8") as file:
-            file.write(text_content)
+    #     # Save the text content as a text file
+    #     with open(file_path, 'w', encoding="utf-8") as file:
+    #         file.write(text_content)
         
-        self.info(f"Webpage content saved to {file_path}")
+    #     self.info(f"Webpage content saved to {file_path}")
+
+
+
+
 
     def rebuild_personalities(self, reload_all=False):
         if reload_all:
@@ -726,6 +730,11 @@ class LOLLMSWebUI(LOLLMSElfServer):
                 ASCIIColors.warning(content)
             else:
                 ASCIIColors.red(content)
+                
+    def refresh_files(self, client_id=None):
+        run_async(partial(self.sio.emit,'refresh_files', to=client_id
+                            )
+        )
 
 
     def new_message(self, 

web/dist/index.html

@@ -6,8 +6,8 @@
     
     <meta name="viewport" content="width=device-width, initial-scale=1.0">
     <title>LoLLMS WebUI - Welcome</title>
-    <script type="module" crossorigin src="/assets/index-f3eff979.js"></script>
-    <link rel="stylesheet" href="/assets/index-2184e400.css">
+    <script type="module" crossorigin src="/assets/index-4b4c8da2.js"></script>
+    <link rel="stylesheet" href="/assets/index-45c73858.css">
   </head>
   <body>
     <div id="app"></div>

web/src/views/DiscussionsView.vue

@@ -2044,6 +2044,9 @@ export default {
         // this.host = `${serverAddress}`; // Construct the full server address dynamically
         // axios.defaults.baseURL = serverAddress
         //console.log('chatbox mnt',this.$refs)
+        socket.on('refresh_files',()=>{
+            this.recoverFiles()
+        })
         this.$nextTick(() => {
             feather.replace();
         });