upgraded

added web page selection for discussion
2025-02-20 17:22:47 +00:00 · 2023-12-16 01:24:25 +01:00 · 2023-12-16 01:24:25 +01:00 · 8865a3318b
commit 8865a3318b
parent 282a873c7a
14 changed files with 398 additions and 258 deletions
--- a/api/init.py
+++ b/api/init.py
@ -38,6 +38,15 @@ from datetime import datetime
 from typing import List, Tuple
 import time

+from lollms.utilities import find_first_available_file_index
+
+if not PackageManager.check_package_installed("requests"):
+    PackageManager.install_package("requests")
+if not PackageManager.check_package_installed("bs4"):
+    PackageManager.install_package("beautifulsoup4")
+import requests
+from bs4 import BeautifulSoup
+
 def terminate_thread(thread):
    if thread:
        if not thread.is_alive():
@ -223,6 +232,26 @@ class LoLLMsAPI(LollmsApplication):
            
            ASCIIColors.error(f'Client {request.sid} disconnected')

+        @socketio.on('add_webpage')
+        def add_webpage(data):
+            ASCIIColors.yellow("Scaping web page")
+            url = data['url']
+            index =  find_first_available_file_index(self.lollms_paths.personal_uploads_path,"web_",".txt")
+            file_path=self.lollms_paths.personal_uploads_path/f"web_{index}.txt"
+            self.scrape_and_save(url=url, file_path=file_path)
+            try:
+                if not self.personality.processor is None:
+                    self.personality.processor.add_file(file_path, partial(self.process_chunk, client_id = request.sid))
+                    # File saved successfully
+                    socketio.emit('web_page_added', {'status':True,})
+                else:
+                    self.personality.add_file(file_path, partial(self.process_chunk, client_id = request.sid))
+                    # File saved successfully
+                    socketio.emit('web_page_added', {'status':True})
+            except Exception as e:
+                # Error occurred while saving the file
+                socketio.emit('web_page_added', {'status':False})
+
        @socketio.on('take_picture')
        def take_picture():
            try:
@ -1097,6 +1126,25 @@ class LoLLMsAPI(LollmsApplication):
        else:
            if output["text"].lower()=="lollms":
                self.summoned = True
+    def scrape_and_save(self, url, file_path):
+        # Send a GET request to the URL
+        response = requests.get(url)
+        
+        # Parse the HTML content using BeautifulSoup
+        soup = BeautifulSoup(response.content, 'html.parser')
+        
+        # Find all the text content in the webpage
+        text_content = soup.get_text()
+        
+        # Remove extra returns and spaces
+        text_content = ' '.join(text_content.split())
+        
+        # Save the text content as a text file
+        with open(file_path, 'w', encoding="utf-8") as file:
+            file.write(text_content)
+        
+        self.info(f"Webpage content saved to {file_path}")
+
    def rebuild_personalities(self, reload_all=False):
        if reload_all:
            self.mounted_personalities=[]
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 50637a80367fd5b26b1d76cab258c370cdd4b883
+Subproject commit dd713c24cd723bb77277f3b86f2c4cf447b4c62e
--- a/requirements.txt
+++ b/requirements.txt
@ -16,3 +16,4 @@ setuptools
 numpy
 flask_compress
 ascii_colors>=0.1.4
+beautifulsoup4
--- a/web/dist/assets/index-2895a271.js
+++ b/web/dist/assets/index-2895a271.js
--- a/web/dist/assets/index-44523b05.css
+++ b/web/dist/assets/index-44523b05.css
--- a/web/dist/index.html
+++ b/web/dist/index.html
@ -6,8 +6,8 @@
    
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>LoLLMS WebUI - Welcome</title>
-    <script type="module" crossorigin src="/assets/index-715d9baa.js"></script>
-    <link rel="stylesheet" href="/assets/index-f3c97c3f.css">
+    <script type="module" crossorigin src="/assets/index-2895a271.js"></script>
+    <link rel="stylesheet" href="/assets/index-44523b05.css">
  </head>
  <body>
    <div id="app"></div>
--- a/web/src/components/ChatBox.vue
+++ b/web/src/components/ChatBox.vue
@ -212,30 +212,38 @@


                            </textarea>
-                            <input type="file" ref="fileDialog" style="display: none" @change="addFiles" multiple />
-                            <button type="button" @click.stop="$refs.fileDialog.click()" title="Add files"
-                                class="absolute inset-y-0 right-0 flex items-center mr-2 w-6 hover:text-secondary duration-75 active:scale-90">
-
-                                <i data-feather="file-plus"></i>
-
-                            </button>
                        </div>
                        <!-- BUTTONS -->
                        <div class="inline-flex justify-center  rounded-full ">
                            <button v-if="!loading" 
                                type="button"
                                @click="startSpeechRecognition"
+                                title="Press and talk"
                                :class="{ 'text-red-500': isLesteningToVoice }"
                                class="w-6 hover:text-secondary duration-75 active:scale-90 cursor-pointer"
                            >
                            <i data-feather="mic"></i>
                            </button>
+
+                            <input type="file" ref="fileDialog" style="display: none" @change="addFiles" multiple />
+                            <button type="button" @click.stop="$refs.fileDialog.click()" title="Add files"
+                                class="m-1 w-6 hover:text-secondary duration-75 active:scale-90">
+
+                                <i data-feather="file-plus"></i>
+
+                            </button>
                            <button type="button" @click.stop="takePicture" title="take a shot from camera"
-                                class=" w-6 hover:text-secondary duration-75 active:scale-90">
+                                class="m-1 w-6 hover:text-secondary duration-75 active:scale-90">

                                <i data-feather="camera"></i>

                            </button>
+                            <button type="button" @click.stop="addWebLink" title="add web link"
+                                class="m-1 w-6 hover:text-secondary duration-75 active:scale-90">
+
+                                <i data-feather="globe"></i>
+
+                            </button>                            
                            <button v-if="!loading" type="button" @click="makeAnEmptyUserMessage" title="New empty user message"
                                class=" w-6 text-blue-400 hover:text-secondary duration-75 active:scale-90">

@ -251,7 +259,7 @@

                                <span class="sr-only">New empty message</span>
                            </button>
-                            <button v-if="!loading" type="button" @click="submit"
+                            <button v-if="!loading" type="button" @click="submit" title="Send"
                                class=" w-6 hover:text-secondary duration-75 active:scale-90">

                                <i data-feather="send"></i>
@ -330,7 +338,7 @@ console.log("modelImgPlaceholder:",modelImgPlaceholder)
 const bUrl = import.meta.env.VITE_LOLLMS_API_BASEURL
 export default {
    name: 'ChatBox',
-    emits: ["messageSentEvent", "sendCMDEvent", "stopGenerating", "loaded", "createEmptyUserMessage", "createEmptyAIMessage", "personalitySelected"],
+    emits: ["messageSentEvent", "sendCMDEvent", "stopGenerating", "loaded", "createEmptyUserMessage", "createEmptyAIMessage", "personalitySelected","addWebLink"],
    props: {
        onTalk: Function,
        discussionList: Array,
@ -769,6 +777,10 @@ export default {
        sendCMDEvent(cmd){
            this.$emit('sendCMDEvent', cmd)
        },
+        addWebLink(){
+            console.log("Emitting addWebLink")
+            this.$emit('addWebLink')
+        },
        takePicture(){
            socket.emit('take_picture')
            socket.on('picture_taken',()=>{
--- a/web/src/components/TopBar.vue
+++ b/web/src/components/TopBar.vue
@ -87,7 +87,7 @@
        <div v-show="progress_visibility" role="status" class="fixed m-0 p-2 left-2 bottom-2  min-w-[24rem] max-w-[24rem] h-20 flex flex-col justify-center items-center pb-4 bg-blue-500 rounded-lg shadow-lg z-50 background-a">
            <ProgressBar ref="progress" :progress="progress_value" class="w-full h-4"></ProgressBar>
            <p class="text-2xl animate-pulse mt-2 text-white">{{ loading_infos }} ...</p>
-        </div>        
+        </div>     
    </header>

    <body>
@ -101,6 +101,7 @@ import Toast from '../components/Toast.vue'
 import MessageBox from "@/components/MessageBox.vue";
 import ProgressBar from "@/components/ProgressBar.vue";

+
 import { RouterLink } from 'vue-router'
 import Navigation from './Navigation.vue'
 import { nextTick } from 'vue'
@ -129,7 +130,7 @@ export default {
        Discussion,
        Toast,
        MessageBox,
-        ProgressBar       
+        ProgressBar,
    },
    watch:{
        isConnected(){
@ -176,6 +177,9 @@ export default {
            event.preventDefault();
            window.location.reload();
        },
+        handleOk(inputText) {
+            console.log("Input text:", inputText);
+        },
        // codeBlockTheme(theme) {
        //     const styleDark = document.createElement('link');
        //     styleDark.type = "text/css";
--- a/web/src/components/input_box.vue
+++ b/web/src/components/input_box.vue
@ -0,0 +1,48 @@
+<template>
+    <div>
+      <div v-if="show" class="fixed top-0 left-0 w-full h-full flex justify-center items-center bg-black bg-opacity-50">
+        <div class="bg-white p-8 rounded">
+          <h2 class="text-xl font-bold mb-4">{{ promptText }}</h2>
+          <input type="text" v-model="inputText" class="border border-gray-300 px-4 py-2 rounded mb-4" />
+          <button @click="ok" class="bg-blue-500 text-white px-4 py-2 rounded mr-2">OK</button>
+          <button @click="cancel" class="bg-gray-500 text-white px-4 py-2 rounded">Cancel</button>
+        </div>
+      </div>
+    </div>
+</template>
+
+<script>
+    export default {
+        data() {
+            return {
+            show: false,
+            prompt: "",
+            inputText: ""
+            };
+        },
+        methods: {
+            showPanel() {
+            this.show = true;
+            },
+            ok() {
+            this.show = false;
+            this.$emit("ok", this.inputText);
+            },
+            cancel() {
+            this.show = false;
+            this.inputText = "";
+            }
+        },
+        props: {
+            promptText: {
+            type: String,
+            required: true
+            }
+        },
+        watch: {
+            promptText(newPrompt) {
+            this.prompt = newPrompt;
+            }
+        }
+    };
+</script>
--- a/web/src/views/DiscussionsView.vue
+++ b/web/src/views/DiscussionsView.vue
@ -261,9 +261,11 @@
                    :discussionList="discussionArr" 
                    :on-show-toast-message="showToastMessage"
                    :on-talk="talk"
+
                    @personalitySelected="recoverFiles"
                    @messageSentEvent="sendMsg" 
                    @sendCMDEvent="sendCmd"
+                    @addWebLink="add_webpage"
                    @createEmptyUserMessage="createEmptyUserMessage"
                    @createEmptyAIMessage="createEmptyAIMessage"
                    @stopGenerating="stopGenerating" 
@ -287,6 +289,8 @@
        <ProgressBar ref="progress" :progress="progress_value" class="w-full h-4"></ProgressBar>
        <p class="text-2xl animate-pulse mt-2 text-white">{{ loading_infos }} ...</p>
    </div>
+    <InputBox prompt-text="Enter the url to the page to use as discussion support" @ok="handleOk" ref="web_url_input_box"></InputBox>   
+
 </template>


@ -440,6 +444,24 @@ export default {
        }
    },
    methods: { 
+        add_webpage(){
+            console.log("addWebLink received")
+            this.$refs.web_url_input_box.showPanel();
+        },
+        handleOk(){
+            console.log("OK")
+            socket.on('web_page_added',()=>{
+                axios.get('/get_current_personality_files_list').then(res=>{
+                    this.filesList = res.data.files;
+                    console.log("this.filesList",this.filesList)
+                    this.isFileSentList= res.data.files.map(file => {
+                        return true;
+                    });
+                    console.log(`Files recovered: ${this.filesList}`)
+                })
+            });
+            socket.emit('add_webpage',{'url':this.$refs.web_url_input_box.inputText})
+        },
        show_progress(data){
            this.progress_visibility_val = true;
        },
@ -1930,7 +1952,8 @@ export default {
        WelcomeComponent,
        ChoiceDialog,
        MessageBox,
-        ProgressBar       
+        ProgressBar,
+        InputBox    
    },
    watch: {  
        progress_visibility_val(newVal) {
@ -2064,6 +2087,8 @@ import Discussion from '../components/Discussion.vue'
 import ChoiceDialog from '@/components/ChoiceDialog.vue'
 import MessageBox from "@/components/MessageBox.vue";
 import ProgressBar from "@/components/ProgressBar.vue";
+import InputBox from "@/components/input_box.vue";
+

 import Message from '../components/Message.vue'
 import ChatBox from '../components/ChatBox.vue'
--- a/web/src/views/InteractiveView.vue
+++ b/web/src/views/InteractiveView.vue
@ -1,20 +1,22 @@
 <template>
-      <div class="flex-col w-[800]px y-overflow scrollbar-thin scrollbar-track-bg-light-tone scrollbar-thumb-bg-light-tone-panel hover:scrollbar-thumb-primary dark:scrollbar-track-bg-dark-tone dark:scrollbar-thumb-bg-dark-tone-panel dark:hover:scrollbar-thumb-primary active:scrollbar-thumb-secondary">
-        <div v-if="!activePersonality || !activePersonality.scene_path" class="text-center">
-          <!-- Display text when there's no scene_path or empty avatar -->
-          Personality does not have a 3d avatar.
+    <div class="flex-row w-[800]px y-overflow scrollbar-thin scrollbar-track-bg-light-tone scrollbar-thumb-bg-light-tone-panel hover:scrollbar-thumb-primary dark:scrollbar-track-bg-dark-tone dark:scrollbar-thumb-bg-dark-tone-panel dark:hover:scrollbar-thumb-primary active:scrollbar-thumb-secondary">
+        <div ref="webglContainer">
+        <div class="flex-col y-overflow scrollbar-thin scrollbar-track-bg-light-tone scrollbar-thumb-bg-light-tone-panel hover:scrollbar-thumb-primary dark:scrollbar-track-bg-dark-tone dark:scrollbar-thumb-bg-dark-tone-panel dark:hover:scrollbar-thumb-primary active:scrollbar-thumb-secondary">
+          <div v-if="!activePersonality || !activePersonality.scene_path" class="text-center">
+            <!-- Display text when there's no scene_path or empty avatar -->
+            Personality does not have a 3d avatar.
+          </div>
+          <div v-if="!activePersonality || (!activePersonality.avatar || activePersonality.avatar === '')" class="text-center">
+            Personality does not have an avatar.
+          </div>
+          <FloatingFrame />
+          <AudioFrame />
+          <div class="floating-frame2">
+            <div v-html="htmlContent"></div>
+          </div>
        </div>
-        <div v-if="!activePersonality || (!activePersonality.avatar || activePersonality.avatar === '')" class="text-center">
-          Personality does not have an avatar.
        </div>
-        <FloatingFrame />
-        <AudioFrame />
-        <div class="floating-frame2">
-          <div v-html="htmlContent"></div>
-        </div>
-      </div>
-      <div ref="webglContainer">
-      </div>
+    </div>
 </template>
  
  <script>
--- a/zoos/bindings_zoo
+++ b/zoos/bindings_zoo
@ -1 +1 @@
-Subproject commit 71b59745cdd3153aca273cac10edbb3fa28b9c6d
+Subproject commit dcb50644f0aa399490fe2d1fe9c04192ae719957
--- a/zoos/models_zoo
+++ b/zoos/models_zoo
@ -1 +1 @@
-Subproject commit 2796e693443c735850206ee5089915a1143cddb0
+Subproject commit 6a1e86d28ac82d6732424948a3707821cfaa5e5e
--- a/zoos/personalities_zoo
+++ b/zoos/personalities_zoo
@ -1 +1 @@
-Subproject commit f2fab08ff81e80d9133c23924a6cff70f5d4bcec
+Subproject commit 0d8341c54937d57513dfae062fd9731812e96f5e