upgraded stt ui

This commit is contained in:
Saifeddine ALOUI 2024-05-27 00:36:45 +02:00
parent 2907446e34
commit 4cbcae569f
15 changed files with 311 additions and 589 deletions

View File

@ -1,5 +1,5 @@
# =================== Lord Of Large Language Multimodal Systems Configuration file ===========================
version: 101
version: 102
binding_name: null
model_name: null
model_variant: null
@ -92,7 +92,7 @@ active_ttm_service: "None" # musicgen (offline)
stt_input_device: 0
# TTS service
# STT service
stt_listening_threshold: 1000
stt_silence_duration: 2
stt_sound_threshold_percentage: 10
@ -101,6 +101,9 @@ stt_rate: 44100
stt_channels: 1
stt_buffer_size: 10
stt_activate_word_detection: false
stt_word_detection_file: null
# ASR STT service

View File

@ -479,31 +479,15 @@ def start_recording(data:Identification):
lollmsElfServer.info("Starting audio capture")
try:
from lollms.media import RTCom
from lollms.media import AudioNinja
lollmsElfServer.rec_output_folder = lollmsElfServer.lollms_paths.personal_outputs_path/"audio_rec"
lollmsElfServer.rec_output_folder.mkdir(exist_ok=True, parents=True)
lollmsElfServer.summoned = False
lollmsElfServer.rt_com = RTCom(
lollmsElfServer,
lollmsElfServer.sio,
lollmsElfServer.personality,
client=client,
threshold=lollmsElfServer.config.stt_listening_threshold,
silence_duration=lollmsElfServer.config.stt_silence_duration,
sound_threshold_percentage=lollmsElfServer.config.stt_sound_threshold_percentage,
gain=lollmsElfServer.config.stt_gain,
rate=lollmsElfServer.config.stt_rate,
channels=lollmsElfServer.config.stt_channels,
buffer_size=lollmsElfServer.config.stt_buffer_size,
model=lollmsElfServer.config.whisper_model,
snd_input_device=lollmsElfServer.config.stt_input_device,
snd_output_device=lollmsElfServer.config.tts_output_device,
logs_folder=lollmsElfServer.rec_output_folder,
voice=None,
block_while_talking=True,
context_size=4096
lollmsElfServer.audioNinja = AudioNinja(
lollmsElfServer,
logs_folder=lollmsElfServer.rec_output_folder
)
lollmsElfServer.rt_com.start_recording()
lollmsElfServer.audioNinja.start_recording()
except:
lollmsElfServer.InfoMessage("Couldn't load media library.\nYou will not be able to perform any of the media linked operations. please verify the logs and install any required installations")
@ -519,23 +503,9 @@ def stop_recording(data:Identification):
return {"status":False,"error":"Stop recording is blocked when the server is exposed outside for very obvious reasons!"}
lollmsElfServer.info("Stopping audio capture")
text = lollmsElfServer.rt_com.stop_recording()
# ai_text = lollmsElfServer.receive_and_generate(text, client, n_predict=lollmsElfServer.config, callback= lollmsElfServer.tasks_library.sink)
# if lollmsElfServer.tts and lollmsElfServer.tts.ready:
# personality_audio:Path = lollmsElfServer.personality.personality_package_path/"audio"
# voice=lollmsElfServer.config.xtts_current_voice
# if personality_audio.exists() and len([v for v in personality_audio.iterdir()])>0:
# voices_folder = personality_audio
# elif voice!="main_voice":
# voices_folder = lollmsElfServer.lollms_paths.custom_voices_path
# else:
# voices_folder = Path(__file__).parent.parent.parent/"services/xtts/voices"
# language = lollmsElfServer.config.xtts_current_language# convert_language_name()
# lollmsElfServer.tts.set_speaker_folder(voices_folder)
# preprocessed_text= add_period(ai_text)
# voice_file = [v for v in voices_folder.iterdir() if v.stem==voice and v.suffix==".wav"]
# lollmsElfServer.tts.tts_audio(preprocessed_text, voice_file[0].name, language=language)
return text
fn = lollmsElfServer.audioNinja.stop_recording()
lollmsElfServer.audioNinja = None
if lollmsElfServer.stt:
text = lollmsElfServer.stt.transcribe(fn)
return text

View File

@ -54,8 +54,8 @@ def add_events(sio:socketio):
lollmsElfServer.info("Stopping video capture")
lollmsElfServer.webcam.stop_capture()
@sio.on('start_audio_stream')
def start_audio_stream(sid):
@sio.on('start_bidirectional_audio_stream')
def start_bidirectional_audio_stream(sid):
client = check_access(lollmsElfServer, sid)
if lollmsElfServer.config.headless_server_mode:
return {"status":False,"error":"Start recording is blocked when in headless mode for obvious security reasons!"}
@ -73,6 +73,10 @@ def add_events(sio:socketio):
return {"status":False,"error":"TTS not ready"}
if lollmsElfServer.rt_com:
lollmsElfServer.info("audio_mode is already on\nTurning it off")
lollmsElfServer.info("Stopping audio capture")
lollmsElfServer.rt_com.stop_recording()
lollmsElfServer.rt_com = None
return {"status":False,"error":"Already running"}
try:
@ -92,23 +96,22 @@ def add_events(sio:socketio):
rate=lollmsElfServer.config.stt_rate,
channels=lollmsElfServer.config.stt_channels,
buffer_size=lollmsElfServer.config.stt_buffer_size,
model=lollmsElfServer.config.whisper_model,
snd_input_device=lollmsElfServer.config.stt_input_device,
snd_output_device=lollmsElfServer.config.tts_output_device,
logs_folder=lollmsElfServer.rec_output_folder,
voice=None,
block_while_talking=True,
context_size=4096
)
)
lollmsElfServer.rt_com.start_recording()
lollmsElfServer.emit_socket_io_info("rtcom_status_changed",{"status":True}, client.client_id)
except Exception as ex:
trace_exception(ex)
lollmsElfServer.InfoMessage("Couldn't load media library.\nYou will not be able to perform any of the media linked operations. please verify the logs and install any required installations")
lollmsElfServer.emit_socket_io_info("rtcom_status_changed",{"status":False}, client.client_id)
@sio.on('stop_audio_stream')
def stop_audio_stream(sid):
@sio.on('stop_bidirectional_audio_stream')
def stop_bidirectional_audio_stream(sid):
client = check_access(lollmsElfServer, sid)
lollmsElfServer.info("Stopping audio capture")
lollmsElfServer.rt_com.stop_recording()

@ -1 +1 @@
Subproject commit 2d0c4e76be93195836ecd0948027e791b8a2626f
Subproject commit 9d458ac979f0afbd427f188ee75747b13d2fd4e8

View File

@ -72,7 +72,7 @@ def terminate_thread(thread):
else:
ASCIIColors.yellow("Canceled successfully")# The current version of the webui
lollms_webui_version="9.8 (β)"
lollms_webui_version="9.8 (γ)"
@ -734,6 +734,9 @@ class LOLLMSWebUI(LOLLMSElfServer):
if callback:
callback(full_text, MSG_TYPE.MSG_TYPE_FULL)
def emit_socket_io_info(self, name, data, client_id):
run_async(partial(self.sio.emit,name, data, to=client_id))
def notify(
self,
content,

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

4
web/dist/index.html vendored
View File

@ -6,8 +6,8 @@
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>LoLLMS WebUI - Welcome</title>
<script type="module" crossorigin src="/assets/index-f88bca23.js"></script>
<link rel="stylesheet" href="/assets/index-1ba2d604.css">
<script type="module" crossorigin src="/assets/index-ae79f596.js"></script>
<link rel="stylesheet" href="/assets/index-0469c5c8.css">
</head>
<body>
<div id="app"></div>

View File

@ -927,19 +927,19 @@ export default {
this.$emit('createEmptyAIMessage')
},
startRTCom(){
socket.emit('start_audio_stream', ()=>{this.isAudioActive = true;});
socket.emit('start_bidirectional_audio_stream');
nextTick(() => {
feather.replace()
}
)
},
stopRTCom(){
socket.emit('stop_audio_stream', ()=>{this.isAudioActive = true;});
socket.emit('stop_bidirectional_audio_stream');
nextTick(() => {
feather.replace()
}
)
this.$store.state.is_rt_on = false;
},
startSpeechRecognition() {
if ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window) {
@ -1182,6 +1182,15 @@ export default {
nextTick(() => {
feather.replace()
})
console.log("Chatbar mounted")
socket.on('rtcom_status_changed', (data)=>{
console.log("rtcom_status_changed")
console.log("rtcom_status_changed: ",data.status)
this.$store.dispatch('fetchisRTOn');
console.log("active_tts_service: ",this.$store.state.config.active_tts_service)
console.log("is_rt_on: ",this.$store.state.is_rt_on)
this.isAudioActive = data.status;
});
},
activated() {
nextTick(() => {

View File

@ -1,195 +0,0 @@
<template>
<div
class=" min-w-96 items-start p-4 hover:bg-primary-light rounded-lg mb-2 shadow-lg border-2 cursor-pointer select-none"
tabindex="-1"
:class="selected_computed ? 'border-2 border-primary-light' : 'border-transparent', isMounted ? 'bg-blue-200 dark:bg-blue-700':''"
:title="!extension.installed ? 'Not installed' : ''">
<div :class="!extension.installed ? 'border-red-500' : ''">
<div class="flex flex-row items-center flex-shrink-0 gap-3">
<img @click="toggleSelected" ref="imgElement" :src="getImgUrl()" @error="defaultImg($event)"
class="w-10 h-10 rounded-full object-fill text-red-700 cursor-pointer">
<!-- :class="extension.installed ? 'grayscale-0':'grayscale'" -->
<h3 @click="toggleSelected" class="font-bold font-large text-lg line-clamp-3 cursor-pointer">
{{ extension.name }}
</h3>
</div>
<div class="">
<div class="">
<div class="flex items-center">
<i data-feather="user" class="w-5 m-1"></i>
<b>Author:&nbsp;</b>
{{ extension.author }}
</div>
<div class="flex items-center">
<i data-feather="bookmark" class="w-5 m-1"></i>
<b>Based on:&nbsp;</b>
{{ extension.based_on }}
</div>
<div v-if="extension.languages && select_language" class="flex items-center">
<i data-feather="globe" class="w-5 m-1"></i>
<b>Languages:&nbsp;</b>
<select id="languages" v-model ="extension.language"
class="bg-gray-50 border border-gray-300 text-gray-900 text-sm rounded-lg focus:ring-blue-500 focus:border-blue-500 block w-full p-2.5 dark:bg-gray-700 dark:border-gray-600 dark:placeholder-gray-400 dark:text-white dark:focus:ring-blue-500 dark:focus:border-blue-500">
<option v-for="(item, index) in extension.languages" :key="index"
:selected="item == extension.languages[0]">{{
item
}}
</option>
</select>
</div>
<div v-if="extension.language" class="flex items-center">
<i data-feather="globe" class="w-5 m-1"></i>
<b>Language:&nbsp;</b>
{{ extension.language }}
</div>
<div class="flex items-center">
<i data-feather="bookmark" class="w-5 m-1"></i>
<b>Category:&nbsp;</b>
{{ extension.category }}
</div>
</div>
<div class="flex items-center">
<i data-feather="info" class="w-5 m-1"></i>
<b>Description:&nbsp;</b><br>
</div>
<p class="mx-1 opacity-80 h-20 overflow-y-auto scrollbar-thin scrollbar-track-bg-light-tone scrollbar-thumb-bg-light-tone-panel hover:scrollbar-thumb-primary dark:scrollbar-track-bg-dark-tone dark:scrollbar-thumb-bg-dark-tone-panel dark:hover:scrollbar-thumb-primary active:scrollbar-thumb-secondary" :title="extension.description" v-html="extension.description"></p>
</div>
<div class="rounded bg-blue-300">
<button v-if="isMounted" type="button" title="Select"
@click="toggleSelected"
class="hover:text-secondary duration-75 active:scale-90 font-medium rounded-lg text-sm p-2 text-center inline-flex items-center " @click.stop="">
<i
data-feather="check"
></i>
<span class="sr-only">Select</span>
</button>
<button v-if="isMounted" type="button" title="Talk"
@click="toggleTalk"
class="hover:text-secondary duration-75 active:scale-90 font-medium rounded-lg text-sm p-2 text-center inline-flex items-center " @click.stop="">
<i data-feather="send" class="w-5"></i>
<span class="sr-only">Talk</span>
</button>
<InteractiveMenu :commands="commandsList" :force_position=2 title="Menu">
</InteractiveMenu>
</div>
</div>
</div>
</template>
<script>
import { nextTick } from 'vue'
import feather from 'feather-icons'
import botImgPlaceholder from "../assets/logo.svg"
import userImgPlaceholder from "../assets/default_user.svg"
import InteractiveMenu from "@/components/InteractiveMenu.vue"
const bUrl = import.meta.env.VITE_LOLLMS_API_BASEURL
export default {
props: {
extension: {},
select_language: Boolean,
selected: Boolean,
full_path: String,
onTalk:Function,
onSelected: Function,
onMount: Function,
onUnMount: Function,
onRemount: Function,
onReinstall: Function,
onSettings: Function
},
components:{
InteractiveMenu
},
data() {
return {
isMounted: false,
name: this.extension.name,
};
},
computed:{
commandsList(){
let main_menu = [
{name:this.isMounted?"unmount":"mount", icon: "feather:settings", is_file:false, value:this.isMounted?this.unmount:this.mount},
{name:"reinstall", icon: "feather:terminal", is_file:false, value:this.toggleReinstall},
];
if(this.isMounted){
main_menu.push({name:"remount", icon: "feather:refresh-ccw", is_file:false, value:this.reMount})
}
main_menu.push({name:"settings", icon: "feather:settings", is_file:false, value:this.toggleSettings})
return main_menu
},
selected_computed(){
return this.selected
}
},
mounted() {
this.isMounted = this.extension.isMounted
nextTick(() => {
feather.replace()
})
},
methods: {
getImgUrl() {
return bUrl + this.extension.avatar
},
defaultImg(event) {
event.target.src = botImgPlaceholder
},
toggleTalk() {
this.onTalk(this)
},
toggleSelected() {
if(this.isMounted){
this.onSelected(this)
}
},
reMount(){
this.onRemount(this)
},
mount() {
console.log("Mounting")
this.onMount(this)
},
unmount() {
console.log("Unmounting")
console.log(this.onUnMount)
this.onUnMount(this)
},
toggleSettings() {
this.onSettings(this)
},
toggleReinstall() {
this.onReinstall(this)
},
},
watch: {
selected() {
nextTick(() => {
feather.replace()
})
}
}
};
</script>

View File

@ -819,7 +819,6 @@ export default {
time_spent() {
const startTime = new Date(Date.parse(this.message.started_generating_at))
const endTime = new Date(Date.parse(this.message.finished_generating_at))
console.log("Computing the generation duration, ", startTime," -> ", endTime)
//const spentTime = new Date(endTime - startTime)
const same = endTime.getTime() === startTime.getTime();
@ -890,7 +889,6 @@ export default {
const startTime = new Date(Date.parse(this.message.started_generating_at))
const endTime = new Date(Date.parse(this.message.finished_generating_at))
const nb_tokens = this.message.nb_tokens
console.log("Computing the generation rate, ", nb_tokens, " in ", startTime," -> ", endTime)
//const spentTime = new Date(endTime - startTime)
const same = endTime.getTime() === startTime.getTime();
if (same) {

View File

@ -228,7 +228,12 @@ export default {
},
is_fun_mode(){
try{
return this.$store.state.config.fun_mode
if (this.$store.state.config){
return this.$store.state.config.fun_mode;
}
else{
return false;
}
}
catch(error){
console.error("Oopsie! Looks like we hit a snag: ", error);
@ -258,7 +263,6 @@ export default {
watch:{
'$store.state.config.fun_mode': function(newVal, oldVal) {
console.log(`Fun mode changed from ${oldVal} to ${newVal}! 🎉`);
this.updateIcon();
},
'$store.state.isConnected': function(newVal, oldVal) {
if (!this.isConnected){

View File

@ -66,7 +66,6 @@ export const store = createStore({
config:null,
mountedPers:null,
mountedPersArr:[],
mountedExtensions:[],
bindingsZoo:[],
modelsArr:[],
selectedModel:null,
@ -79,7 +78,6 @@ export const store = createStore({
installedBindings:[],
currentModel:null,
currentBinding:null,
extensionsZoo:[],
databases:[],
}
},
@ -122,9 +120,6 @@ export const store = createStore({
setMountedPersArr(state, mountedPersArr) {
state.mountedPersArr = mountedPersArr;
},
setMountedExtensions(state, mountedExtensions) {
state.mountedExtensions = mountedExtensions;
},
setbindingsZoo(state, bindingsZoo) {
state.bindingsZoo = bindingsZoo;
},
@ -153,9 +148,6 @@ export const store = createStore({
state.currentModel = currentModel;
},
setExtensionsZoo(state, extensionsZoo) {
state.extensionsZoo = extensionsZoo;
},
setDatabases(state, databases) {
state.databases = databases;
},
@ -198,9 +190,6 @@ export const store = createStore({
getMountedPersArr(state) {
return state.mountedPersArr;
},
getmmountedExtensions(state) {
return state.mountedExtensions;
},
getMountedPers(state) {
return state.mountedPers;
},
@ -232,9 +221,6 @@ export const store = createStore({
getCurrentModel(state) {
return state.currentModel;
},
getExtensionsZoo(state) {
return state.extensionsZoo;
},
},
actions: {
async getVersion(){
@ -263,12 +249,6 @@ export const store = createStore({
configFile.personality_category = personality_path_infos[0]
configFile.personality_folder = personality_path_infos[1]
if (configFile.extensions.length>0){
configFile.extension_category = configFile.extensions[-1]
}
else{
configFile.extension_category = "ai_sensors"
}
console.log("Recovered config")
console.log(configFile)
console.log("Committing config");
@ -292,7 +272,7 @@ export const store = createStore({
);
console.log("response", response)
const is_rt_on = response.data;
const is_rt_on = response.data.status;
console.log("languages", is_rt_on)
commit('setRTOn', is_rt_on);
},
@ -515,77 +495,6 @@ export const store = createStore({
commit('setCurrentModel',this.state.modelsZoo[index])
}
},
async refreshExtensionsZoo({ commit }) {
let extensions = []
let catdictionary = await api_get_req("list_extensions")
const catkeys = Object.keys(catdictionary); // returns categories
console.log("Extensions recovered:"+catdictionary)
for (let j = 0; j < catkeys.length; j++) {
const catkey = catkeys[j];
const extensionsArray = catdictionary[catkey];
const modExtArr = extensionsArray.map((item) => {
let isMounted = false;
for(const extension of this.state.config.extensions){
if(extension.includes(catkey + '/' + item.folder)){
isMounted = true;
}
}
// if (isMounted) {
// console.log(item)
// }
let newItem = {}
newItem = item
newItem.category = catkey // add new props to items
newItem.full_path = catkey + '/' + item.folder // add new props to items
newItem.isMounted = isMounted // add new props to items
return newItem
})
if (extensions.length == 0) {
extensions = modExtArr
} else {
extensions = extensions.concat(modExtArr)
}
}
extensions.sort((a, b) => a.name.localeCompare(b.name))
//commit('setActiveExtensions', this.state.config.extensions);
console.log("Done loading extensions")
commit('setExtensionsZoo',extensions)
},
refreshmountedExtensions({ commit }) {
console.log("Mounting extensions")
let mountedExtensions = []
// console.log('perrs listo',this.state.personalities)
const indicesToRemove = [];
for (let i = 0; i < this.state.config.extensions.length; i++) {
const full_path_item = this.state.config.extensions[i]
const index = this.state.extensionsZoo.findIndex(item => item.full_path == full_path_item)
if(index>=0){
let ext = copyObject(this.state.config.extensions[index])
if (ext) {
mountedExtensions.push(ext)
}
}
else{
indicesToRemove.push(i)
console.log("Couldn't load extension : ",full_path_item)
}
}
// Remove the broken extensions using the collected indices
for (let i = indicesToRemove.length - 1; i >= 0; i--) {
console.log("Removing extensions : ",this.state.config.extensions[indicesToRemove[i]])
this.state.config.extensions.splice(indicesToRemove[i], 1);
}
commit('setMountedExtensions', mountedExtensions);
},
async refreshDiskUsage({ commit }) {
this.state.diskUsage = await api_get_req("disk_usage")
},
@ -731,23 +640,7 @@ app.mixin({
catch (ex){
console.log("Error cought:", ex)
}
try{
this.$store.state.loading_infos = "Getting extensions zoo"
this.$store.state.loading_progress = 50
await this.$store.dispatch('refreshExtensionsZoo');
}
catch (ex){
console.log("Error cought:", ex)
}
try{
this.$store.state.loading_infos = "Getting mounted extensions"
this.$store.state.loading_progress = 60
await this.$store.dispatch('refreshmountedExtensions');
}
catch (ex){
console.log("Error cought:", ex)
}
try{
this.$store.state.loading_infos = "Getting personalities zoo"
this.$store.state.loading_progress = 70

View File

@ -1333,7 +1333,40 @@
>
</td>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="stt_activate_word_detection" class="text-sm font-bold" style="margin-right: 1rem;">Activate word detection:</label>
</td>
<td>
<div class="flex flex-row">
<input
type="checkbox"
id="stt_activate_word_detection"
required
v-model="configFile.stt_activate_word_detection"
@change="settingsChanged=true"
class="mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
</div>
</td>
</tr>
<tr>
<td style="min-width: 200px;">
<label for="stt_word_detection_file" class="text-sm font-bold" style="margin-right: 1rem;">Word detection wav file:</label>
</td>
<td>
<div class="flex flex-row">
<input
type="text"
id="stt_word_detection_file"
required
v-model="configFile.stt_word_detection_file"
@change="settingsChanged=true"
class="mt-1 px-2 py-1 border border-gray-300 rounded dark:bg-gray-600"
>
</div>
</td>
</tr>
</table>
</Card>
<Card title="Audio devices settings" :is_subcard="true" class="pb-2 m-2">

1
zoos/extensions_zoo Submodule

@ -0,0 +1 @@
Subproject commit 7f6b99c1e5fc01050b6c60b30706dd141a18d734