mirror of
https://github.com/ParisNeo/lollms-webui.git
synced 2024-12-18 20:17:50 +00:00
real time
This commit is contained in:
Saifeddine ALOUI
parent
b0be864770
commit
6e92575777
@ -454,6 +454,9 @@ async def open_personality_folder(request: PersonalityFolderRequest):
|
||||
lollmsElfServer.error(ex)
|
||||
return {"status": False, "error": "An error occurred while processing the request"}
|
||||
|
||||
@router.get("/is_rt_on")
|
||||
def is_rt_on():
|
||||
return {"status": lollmsElfServer.rt_com is not None}
|
||||
|
||||
@router.post("/start_recording")
|
||||
def start_recording(data:Identification):
|
||||
@ -467,11 +470,11 @@ def start_recording(data:Identification):
|
||||
|
||||
lollmsElfServer.info("Starting audio capture")
|
||||
try:
|
||||
from lollms.media import AudioRecorder
|
||||
from lollms.media import RTCom
|
||||
lollmsElfServer.rec_output_folder = lollmsElfServer.lollms_paths.personal_outputs_path/"audio_rec"
|
||||
lollmsElfServer.rec_output_folder.mkdir(exist_ok=True, parents=True)
|
||||
lollmsElfServer.summoned = False
|
||||
lollmsElfServer.audio_cap = AudioRecorder(
|
||||
lollmsElfServer.rt_com = RTCom(
|
||||
lollmsElfServer,
|
||||
lollmsElfServer.sio,
|
||||
lollmsElfServer.personality,
|
||||
@ -483,14 +486,14 @@ def start_recording(data:Identification):
|
||||
rate=44100,
|
||||
channels=1,
|
||||
buffer_size=10,
|
||||
model="small.en",
|
||||
model=lollmsElfServer.config.whisper_model,
|
||||
snd_device=None,
|
||||
logs_folder="logs",
|
||||
voice=None,
|
||||
block_while_talking=True,
|
||||
context_size=4096
|
||||
)
|
||||
lollmsElfServer.audio_cap.start_recording()
|
||||
lollmsElfServer.rt_com.start_recording()
|
||||
except:
|
||||
lollmsElfServer.InfoMessage("Couldn't load media library.\nYou will not be able to perform any of the media linked operations. please verify the logs and install any required installations")
|
||||
|
||||
@ -506,7 +509,7 @@ def stop_recording(data:Identification):
|
||||
return {"status":False,"error":"Stop recording is blocked when the server is exposed outside for very obvious reasons!"}
|
||||
|
||||
lollmsElfServer.info("Stopping audio capture")
|
||||
text = lollmsElfServer.audio_cap.stop_recording()
|
||||
text = lollmsElfServer.rt_com.stop_recording()
|
||||
|
||||
# ai_text = lollmsElfServer.receive_and_generate(text, client, n_predict=lollmsElfServer.config, callback= lollmsElfServer.tasks_library.sink)
|
||||
# if lollmsElfServer.tts and lollmsElfServer.tts.ready:
|
||||
|
||||
@ -65,11 +65,11 @@ def add_events(sio:socketio):
|
||||
|
||||
lollmsElfServer.info("Starting audio capture")
|
||||
try:
|
||||
from lollms.media import AudioRecorder
|
||||
from lollms.media import RTCom
|
||||
lollmsElfServer.rec_output_folder = lollmsElfServer.lollms_paths.personal_outputs_path/"audio_rec"
|
||||
lollmsElfServer.rec_output_folder.mkdir(exist_ok=True, parents=True)
|
||||
lollmsElfServer.summoned = False
|
||||
lollmsElfServer.audio_cap = AudioRecorder(
|
||||
lollmsElfServer.rt_com = RTCom(
|
||||
lollmsElfServer,
|
||||
lollmsElfServer.sio,
|
||||
lollmsElfServer.personality,
|
||||
@ -81,14 +81,14 @@ def add_events(sio:socketio):
|
||||
rate=44100,
|
||||
channels=1,
|
||||
buffer_size=10,
|
||||
model="small.en",
|
||||
model=lollmsElfServer.config.whisper_model,
|
||||
snd_device=None,
|
||||
logs_folder="logs",
|
||||
voice=None,
|
||||
block_while_talking=True,
|
||||
context_size=4096
|
||||
)
|
||||
lollmsElfServer.audio_cap.start_recording()
|
||||
lollmsElfServer.rt_com.start_recording()
|
||||
except Exception as ex:
|
||||
trace_exception(ex)
|
||||
lollmsElfServer.InfoMessage("Couldn't load media library.\nYou will not be able to perform any of the media linked operations. please verify the logs and install any required installations")
|
||||
@ -99,7 +99,8 @@ def add_events(sio:socketio):
|
||||
def stop_audio_stream(sid):
|
||||
client = check_access(lollmsElfServer, sid)
|
||||
lollmsElfServer.info("Stopping audio capture")
|
||||
lollmsElfServer.audio_cap.stop_recording()
|
||||
lollmsElfServer.rt_com.stop_recording()
|
||||
lollmsElfServer.rt_com = None
|
||||
|
||||
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
Subproject commit 5e0f38da1bf4620713f196a24ba1332499fcad72
|
||||
Subproject commit 124f614480049b8f67d55bbe0d10e13bf441c55d
|
||||
@ -826,7 +826,45 @@ class LOLLMSWebUI(LOLLMSElfServer):
|
||||
}, to=client_id
|
||||
)
|
||||
)
|
||||
|
||||
def new_block( self,
|
||||
client_id,
|
||||
sender=None,
|
||||
content="",
|
||||
parameters=None,
|
||||
metadata=None,
|
||||
ui=None,
|
||||
message_type:MSG_TYPE=MSG_TYPE.MSG_TYPE_FULL,
|
||||
sender_type:SENDER_TYPES=SENDER_TYPES.SENDER_TYPES_AI,
|
||||
open=False
|
||||
):
|
||||
# like new_message but without adding the information to the database
|
||||
client = self.session.get_client(client_id)
|
||||
run_async(partial(
|
||||
self.sio.emit,'new_message',
|
||||
{
|
||||
"sender": sender,
|
||||
"message_type": message_type.value,
|
||||
"sender_type": SENDER_TYPES.SENDER_TYPES_AI.value,
|
||||
"content": content,
|
||||
"parameters": parameters,
|
||||
"metadata": metadata,
|
||||
"ui": ui,
|
||||
"id": 0,
|
||||
"parent_message_id": 0,
|
||||
|
||||
'binding': self.config["binding_name"],
|
||||
'model' : self.config["model_name"],
|
||||
'personality': self.config["personalities"][self.config["active_personality_id"]],
|
||||
|
||||
'created_at': client.discussion.current_message.created_at,
|
||||
'started_generating_at': client.discussion.current_message.started_generating_at,
|
||||
'finished_generating_at': client.discussion.current_message.finished_generating_at,
|
||||
'nb_tokens': client.discussion.current_message.nb_tokens,
|
||||
|
||||
'open': open
|
||||
}, to=client_id
|
||||
)
|
||||
)
|
||||
def send_refresh(self, client_id):
|
||||
client = self.session.get_client(client_id)
|
||||
run_async(
|
||||
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
4
web/dist/index.html
vendored
4
web/dist/index.html
vendored
@ -6,8 +6,8 @@
|
||||
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>LoLLMS WebUI - Welcome</title>
|
||||
<script type="module" crossorigin src="/assets/index-d452ede2.js"></script>
|
||||
<link rel="stylesheet" href="/assets/index-1f5d7ed5.css">
|
||||
<script type="module" crossorigin src="/assets/index-a89a59fe.js"></script>
|
||||
<link rel="stylesheet" href="/assets/index-a5441302.css">
|
||||
</head>
|
||||
<body>
|
||||
<div id="app"></div>
|
||||
|
||||
@ -250,13 +250,35 @@
|
||||
<button v-if="!loading"
|
||||
type="button"
|
||||
@click="startSpeechRecognition"
|
||||
:class="{ 'text-red-500': isLesteningToVoice }"
|
||||
:class="{ 'text-red-500': isListeningToVoice }"
|
||||
class="w-6 hover:text-secondary duration-75 active:scale-90 cursor-pointer transform transition-transform hover:translate-y-[-5px] active:scale-90"
|
||||
>
|
||||
<i data-feather="mic"></i>
|
||||
</button>
|
||||
<div class="pointer-events-none absolute -top-20 left-1/2 w-max -translate-x-1/2 rounded-md bg-gray-100 p-2 opacity-0 transition-opacity group-hover:opacity-100 dark:bg-gray-800"><p class="max-w-sm text-sm text-gray-800 dark:text-gray-200">Press and talk.</p></div>
|
||||
</div>
|
||||
<div class="group relative w-max">
|
||||
<button v-if="$store.state.config.active_tts_service!='None' && !$store.state.is_rt_on"
|
||||
type="button"
|
||||
@click="startRTCom"
|
||||
:class="{ 'text-red-500': isListeningToVoice }"
|
||||
class="w-6 hover:text-secondary duration-75 active:scale-90 cursor-pointer transform transition-transform hover:translate-y-[-5px] active:scale-90 bg-green-500 border border-green-700 rounded-md"
|
||||
>
|
||||
<i data-feather="mic"></i>
|
||||
</button>
|
||||
<div class="pointer-events-none absolute -top-20 left-1/2 w-max -translate-x-1/2 rounded-md bg-gray-100 p-2 opacity-0 transition-opacity group-hover:opacity-100 dark:bg-gray-800"><p class="max-w-sm text-sm text-gray-800 dark:text-gray-200">Real time bidirectional audio mode.</p></div>
|
||||
<button v-if="$store.state.config.active_tts_service!='None' && $store.state.is_rt_on"
|
||||
type="button"
|
||||
@click="stopRTCom"
|
||||
:class="{ 'text-red-500': isListeningToVoice }"
|
||||
class="w-6 hover:text-secondary duration-75 active:scale-90 cursor-pointer transform transition-transform hover:translate-y-[-5px] active:scale-90 bg-red-500 border border-red-700 rounded-md"
|
||||
>
|
||||
<i data-feather="mic"></i>
|
||||
</button>
|
||||
<div class="pointer-events-none absolute -top-20 left-1/2 w-max -translate-x-1/2 rounded-md bg-gray-100 p-2 opacity-0 transition-opacity group-hover:opacity-100 dark:bg-gray-800"><p class="max-w-sm text-sm text-gray-800 dark:text-gray-200">Real time bidirectional audio mode.</p></div>
|
||||
|
||||
</div>
|
||||
|
||||
<div v-if="!loading" class="group relative w-max">
|
||||
<input type="file" ref="fileDialog" style="display: none" @change="addFiles" multiple />
|
||||
<button type="button" @click.prevent="add_file"
|
||||
@ -389,7 +411,7 @@ export default {
|
||||
selecting_binding:false,
|
||||
selecting_model:false,
|
||||
selectedModel:'',
|
||||
isLesteningToVoice:false,
|
||||
isListeningToVoice:false,
|
||||
filesList: [],
|
||||
isFileSentList: [],
|
||||
totalSize: 0,
|
||||
@ -639,6 +661,8 @@ export default {
|
||||
await this.$store.dispatch('refreshConfig');
|
||||
await this.$store.dispatch('fetchLanguages');
|
||||
await this.$store.dispatch('fetchLanguage');
|
||||
await this.$store.dispatch('fetchisRTOn');
|
||||
|
||||
console.log('pers is mounted', res)
|
||||
|
||||
if (res && res.status && res.active_personality_id > -1) {
|
||||
@ -829,6 +853,22 @@ export default {
|
||||
makeAnEmptyAIMessage() {
|
||||
this.$emit('createEmptyAIMessage')
|
||||
},
|
||||
startRTCom(){
|
||||
socket.emit('start_audio_stream', ()=>{this.isAudioActive = true;});
|
||||
nextTick(() => {
|
||||
feather.replace()
|
||||
}
|
||||
)
|
||||
this.$store.state.is_rt_on = true;
|
||||
},
|
||||
stopRTCom(){
|
||||
socket.emit('stop_audio_stream', ()=>{this.isAudioActive = true;});
|
||||
nextTick(() => {
|
||||
feather.replace()
|
||||
}
|
||||
)
|
||||
this.$store.state.is_rt_on = false;
|
||||
},
|
||||
startSpeechRecognition() {
|
||||
if ('SpeechRecognition' in window || 'webkitSpeechRecognition' in window) {
|
||||
this.recognition = new (window.SpeechRecognition || window.webkitSpeechRecognition)();
|
||||
@ -836,7 +876,7 @@ export default {
|
||||
this.recognition.interimResults = true; // Enable interim results to get real-time updates
|
||||
|
||||
this.recognition.onstart = () => {
|
||||
this.isLesteningToVoice = true;
|
||||
this.isListeningToVoice = true;
|
||||
this.silenceTimer = setTimeout(() => {
|
||||
this.recognition.stop();
|
||||
}, this.silenceTimeout); // Set the silence timeout to stop recognition
|
||||
@ -856,13 +896,13 @@ export default {
|
||||
|
||||
this.recognition.onerror = (event) => {
|
||||
console.error('Speech recognition error:', event.error);
|
||||
this.isLesteningToVoice = false;
|
||||
this.isListeningToVoice = false;
|
||||
clearTimeout(this.silenceTimer); // Clear the silence timeout on error
|
||||
};
|
||||
|
||||
this.recognition.onend = () => {
|
||||
console.log('Speech recognition ended.');
|
||||
this.isLesteningToVoice = false;
|
||||
this.isListeningToVoice = false;
|
||||
clearTimeout(this.silenceTimer); // Clear the silence timeout when recognition ends normally
|
||||
this.submit(); // Call the function to handle form submission or action once speech recognition ends
|
||||
};
|
||||
|
||||
@ -24,6 +24,7 @@ function copyObject(obj) {
|
||||
export const store = createStore({
|
||||
state () {
|
||||
return {
|
||||
is_rt_on:false,
|
||||
language: "english",
|
||||
languages: [],
|
||||
currentTheme: '',
|
||||
@ -83,6 +84,11 @@ export const store = createStore({
|
||||
}
|
||||
},
|
||||
mutations: {
|
||||
|
||||
setisRTOn(state, is_rt_on) {
|
||||
state.is_rt_on = is_rt_on;
|
||||
},
|
||||
|
||||
setLanguages(state, languages) {
|
||||
state.languages = languages;
|
||||
},
|
||||
@ -162,6 +168,10 @@ export const store = createStore({
|
||||
}
|
||||
},
|
||||
getters: {
|
||||
|
||||
getisRTOn(state) {
|
||||
return state.is_rt_on;
|
||||
},
|
||||
getLanguages(state) {
|
||||
return state.languages;
|
||||
},
|
||||
@ -275,7 +285,17 @@ export const store = createStore({
|
||||
console.log("databases:",databases)
|
||||
commit('setDatabases', databases);
|
||||
},
|
||||
|
||||
async fetchisRTOn({ commit }) {
|
||||
console.log("is_rt_on", this.state.client_id)
|
||||
const response = await axios.get(
|
||||
'/is_rt_on'
|
||||
);
|
||||
|
||||
console.log("response", response)
|
||||
const is_rt_on = response.data;
|
||||
console.log("languages", is_rt_on)
|
||||
commit('setRTOn', is_rt_on);
|
||||
},
|
||||
async fetchLanguages({ commit }) {
|
||||
console.log("get_personality_languages_list", this.state.client_id)
|
||||
const response = await axios.post(
|
||||
@ -770,7 +790,14 @@ app.mixin({
|
||||
catch (ex){
|
||||
console.log("Error cought:", ex)
|
||||
}
|
||||
|
||||
try{
|
||||
await this.$store.dispatch('fetchisRTOn');
|
||||
}
|
||||
catch (ex){
|
||||
console.log("Error cought:", ex)
|
||||
}
|
||||
|
||||
|
||||
this.$store.state.ready = true;
|
||||
}
|
||||
|
||||
|
||||
@ -1 +1 @@
|
||||
Subproject commit 4107557398450addb7500e06d80f26b37f572805
|
||||
Subproject commit 7f6b99c1e5fc01050b6c60b30706dd141a18d734
|
||||
Loading…
Reference in New Issue
Block a user