ExternalVendorCode/lollms
1
0
Fork 0
mirror of https://github.com/ParisNeo/lollms.git synced 2024-12-22 05:57:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Realtime interaction added

Browse Source
This commit is contained in:
Saifeddine ALOUI 2024-05-18 23:07:48 +02:00
parent 443bdef3ef
commit f98fb13363
1 changed files with 12 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lollms/media.py
View File

@ -9,6 +9,7 @@ License: Apache 2.0
from lollms.utilities import PackageManager from lollms.utilities import PackageManager
from lollms.com import LoLLMsCom from lollms.com import LoLLMsCom
from lollms.utilities import trace_exception, run_async from lollms.utilities import trace_exception, run_async
from lollms.types import MSG_TYPE, SENDER_TYPES
from lollms.client_session import Session from lollms.client_session import Session
from ascii_colors import ASCIIColors from ascii_colors import ASCIIColors
import platform import platform
@ -98,9 +99,12 @@ class AudioRecorder:
self, self,
lc:LollmsApplication, lc:LollmsApplication,
sio:socketio.Client, sio:socketio.Client,
personality:AIPersonality, threshold=1000, silence_duration=2, sound_threshold_percentage=10, gain=1.0, rate=44100, channels=1, buffer_size=10, model="small.en", snd_device=None, logs_folder="logs", voice=None, block_while_talking=True, context_size=4096): personality:AIPersonality,
discussion_database:DiscussionsDB,
threshold=1000, silence_duration=2, sound_threshold_percentage=10, gain=1.0, rate=44100, channels=1, buffer_size=10, model="small.en", snd_device=None, logs_folder="logs", voice=None, block_while_talking=True, context_size=4096):
self.sio = sio self.sio = sio
self.lc = lc self.lc = lc
self.discussion_database = discussion_database
self.tts = LollmsTTS(self.lc) self.tts = LollmsTTS(self.lc)
self.tl = TasksLibrary(self.lc) self.tl = TasksLibrary(self.lc)
self.fn = FunctionCalling_Library(self.tl) self.fn = FunctionCalling_Library(self.tl)
@ -172,7 +176,7 @@ class AudioRecorder:
self.model = model self.model = model
self.whisper = whisper.load_model(model) self.whisper = whisper.load_model(model)
ASCIIColors.success("OK") ASCIIColors.success("OK")
self.discussion = DiscussionsDB.create_discussion("RT_chat") self.discussion = discussion_database.create_discussion("RT_chat")
def get_date_time(self): def get_date_time(self):
now = datetime.now() now = datetime.now()
@ -297,7 +301,8 @@ class AudioRecorder:
def _save_wav(self, frames): def _save_wav(self, frames):
ASCIIColors.green("<<SEGMENT_RECOVERED>>") ASCIIColors.green("<<SEGMENT_RECOVERED>>")
self.transcription_signal.update_status.emit("Segment detected and saved") # Todo annouce
# self.transcription_signal.update_status.emit("Segment detected and saved")
filename = f"recording_{self.file_index}.wav" filename = f"recording_{self.file_index}.wav"
self.file_index += 1 self.file_index += 1
@ -345,7 +350,8 @@ class AudioRecorder:
self.block_listening = True self.block_listening = True
try: try:
if filename: if filename:
self.transcription_signal.update_status.emit("Transcribing") # TODO: send signal
# self.transcription_signal.update_status.emit("Transcribing")
ASCIIColors.green("<<TRANSCRIBING>>") ASCIIColors.green("<<TRANSCRIBING>>")
result = self.whisper.transcribe(str(Path(self.logs_folder)/filename)) result = self.whisper.transcribe(str(Path(self.logs_folder)/filename))
transcription_fn = str(Path(self.logs_folder)/filename) + ".txt" transcription_fn = str(Path(self.logs_folder)/filename) + ".txt"
@ -358,7 +364,7 @@ class AudioRecorder:
if result["text"]!="": if result["text"]!="":
# TODO : send the output # TODO : send the output
# self.transcription_signal.new_user_transcription.emit(filename, result["text"]) # self.transcription_signal.new_user_transcription.emit(filename, result["text"])
self.discussion.add_message("user",result["text"]) self.discussion.add_message(MSG_TYPE.MSG_TYPE_FULL, SENDER_TYPES.SENDER_TYPES_USER, "user",result["text"])
discussion = self.discussion.format_discussion(self.context_size) discussion = self.discussion.format_discussion(self.context_size)
full_context = '!@>system:' + self.personality.personality_conditioning +"\n" + discussion+"\n!@>lollms:" full_context = '!@>system:' + self.personality.personality_conditioning +"\n" + discussion+"\n!@>lollms:"
ASCIIColors.red(" ---------------- Discussion ---------------------") ASCIIColors.red(" ---------------- Discussion ---------------------")
@ -375,7 +381,7 @@ class AudioRecorder:
else: else:
lollms_text = self.lc.generate(full_context+"!@>lollms: "+ lollms_text + "\n!@>functions outputs:\n"+ "\n".join(responses) +"!@>lollms:") lollms_text = self.lc.generate(full_context+"!@>lollms: "+ lollms_text + "\n!@>functions outputs:\n"+ "\n".join(responses) +"!@>lollms:")
lollms_text = self.fix_string_for_xtts(lollms_text) lollms_text = self.fix_string_for_xtts(lollms_text)
self.discussion.add_message("lollms",lollms_text) self.discussion.add_message(MSG_TYPE.MSG_TYPE_FULL, SENDER_TYPES.SENDER_TYPES_AI, "lollms",lollms_text)
ASCIIColors.red(" -------------- LOLLMS answer -------------------") ASCIIColors.red(" -------------- LOLLMS answer -------------------")
ASCIIColors.yellow(lollms_text) ASCIIColors.yellow(lollms_text)
ASCIIColors.red(" -------------------------------------------------") ASCIIColors.red(" -------------------------------------------------")