From f5fdf189d2034b7bec90d3f7c2e0224f2e5876c0 Mon Sep 17 00:00:00 2001 From: Saifeddine ALOUI Date: Fri, 14 Apr 2023 02:10:22 +0200 Subject: [PATCH] First working AUdio in and audio out --- app.py | 17 ++++ configs/default.yaml | 3 +- static/js/audio.js | 202 ++++++++++++++++++++++++++++++++++++++++ static/js/chat.js | 2 + static/js/settings.js | 28 ++++++ static/js/tabs.js | 3 + templates/chat.html | 1 + templates/settings.html | 10 ++ 8 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 static/js/audio.js diff --git a/app.py b/app.py index 3a85ebdd..94d94f64 100644 --- a/app.py +++ b/app.py @@ -61,6 +61,10 @@ class Gpt4AllWebUI: "/list_personalities", "list_personalities", self.list_personalities, methods=["GET"] ) + self.add_endpoint( + "/list_languages", "list_languages", self.list_languages, methods=["GET"] + ) + self.add_endpoint( "/list_discussions", "list_discussions", self.list_discussions, methods=["GET"] ) @@ -138,6 +142,19 @@ class Gpt4AllWebUI: personalities = [f.name for f in personalities_dir.glob('*.yaml')] return jsonify(personalities) + def list_languages(self): + lanuguages= [ + { "value": "en-US", "label": "English" }, + { "value": "fr-FR", "label": "Français" }, + { "value": "ar-AR", "label": "العربية" }, + { "value": "it-IT", "label": "Italiano" }, + { "value": "de-DE", "label": "Deutsch" }, + { "value": "nl-XX", "label": "Dutch" }, + { "value": "zh-CN", "label": "中國人" } + ] + return jsonify(lanuguages) + + def list_discussions(self): discussions = self.db.get_discussions() return jsonify(discussions) diff --git a/configs/default.yaml b/configs/default.yaml index 8e35ecef..24d71d0a 100644 --- a/configs/default.yaml +++ b/configs/default.yaml @@ -12,4 +12,5 @@ host: "localhost" port: 9600 db_path: "database.db" nb_messages_to_remember: 5 -personality: "gpt4all_chatbot" \ No newline at end of file +personality: "gpt4all_chatbot" +language: "en_XX" \ No newline at end of file diff --git a/static/js/audio.js b/static/js/audio.js new file mode 100644 index 00000000..d8b2ef6e --- /dev/null +++ b/static/js/audio.js @@ -0,0 +1,202 @@ +isStarted = false; +isSpeaking = false; +const SpeechRecognition = window.SpeechRecognition || webkitSpeechRecognition; +const recognition = new SpeechRecognition(); +const synth = window.speechSynthesis || webkitspeechSynthesis; +var voices = synth.getVoices(); +function prepre_audio(){ + recognition.continuous = true; + recognition.interimResults = true; + recognition.maxAlternatives = 10; + language_select = document.getElementById("language") +} +voices = []; +function populateVoicesList() { + voices = synth.getVoices(); + voice_select = document.getElementById("voice") + voice_select.innerHTML=""; + for (let i = 0; i < voices.length; i++) { + if ( + voices[i].lang.startsWith( + language_select.value.substring(0, 2) + ) + ) { + const option = document.createElement("option"); + option.textContent = `${voices[i].name} (${voices[i].lang})`; + + if (voices[i].default) { + option.textContent += " — DEFAULT"; + } + + option.setAttribute("data-lang", voices[i].lang); + option.setAttribute("data-name", voices[i].name); + voice_select.appendChild(option); + } + } + voice_select.addEventListener("change", function () { + }); +} +// Audio code +function splitString(string, maxLength) { + const sentences = string.match(/[^.!?]+[.!?]/g); + const strings = []; + let currentString = ""; + + if (sentences) { + for (const sentence of sentences) { + if (currentString.length + sentence.length > maxLength) { + strings.push(currentString); + currentString = ""; + } + + currentString += `${sentence} `; + } + } else { + strings.push(string); + } + + if (currentString) { + strings.push(currentString); + } + + return strings; +} +function addListeners(button, utterThis) { + utterThis.onstart = (event) => { + isSpeaking = true; + button.style.backgroundColor = "red"; + button.style.boxShadow = "2px 2px 0.5px #808080"; + }; + + utterThis.onend = (event) => { + isSpeaking = false; + button.style.backgroundColor = ""; + button.style.boxShadow = ""; + }; +} + +function attachAudio_modules(div) { + if (div.parentNode.getElementsByClassName("audio-out-button").length > 0) { + return; + } + const audio_out_button = document.createElement("button"); + audio_out_button.id = "audio-out-button"; + audio_out_button.classList.add("audio_btn"); + audio_out_button.innerHTML = "🕪"; + div.classList.add("flex-1"); + audio_out_button.classList.add("audio-out-button"); + div.appendChild(audio_out_button); + + function play_audio() { + if (isSpeaking) { + + audio_out_button.style.backgroundColor = ""; + audio_out_button.style.boxShadow = ""; + synth.cancel(); + isSpeaking = false; + } else { + isSpeaking = true; + text = audio_out_button.previousSibling.textContent; + + const selectedOption = + voice_select.selectedOptions[0].getAttribute("data-name"); + var selectedVoice = null; + for (let i = 0; i < voices.length; i++) { + if (voices[i].name === selectedOption) { + selectedVoice = voices[i]; + } + } + if (selectedVoice && selectedVoice.voiceURI === "native") { + const utterThis = new SpeechSynthesisUtterance(text); + utterThis.voice = selectedVoice; + addListeners(audio_out_button, utterThis); + synth.speak(utterThis); + } else { + texts = splitString(text, 200); + texts.forEach((text) => { + const utterThis = new SpeechSynthesisUtterance(text); + utterThis.voice = selectedVoice; + addListeners(audio_out_button, utterThis); + synth.speak(utterThis); + }); + } + } + } + audio_out_button.addEventListener("click", () => { + play_audio(); + }); + // TODO : activate using configuration file + //if (global["auto_audio"]) { + // play_audio(); + //} +} + +function add_audio_in_ui() { + const inputs = document.querySelectorAll("#user-input"); + inputs.forEach((input) => { + // const wrapper = document.createElement("div"); + // wrapper.classList.add("flex", "items-center"); + var btn = document.querySelectorAll("#audio_in_tool"); + + var found = false; + // Iterate through the children + for (var i = 0; i < btn.length; i++) { + var child = btn[i]; + // Check if the wrapper element contains the current child element + if (input.parentNode.parentNode.contains(child)) { + found = true; + } + } + + + if (!found) { + const audio_in_button = document.createElement("button"); + audio_in_button.id = "audio_in_tool"; + audio_in_button.classList.add("audio_btn"); + audio_in_button.innerHTML = "🎤"; + + input.parentNode.insertBefore( + audio_in_button, + input + ); + + input.classList.add("flex-1"); + audio_in_button.classList.add("ml-2"); + //wrapper.appendChild(audio_in_button); + //input.parentNode.parentNode.insertBefore(wrapper, input); + //input.parentNode.removeChild(input); + //wrapper.appendChild(input); + + audio_in_button.addEventListener("click", () => { + if (isStarted) { + recognition.stop(); + isStarted = false; + } else { + recognition.lang = language_select.value; + recognition.start(); + isStarted = true; + } + }); + + recognition.addEventListener("result", (event) => { + let transcript = ""; + for (const result of event.results) { + transcript += result[0].transcript; + } + if (transcript != "") { + input.value = transcript; + } + }); + + recognition.addEventListener("start", () => { + audio_in_button.style.backgroundColor = "red"; + audio_in_button.style.boxShadow = "2px 2px 0.5px #808080"; + }); + + recognition.addEventListener("end", () => { + audio_in_button.style.backgroundColor = ""; + audio_in_button.style.boxShadow = ""; + }); + } + }); + } \ No newline at end of file diff --git a/static/js/chat.js b/static/js/chat.js index 948a8bb3..83884e9e 100644 --- a/static/js/chat.js +++ b/static/js/chat.js @@ -319,6 +319,8 @@ function addMessage(sender, message, id, rank=0, can_edit=false) { } chatWindow.appendChild(messageElement); chatWindow.appendChild(hiddenElement); + + attachAudio_modules(messageTextElement); // scroll to bottom of chat window chatWindow.scrollTop = chatWindow.scrollHeight; diff --git a/static/js/settings.js b/static/js/settings.js index acdb8beb..aa1001a0 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -5,6 +5,8 @@ fetch('/settings') document.getElementById('settings').innerHTML = html; modelInput = document.getElementById('model'); + personalityInput = document.getElementById('personalities'); + languageInput = document.getElementById('language'); seedInput = document.getElementById('seed'); tempInput = document.getElementById('temp'); nPredictInput = document.getElementById('n-predict'); @@ -52,6 +54,8 @@ fetch('/settings') .then((data) => { console.log(data); modelInput.value = data["model"] + personalityInput.value = data["personality"] + languageInput.value = data["language"] seedInput.value = data["seed"] tempInput.value = data["temp"] nPredictInput.value = data["n_predict"] @@ -168,6 +172,30 @@ function populate_models(){ } }); + // Fetch the list of .yaml files from the models subfolder + fetch('/list_languages') + .then(response => response.json()) + .then(data => { + if (Array.isArray(data)) { + // data is an array + const selectElement = document.getElementById('language'); + data.forEach(row => { + const optionElement = document.createElement('option'); + optionElement.value = row.value; + optionElement.innerHTML = row.label; + selectElement.appendChild(optionElement); + }); + + // fetch('/get_args') + // .then(response=> response.json()) + // .then(data=>{ + + // }) + } else { + console.error('Expected an array, but received:', data); + } + }); + } populate_models() diff --git a/static/js/tabs.js b/static/js/tabs.js index 6324b0af..459cd98c 100644 --- a/static/js/tabs.js +++ b/static/js/tabs.js @@ -32,6 +32,9 @@ fetch('/main') load_discussion(); update_main(); db_export(); + prepre_audio(); + add_audio_in_ui(); + populateVoicesList(); }) .catch(error => { diff --git a/templates/chat.html b/templates/chat.html index 69b98c88..07a9c14f 100644 --- a/templates/chat.html +++ b/templates/chat.html @@ -55,6 +55,7 @@ + diff --git a/templates/settings.html b/templates/settings.html index 7d24947d..419c8c79 100644 --- a/templates/settings.html +++ b/templates/settings.html @@ -10,6 +10,16 @@ +
+ + +
+
+ + +