From 1fd916a3eb50181fac2b6ad6f49f57afd661f2d7 Mon Sep 17 00:00:00 2001 From: Patrick_Pluto Date: Mon, 30 Sep 2024 12:45:19 +0200 Subject: [PATCH] WOHOOOOOOOO --- app/backend/InputOutputHandler.tsx | 79 ++++++++++++++++-------------- app/backend/voice_backend.ts | 38 ++++++-------- app/components/InputFrontend.tsx | 10 ++-- 3 files changed, 66 insertions(+), 61 deletions(-) diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index d4614fd..8529202 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -2,7 +2,7 @@ import React, { use, useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; -import VoiceSend from "./voice_backend" +import { sendToVoiceRecognition } from "./voice_backend" import { AudioRecorder } from "./AudioRecorder"; import axios from "axios"; import { resolve } from "path"; @@ -33,12 +33,13 @@ const InputOutputBackend: React.FC = () => { setTimeZone(localStorage.getItem("timeZone")) setDateFormat(localStorage.getItem("dateFormat")) }, [preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat]) - + const [copyClicked, setCopyClicked] = useState(false) const [accessToken, setAccessToken] = useState("") const postWorkerRef = useRef(null) const getWorkerRef = useRef(null) - const [messages, setMessages] = useState([{ role: "system", + const [messages, setMessages] = useState([{ + role: "system", content: `You are in the timezone: ${timeZone}. You use the time format ${timeFormat}. You use the date format ${dateFormat} for all references of dates. @@ -46,12 +47,12 @@ const InputOutputBackend: React.FC = () => { You will only answer in the language (you will receive the country code) ${preferredLanguage}. But in the case the user specifically states to answer in an other language do that speaking in a nother language is not stating you should answer in that language. Additionally do not translate your answer into multiple languages` - },{ role: "assistant", content: "Hello! How can I help you?" }]) + }, { role: "assistant", content: "Hello! How can I help you?" }]) const [liveMessage, setLiveMessage] = useState("") const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) const [isRecording, setIsRecording] = useState(false) - const mediaRecorderRef = useRef(null) + const mediaRecorderRef = useRef(null) const audioChunks = useRef([]) @@ -169,40 +170,46 @@ const InputOutputBackend: React.FC = () => { } } - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - const mediaRecorder = new MediaRecorder(stream) - mediaRecorderRef.current = mediaRecorder - - mediaRecorder.ondataavailable = (event) => { - audioChunks.current.push(event.data) - } - - mediaRecorder.onstop = async () => { - const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }) - audioChunks.current = [] - // console.log(audioBlob); - // const url = URL.createObjectURL(audioBlob) - // const audio = new Audio(url); - // audio.play().catch(error => console.error("Error playing audio:", error)); + const startRecording = async (): Promise => { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mediaRecorder = new MediaRecorder(stream); + mediaRecorderRef.current = mediaRecorder; - const remote = new VoiceSend() - remote.sendToVoiceRecognition(audioBlob) - } + audioChunks.current = []; // Initialize audio chunks + + // Create a promise that resolves when the onstop event is done + const stopRecordingPromise = new Promise((resolve) => { + mediaRecorder.ondataavailable = (event) => { + audioChunks.current.push(event.data); + }; + + mediaRecorder.onstop = async () => { + const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }); + audioChunks.current = []; + + const text_voice = await sendToVoiceRecognition(audioBlob); + console.log(text_voice); + resolve(text_voice); // Resolve the promise with the recognized text + }; + }); + + mediaRecorder.start(); + setIsRecording(true); + + // Wait for the recording to stop and get the recognized text + return stopRecordingPromise; + }; - mediaRecorder.start() - setIsRecording(true) - } - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } + mediaRecorderRef.current?.stop(); + setIsRecording(false); + }; - - const handleMicClick = () => { + const handleMicClick = async () => { if (!isRecording) { - startRecording(); + const recognizedText = await startRecording(); + setInputMessage(recognizedText); // Set the recognized text after recording + console.log("Set!") } else { stopRecording(); } @@ -246,7 +253,7 @@ const InputOutputBackend: React.FC = () => { await wait(1000) setCopyClicked(false) } - + return ( <> @@ -263,7 +270,7 @@ const InputOutputBackend: React.FC = () => { onMicClick={handleMicClick} inputDisabled={inputDisabled} isRecording={isRecording} - /> + /> ) } diff --git a/app/backend/voice_backend.ts b/app/backend/voice_backend.ts index 3c4193b..ca8a998 100644 --- a/app/backend/voice_backend.ts +++ b/app/backend/voice_backend.ts @@ -1,26 +1,20 @@ import axios from "axios"; +export const sendToVoiceRecognition = (audio_data: Blob): Promise => { + console.log("sending recording..."); -class VoiceSend { - sendToVoiceRecognition(audio_data: Blob) { - console.log("sending recording..."); + const formdata = new FormData() + formdata.append("audio", audio_data) - const formdata = new FormData() - formdata.append("audio", audio_data) - - const dataSend = { option:"offline", type:"basic",audio:audio_data } - axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata) - .then((response) => { - console.log(response.data) - return response.data.response - }) - .catch(error => { - console.log("Error calling API:", error) - postMessage({ status: 500 }) - }) - } - -} - - -export default VoiceSend; \ No newline at end of file + const dataSend = { option: "offline", type: "basic", audio: audio_data } + return axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata) + .then((response) => { + console.log(response.data) + return response.data.response + }) + .catch(error => { + console.log("Error calling API:", error) + postMessage({ status: 500 }) + return "Error" + }) +} \ No newline at end of file diff --git a/app/components/InputFrontend.tsx b/app/components/InputFrontend.tsx index f02442b..5d51834 100644 --- a/app/components/InputFrontend.tsx +++ b/app/components/InputFrontend.tsx @@ -6,13 +6,17 @@ interface InputProps { onSendClick: (message: string, override: boolean) => void; onMicClick: () => void; inputDisabled: boolean; - isRecording:boolean + isRecording: boolean } const InputFrontend = React.forwardRef( - ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef) => { + ({ message, onSendClick, onMicClick, inputDisabled, isRecording }, ref: ForwardedRef) => { const [inputValue, setInputValue] = useState(''); + useEffect(() => { + setInputValue(message); + }, [message]); + const handleInputChange = (e: React.ChangeEvent) => { setInputValue(e.target.value); }; @@ -40,7 +44,7 @@ const InputFrontend = React.forwardRef( -