diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 02a227b..3b00707 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -2,9 +2,11 @@ import React, { useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; +import VoiceSend from "./voice_backend" import axios from "axios"; import { skip } from "node:test"; + const InputOutputBackend: React.FC = () => { type Message = { role: string @@ -19,6 +21,10 @@ const InputOutputBackend: React.FC = () => { const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) const [lastMessage, setLastMessage] = useState({ role: "user", content: "Not supposed to happen." }) + const [isRecording, setIsRecording] = useState(false); + const mediaRecorderRef = useRef(null); + const audioChunksRef = useRef([]); + console.log(messages); @@ -136,9 +142,47 @@ const InputOutputBackend: React.FC = () => { } } + const startRecording = async () => { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + mediaRecorderRef.current = new MediaRecorder(stream); + + mediaRecorderRef.current.ondataavailable = (event) => { + audioChunksRef.current.push(event.data); + }; + + mediaRecorderRef.current.onstop = () => { + const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/wav' }); + audioChunksRef.current = []; // Clear the chunks for the next recording + // Call your existing function to send the audioBlob + // Example: sendAudioToApi(audioBlob); + }; + + mediaRecorderRef.current.start(); + setIsRecording(true); + + // Automatically stop recording after 10 seconds + setTimeout(() => { + stopRecording(); + }, 10000); + }; + + const stopRecording = () => { + if (mediaRecorderRef.current) { + mediaRecorderRef.current.stop(); + setIsRecording(false); + var remote = new VoiceSend() + remote.sendToVoiceRecognition(new Blob(audioChunksRef.current, { type: 'audio/wav' }), remote.voiceDataTemplate); + } + }; + + const handleMicClick = () => { - // do stuff - } + if (!isRecording) { + startRecording(); + } else { + stopRecording(); + } + }; const handleResendClick = () => { var temporary_message = messages[messages.length - 2]['content'] diff --git a/app/backend/voice_backend.ts b/app/backend/voice_backend.ts new file mode 100644 index 0000000..650afeb --- /dev/null +++ b/app/backend/voice_backend.ts @@ -0,0 +1,28 @@ +import axios from "axios"; + +class VoiceSend { + + voiceDataTemplate = { + type: "basic", + audio_data: null, + option: "offline" + } + + sendToVoiceRecognition(audio_data: Blob, data: any) { + var dataSend = data + dataSend['audio_data'] = audio_data + axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", dataSend) + .then((response: any) => { + console.log(response['response']) + return response['response'] + }) + .catch(error => { + console.log("Error calling API:", error) + postMessage({ status: 500 }) + }) + } + +} + + +export default VoiceSend; \ No newline at end of file