diff --git a/app/backend/AudioRecorder(not yet).tsx b/app/backend/AudioRecorder(not yet).tsx deleted file mode 100644 index af6a941..0000000 --- a/app/backend/AudioRecorder(not yet).tsx +++ /dev/null @@ -1,39 +0,0 @@ -// import React, { useState, useRef } from 'react' - -// const AudioRecorder: React.FC = () => { -// const [isRecording, setIsRecording] = useState(false) -// const [audioURL, setAudioURL] = useState(null) -// const medaRecorderRef = useRef(null) -// const audioChunks = useRef([]) - -// const startRecording = async () => { -// const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) -// const mediaRecorder = new MediaRecorder(stream) -// medaRecorderRef.current = mediaRecorder - -// mediaRecorder.ondataavailable = (event) => { -// audioChunks.current.push(event.data) -// } - -// mediaRecorder.onstop = () => { -// const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) -// const url = URL.createObjectURL(audioBlob) -// setAudioURL(url) -// audioChunks.current = [] -// } - -// mediaRecorder.start() -// setIsRecording(true) - -// const stopRecording = () => { -// medaRecorderRef.current?.stop() -// setIsRecording(false) -// } - -// return ( -//
-// ) -// } -// } - -// export default AudioRecorder \ No newline at end of file diff --git a/app/backend/AudioRecorder.ts b/app/backend/AudioRecorder.ts new file mode 100644 index 0000000..459674e --- /dev/null +++ b/app/backend/AudioRecorder.ts @@ -0,0 +1,34 @@ +import React, { useState, useRef } from 'react' + + export const AudioRecorder= () => { + const [isRecording, setIsRecording] = useState(false) + const [audioURL, setAudioURL] = useState(null) + const mediaRecorderRef = useRef(null) + const audioChunks = useRef([]) + + const startRecording = async () => { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + const mediaRecorder = new MediaRecorder(stream) + mediaRecorderRef.current = mediaRecorder + + mediaRecorder.ondataavailable = (event) => { + audioChunks.current.push(event.data) + } + + mediaRecorder.onstop = () => { + const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) + const url = URL.createObjectURL(audioBlob) + setAudioURL(url) + audioChunks.current = [] + } + + mediaRecorder.start() + setIsRecording(true) + + } + + const stopRecording = () => { + mediaRecorderRef.current?.stop() + setIsRecording(false) + } + } \ No newline at end of file diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 3b00707..36b3fac 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -3,8 +3,8 @@ import React, { useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; import VoiceSend from "./voice_backend" +import { AudioRecorder } from "./AudioRecorder"; import axios from "axios"; -import { skip } from "node:test"; const InputOutputBackend: React.FC = () => { @@ -20,10 +20,10 @@ const InputOutputBackend: React.FC = () => { const [liveMessage, setLiveMessage] = useState("") const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) - const [lastMessage, setLastMessage] = useState({ role: "user", content: "Not supposed to happen." }) - const [isRecording, setIsRecording] = useState(false); - const mediaRecorderRef = useRef(null); - const audioChunksRef = useRef([]); + const [isRecording, setIsRecording] = useState(false) + const [audioURL, setAudioURL] = useState(null) + const mediaRecorderRef = useRef(null) + const audioChunks = useRef([]) console.log(messages); @@ -129,7 +129,6 @@ const InputOutputBackend: React.FC = () => { } const handleSendClick = (inputValue: string, override: boolean) => { if (inputValue != "") { - console.log(inputDisabled) if (!inputDisabled || override) { setInputDisabled(true) if (postWorkerRef.current) { @@ -143,37 +142,33 @@ const InputOutputBackend: React.FC = () => { } const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); - mediaRecorderRef.current = new MediaRecorder(stream); - - mediaRecorderRef.current.ondataavailable = (event) => { - audioChunksRef.current.push(event.data); - }; - - mediaRecorderRef.current.onstop = () => { - const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/wav' }); - audioChunksRef.current = []; // Clear the chunks for the next recording - // Call your existing function to send the audioBlob - // Example: sendAudioToApi(audioBlob); - }; - - mediaRecorderRef.current.start(); - setIsRecording(true); - - // Automatically stop recording after 10 seconds - setTimeout(() => { - stopRecording(); - }, 10000); - }; - - const stopRecording = () => { - if (mediaRecorderRef.current) { - mediaRecorderRef.current.stop(); - setIsRecording(false); - var remote = new VoiceSend() - remote.sendToVoiceRecognition(new Blob(audioChunksRef.current, { type: 'audio/wav' }), remote.voiceDataTemplate); + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) + const mediaRecorder = new MediaRecorder(stream) + mediaRecorderRef.current = mediaRecorder + + mediaRecorder.ondataavailable = (event) => { + audioChunks.current.push(event.data) + } + + mediaRecorder.onstop = () => { + const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) + const url = URL.createObjectURL(audioBlob) + console.log(url); + setAudioURL(url) + audioChunks.current = [] + const remote = new VoiceSend() + remote.sendToVoiceRecognition(audioBlob,) + } + + mediaRecorder.start() + setIsRecording(true) + + } + + const stopRecording = () => { + mediaRecorderRef.current?.stop() + setIsRecording(false) } - }; const handleMicClick = () => { @@ -224,6 +219,7 @@ const InputOutputBackend: React.FC = () => { onSendClick={handleSendClick} onMicClick={handleMicClick} inputDisabled={inputDisabled} + isRecording={isRecording} /> ) diff --git a/app/backend/voice_backend.ts b/app/backend/voice_backend.ts index 650afeb..f809a5c 100644 --- a/app/backend/voice_backend.ts +++ b/app/backend/voice_backend.ts @@ -1,16 +1,10 @@ import axios from "axios"; class VoiceSend { - - voiceDataTemplate = { - type: "basic", - audio_data: null, - option: "offline" - } - - sendToVoiceRecognition(audio_data: Blob, data: any) { - var dataSend = data - dataSend['audio_data'] = audio_data + sendToVoiceRecognition(audio_data: Blob) { + console.log("sending recording..."); + + const dataSend = { audio_data } axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", dataSend) .then((response: any) => { console.log(response['response']) diff --git a/app/components/InputFrontend.tsx b/app/components/InputFrontend.tsx index e50e916..ce8ac49 100644 --- a/app/components/InputFrontend.tsx +++ b/app/components/InputFrontend.tsx @@ -4,11 +4,12 @@ interface InputProps { message: string; onSendClick: (message: string, override: boolean) => void; onMicClick: () => void; - inputDisabled: boolean + inputDisabled: boolean; + isRecording:boolean } const InputFrontend = React.forwardRef( - ({ message, onSendClick, onMicClick, inputDisabled }, ref: ForwardedRef) => { + ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef) => { const [inputValue, setInputValue] = useState(''); useEffect(() => { @@ -42,7 +43,7 @@ const InputFrontend = React.forwardRef( -