Merge branch 'main' into main

app/backend/InputOutputHandler.tsx

@@ -2,9 +2,11 @@
 import React, { useEffect, useRef, useState } from "react";
 import ConversationFrontend from "../components/ConversationFrontend";
 import InputFrontend from "../components/InputFrontend";
+import VoiceSend from "./voice_backend"
 import axios from "axios";
 import { skip } from "node:test";
 
+
 const InputOutputBackend: React.FC = () => {
   type Message = {
     role: string
@@ -19,6 +21,10 @@ const InputOutputBackend: React.FC = () => {
   const [inputMessage, setInputMessage] = useState<string>("")
   const [inputDisabled, setInputDisabled] = useState(false)
   const [lastMessage, setLastMessage] = useState<Message>({ role: "user", content: "Not supposed to happen." })
+  const [isRecording, setIsRecording] = useState(false);
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null);
+  const audioChunksRef = useRef<Blob[]>([]);
+
 
   console.log(messages);
 
@@ -136,9 +142,47 @@ const InputOutputBackend: React.FC = () => {
     }
   }
 
-  const handleMicClick = () => {
-    // do stuff
+  const startRecording = async () => {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    mediaRecorderRef.current = new MediaRecorder(stream);
+
+    mediaRecorderRef.current.ondataavailable = (event) => {
+      audioChunksRef.current.push(event.data);
+    };
+
+    mediaRecorderRef.current.onstop = () => {
+      const audioBlob = new Blob(audioChunksRef.current, { type: 'audio/wav' });
+      audioChunksRef.current = []; // Clear the chunks for the next recording
+      // Call your existing function to send the audioBlob
+      // Example: sendAudioToApi(audioBlob);
+    };
+
+    mediaRecorderRef.current.start();
+    setIsRecording(true);
+
+    // Automatically stop recording after 10 seconds
+    setTimeout(() => {
+      stopRecording();
+    }, 10000);
+  };
+
+  const stopRecording = () => {
+    if (mediaRecorderRef.current) {
+      mediaRecorderRef.current.stop();
+      setIsRecording(false);
+      var remote = new VoiceSend()
+      remote.sendToVoiceRecognition(new Blob(audioChunksRef.current, { type: 'audio/wav' }), remote.voiceDataTemplate);
     }
+  };
+
+
+  const handleMicClick = () => {
+    if (!isRecording) {
+      startRecording();
+    } else {
+      stopRecording();
+    }
+  };
 
   const handleResendClick = () => {
     var temporary_message = messages[messages.length - 2]['content']

app/backend/voice_backend.ts

@@ -0,0 +1,28 @@
+import axios from "axios";
+
+class VoiceSend {
+
+    voiceDataTemplate = {
+        type: "basic",
+        audio_data: null,
+        option: "offline"
+    }
+
+    sendToVoiceRecognition(audio_data: Blob, data: any) {
+        var dataSend = data
+        dataSend['audio_data'] = audio_data
+        axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", dataSend)
+            .then((response: any) => {
+                console.log(response['response'])
+                return response['response']
+            })
+            .catch(error => {
+                console.log("Error calling API:", error)
+                postMessage({ status: 500 })
+            })
+    }
+
+}
+
+
+export default VoiceSend;