WOHOOOOOOOO

app/backend/InputOutputHandler.tsx

@@ -2,7 +2,7 @@
 import React, { use, useEffect, useRef, useState } from "react";
 import ConversationFrontend from "../components/ConversationFrontend";
 import InputFrontend from "../components/InputFrontend";
-import VoiceSend from "./voice_backend"
+import { sendToVoiceRecognition } from "./voice_backend"
 import { AudioRecorder } from "./AudioRecorder";
 import axios from "axios";
 import { resolve } from "path";
@@ -33,12 +33,13 @@ const InputOutputBackend: React.FC = () => {
     setTimeZone(localStorage.getItem("timeZone"))
     setDateFormat(localStorage.getItem("dateFormat"))
   }, [preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat])
-  
+
   const [copyClicked, setCopyClicked] = useState(false)
   const [accessToken, setAccessToken] = useState("")
   const postWorkerRef = useRef<Worker | null>(null)
   const getWorkerRef = useRef<Worker | null>(null)
-  const [messages, setMessages] = useState<Message[]>([{ role: "system", 
+  const [messages, setMessages] = useState<Message[]>([{
+    role: "system",
     content: `You are in the timezone: ${timeZone}. 
     You use the time format ${timeFormat}. 
     You use the date format ${dateFormat} for all references of dates. 
@@ -46,12 +47,12 @@ const InputOutputBackend: React.FC = () => {
     You will only answer in the language (you will receive the country code) ${preferredLanguage}. 
     But in the case the user specifically states to answer in an other language do that speaking in a
     nother language is not stating you should answer in that language. Additionally do not translate your answer into multiple languages`
-  },{ role: "assistant", content: "Hello! How can I help you?" }])
+  }, { role: "assistant", content: "Hello! How can I help you?" }])
   const [liveMessage, setLiveMessage] = useState("")
   const [inputMessage, setInputMessage] = useState<string>("")
   const [inputDisabled, setInputDisabled] = useState(false)
   const [isRecording, setIsRecording] = useState(false)
-  const mediaRecorderRef = useRef<MediaRecorder | null>(null) 
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null)
   const audioChunks = useRef<Blob[]>([])
 
 
@@ -169,40 +170,46 @@ const InputOutputBackend: React.FC = () => {
     }
   }
 
-  const startRecording = async () => {
+  const startRecording = async (): Promise<string> => {
-    const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
-    const mediaRecorder = new MediaRecorder(stream)
+    const mediaRecorder = new MediaRecorder(stream);
-    mediaRecorderRef.current = mediaRecorder
+    mediaRecorderRef.current = mediaRecorder;
-    
-    mediaRecorder.ondataavailable = (event) => {
-      audioChunks.current.push(event.data)
-    }
-        
-    mediaRecorder.onstop = async () => {
-      const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" })
-      audioChunks.current = []
-      // console.log(audioBlob);
-      // const url = URL.createObjectURL(audioBlob)
-      // const audio = new Audio(url);
-      // audio.play().catch(error => console.error("Error playing audio:", error));
 
-      const remote = new VoiceSend()
+    audioChunks.current = []; // Initialize audio chunks
-      remote.sendToVoiceRecognition(audioBlob)
+
-    }
+    // Create a promise that resolves when the onstop event is done
+    const stopRecordingPromise = new Promise<string>((resolve) => {
+      mediaRecorder.ondataavailable = (event) => {
+        audioChunks.current.push(event.data);
+      };
+
+      mediaRecorder.onstop = async () => {
+        const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" });
+        audioChunks.current = [];
+
+        const text_voice = await sendToVoiceRecognition(audioBlob);
+        console.log(text_voice);
+        resolve(text_voice); // Resolve the promise with the recognized text
+      };
+    });
+
+    mediaRecorder.start();
+    setIsRecording(true);
+
+    // Wait for the recording to stop and get the recognized text
+    return stopRecordingPromise;
+  };
 
-    mediaRecorder.start()
-    setIsRecording(true)
-  }
-    
   const stopRecording = () => {
-    mediaRecorderRef.current?.stop()
+    mediaRecorderRef.current?.stop();
-    setIsRecording(false)
+    setIsRecording(false);
-  }
+  };
 
-
+  const handleMicClick = async () => {
-  const handleMicClick = () => {
     if (!isRecording) {
-      startRecording();
+      const recognizedText = await startRecording();
+      setInputMessage(recognizedText); // Set the recognized text after recording
+      console.log("Set!")
     } else {
       stopRecording();
     }
@@ -246,7 +253,7 @@ const InputOutputBackend: React.FC = () => {
     await wait(1000)
     setCopyClicked(false)
   }
-  
+
 
   return (
     <>
@@ -263,7 +270,7 @@ const InputOutputBackend: React.FC = () => {
         onMicClick={handleMicClick}
         inputDisabled={inputDisabled}
         isRecording={isRecording}
-      /> 
+      />
     </>
   )
 }

app/backend/voice_backend.ts

@@ -1,26 +1,20 @@
 import axios from "axios";
 
+export const sendToVoiceRecognition = (audio_data: Blob): Promise<string> => {
+    console.log("sending recording...");
 
-class VoiceSend {
+    const formdata = new FormData()
-    sendToVoiceRecognition(audio_data: Blob) {
+    formdata.append("audio", audio_data)
-        console.log("sending recording...");
 
-        const formdata = new FormData()
+    const dataSend = { option: "offline", type: "basic", audio: audio_data }
-        formdata.append("audio", audio_data)
+    return axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata)
-
+        .then((response) => {
-        const dataSend = { option:"offline", type:"basic",audio:audio_data }
+            console.log(response.data)
-        axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata)
+            return response.data.response
-            .then((response) => {
+        })
-                console.log(response.data)
+        .catch(error => {
-                return response.data.response
+            console.log("Error calling API:", error)
-            })
+            postMessage({ status: 500 })
-            .catch(error => {
+            return "Error"
-                console.log("Error calling API:", error)
+        })
-                postMessage({ status: 500 })
+}
-            })
-    }
-
-}
-
-
-export default VoiceSend;

app/components/InputFrontend.tsx

@@ -6,13 +6,17 @@ interface InputProps {
   onSendClick: (message: string, override: boolean) => void;
   onMicClick: () => void;
   inputDisabled: boolean;
-  isRecording:boolean
+  isRecording: boolean
 }
 
 const InputFrontend = React.forwardRef<HTMLDivElement, InputProps>(
-  ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef<HTMLDivElement>) => {
+  ({ message, onSendClick, onMicClick, inputDisabled, isRecording }, ref: ForwardedRef<HTMLDivElement>) => {
     const [inputValue, setInputValue] = useState('');
 
+    useEffect(() => {
+      setInputValue(message);
+    }, [message]);
+
     const handleInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
       setInputValue(e.target.value);
     };
@@ -40,7 +44,7 @@ const InputFrontend = React.forwardRef<HTMLDivElement, InputProps>(
         <button type="button" onClick={() => onSendClick(inputValue, false)} disabled={inputDisabled ? true : false}>
           <img src="/img/send.svg" alt="send" />
         </button>
-        <button className={`microphone-button ${isRecording ? "red": "var(--input-button-color)"}`} type="button" onClick={onMicClick}>
+        <button className={`microphone-button ${isRecording ? "red" : "var(--input-button-color)"}`} type="button" onClick={onMicClick}>
           <img src="/img/microphone.svg" alt="microphone" />
         </button>
       </div>