From 460f22abd8bce615b7a44bfa836132cb2b1e5553 Mon Sep 17 00:00:00 2001
From: sageTheDM <info@photofuel.tech>
Date: Mon, 30 Sep 2024 12:52:58 +0200
Subject: [PATCH] Bug fixes merge

---
 app/backend/InputOutputHandler.tsx | 127 +++++++++++++++--------------
 1 file changed, 65 insertions(+), 62 deletions(-)

diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx
index 2a33591..92fa6df 100644
--- a/app/backend/InputOutputHandler.tsx
+++ b/app/backend/InputOutputHandler.tsx
@@ -2,7 +2,7 @@
 import React, { use, useEffect, useRef, useState } from "react";
 import ConversationFrontend from "../components/ConversationFrontend";
 import InputFrontend from "../components/InputFrontend";
-import VoiceSend from "./voice_backend"
+import { sendToVoiceRecognition } from "./voice_backend"
 import { AudioRecorder } from "./AudioRecorder";
 import axios from "axios";
 import { resolve } from "path";
@@ -11,6 +11,7 @@ import { fetchFile, toBlobURL } from "@ffmpeg/util"
 
 
 const InputOutputBackend: React.FC = () => {
+  // # variables
   type Message = {
     role: string
     content: string
@@ -23,22 +24,22 @@ const InputOutputBackend: React.FC = () => {
   const [timeZone, setTimeZone] = useState<string | null>(null);
   const [dateFormat, setDateFormat] = useState<string | null>(null);
   const [messages, setMessages] = useState<Message[]>([]);
-    
-    useEffect(() => {
-      setPreferredCurrency(localStorage.getItem("preferredCurrency"));
-      setPreferredLanguage(localStorage.getItem("preferredLanguage"));
-      setTimeFormat(localStorage.getItem("timeFormat"));
-      setPreferredMeasurement(localStorage.getItem("preferredMeasurement"));
-      setTimeZone(localStorage.getItem("timeZone"));
-      setDateFormat(localStorage.getItem("dateFormat"));
-    }, []);
-  
-    useEffect(() => {
-      if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) {
-        setMessages([
-          {
-            role: "system",
-            content: `You are in the timezone: ${timeZone}. 
+
+  useEffect(() => {
+    setPreferredCurrency(localStorage.getItem("preferredCurrency"));
+    setPreferredLanguage(localStorage.getItem("preferredLanguage"));
+    setTimeFormat(localStorage.getItem("timeFormat"));
+    setPreferredMeasurement(localStorage.getItem("preferredMeasurement"));
+    setTimeZone(localStorage.getItem("timeZone"));
+    setDateFormat(localStorage.getItem("dateFormat"));
+  }, []);
+
+  useEffect(() => {
+    if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) {
+      setMessages([
+        {
+          role: "system",
+          content: `You are in the timezone: ${timeZone}. 
             You use the time format ${timeFormat}. 
             You use the date format ${dateFormat} for all references of dates. 
             You use the ${preferredMeasurement} system. 
@@ -47,12 +48,13 @@ const InputOutputBackend: React.FC = () => {
             But in the case the user specifically states to answer in another language, do that. Speaking in 
             another language is not stating you should answer in that language. 
             Additionally, under no circumstances translate your answer into multiple languages.`,
-          },
-          { role: "assistant", content: "Hello! How can I help you?" },
-        ]);
-      }
-    }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);
-    
+        },
+        { role: "assistant", content: "Hello! How can I help you?" },
+      ]);
+    }
+  }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);
+
+
   const [copyClicked, setCopyClicked] = useState(false)
   const [accessToken, setAccessToken] = useState("")
   const postWorkerRef = useRef<Worker | null>(null)
@@ -61,7 +63,7 @@ const InputOutputBackend: React.FC = () => {
   const [inputMessage, setInputMessage] = useState<string>("")
   const [inputDisabled, setInputDisabled] = useState(false)
   const [isRecording, setIsRecording] = useState(false)
-  const mediaRecorderRef = useRef<MediaRecorder | null>(null) 
+  const mediaRecorderRef = useRef<MediaRecorder | null>(null)
   const audioChunks = useRef<Blob[]>([])
 
 
@@ -162,11 +164,6 @@ const InputOutputBackend: React.FC = () => {
     });
   };
 
-
-  useEffect(() => {
-    
-  },[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat])
-
   const addMessage = (role: string, content: string) => {
     setMessages(previous => [...previous, { role, content }])
   }
@@ -184,40 +181,46 @@ const InputOutputBackend: React.FC = () => {
     }
   }
 
-  const startRecording = async () => {
-    const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
-    const mediaRecorder = new MediaRecorder(stream)
-    mediaRecorderRef.current = mediaRecorder
-    
-    mediaRecorder.ondataavailable = (event) => {
-      audioChunks.current.push(event.data)
-    }
-        
-    mediaRecorder.onstop = async () => {
-      const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" })
-      audioChunks.current = []
-      // console.log(audioBlob);
-      // const url = URL.createObjectURL(audioBlob)
-      // const audio = new Audio(url);
-      // audio.play().catch(error => console.error("Error playing audio:", error));
+  const startRecording = async (): Promise<string> => {
+    const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    const mediaRecorder = new MediaRecorder(stream);
+    mediaRecorderRef.current = mediaRecorder;
 
-      const remote = new VoiceSend()
-      remote.sendToVoiceRecognition(audioBlob)
-    }
+    audioChunks.current = []; // Initialize audio chunks
+
+    // Create a promise that resolves when the onstop event is done
+    const stopRecordingPromise = new Promise<string>((resolve) => {
+      mediaRecorder.ondataavailable = (event) => {
+        audioChunks.current.push(event.data);
+      };
+
+      mediaRecorder.onstop = async () => {
+        const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" });
+        audioChunks.current = [];
+
+        const text_voice = await sendToVoiceRecognition(audioBlob);
+        console.log(text_voice);
+        resolve(text_voice); // Resolve the promise with the recognized text
+      };
+    });
+
+    mediaRecorder.start();
+    setIsRecording(true);
+
+    // Wait for the recording to stop and get the recognized text
+    return stopRecordingPromise;
+  };
 
-    mediaRecorder.start()
-    setIsRecording(true)
-  }
-    
   const stopRecording = () => {
-    mediaRecorderRef.current?.stop()
-    setIsRecording(false)
-  }
+    mediaRecorderRef.current?.stop();
+    setIsRecording(false);
+  };
 
-
-  const handleMicClick = () => {
+  const handleMicClick = async () => {
     if (!isRecording) {
-      startRecording();
+      const recognizedText = await startRecording();
+      setInputMessage(recognizedText); // Set the recognized text after recording
+      console.log("Set!")
     } else {
       stopRecording();
     }
@@ -261,10 +264,10 @@ const InputOutputBackend: React.FC = () => {
     await wait(1000)
     setCopyClicked(false)
   }
-  
+
 
   return (
-    <div>
+    <>
       <ConversationFrontend
         messages={messages}
         onResendClick={handleResendClick}
@@ -278,8 +281,8 @@ const InputOutputBackend: React.FC = () => {
         onMicClick={handleMicClick}
         inputDisabled={inputDisabled}
         isRecording={isRecording}
-      /> 
-    </div>
+      />
+    </>
   )
 }