app/backend/AudioRecorder(not yet).tsx

@@ -0,0 +1,39 @@
+// import React, { useState, useRef } from 'react'
+
+// const AudioRecorder: React.FC = () => {
+//     const [isRecording, setIsRecording] = useState(false)
+//     const [audioURL, setAudioURL] = useState<string | null>(null)
+//     const medaRecorderRef = useRef<MediaRecorder | null>(null)
+//     const audioChunks = useRef<Blob[]>([])
+
+//     const startRecording = async () => {
+//         const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+//         const mediaRecorder = new MediaRecorder(stream)
+//         medaRecorderRef.current = mediaRecorder
+
+//         mediaRecorder.ondataavailable = (event) => {
+//             audioChunks.current.push(event.data)
+//         }
+
+//         mediaRecorder.onstop = () => {
+//             const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" })
+//             const url = URL.createObjectURL(audioBlob)
+//             setAudioURL(url)
+//             audioChunks.current = []
+//         }
+
+//         mediaRecorder.start()
+//         setIsRecording(true)
+
+//         const stopRecording = () => {
+//             medaRecorderRef.current?.stop()
+//             setIsRecording(false)
+//         }
+
+//         return (
+//             <div></div>
+//         )
+//     }
+// }
+
+// export default AudioRecorder

app/backend/AudioRecorder.ts

@@ -1,34 +0,0 @@
-import React, { useState, useRef } from 'react'
-
-    export const AudioRecorder= () => {
-        const [isRecording, setIsRecording] = useState(false)
-        const [audioURL, setAudioURL] = useState<string | null>(null)
-        const mediaRecorderRef = useRef<MediaRecorder | null>(null)
-        const audioChunks = useRef<Blob[]>([])
-    
-        const startRecording = async () => {
-            const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
-            const mediaRecorder = new MediaRecorder(stream)
-            mediaRecorderRef.current = mediaRecorder
-        
-            mediaRecorder.ondataavailable = (event) => {
-                audioChunks.current.push(event.data)
-            }
-        
-            mediaRecorder.onstop = () => {
-                const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" })
-                const url = URL.createObjectURL(audioBlob)
-                setAudioURL(url)
-                audioChunks.current = []
-            }
-
-            mediaRecorder.start()
-            setIsRecording(true)
-        
-        }
-    
-        const stopRecording = () => {
-            mediaRecorderRef.current?.stop()
-           setIsRecording(false)
-        }
-    }

app/backend/InputOutputHandler.tsx

@@ -2,10 +2,8 @@
 import React, { useEffect, useRef, useState } from "react";
 import ConversationFrontend from "../components/ConversationFrontend";
 import InputFrontend from "../components/InputFrontend";
-import VoiceSend from "./voice_backend"
-import { AudioRecorder } from "./AudioRecorder";
 import axios from "axios";
-
+import { skip } from "node:test";
 
 const InputOutputBackend: React.FC = () => {
   type Message = {
@@ -20,11 +18,7 @@ const InputOutputBackend: React.FC = () => {
   const [liveMessage, setLiveMessage] = useState("")
   const [inputMessage, setInputMessage] = useState<string>("")
   const [inputDisabled, setInputDisabled] = useState(false)
-  const [isRecording, setIsRecording] = useState(false)
+  const [lastMessage, setLastMessage] = useState<Message>({ role: "user", content: "Not supposed to happen." })
-  const [audioURL, setAudioURL] = useState<string | null>(null)
-  const mediaRecorderRef = useRef<MediaRecorder | null>(null)
-  const audioChunks = useRef<Blob[]>([])
-
 
   console.log(messages);
 
@@ -129,6 +123,7 @@ const InputOutputBackend: React.FC = () => {
   }
   const handleSendClick = (inputValue: string, override: boolean) => {
     if (inputValue != "") {
+      console.log(inputDisabled)
       if (!inputDisabled || override) {
         setInputDisabled(true)
         if (postWorkerRef.current) {
@@ -141,43 +136,9 @@ const InputOutputBackend: React.FC = () => {
     }
   }
 
-  const startRecording = async () => {
-    const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
-    const mediaRecorder = new MediaRecorder(stream)
-    mediaRecorderRef.current = mediaRecorder
-    
-    mediaRecorder.ondataavailable = (event) => {
-      audioChunks.current.push(event.data)
-    }
-        
-    mediaRecorder.onstop = () => {
-      const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" })
-      const url = URL.createObjectURL(audioBlob)
-      console.log(url);
-      setAudioURL(url)
-      audioChunks.current = []
-      const remote = new VoiceSend()
-      remote.sendToVoiceRecognition(audioBlob,)
-    }
-
-    mediaRecorder.start()
-    setIsRecording(true)
-        
-    }
-    
-    const stopRecording = () => {
-      mediaRecorderRef.current?.stop()
-      setIsRecording(false)
-    }
-
-
   const handleMicClick = () => {
-    if (!isRecording) {
+    // do stuff
-      startRecording();
-    } else {
-      stopRecording();
   }
-  };
 
   const handleResendClick = () => {
     var temporary_message = messages[messages.length - 2]['content']
@@ -219,7 +180,6 @@ const InputOutputBackend: React.FC = () => {
         onSendClick={handleSendClick}
         onMicClick={handleMicClick}
         inputDisabled={inputDisabled}
-        isRecording={isRecording}
       />
     </div>
   )

app/backend/voice_backend.ts

@@ -1,30 +0,0 @@
-import axios from "axios";
-
-
-class VoiceSend {
-    sendToVoiceRecognition(audio_data: Blob) {
-        console.log("sending recording...");
-        console.log(typeof (audio_data));
-        console.log(audio_data instanceof Blob);
-
-        const formdata = new FormData()
-        formdata.append("audio", audio_data)
-        formdata.append("option", "offline")
-        formdata.append("type", "basic")
-
-        const dataSend = { option:"offline", type:"basic",audio:audio_data }
-        axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata)
-            .then((response) => {
-                console.log(response.data)
-                return response.data.response
-            })
-            .catch(error => {
-                console.log("Error calling API:", error)
-                postMessage({ status: 500 })
-            })
-    }
-
-}
-
-
-export default VoiceSend;

app/components/InputFrontend.tsx

@@ -1,16 +1,14 @@
 import React, { useState, ForwardedRef, useEffect } from 'react';
-import "../styles/variables.css"
 
 interface InputProps {
   message: string;
   onSendClick: (message: string, override: boolean) => void;
   onMicClick: () => void;
-  inputDisabled: boolean;
+  inputDisabled: boolean
-  isRecording:boolean
 }
 
 const InputFrontend = React.forwardRef<HTMLDivElement, InputProps>(
-  ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef<HTMLDivElement>) => {
+  ({ message, onSendClick, onMicClick, inputDisabled }, ref: ForwardedRef<HTMLDivElement>) => {
     const [inputValue, setInputValue] = useState('');
 
     useEffect(() => {
@@ -31,10 +29,6 @@ const InputFrontend = React.forwardRef<HTMLDivElement, InputProps>(
       }
     };
 
-    const styles = {
-      
-    }
-
     return (
       <div className="input" id="inputForm" ref={ref}>
         <input
@@ -48,7 +42,7 @@ const InputFrontend = React.forwardRef<HTMLDivElement, InputProps>(
         <button type="button" onClick={() => onSendClick(inputValue, false)} disabled={inputDisabled ? true : false}>
           <img src="/img/send.svg" alt="send" />
         </button>
-        <button className={`microphone-button ${isRecording ? "red":"green"}`} type="button" onClick={onMicClick}>
+        <button type="button" onClick={onMicClick}>
           <img src="/img/microphone.svg" alt="microphone" />
         </button>
       </div>

app/styles/input.css

@@ -59,19 +59,3 @@
     background-color: var(--input-button-hover-color);
     box-shadow: 0 6px 15px rgba(0, 0, 0, 0.2);
 }
-
-.microphone-button.red{
-    background-color: var(--close-button-color);
-}
-
-.microphone-button.green{
-    background-color: var(--button-background-color);
-}
-
-.microphone-button.red:hover{
-    background-color: var(--close-button-hover-color);
-}
-
-.microphone-button.green:hover{
-    background-color: var(--input-button-hover-color);
-}

app/styles/variables.css

@@ -19,7 +19,6 @@
     --conversation-background-color: #79832e; /* Background color for conversation container */
     --doc-background-color: #ffffff; /* Background color for documents */
     --close-button-color: red;
-    --close-button-hover-color: #9e0101; /*NEW*/
     --burger-menu-background-color: #79832e;  /*NEW*/
     --overlay-text-color:white; /*NEW*/
     

py/api.py

@@ -6,7 +6,6 @@ from ai import AI
 from db import DB
 from weather import Weather
 from voice import VoiceRecognition
-from tts import TTS
 
 
 class API:
@@ -18,7 +17,6 @@ class API:
         self.db = DB()
         self.weather = Weather()
         self.voice = VoiceRecognition()
-        self.tts = TTS()
         self.db.load_database()
         self.ai_response_lock = threading.Lock()
         CORS(self.app)
@@ -99,15 +97,12 @@ class API:
 
         @self.app.route('/interstellar_ai/api/voice_recognition', methods=['POST'])
         def voice_recognition():
-            print(request.args)
+            recognition_type = request.args.get('type')
-            recog_type = request.form.get('type')
+            audio = request.args.get('audio_data')
-            print(recog_type)
+            option = request.args.get('option')
-            audio = request.files.get('audio')
+            if recognition_type == "basic":
-            option = request.form.get('option')
+                return jsonify({'status': 200, 'response': self.voice.basic_recognition(audio, option)})
-            if recog_type == "basic":
+
-                text = self.voice.basic_recognition(audio, option)
-                return jsonify({'status': 200, 'response': text})
-            else:
             return jsonify({'status': 401, 'response': "Invalid type"})
 
         @self.app.route('/interstellar_ai/api/weather', methods=['POST'])
@@ -118,11 +113,6 @@ class API:
 
         self.app.run(debug=True, host='0.0.0.0', port=5000)
 
-        @self.app.route('/interstellar_ai/api/tts', methods=['POST'])
-        def tts():
-            text = request.args.get('text')
-            return jsonify({'status': 200, 'response': self.tts.gen_tts(text)})
-
 
 api = API()
 api.run()

py/requirements.txt

@@ -11,6 +11,3 @@ google-cloud-speech
 google-generativeai
 python-weather
 pycouchdb
-pyttsx3
-pip-licenses
-openai-whisper

py/tts.py

@@ -1,10 +0,0 @@
-import pyttsx3
-
-
-class TTS:
-    @staticmethod
-    def gen_tts(text):
-        engine = pyttsx3.init()
-        engine.setProperty('rate', 70)
-        engine.say(text)
-        engine.runAndWait()

py/voice.py

@@ -4,21 +4,10 @@ import speech_recognition as sr
 class VoiceRecognition:
     @staticmethod
     def basic_recognition(audio, option):
-        print(type(audio))
-        print("preparing")
         r = sr.Recognizer()
         if option == "online":
-            with sr.AudioFile(audio) as source:
+            return r.recognize_google_cloud(audio)
-                print("online")
-                text = r.recognize_google_cloud(source)
-                print("recognized as: " + text)
-                return text
         elif option == "offline":
-            with sr.AudioFile(audio) as source:
+            return r.recognize_sphinx(audio)
-                print("offline")
-                text = r.recognize_sphinx(source)
-                print("recognized as: " + text)
-                return text
 
-        print("nothing")
         return False