diff --git a/app/backend/AudioRecorder(not yet).tsx b/app/backend/AudioRecorder(not yet).tsx new file mode 100644 index 0000000..af6a941 --- /dev/null +++ b/app/backend/AudioRecorder(not yet).tsx @@ -0,0 +1,39 @@ +// import React, { useState, useRef } from 'react' + +// const AudioRecorder: React.FC = () => { +// const [isRecording, setIsRecording] = useState(false) +// const [audioURL, setAudioURL] = useState(null) +// const medaRecorderRef = useRef(null) +// const audioChunks = useRef([]) + +// const startRecording = async () => { +// const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) +// const mediaRecorder = new MediaRecorder(stream) +// medaRecorderRef.current = mediaRecorder + +// mediaRecorder.ondataavailable = (event) => { +// audioChunks.current.push(event.data) +// } + +// mediaRecorder.onstop = () => { +// const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) +// const url = URL.createObjectURL(audioBlob) +// setAudioURL(url) +// audioChunks.current = [] +// } + +// mediaRecorder.start() +// setIsRecording(true) + +// const stopRecording = () => { +// medaRecorderRef.current?.stop() +// setIsRecording(false) +// } + +// return ( +//
+// ) +// } +// } + +// export default AudioRecorder \ No newline at end of file diff --git a/app/backend/AudioRecorder.ts b/app/backend/AudioRecorder.ts deleted file mode 100644 index 459674e..0000000 --- a/app/backend/AudioRecorder.ts +++ /dev/null @@ -1,34 +0,0 @@ -import React, { useState, useRef } from 'react' - - export const AudioRecorder= () => { - const [isRecording, setIsRecording] = useState(false) - const [audioURL, setAudioURL] = useState(null) - const mediaRecorderRef = useRef(null) - const audioChunks = useRef([]) - - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - const mediaRecorder = new MediaRecorder(stream) - mediaRecorderRef.current = mediaRecorder - - mediaRecorder.ondataavailable = (event) => { - audioChunks.current.push(event.data) - } - - mediaRecorder.onstop = () => { - const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) - const url = URL.createObjectURL(audioBlob) - setAudioURL(url) - audioChunks.current = [] - } - - mediaRecorder.start() - setIsRecording(true) - - } - - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } - } \ No newline at end of file diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 36b3fac..02a227b 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -2,10 +2,8 @@ import React, { useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; -import VoiceSend from "./voice_backend" -import { AudioRecorder } from "./AudioRecorder"; import axios from "axios"; - +import { skip } from "node:test"; const InputOutputBackend: React.FC = () => { type Message = { @@ -20,11 +18,7 @@ const InputOutputBackend: React.FC = () => { const [liveMessage, setLiveMessage] = useState("") const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) - const [isRecording, setIsRecording] = useState(false) - const [audioURL, setAudioURL] = useState(null) - const mediaRecorderRef = useRef(null) - const audioChunks = useRef([]) - + const [lastMessage, setLastMessage] = useState({ role: "user", content: "Not supposed to happen." }) console.log(messages); @@ -129,6 +123,7 @@ const InputOutputBackend: React.FC = () => { } const handleSendClick = (inputValue: string, override: boolean) => { if (inputValue != "") { + console.log(inputDisabled) if (!inputDisabled || override) { setInputDisabled(true) if (postWorkerRef.current) { @@ -141,43 +136,9 @@ const InputOutputBackend: React.FC = () => { } } - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - const mediaRecorder = new MediaRecorder(stream) - mediaRecorderRef.current = mediaRecorder - - mediaRecorder.ondataavailable = (event) => { - audioChunks.current.push(event.data) - } - - mediaRecorder.onstop = () => { - const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) - const url = URL.createObjectURL(audioBlob) - console.log(url); - setAudioURL(url) - audioChunks.current = [] - const remote = new VoiceSend() - remote.sendToVoiceRecognition(audioBlob,) - } - - mediaRecorder.start() - setIsRecording(true) - - } - - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } - - const handleMicClick = () => { - if (!isRecording) { - startRecording(); - } else { - stopRecording(); - } - }; + // do stuff + } const handleResendClick = () => { var temporary_message = messages[messages.length - 2]['content'] @@ -219,7 +180,6 @@ const InputOutputBackend: React.FC = () => { onSendClick={handleSendClick} onMicClick={handleMicClick} inputDisabled={inputDisabled} - isRecording={isRecording} /> ) diff --git a/app/backend/voice_backend.ts b/app/backend/voice_backend.ts deleted file mode 100644 index a93fd89..0000000 --- a/app/backend/voice_backend.ts +++ /dev/null @@ -1,30 +0,0 @@ -import axios from "axios"; - - -class VoiceSend { - sendToVoiceRecognition(audio_data: Blob) { - console.log("sending recording..."); - console.log(typeof (audio_data)); - console.log(audio_data instanceof Blob); - - const formdata = new FormData() - formdata.append("audio", audio_data) - formdata.append("option", "offline") - formdata.append("type", "basic") - - const dataSend = { option:"offline", type:"basic",audio:audio_data } - axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata) - .then((response) => { - console.log(response.data) - return response.data.response - }) - .catch(error => { - console.log("Error calling API:", error) - postMessage({ status: 500 }) - }) - } - -} - - -export default VoiceSend; \ No newline at end of file diff --git a/app/components/InputFrontend.tsx b/app/components/InputFrontend.tsx index c84124b..e50e916 100644 --- a/app/components/InputFrontend.tsx +++ b/app/components/InputFrontend.tsx @@ -1,16 +1,14 @@ import React, { useState, ForwardedRef, useEffect } from 'react'; -import "../styles/variables.css" interface InputProps { message: string; onSendClick: (message: string, override: boolean) => void; onMicClick: () => void; - inputDisabled: boolean; - isRecording:boolean + inputDisabled: boolean } const InputFrontend = React.forwardRef( - ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef) => { + ({ message, onSendClick, onMicClick, inputDisabled }, ref: ForwardedRef) => { const [inputValue, setInputValue] = useState(''); useEffect(() => { @@ -31,10 +29,6 @@ const InputFrontend = React.forwardRef( } }; - const styles = { - - } - return (
( -
diff --git a/app/styles/input.css b/app/styles/input.css index ffbffeb..ceeb0b3 100644 --- a/app/styles/input.css +++ b/app/styles/input.css @@ -59,19 +59,3 @@ background-color: var(--input-button-hover-color); box-shadow: 0 6px 15px rgba(0, 0, 0, 0.2); } - -.microphone-button.red{ - background-color: var(--close-button-color); -} - -.microphone-button.green{ - background-color: var(--button-background-color); -} - -.microphone-button.red:hover{ - background-color: var(--close-button-hover-color); -} - -.microphone-button.green:hover{ - background-color: var(--input-button-hover-color); -} \ No newline at end of file diff --git a/app/styles/variables.css b/app/styles/variables.css index 795b4fc..72c464a 100644 --- a/app/styles/variables.css +++ b/app/styles/variables.css @@ -19,7 +19,6 @@ --conversation-background-color: #79832e; /* Background color for conversation container */ --doc-background-color: #ffffff; /* Background color for documents */ --close-button-color: red; - --close-button-hover-color: #9e0101; /*NEW*/ --burger-menu-background-color: #79832e; /*NEW*/ --overlay-text-color:white; /*NEW*/ diff --git a/py/api.py b/py/api.py index 0951717..01e99b1 100644 --- a/py/api.py +++ b/py/api.py @@ -6,7 +6,6 @@ from ai import AI from db import DB from weather import Weather from voice import VoiceRecognition -from tts import TTS class API: @@ -18,7 +17,6 @@ class API: self.db = DB() self.weather = Weather() self.voice = VoiceRecognition() - self.tts = TTS() self.db.load_database() self.ai_response_lock = threading.Lock() CORS(self.app) @@ -99,16 +97,13 @@ class API: @self.app.route('/interstellar_ai/api/voice_recognition', methods=['POST']) def voice_recognition(): - print(request.args) - recog_type = request.form.get('type') - print(recog_type) - audio = request.files.get('audio') - option = request.form.get('option') - if recog_type == "basic": - text = self.voice.basic_recognition(audio, option) - return jsonify({'status': 200, 'response': text}) - else: - return jsonify({'status': 401, 'response': "Invalid type"}) + recognition_type = request.args.get('type') + audio = request.args.get('audio_data') + option = request.args.get('option') + if recognition_type == "basic": + return jsonify({'status': 200, 'response': self.voice.basic_recognition(audio, option)}) + + return jsonify({'status': 401, 'response': "Invalid type"}) @self.app.route('/interstellar_ai/api/weather', methods=['POST']) def get_weather(): @@ -118,11 +113,6 @@ class API: self.app.run(debug=True, host='0.0.0.0', port=5000) - @self.app.route('/interstellar_ai/api/tts', methods=['POST']) - def tts(): - text = request.args.get('text') - return jsonify({'status': 200, 'response': self.tts.gen_tts(text)}) - api = API() api.run() diff --git a/py/requirements.txt b/py/requirements.txt index 8c6a016..6cd3616 100644 --- a/py/requirements.txt +++ b/py/requirements.txt @@ -10,7 +10,4 @@ PocketSphinx google-cloud-speech google-generativeai python-weather -pycouchdb -pyttsx3 -pip-licenses -openai-whisper \ No newline at end of file +pycouchdb \ No newline at end of file diff --git a/py/tts.py b/py/tts.py deleted file mode 100644 index 93f7fa4..0000000 --- a/py/tts.py +++ /dev/null @@ -1,10 +0,0 @@ -import pyttsx3 - - -class TTS: - @staticmethod - def gen_tts(text): - engine = pyttsx3.init() - engine.setProperty('rate', 70) - engine.say(text) - engine.runAndWait() diff --git a/py/voice.py b/py/voice.py index ec6c407..7ead0a5 100644 --- a/py/voice.py +++ b/py/voice.py @@ -4,21 +4,10 @@ import speech_recognition as sr class VoiceRecognition: @staticmethod def basic_recognition(audio, option): - print(type(audio)) - print("preparing") r = sr.Recognizer() if option == "online": - with sr.AudioFile(audio) as source: - print("online") - text = r.recognize_google_cloud(source) - print("recognized as: " + text) - return text + return r.recognize_google_cloud(audio) elif option == "offline": - with sr.AudioFile(audio) as source: - print("offline") - text = r.recognize_sphinx(source) - print("recognized as: " + text) - return text + return r.recognize_sphinx(audio) - print("nothing") return False