diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 1316901..fa8dcf6 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -1,11 +1,13 @@ "use client" -import React, { useEffect, useRef, useState } from "react"; +import React, { use, useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; import VoiceSend from "./voice_backend" import { AudioRecorder } from "./AudioRecorder"; import axios from "axios"; import { resolve } from "path"; +import { FFmpeg } from "@ffmpeg/ffmpeg"; +import { fetchFile, toBlobURL } from "@ffmpeg/util" const InputOutputBackend: React.FC = () => { @@ -125,6 +127,17 @@ const InputOutputBackend: React.FC = () => { }); }; + /* Variables for System-prompt */ + const [preferredCurrency, setPreferredCurrency] = useState(localStorage.getItem("preferredCurrency") || "") + const [preferredLanguage, setPreferredLanguage] = useState(localStorage.getItem("preferredLanguage") || "") + const [timeFormat, setTimeFormat] = useState(localStorage.getItem("timeFormat") || "") + const [preferredMeasurement, setPreferredMeasurement] = useState(localStorage.getItem("preferredMeasurement") || "") + const [timeZone, setTimeZone] = useState(localStorage.getItem("timeZone") || "") + const [dateFormat, setDateFormat] = useState(localStorage.getItem("dateFormat") || "") + + useEffect(() => { + + },[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat]) const addMessage = (role: string, content: string) => { setMessages(previous => [...previous, { role, content }]) @@ -152,25 +165,52 @@ const InputOutputBackend: React.FC = () => { audioChunks.current.push(event.data) } - mediaRecorder.onstop = () => { + mediaRecorder.onstop = async () => { const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }) + console.log(audioBlob); const url = URL.createObjectURL(audioBlob) + const audio = new Audio(url); + audio.play().catch(error => console.error("Error playing audio:", error)); console.log(url); setAudioURL(url) audioChunks.current = [] + const wavBlob = await convertOggToWav(audioBlob) const remote = new VoiceSend() - remote.sendToVoiceRecognition(audioBlob,) + remote.sendToVoiceRecognition(wavBlob) } mediaRecorder.start() setIsRecording(true) - + } + + const ffmpegRef = useRef(null) + const audioRef = useRef("") + + const loadFFmpeg = async () => { + if (!ffmpegRef.current) { + ffmpegRef.current = new FFmpeg() + await ffmpegRef.current.load() } + } + + const convertOggToWav = async (oggFile: File | Blob) => { + await loadFFmpeg() + + const ffmpeg = ffmpegRef.current! + + await ffmpeg.writeFile("input.ogg", await fetchFile(oggFile)) + await ffmpeg.exec(["-i", "input.ogg", "output.wav"]) + const wavData = await ffmpeg.readFile("output.wav") + console.log(wavData); + const wavBlob = new Blob([wavData], { type: "audio/wav" }) + audioRef.current = URL.createObjectURL(wavBlob) + return wavBlob + } - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } + const stopRecording = () => { + mediaRecorderRef.current?.stop() + setIsRecording(false) + } const handleMicClick = () => { diff --git a/app/layout.tsx b/app/layout.tsx index 8070a08..0896d49 100644 --- a/app/layout.tsx +++ b/app/layout.tsx @@ -12,7 +12,7 @@ export default function RootLayout({ children }: { children: ReactNode }) { {metadata.title} {/* Tried adding the favicon here */} - +
{children}