YasinOnm08/interstellar_ai
0
0
Fork 0
forked from React-Group/interstellar_ai
Code Pull requests Activity

voice recognition progress 1

Browse source
This commit is contained in:
YasinOnm08 2024-09-27 10:57:18 +02:00
parent 9663f49dee
commit e1f77c5b17
2 changed files with 49 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
app/backend/InputOutputHandler.tsx
View file

@ -1,11 +1,13 @@
"use client" "use client"
import React, { useEffect, useRef, useState } from "react"; import React, { use, useEffect, useRef, useState } from "react";
import ConversationFrontend from "../components/ConversationFrontend"; import ConversationFrontend from "../components/ConversationFrontend";
import InputFrontend from "../components/InputFrontend"; import InputFrontend from "../components/InputFrontend";
import VoiceSend from "./voice_backend" import VoiceSend from "./voice_backend"
import { AudioRecorder } from "./AudioRecorder"; import { AudioRecorder } from "./AudioRecorder";
import axios from "axios"; import axios from "axios";
import { resolve } from "path"; import { resolve } from "path";
import { FFmpeg } from "@ffmpeg/ffmpeg";
import { fetchFile, toBlobURL } from "@ffmpeg/util"
const InputOutputBackend: React.FC = () => { const InputOutputBackend: React.FC = () => {
@ -125,6 +127,17 @@ const InputOutputBackend: React.FC = () => {
}); });
}; };
/* Variables for System-prompt */
const [preferredCurrency, setPreferredCurrency] = useState(localStorage.getItem("preferredCurrency") || "")
const [preferredLanguage, setPreferredLanguage] = useState(localStorage.getItem("preferredLanguage") || "")
const [timeFormat, setTimeFormat] = useState(localStorage.getItem("timeFormat") || "")
const [preferredMeasurement, setPreferredMeasurement] = useState(localStorage.getItem("preferredMeasurement") || "")
const [timeZone, setTimeZone] = useState(localStorage.getItem("timeZone") || "")
const [dateFormat, setDateFormat] = useState(localStorage.getItem("dateFormat") || "")
useEffect(() => {
},[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat])
const addMessage = (role: string, content: string) => { const addMessage = (role: string, content: string) => {
setMessages(previous => [...previous, { role, content }]) setMessages(previous => [...previous, { role, content }])
@ -152,19 +165,46 @@ const InputOutputBackend: React.FC = () => {
audioChunks.current.push(event.data) audioChunks.current.push(event.data)
} }
mediaRecorder.onstop = () => { mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }) const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" })
console.log(audioBlob);
const url = URL.createObjectURL(audioBlob) const url = URL.createObjectURL(audioBlob)
const audio = new Audio(url);
audio.play().catch(error => console.error("Error playing audio:", error));
console.log(url); console.log(url);
setAudioURL(url) setAudioURL(url)
audioChunks.current = [] audioChunks.current = []
const wavBlob = await convertOggToWav(audioBlob)
const remote = new VoiceSend() const remote = new VoiceSend()
remote.sendToVoiceRecognition(audioBlob,) remote.sendToVoiceRecognition(wavBlob)
} }
mediaRecorder.start() mediaRecorder.start()
setIsRecording(true) setIsRecording(true)
}
const ffmpegRef = useRef<FFmpeg | null>(null)
const audioRef = useRef("")
const loadFFmpeg = async () => {
if (!ffmpegRef.current) {
ffmpegRef.current = new FFmpeg()
await ffmpegRef.current.load()
}
}
const convertOggToWav = async (oggFile: File | Blob) => {
await loadFFmpeg()
const ffmpeg = ffmpegRef.current!
await ffmpeg.writeFile("input.ogg", await fetchFile(oggFile))
await ffmpeg.exec(["-i", "input.ogg", "output.wav"])
const wavData = await ffmpeg.readFile("output.wav")
console.log(wavData);
const wavBlob = new Blob([wavData], { type: "audio/wav" })
audioRef.current = URL.createObjectURL(wavBlob)
return wavBlob
} }
const stopRecording = () => { const stopRecording = () => {

2
app/layout.tsx
View file

@ -12,7 +12,7 @@ export default function RootLayout({ children }: { children: ReactNode }) {
<title>{metadata.title}</title> <title>{metadata.title}</title>
<meta name="description" content={metadata.description} /> <meta name="description" content={metadata.description} />
{/* Tried adding the favicon here */} {/* Tried adding the favicon here */}
<link rel="icon" href="./public/favicon.ico" type="image/x-icon" /> <link rel="icon" href="../public/favicon.ico" type="image/x-icon" />
</head> </head>
<body> <body>
<main>{children}</main> <main>{children}</main>