sageTheDm/interstellar_ai
1
0
Fork 0
forked from React-Group/interstellar_ai
Code Pull requests Activity

Bug fixes merge

Browse source
This commit is contained in:
sageTheDM 2024-09-30 12:52:58 +02:00
parent 5adb79e89d
commit 460f22abd8
1 changed files with 65 additions and 62 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

109
app/backend/InputOutputHandler.tsx
View file

@ -2,7 +2,7 @@
import React, { use, useEffect, useRef, useState } from "react";
import ConversationFrontend from "../components/ConversationFrontend";
import InputFrontend from "../components/InputFrontend";
import VoiceSend from "./voice_backend"
import { sendToVoiceRecognition } from "./voice_backend"
import { AudioRecorder } from "./AudioRecorder";
import axios from "axios";
import { resolve } from "path";
@ -11,6 +11,7 @@ import { fetchFile, toBlobURL } from "@ffmpeg/util"
const InputOutputBackend: React.FC = () => {
// # variables
type Message = {
role: string
content: string
@ -24,21 +25,21 @@ const InputOutputBackend: React.FC = () => {
const [dateFormat, setDateFormat] = useState<string | null>(null);
const [messages, setMessages] = useState<Message[]>([]);
useEffect(() => {
setPreferredCurrency(localStorage.getItem("preferredCurrency"));
setPreferredLanguage(localStorage.getItem("preferredLanguage"));
setTimeFormat(localStorage.getItem("timeFormat"));
setPreferredMeasurement(localStorage.getItem("preferredMeasurement"));
setTimeZone(localStorage.getItem("timeZone"));
setDateFormat(localStorage.getItem("dateFormat"));
}, []);
useEffect(() => {
setPreferredCurrency(localStorage.getItem("preferredCurrency"));
setPreferredLanguage(localStorage.getItem("preferredLanguage"));
setTimeFormat(localStorage.getItem("timeFormat"));
setPreferredMeasurement(localStorage.getItem("preferredMeasurement"));
setTimeZone(localStorage.getItem("timeZone"));
setDateFormat(localStorage.getItem("dateFormat"));
}, []);
useEffect(() => {
if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) {
setMessages([
{
role: "system",
content: `You are in the timezone: ${timeZone}.
useEffect(() => {
if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) {
setMessages([
{
role: "system",
content: `You are in the timezone: ${timeZone}.
You use the time format ${timeFormat}.
You use the date format ${dateFormat} for all references of dates.
You use the ${preferredMeasurement} system.
@ -47,11 +48,12 @@ const InputOutputBackend: React.FC = () => {
But in the case the user specifically states to answer in another language, do that. Speaking in
another language is not stating you should answer in that language.
Additionally, under no circumstances translate your answer into multiple languages.`,
},
{ role: "assistant", content: "Hello! How can I help you?" },
]);
}
}, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);
},
{ role: "assistant", content: "Hello! How can I help you?" },
]);
}
}, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);
const [copyClicked, setCopyClicked] = useState(false)
const [accessToken, setAccessToken] = useState("")
@ -162,11 +164,6 @@ const InputOutputBackend: React.FC = () => {
});
};
useEffect(() => {
},[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat])
const addMessage = (role: string, content: string) => {
setMessages(previous => [...previous, { role, content }])
}
@ -184,40 +181,46 @@ const InputOutputBackend: React.FC = () => {
}
}
const startRecording = async () => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
const mediaRecorder = new MediaRecorder(stream)
mediaRecorderRef.current = mediaRecorder
const startRecording = async (): Promise<string> => {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const mediaRecorder = new MediaRecorder(stream);
mediaRecorderRef.current = mediaRecorder;
mediaRecorder.ondataavailable = (event) => {
audioChunks.current.push(event.data)
}
audioChunks.current = []; // Initialize audio chunks
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" })
audioChunks.current = []
// console.log(audioBlob);
// const url = URL.createObjectURL(audioBlob)
// const audio = new Audio(url);
// audio.play().catch(error => console.error("Error playing audio:", error));
// Create a promise that resolves when the onstop event is done
const stopRecordingPromise = new Promise<string>((resolve) => {
mediaRecorder.ondataavailable = (event) => {
audioChunks.current.push(event.data);
};
const remote = new VoiceSend()
remote.sendToVoiceRecognition(audioBlob)
}
mediaRecorder.onstop = async () => {
const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" });
audioChunks.current = [];
mediaRecorder.start()
setIsRecording(true)
}
const text_voice = await sendToVoiceRecognition(audioBlob);
console.log(text_voice);
resolve(text_voice); // Resolve the promise with the recognized text
};
});
mediaRecorder.start();
setIsRecording(true);
// Wait for the recording to stop and get the recognized text
return stopRecordingPromise;
};
const stopRecording = () => {
mediaRecorderRef.current?.stop()
setIsRecording(false)
}
mediaRecorderRef.current?.stop();
setIsRecording(false);
};
const handleMicClick = () => {
const handleMicClick = async () => {
if (!isRecording) {
startRecording();
const recognizedText = await startRecording();
setInputMessage(recognizedText); // Set the recognized text after recording
console.log("Set!")
} else {
stopRecording();
}
@ -264,7 +267,7 @@ const InputOutputBackend: React.FC = () => {
return (
<div>
<>
<ConversationFrontend
messages={messages}
onResendClick={handleResendClick}
@ -279,7 +282,7 @@ const InputOutputBackend: React.FC = () => {
inputDisabled={inputDisabled}
isRecording={isRecording}
/>
</div>
</>
)
}