forked from React-Group/interstellar_ai
Bug fixes merge
This commit is contained in:
parent
5adb79e89d
commit
460f22abd8
1 changed files with 65 additions and 62 deletions
|
|
@ -2,7 +2,7 @@
|
|||
import React, { use, useEffect, useRef, useState } from "react";
|
||||
import ConversationFrontend from "../components/ConversationFrontend";
|
||||
import InputFrontend from "../components/InputFrontend";
|
||||
import VoiceSend from "./voice_backend"
|
||||
import { sendToVoiceRecognition } from "./voice_backend"
|
||||
import { AudioRecorder } from "./AudioRecorder";
|
||||
import axios from "axios";
|
||||
import { resolve } from "path";
|
||||
|
|
@ -11,6 +11,7 @@ import { fetchFile, toBlobURL } from "@ffmpeg/util"
|
|||
|
||||
|
||||
const InputOutputBackend: React.FC = () => {
|
||||
// # variables
|
||||
type Message = {
|
||||
role: string
|
||||
content: string
|
||||
|
|
@ -23,22 +24,22 @@ const InputOutputBackend: React.FC = () => {
|
|||
const [timeZone, setTimeZone] = useState<string | null>(null);
|
||||
const [dateFormat, setDateFormat] = useState<string | null>(null);
|
||||
const [messages, setMessages] = useState<Message[]>([]);
|
||||
|
||||
useEffect(() => {
|
||||
setPreferredCurrency(localStorage.getItem("preferredCurrency"));
|
||||
setPreferredLanguage(localStorage.getItem("preferredLanguage"));
|
||||
setTimeFormat(localStorage.getItem("timeFormat"));
|
||||
setPreferredMeasurement(localStorage.getItem("preferredMeasurement"));
|
||||
setTimeZone(localStorage.getItem("timeZone"));
|
||||
setDateFormat(localStorage.getItem("dateFormat"));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) {
|
||||
setMessages([
|
||||
{
|
||||
role: "system",
|
||||
content: `You are in the timezone: ${timeZone}.
|
||||
|
||||
useEffect(() => {
|
||||
setPreferredCurrency(localStorage.getItem("preferredCurrency"));
|
||||
setPreferredLanguage(localStorage.getItem("preferredLanguage"));
|
||||
setTimeFormat(localStorage.getItem("timeFormat"));
|
||||
setPreferredMeasurement(localStorage.getItem("preferredMeasurement"));
|
||||
setTimeZone(localStorage.getItem("timeZone"));
|
||||
setDateFormat(localStorage.getItem("dateFormat"));
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) {
|
||||
setMessages([
|
||||
{
|
||||
role: "system",
|
||||
content: `You are in the timezone: ${timeZone}.
|
||||
You use the time format ${timeFormat}.
|
||||
You use the date format ${dateFormat} for all references of dates.
|
||||
You use the ${preferredMeasurement} system.
|
||||
|
|
@ -47,12 +48,13 @@ const InputOutputBackend: React.FC = () => {
|
|||
But in the case the user specifically states to answer in another language, do that. Speaking in
|
||||
another language is not stating you should answer in that language.
|
||||
Additionally, under no circumstances translate your answer into multiple languages.`,
|
||||
},
|
||||
{ role: "assistant", content: "Hello! How can I help you?" },
|
||||
]);
|
||||
}
|
||||
}, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);
|
||||
|
||||
},
|
||||
{ role: "assistant", content: "Hello! How can I help you?" },
|
||||
]);
|
||||
}
|
||||
}, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);
|
||||
|
||||
|
||||
const [copyClicked, setCopyClicked] = useState(false)
|
||||
const [accessToken, setAccessToken] = useState("")
|
||||
const postWorkerRef = useRef<Worker | null>(null)
|
||||
|
|
@ -61,7 +63,7 @@ const InputOutputBackend: React.FC = () => {
|
|||
const [inputMessage, setInputMessage] = useState<string>("")
|
||||
const [inputDisabled, setInputDisabled] = useState(false)
|
||||
const [isRecording, setIsRecording] = useState(false)
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null)
|
||||
const mediaRecorderRef = useRef<MediaRecorder | null>(null)
|
||||
const audioChunks = useRef<Blob[]>([])
|
||||
|
||||
|
||||
|
|
@ -162,11 +164,6 @@ const InputOutputBackend: React.FC = () => {
|
|||
});
|
||||
};
|
||||
|
||||
|
||||
useEffect(() => {
|
||||
|
||||
},[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat])
|
||||
|
||||
const addMessage = (role: string, content: string) => {
|
||||
setMessages(previous => [...previous, { role, content }])
|
||||
}
|
||||
|
|
@ -184,40 +181,46 @@ const InputOutputBackend: React.FC = () => {
|
|||
}
|
||||
}
|
||||
|
||||
const startRecording = async () => {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true })
|
||||
const mediaRecorder = new MediaRecorder(stream)
|
||||
mediaRecorderRef.current = mediaRecorder
|
||||
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
audioChunks.current.push(event.data)
|
||||
}
|
||||
|
||||
mediaRecorder.onstop = async () => {
|
||||
const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" })
|
||||
audioChunks.current = []
|
||||
// console.log(audioBlob);
|
||||
// const url = URL.createObjectURL(audioBlob)
|
||||
// const audio = new Audio(url);
|
||||
// audio.play().catch(error => console.error("Error playing audio:", error));
|
||||
const startRecording = async (): Promise<string> => {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
const mediaRecorder = new MediaRecorder(stream);
|
||||
mediaRecorderRef.current = mediaRecorder;
|
||||
|
||||
const remote = new VoiceSend()
|
||||
remote.sendToVoiceRecognition(audioBlob)
|
||||
}
|
||||
audioChunks.current = []; // Initialize audio chunks
|
||||
|
||||
// Create a promise that resolves when the onstop event is done
|
||||
const stopRecordingPromise = new Promise<string>((resolve) => {
|
||||
mediaRecorder.ondataavailable = (event) => {
|
||||
audioChunks.current.push(event.data);
|
||||
};
|
||||
|
||||
mediaRecorder.onstop = async () => {
|
||||
const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" });
|
||||
audioChunks.current = [];
|
||||
|
||||
const text_voice = await sendToVoiceRecognition(audioBlob);
|
||||
console.log(text_voice);
|
||||
resolve(text_voice); // Resolve the promise with the recognized text
|
||||
};
|
||||
});
|
||||
|
||||
mediaRecorder.start();
|
||||
setIsRecording(true);
|
||||
|
||||
// Wait for the recording to stop and get the recognized text
|
||||
return stopRecordingPromise;
|
||||
};
|
||||
|
||||
mediaRecorder.start()
|
||||
setIsRecording(true)
|
||||
}
|
||||
|
||||
const stopRecording = () => {
|
||||
mediaRecorderRef.current?.stop()
|
||||
setIsRecording(false)
|
||||
}
|
||||
mediaRecorderRef.current?.stop();
|
||||
setIsRecording(false);
|
||||
};
|
||||
|
||||
|
||||
const handleMicClick = () => {
|
||||
const handleMicClick = async () => {
|
||||
if (!isRecording) {
|
||||
startRecording();
|
||||
const recognizedText = await startRecording();
|
||||
setInputMessage(recognizedText); // Set the recognized text after recording
|
||||
console.log("Set!")
|
||||
} else {
|
||||
stopRecording();
|
||||
}
|
||||
|
|
@ -261,10 +264,10 @@ const InputOutputBackend: React.FC = () => {
|
|||
await wait(1000)
|
||||
setCopyClicked(false)
|
||||
}
|
||||
|
||||
|
||||
|
||||
return (
|
||||
<div>
|
||||
<>
|
||||
<ConversationFrontend
|
||||
messages={messages}
|
||||
onResendClick={handleResendClick}
|
||||
|
|
@ -278,8 +281,8 @@ const InputOutputBackend: React.FC = () => {
|
|||
onMicClick={handleMicClick}
|
||||
inputDisabled={inputDisabled}
|
||||
isRecording={isRecording}
|
||||
/>
|
||||
</div>
|
||||
/>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue