From 5adb79e89d28434e6692ed64745707c49271392d Mon Sep 17 00:00:00 2001 From: sageTheDM Date: Mon, 30 Sep 2024 12:45:26 +0200 Subject: [PATCH 1/3] System Prompt update --- app/backend/InputOutputHandler.tsx | 61 ++++++++++++++++++------------ 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 4fa0042..2a33591 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -16,36 +16,47 @@ const InputOutputBackend: React.FC = () => { content: string } - /* Variables for System-prompt */ - const [preferredCurrency, setPreferredCurrency] = useState("") - const [preferredLanguage, setPreferredLanguage] = useState("") - const [timeFormat, setTimeFormat] = useState("") - const [preferredMeasurement, setPreferredMeasurement] = useState("") - const [timeZone, setTimeZone] = useState("") - const [dateFormat, setDateFormat] = useState("") - - useEffect(() => { - setPreferredCurrency(localStorage.getItem("preferredCurrency")) - setPreferredLanguage(localStorage.getItem("preferredLanguage")) - setTimeFormat(localStorage.getItem("timeFormat")) - setPreferredMeasurement(localStorage.getItem("preferredMeasurement")) - setTimeZone(localStorage.getItem("timeZone")) - setDateFormat(localStorage.getItem("dateFormat")) - }) + const [preferredCurrency, setPreferredCurrency] = useState(null); + const [preferredLanguage, setPreferredLanguage] = useState(null); + const [timeFormat, setTimeFormat] = useState(null); + const [preferredMeasurement, setPreferredMeasurement] = useState(null); + const [timeZone, setTimeZone] = useState(null); + const [dateFormat, setDateFormat] = useState(null); + const [messages, setMessages] = useState([]); + + useEffect(() => { + setPreferredCurrency(localStorage.getItem("preferredCurrency")); + setPreferredLanguage(localStorage.getItem("preferredLanguage")); + setTimeFormat(localStorage.getItem("timeFormat")); + setPreferredMeasurement(localStorage.getItem("preferredMeasurement")); + setTimeZone(localStorage.getItem("timeZone")); + setDateFormat(localStorage.getItem("dateFormat")); + }, []); + useEffect(() => { + if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) { + setMessages([ + { + role: "system", + content: `You are in the timezone: ${timeZone}. + You use the time format ${timeFormat}. + You use the date format ${dateFormat} for all references of dates. + You use the ${preferredMeasurement} system. + You use the currency ${preferredCurrency}. + You will only answer in the language (you will receive the country code) ${preferredLanguage}. + But in the case the user specifically states to answer in another language, do that. Speaking in + another language is not stating you should answer in that language. + Additionally, under no circumstances translate your answer into multiple languages.`, + }, + { role: "assistant", content: "Hello! How can I help you?" }, + ]); + } + }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]); + const [copyClicked, setCopyClicked] = useState(false) const [accessToken, setAccessToken] = useState("") const postWorkerRef = useRef(null) const getWorkerRef = useRef(null) - const [messages, setMessages] = useState([{ role: "system", - content: `You are in the timezone: ${timeZone}. - You use the time format ${timeFormat}. - You use the date format ${dateFormat} for all references of dates. - You use the ${preferredMeasurement} system. You use the currency ${preferredCurrency}. - You will only answer in the language (you will receive the country code) ${preferredLanguage}. - But in the case the user specifically states to answer in an other language do that speaking in a - nother language is not stating you should answer in that language. Additionally do not translate your answer into multiple languages` - },{ role: "assistant", content: "Hello! How can I help you?" }]) const [liveMessage, setLiveMessage] = useState("") const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) From 460f22abd8bce615b7a44bfa836132cb2b1e5553 Mon Sep 17 00:00:00 2001 From: sageTheDM Date: Mon, 30 Sep 2024 12:52:58 +0200 Subject: [PATCH 2/3] Bug fixes merge --- app/backend/InputOutputHandler.tsx | 127 +++++++++++++++-------------- 1 file changed, 65 insertions(+), 62 deletions(-) diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 2a33591..92fa6df 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -2,7 +2,7 @@ import React, { use, useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; -import VoiceSend from "./voice_backend" +import { sendToVoiceRecognition } from "./voice_backend" import { AudioRecorder } from "./AudioRecorder"; import axios from "axios"; import { resolve } from "path"; @@ -11,6 +11,7 @@ import { fetchFile, toBlobURL } from "@ffmpeg/util" const InputOutputBackend: React.FC = () => { + // # variables type Message = { role: string content: string @@ -23,22 +24,22 @@ const InputOutputBackend: React.FC = () => { const [timeZone, setTimeZone] = useState(null); const [dateFormat, setDateFormat] = useState(null); const [messages, setMessages] = useState([]); - - useEffect(() => { - setPreferredCurrency(localStorage.getItem("preferredCurrency")); - setPreferredLanguage(localStorage.getItem("preferredLanguage")); - setTimeFormat(localStorage.getItem("timeFormat")); - setPreferredMeasurement(localStorage.getItem("preferredMeasurement")); - setTimeZone(localStorage.getItem("timeZone")); - setDateFormat(localStorage.getItem("dateFormat")); - }, []); - - useEffect(() => { - if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) { - setMessages([ - { - role: "system", - content: `You are in the timezone: ${timeZone}. + + useEffect(() => { + setPreferredCurrency(localStorage.getItem("preferredCurrency")); + setPreferredLanguage(localStorage.getItem("preferredLanguage")); + setTimeFormat(localStorage.getItem("timeFormat")); + setPreferredMeasurement(localStorage.getItem("preferredMeasurement")); + setTimeZone(localStorage.getItem("timeZone")); + setDateFormat(localStorage.getItem("dateFormat")); + }, []); + + useEffect(() => { + if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) { + setMessages([ + { + role: "system", + content: `You are in the timezone: ${timeZone}. You use the time format ${timeFormat}. You use the date format ${dateFormat} for all references of dates. You use the ${preferredMeasurement} system. @@ -47,12 +48,13 @@ const InputOutputBackend: React.FC = () => { But in the case the user specifically states to answer in another language, do that. Speaking in another language is not stating you should answer in that language. Additionally, under no circumstances translate your answer into multiple languages.`, - }, - { role: "assistant", content: "Hello! How can I help you?" }, - ]); - } - }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]); - + }, + { role: "assistant", content: "Hello! How can I help you?" }, + ]); + } + }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]); + + const [copyClicked, setCopyClicked] = useState(false) const [accessToken, setAccessToken] = useState("") const postWorkerRef = useRef(null) @@ -61,7 +63,7 @@ const InputOutputBackend: React.FC = () => { const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) const [isRecording, setIsRecording] = useState(false) - const mediaRecorderRef = useRef(null) + const mediaRecorderRef = useRef(null) const audioChunks = useRef([]) @@ -162,11 +164,6 @@ const InputOutputBackend: React.FC = () => { }); }; - - useEffect(() => { - - },[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat]) - const addMessage = (role: string, content: string) => { setMessages(previous => [...previous, { role, content }]) } @@ -184,40 +181,46 @@ const InputOutputBackend: React.FC = () => { } } - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - const mediaRecorder = new MediaRecorder(stream) - mediaRecorderRef.current = mediaRecorder - - mediaRecorder.ondataavailable = (event) => { - audioChunks.current.push(event.data) - } - - mediaRecorder.onstop = async () => { - const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }) - audioChunks.current = [] - // console.log(audioBlob); - // const url = URL.createObjectURL(audioBlob) - // const audio = new Audio(url); - // audio.play().catch(error => console.error("Error playing audio:", error)); + const startRecording = async (): Promise => { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mediaRecorder = new MediaRecorder(stream); + mediaRecorderRef.current = mediaRecorder; - const remote = new VoiceSend() - remote.sendToVoiceRecognition(audioBlob) - } + audioChunks.current = []; // Initialize audio chunks + + // Create a promise that resolves when the onstop event is done + const stopRecordingPromise = new Promise((resolve) => { + mediaRecorder.ondataavailable = (event) => { + audioChunks.current.push(event.data); + }; + + mediaRecorder.onstop = async () => { + const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }); + audioChunks.current = []; + + const text_voice = await sendToVoiceRecognition(audioBlob); + console.log(text_voice); + resolve(text_voice); // Resolve the promise with the recognized text + }; + }); + + mediaRecorder.start(); + setIsRecording(true); + + // Wait for the recording to stop and get the recognized text + return stopRecordingPromise; + }; - mediaRecorder.start() - setIsRecording(true) - } - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } + mediaRecorderRef.current?.stop(); + setIsRecording(false); + }; - - const handleMicClick = () => { + const handleMicClick = async () => { if (!isRecording) { - startRecording(); + const recognizedText = await startRecording(); + setInputMessage(recognizedText); // Set the recognized text after recording + console.log("Set!") } else { stopRecording(); } @@ -261,10 +264,10 @@ const InputOutputBackend: React.FC = () => { await wait(1000) setCopyClicked(false) } - + return ( -
+ <> { onMicClick={handleMicClick} inputDisabled={inputDisabled} isRecording={isRecording} - /> -
+ /> + ) } From 27b80b6c76687da37fd3ce4867125d8b6d7b52bf Mon Sep 17 00:00:00 2001 From: sageTheDM Date: Mon, 30 Sep 2024 14:37:02 +0200 Subject: [PATCH 3/3] Minor changes to the system prompt --- app/backend/InputOutputHandler.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 92fa6df..7a24036 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -42,15 +42,15 @@ const InputOutputBackend: React.FC = () => { content: `You are in the timezone: ${timeZone}. You use the time format ${timeFormat}. You use the date format ${dateFormat} for all references of dates. - You use the ${preferredMeasurement} system. + You use the ${preferredMeasurement} system. You use the currency ${preferredCurrency}. You will only answer in the language (you will receive the country code) ${preferredLanguage}. But in the case the user specifically states to answer in another language, do that. Speaking in another language is not stating you should answer in that language. - Additionally, under no circumstances translate your answer into multiple languages.`, + Additionally, under no circumstances ever translate your answer into multiple languages.`, }, - { role: "assistant", content: "Hello! How can I help you?" }, - ]); + { role: "assistant", content: "Hello! How may I help you?" }, + ]); } }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]);