diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index 4fa0042..92fa6df 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -2,7 +2,7 @@ import React, { use, useEffect, useRef, useState } from "react"; import ConversationFrontend from "../components/ConversationFrontend"; import InputFrontend from "../components/InputFrontend"; -import VoiceSend from "./voice_backend" +import { sendToVoiceRecognition } from "./voice_backend" import { AudioRecorder } from "./AudioRecorder"; import axios from "axios"; import { resolve } from "path"; @@ -11,46 +11,59 @@ import { fetchFile, toBlobURL } from "@ffmpeg/util" const InputOutputBackend: React.FC = () => { + // # variables type Message = { role: string content: string } - /* Variables for System-prompt */ - const [preferredCurrency, setPreferredCurrency] = useState("") - const [preferredLanguage, setPreferredLanguage] = useState("") - const [timeFormat, setTimeFormat] = useState("") - const [preferredMeasurement, setPreferredMeasurement] = useState("") - const [timeZone, setTimeZone] = useState("") - const [dateFormat, setDateFormat] = useState("") + const [preferredCurrency, setPreferredCurrency] = useState(null); + const [preferredLanguage, setPreferredLanguage] = useState(null); + const [timeFormat, setTimeFormat] = useState(null); + const [preferredMeasurement, setPreferredMeasurement] = useState(null); + const [timeZone, setTimeZone] = useState(null); + const [dateFormat, setDateFormat] = useState(null); + const [messages, setMessages] = useState([]); useEffect(() => { - setPreferredCurrency(localStorage.getItem("preferredCurrency")) - setPreferredLanguage(localStorage.getItem("preferredLanguage")) - setTimeFormat(localStorage.getItem("timeFormat")) - setPreferredMeasurement(localStorage.getItem("preferredMeasurement")) - setTimeZone(localStorage.getItem("timeZone")) - setDateFormat(localStorage.getItem("dateFormat")) - }) - + setPreferredCurrency(localStorage.getItem("preferredCurrency")); + setPreferredLanguage(localStorage.getItem("preferredLanguage")); + setTimeFormat(localStorage.getItem("timeFormat")); + setPreferredMeasurement(localStorage.getItem("preferredMeasurement")); + setTimeZone(localStorage.getItem("timeZone")); + setDateFormat(localStorage.getItem("dateFormat")); + }, []); + + useEffect(() => { + if (preferredCurrency && preferredLanguage && timeFormat && dateFormat && preferredMeasurement && timeZone) { + setMessages([ + { + role: "system", + content: `You are in the timezone: ${timeZone}. + You use the time format ${timeFormat}. + You use the date format ${dateFormat} for all references of dates. + You use the ${preferredMeasurement} system. + You use the currency ${preferredCurrency}. + You will only answer in the language (you will receive the country code) ${preferredLanguage}. + But in the case the user specifically states to answer in another language, do that. Speaking in + another language is not stating you should answer in that language. + Additionally, under no circumstances translate your answer into multiple languages.`, + }, + { role: "assistant", content: "Hello! How can I help you?" }, + ]); + } + }, [preferredCurrency, preferredLanguage, timeFormat, dateFormat, preferredMeasurement, timeZone]); + + const [copyClicked, setCopyClicked] = useState(false) const [accessToken, setAccessToken] = useState("") const postWorkerRef = useRef(null) const getWorkerRef = useRef(null) - const [messages, setMessages] = useState([{ role: "system", - content: `You are in the timezone: ${timeZone}. - You use the time format ${timeFormat}. - You use the date format ${dateFormat} for all references of dates. - You use the ${preferredMeasurement} system. You use the currency ${preferredCurrency}. - You will only answer in the language (you will receive the country code) ${preferredLanguage}. - But in the case the user specifically states to answer in an other language do that speaking in a - nother language is not stating you should answer in that language. Additionally do not translate your answer into multiple languages` - },{ role: "assistant", content: "Hello! How can I help you?" }]) const [liveMessage, setLiveMessage] = useState("") const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) const [isRecording, setIsRecording] = useState(false) - const mediaRecorderRef = useRef(null) + const mediaRecorderRef = useRef(null) const audioChunks = useRef([]) @@ -151,11 +164,6 @@ const InputOutputBackend: React.FC = () => { }); }; - - useEffect(() => { - - },[preferredCurrency, preferredLanguage, timeFormat, preferredMeasurement, timeZone, dateFormat]) - const addMessage = (role: string, content: string) => { setMessages(previous => [...previous, { role, content }]) } @@ -173,40 +181,46 @@ const InputOutputBackend: React.FC = () => { } } - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - const mediaRecorder = new MediaRecorder(stream) - mediaRecorderRef.current = mediaRecorder - - mediaRecorder.ondataavailable = (event) => { - audioChunks.current.push(event.data) - } - - mediaRecorder.onstop = async () => { - const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }) - audioChunks.current = [] - // console.log(audioBlob); - // const url = URL.createObjectURL(audioBlob) - // const audio = new Audio(url); - // audio.play().catch(error => console.error("Error playing audio:", error)); + const startRecording = async (): Promise => { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const mediaRecorder = new MediaRecorder(stream); + mediaRecorderRef.current = mediaRecorder; - const remote = new VoiceSend() - remote.sendToVoiceRecognition(audioBlob) - } + audioChunks.current = []; // Initialize audio chunks + + // Create a promise that resolves when the onstop event is done + const stopRecordingPromise = new Promise((resolve) => { + mediaRecorder.ondataavailable = (event) => { + audioChunks.current.push(event.data); + }; + + mediaRecorder.onstop = async () => { + const audioBlob = new Blob(audioChunks.current, { type: "audio/ogg" }); + audioChunks.current = []; + + const text_voice = await sendToVoiceRecognition(audioBlob); + console.log(text_voice); + resolve(text_voice); // Resolve the promise with the recognized text + }; + }); + + mediaRecorder.start(); + setIsRecording(true); + + // Wait for the recording to stop and get the recognized text + return stopRecordingPromise; + }; - mediaRecorder.start() - setIsRecording(true) - } - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } + mediaRecorderRef.current?.stop(); + setIsRecording(false); + }; - - const handleMicClick = () => { + const handleMicClick = async () => { if (!isRecording) { - startRecording(); + const recognizedText = await startRecording(); + setInputMessage(recognizedText); // Set the recognized text after recording + console.log("Set!") } else { stopRecording(); } @@ -250,10 +264,10 @@ const InputOutputBackend: React.FC = () => { await wait(1000) setCopyClicked(false) } - + return ( -
+ <> { onMicClick={handleMicClick} inputDisabled={inputDisabled} isRecording={isRecording} - /> -
+ /> + ) } diff --git a/app/backend/voice_backend.ts b/app/backend/voice_backend.ts index 3c4193b..ca8a998 100644 --- a/app/backend/voice_backend.ts +++ b/app/backend/voice_backend.ts @@ -1,26 +1,20 @@ import axios from "axios"; +export const sendToVoiceRecognition = (audio_data: Blob): Promise => { + console.log("sending recording..."); -class VoiceSend { - sendToVoiceRecognition(audio_data: Blob) { - console.log("sending recording..."); + const formdata = new FormData() + formdata.append("audio", audio_data) - const formdata = new FormData() - formdata.append("audio", audio_data) - - const dataSend = { option:"offline", type:"basic",audio:audio_data } - axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata) - .then((response) => { - console.log(response.data) - return response.data.response - }) - .catch(error => { - console.log("Error calling API:", error) - postMessage({ status: 500 }) - }) - } - -} - - -export default VoiceSend; \ No newline at end of file + const dataSend = { option: "offline", type: "basic", audio: audio_data } + return axios.post("http://localhost:5000/interstellar_ai/api/voice_recognition", formdata) + .then((response) => { + console.log(response.data) + return response.data.response + }) + .catch(error => { + console.log("Error calling API:", error) + postMessage({ status: 500 }) + return "Error" + }) +} \ No newline at end of file diff --git a/app/components/AI.tsx b/app/components/AI.tsx index e1d244b..824f6d6 100644 --- a/app/components/AI.tsx +++ b/app/components/AI.tsx @@ -4,11 +4,9 @@ import InputOutputBackend from '../backend/InputOutputHandler'; const AI: React.FC = () => { return ( -
-
); }; diff --git a/app/components/ConversationFrontend.tsx b/app/components/ConversationFrontend.tsx index 3294d09..fd17324 100644 --- a/app/components/ConversationFrontend.tsx +++ b/app/components/ConversationFrontend.tsx @@ -17,13 +17,6 @@ const ConversationFrontend = React.forwardRef ({ messages, onResendClick, onEditClick, onCopyClick, isClicked}, ref: ForwardedRef) => { const endOfMessagesRef = useRef(null); - // Auto-scroll to the bottom of the conversation whenever a new message is added - useEffect(() => { - if (endOfMessagesRef.current) { - endOfMessagesRef.current.scrollIntoView({ behavior: 'smooth' }); - } - }, [messages]); // Triggers the effect whenever the 'messages' array changes - useEffect(() => { console.log(isClicked); diff --git a/app/components/InputFrontend.tsx b/app/components/InputFrontend.tsx index f02442b..5d51834 100644 --- a/app/components/InputFrontend.tsx +++ b/app/components/InputFrontend.tsx @@ -6,13 +6,17 @@ interface InputProps { onSendClick: (message: string, override: boolean) => void; onMicClick: () => void; inputDisabled: boolean; - isRecording:boolean + isRecording: boolean } const InputFrontend = React.forwardRef( - ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef) => { + ({ message, onSendClick, onMicClick, inputDisabled, isRecording }, ref: ForwardedRef) => { const [inputValue, setInputValue] = useState(''); + useEffect(() => { + setInputValue(message); + }, [message]); + const handleInputChange = (e: React.ChangeEvent) => { setInputValue(e.target.value); }; @@ -40,7 +44,7 @@ const InputFrontend = React.forwardRef( - diff --git a/app/components/Models.tsx b/app/components/Models.tsx index 894623a..1f9dc56 100644 --- a/app/components/Models.tsx +++ b/app/components/Models.tsx @@ -1,7 +1,143 @@ "use client"; import React, { useState, useEffect } from 'react'; -// Define the available model options +// Define all models that should be available. +const modelList = { + 'Offline Fast': { + 'model_type': 'local', + 'Math': 'qwen2-math:1.5b', + 'Code': 'starcoder2', + 'Language': 'llama3.2', + 'Character': 'dolphin-phi', + 'Finance': 'qwen2-math:1.5b', + 'Weather': 'llama3.2', + 'Time': 'llama3.2', + 'Image': 'llava-phi3' + }, + 'Offline Slow': { + 'model_type': 'local', + 'Math': 'wizard-math', + 'Code': 'starcoder2:7b', + 'Language': 'llama3.1', + 'Character': 'dolphin-llama3', + 'Finance': 'wizard-math', + 'Weather': 'llama3.1', + 'Time': 'llama3.1', + 'Image': 'llava' + }, + 'Offline Fast (FOSS)': { + 'model_type': 'local', + 'Math': 'qwen2-math:1.5b', + 'Code': 'qwen2.5-coder:1.5b', + 'Language': 'phi3.5', + 'Character': 'dolphin-mistral', + 'Finance': 'qwen2-math:1.5b', + 'Weather': 'phi3.5', + 'Time': 'phi3.5', + 'Image': 'llava' + }, + 'Offline Slow (FOSS)': { + 'model_type': 'local', + 'Math': 'mathstral', + 'Code': 'qwen2.5-coder', + 'Language': 'qwen2.5', + 'Character': 'dolphin-mistral', + 'Finance': 'mathstral', + 'Weather': 'qwen2.5', + 'Time': 'qwen2.5', + 'Image': 'llava' + }, + 'Online Cheap (OpenAI)': { + 'model_type': 'openai', + 'Math': 'gpt-4o-mini', + 'Code': 'gpt-4o-mini', + 'Language': 'gpt-4o-mini', + 'Character': 'gpt-4o-mini', + 'Finance': 'gpt-4o-mini', + 'Weather': 'gpt-4o-mini', + 'Time': 'gpt-4o-mini', + 'Image': 'gpt-4o-mini' + }, + 'Online Expensive (OpenAI)': { + 'model_type': 'openai', + 'Math': 'gpt-4o', + 'Code': 'gpt-4o', + 'Language': 'gpt-4o', + 'Character': 'gpt-4o', + 'Finance': 'gpt-4o', + 'Weather': 'gpt-4o', + 'Time': 'gpt-4o', + 'Image': 'gpt-4o' + }, + 'Online Cheap (Anthropic)': { + 'model_type': 'anthropic', + 'Math': 'claude-3-haiku', + 'Code': 'claude-3-haiku', + 'Language': 'claude-3-haiku', + 'Character': 'claude-3-haiku', + 'Finance': 'claude-3-haiku', + 'Weather': 'claude-3-haiku', + 'Time': 'claude-3-haiku', + 'Image': 'claude-3-haiku' + }, + 'Online Expensive (Anthropic)': { + 'model_type': 'anthropic', + 'Math': 'claude-3-5-sonnet', + 'Code': 'claude-3-5-sonnet', + 'Language': 'claude-3-5-sonnet', + 'Character': 'claude-3-5-sonnet', + 'Finance': 'claude-3-5-sonnet', + 'Weather': 'claude-3-5-sonnet', + 'Time': 'claude-3-5-sonnet', + 'Image': 'claude-3-5-sonnet' + }, + 'Online Cheap (Google)': { + 'model_type': 'google', + 'Math': 'gemini-1.5-flash-latest', + 'Code': 'gemini-1.5-flash-latest', + 'Language': 'gemini-1.5-flash-latest', + 'Character': 'gemini-1.5-flash-latest', + 'Finance': 'gemini-1.5-flash-latest', + 'Weather': 'gemini-1.5-flash-latest', + 'Time': 'gemini-1.5-flash-latest', + 'Image': 'gemini-1.5-flash-latest' + }, + 'Online Expensive (Google)': { + 'model_type': 'google', + 'Math': 'gemini-1.5-pro-latest', + 'Code': 'gemini-1.5-pro-latest', + 'Language': 'gemini-1.5-pro-latest', + 'Character': 'gemini-1.5-pro-latest', + 'Finance': 'gemini-1.5-pro-latest', + 'Weather': 'gemini-1.5-pro-latest', + 'Time': 'gemini-1.5-pro-latest', + 'Image': 'gemini-1.5-pro-latest' + }, + 'Online (La Plateforme)': { + 'model_type': 'mistral', + 'Math': 'open-mistral-nemo', + 'Code': 'codestral-latest', + 'Language': 'mistral-small-latest', + 'Character': 'mistral-large-latest', + 'Finance': 'open-mistral-nemo', + 'Weather': 'mistral-small-latest', + 'Time': 'mistral-small-latest', + 'Image': 'pixtral-12b-2409' + }, + 'Online (FOSS) (La Plateforme)': { + 'model_type': 'mistral', + 'Math': 'open-mistral-nemo', + 'Code': 'open-codestral-mamba', + 'Language': 'open-mistral-nemo', + 'Character': 'open-mixtral-8x22b', + 'Finance': 'open-mixtral-8x22b', + 'Weather': 'open-mistral-nemo', + 'Time': 'open-mistral-nemo', + 'Image': 'pixtral-12b-2409' + } +} + +// Define the available category options const modelDropdown = { offlineWithoutFoss: ['Offline Fast', 'Offline Slow'], offlineFoss: ['Offline Fast (FOSS)', 'Offline Slow (FOSS)'], @@ -12,6 +148,7 @@ const modelDropdown = { 'Online Expensive (Anthropic)', 'Online Cheap (Google)', 'Online Expensive (Google)', + 'Online (La Plateforme)' ], onlineFoss: ['Online (FOSS) (La Plateforme)'], }; @@ -20,7 +157,7 @@ const Models: React.FC = () => { // Initialize state with value from localStorage or default to '' const [selectedModel, setSelectedModel] = useState(''); const [radioSelection, setRadioSelection] = useState("") - + useEffect(() => { setRadioSelection(localStorage.getItem('radioSelection')) const handleStorageChange = () => { @@ -98,6 +235,13 @@ const Models: React.FC = () => { const isOfflineModel = (model: string) => modelDropdown.offlineWithoutFoss.includes(model) || modelDropdown.offlineFoss.includes(model); + const modelClicked = (model: string) => { + const category = selectedModel as keyof typeof modelList; + console.log(model) + console.log(category) + console.log(modelList[category][model as keyof typeof modelList[typeof category]]); + } + return (
@@ -122,7 +266,7 @@ const Models: React.FC = () => {
{['Code', 'Math', 'Language', 'Character', 'Finance', 'Weather', 'Time', 'Image', 'Custom1', 'Custom2'].map( (category) => ( -