diff --git a/app/backend/AudioRecorder.ts b/app/backend/AudioRecorder.ts deleted file mode 100644 index 459674e..0000000 --- a/app/backend/AudioRecorder.ts +++ /dev/null @@ -1,34 +0,0 @@ -import React, { useState, useRef } from 'react' - - export const AudioRecorder= () => { - const [isRecording, setIsRecording] = useState(false) - const [audioURL, setAudioURL] = useState(null) - const mediaRecorderRef = useRef(null) - const audioChunks = useRef([]) - - const startRecording = async () => { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }) - const mediaRecorder = new MediaRecorder(stream) - mediaRecorderRef.current = mediaRecorder - - mediaRecorder.ondataavailable = (event) => { - audioChunks.current.push(event.data) - } - - mediaRecorder.onstop = () => { - const audioBlob = new Blob(audioChunks.current, { type: "audio/wav" }) - const url = URL.createObjectURL(audioBlob) - setAudioURL(url) - audioChunks.current = [] - } - - mediaRecorder.start() - setIsRecording(true) - - } - - const stopRecording = () => { - mediaRecorderRef.current?.stop() - setIsRecording(false) - } - } \ No newline at end of file diff --git a/app/backend/ChatHistory.ts b/app/backend/ChatHistory.ts index 8d768ed..dc12cb1 100644 --- a/app/backend/ChatHistory.ts +++ b/app/backend/ChatHistory.ts @@ -1,4 +1,4 @@ -type ChatMessage = { +/* type ChatMessage = { name: string; messages: any; timestamp: number; @@ -25,3 +25,4 @@ function removeMessageFromHistory(timestamp: number): void { console.log(`Message not found with timestamp: ${timestamp}`); } } + */ \ No newline at end of file diff --git a/app/backend/InputOutputHandler.tsx b/app/backend/InputOutputHandler.tsx index aeb4e3b..6201ca9 100644 --- a/app/backend/InputOutputHandler.tsx +++ b/app/backend/InputOutputHandler.tsx @@ -1,12 +1,9 @@ "use client" -import React, { use, useEffect, useRef, useState } from "react"; +import React, { useEffect, useRef, useState } from "react"; import ConversationFrontend from '../components/ConversationFrontend'; import InputFrontend from "../components/InputFrontend"; import { sendToVoiceRecognition } from "./voice_backend" import axios from "axios"; -import { resolve } from "path"; -import { FFmpeg } from "@ffmpeg/ffmpeg"; -import { fetchFile, toBlobURL } from "@ffmpeg/util" const InputOutputBackend: React.FC = () => { // # variables @@ -66,7 +63,6 @@ const InputOutputBackend: React.FC = () => { const [accessToken, setAccessToken] = useState("") const postWorkerRef = useRef(null) const getWorkerRef = useRef(null) - const [liveMessage, setLiveMessage] = useState("") const [inputMessage, setInputMessage] = useState("") const [inputDisabled, setInputDisabled] = useState(false) const [isRecording, setIsRecording] = useState(false) @@ -128,7 +124,7 @@ const InputOutputBackend: React.FC = () => { const data = event.data if (event.data == "error") { - setLiveMessage("error getting AI response: " + data.error) + console.log("Error getting ai message.") } else { console.log("Received data:", data); editLastMessage(data.response) @@ -176,7 +172,7 @@ const InputOutputBackend: React.FC = () => { if (postWorkerRef.current) { addMessage("user", inputValue) const type = localStorage.getItem('type') - var api_key: string = "" + let api_key: string = "" if (type != null && type != 'local') { const try_key = localStorage.getItem(type) if (try_key) { @@ -241,7 +237,7 @@ const InputOutputBackend: React.FC = () => { } const handleResendClick = () => { - var temporary_message = messages[messages.length - 2]['content'] + const temporary_message = messages[messages.length - 2]['content'] const updatedMessages = messages.slice(0, -2) setMessages(updatedMessages) endGetWorker() @@ -251,7 +247,7 @@ const InputOutputBackend: React.FC = () => { } const handleEditClick = () => { - let newestMessage = messages[messages.length - 2].content + const newestMessage = messages[messages.length - 2].content setInputMessage(newestMessage) const updatedMessages = messages.slice(0, messages.length - 2) setMessages(updatedMessages) diff --git a/app/backend/ProcessMemory.ts b/app/backend/ProcessMemory.ts index d66d8e5..5765138 100644 --- a/app/backend/ProcessMemory.ts +++ b/app/backend/ProcessMemory.ts @@ -1,5 +1,4 @@ -import { Settings } from 'electron' -import React from 'react' +/* import { Settings } from 'electron' type Message = { role: string @@ -14,7 +13,7 @@ type Chat = { type Data = { chats: Chat[] settings: Settings[] -} +} */ diff --git a/app/backend/database.ts b/app/backend/database.ts index 559179a..f4f085c 100644 --- a/app/backend/database.ts +++ b/app/backend/database.ts @@ -21,7 +21,7 @@ to check if the request was accepted or declined, check response.data.response, const apiURL = new URL("http://localhost:5000/interstellar_ai/db") apiURL.hostname = window.location.hostname; -export const sendToDatabase = async (data: any): Promise => { +export const sendToDatabase = async (data: object): Promise => { try { const response = await axios.post(apiURL.href, data); const status = response.data.status; @@ -30,11 +30,12 @@ export const sendToDatabase = async (data: any): Promise => { return success; } catch (error) { postMessage({ status: 500, success: false }); + console.log(error) return false; } }; -export const sendToDatabaseAndGetString = async (data: any): Promise => { +export const sendToDatabaseAndGetString = async (data: object): Promise => { try { const response = await axios.post(apiURL.href, data); const status = response.data.status; @@ -43,6 +44,7 @@ export const sendToDatabaseAndGetString = async (data: any): Promise => return success; } catch (error) { postMessage({ status: 500, success: false }); + console.log(error) return "false"; } }; @@ -99,7 +101,7 @@ export const getName = async (usernameOrEmail: string, password: string): Promis return await sendToDatabaseAndGetString(data); }; -export const changeData = async (usernameOrEmail: string, password: string, newData: any) => { +export const changeData = async (usernameOrEmail: string, password: string, newData: object) => { const data = { action: "change_settings", username: usernameOrEmail.includes('@') ? undefined : usernameOrEmail, @@ -117,7 +119,7 @@ export const checkCredentials = async (usernameOrEmail: string, password: string email: usernameOrEmail.includes('@') ? usernameOrEmail : undefined, password, }; - var sendBack = await sendToDatabase(data); + const sendBack = await sendToDatabase(data); if (sendBack) { localStorage.setItem("accountEmail", await getEmail(usernameOrEmail, password)) localStorage.setItem("accountName", await getName(usernameOrEmail, password)) diff --git a/app/components/ConversationFrontend.tsx b/app/components/ConversationFrontend.tsx index 6e0a47e..b120f06 100644 --- a/app/components/ConversationFrontend.tsx +++ b/app/components/ConversationFrontend.tsx @@ -38,7 +38,7 @@ const ConversationFrontend = React.forwardRef threshold: 1.0, // Ensure the whole element is visible } ); - + const endOfMessages = messagesEndRef.current; if (endOfMessages) { observer.observe(endOfMessages); @@ -80,36 +80,38 @@ const ConversationFrontend = React.forwardRef
+ Stop
Resend
Edit
- {isClicked?"Copied!": "Copy" } + {isClicked ? "Copied!" : "Copy"}
-
+
); - } + } ); +ConversationFrontend.displayName = "ConversationFrontend"; + export default ConversationFrontend; diff --git a/app/components/Documentation.tsx b/app/components/Documentation.tsx index 81df5e9..ba34d33 100644 --- a/app/components/Documentation.tsx +++ b/app/components/Documentation.tsx @@ -3,120 +3,120 @@ import React from 'react'; const Documentation = () => { return (
-
-

AI Virtual Assistant - Internship Students 2024

-

General planning:

-

- We are currently in the process of developing a Python application that leverages HTML and CSS to create an intuitive graphical user interface (GUI). This application will integrate multiple AI models through API calls, enabling users to effortlessly switch between different models tailored for specific tasks, such as coding, mathematics, and language processing. A key feature of our design is that the application will be capable of running locally, ensuring that users can access its functionality without the need for an internet connection. -

-

- Upon receiving our assignment, we initiated the project by outlining a detailed timeline for each task to ensure a structured approach to development. This timeline serves as a roadmap, helping us allocate resources effectively and track our progress. Following this planning phase, Patrick_Pluto took the lead in creating the repository, establishing a centralized location for our codebase. Meanwhile, Yasin and sageTheDM forked the repository to begin their contributions, allowing them to work on specific features and enhancements independently. -

-

- As we move forward, our focus will be on refining the user experience, optimizing the integration of AI models, and ensuring that the application is robust and user-friendly. We are excited about the potential of this project and are committed to delivering a high-quality application that meets the needs of our users. -

+
+

AI Virtual Assistant - Internship Students 2024

+

General planning:

+

+ We are currently in the process of developing a Python application that leverages HTML and CSS to create an intuitive graphical user interface (GUI). This application will integrate multiple AI models through API calls, enabling users to effortlessly switch between different models tailored for specific tasks, such as coding, mathematics, and language processing. A key feature of our design is that the application will be capable of running locally, ensuring that users can access its functionality without the need for an internet connection. +

+

+ Upon receiving our assignment, we initiated the project by outlining a detailed timeline for each task to ensure a structured approach to development. This timeline serves as a roadmap, helping us allocate resources effectively and track our progress. Following this planning phase, Patrick_Pluto took the lead in creating the repository, establishing a centralized location for our codebase. Meanwhile, Yasin and sageTheDM forked the repository to begin their contributions, allowing them to work on specific features and enhancements independently. +

+

+ As we move forward, our focus will be on refining the user experience, optimizing the integration of AI models, and ensuring that the application is robust and user-friendly. We are excited about the potential of this project and are committed to delivering a high-quality application that meets the needs of our users. +

-

Frontend planning:

-

- We are committed to designing a visually appealing graphical user interface (GUI) that significantly enhances user experience and engagement. The GUI will prominently feature a chat function, allowing users to interact seamlessly with the AI Assistant. This conversational interface will facilitate easy communication, making it intuitive for users to ask questions and receive responses. -

-

- To further improve usability, we will incorporate distinct buttons that enable users to switch effortlessly between online and offline modes. This functionality ensures that users can access the AI's capabilities regardless of their internet connectivity, making the application versatile and accommodating to various user environments. -

-

- Additionally, the interface will provide options for users to select from a range of AI models, each tailored for specific use cases such as coding assistance, mathematical problem-solving, and language translation. This feature empowers users to choose the most suitable AI for their needs, thereby enhancing the overall effectiveness and relevance of the application. -

-

- We will also prioritize creating an intuitive layout, ensuring that all elements are easily accessible and visually coherent. By focusing on user experience in our design, we aim to make the application not only functional but also enjoyable to use. Our goal is to create a user-friendly environment that encourages exploration and interaction, ultimately leading to a more satisfying and productive experience for all users. -

+

Frontend planning:

+

+ We are committed to designing a visually appealing graphical user interface (GUI) that significantly enhances user experience and engagement. The GUI will prominently feature a chat function, allowing users to interact seamlessly with the AI Assistant. This conversational interface will facilitate easy communication, making it intuitive for users to ask questions and receive responses. +

+

+ To further improve usability, we will incorporate distinct buttons that enable users to switch effortlessly between online and offline modes. This functionality ensures that users can access the AI's capabilities regardless of their internet connectivity, making the application versatile and accommodating to various user environments. +

+

+ Additionally, the interface will provide options for users to select from a range of AI models, each tailored for specific use cases such as coding assistance, mathematical problem-solving, and language translation. This feature empowers users to choose the most suitable AI for their needs, thereby enhancing the overall effectiveness and relevance of the application. +

+

+ We will also prioritize creating an intuitive layout, ensuring that all elements are easily accessible and visually coherent. By focusing on user experience in our design, we aim to make the application not only functional but also enjoyable to use. Our goal is to create a user-friendly environment that encourages exploration and interaction, ultimately leading to a more satisfying and productive experience for all users. +

-

Draw.io:

- Live Message Skizze +

Draw.io:

+ Live Message Skizze -

Inspiration:

-

We got our inspiration from Huggingchat.

+

Inspiration:

+

We got our inspiration from Huggingchat.

-

First prototype:

-

- After prototyping the Website we started working on optimizing the css and html structure, and combining the front and the backend together with each other using Flask. Since we have never once done that once it was more learning by doing than planning that is why we have not planned this step but documented it. -

+

First prototype:

+

+ After prototyping the Website we started working on optimizing the css and html structure, and combining the front and the backend together with each other using Flask. Since we have never once done that once it was more learning by doing than planning that is why we have not planned this step but documented it. +

-

Web APP

-

QtWebEngine 5

-

- We decided on going with QtWebEngine, because Qt is cross platform, and easy to use. QtWebEngine is basically a slimmed down version of Chromium that runs on the Qt Widget Framework. It looked pretty good, but the browser part is very barebones, so it broke a lot of styling. -

- -

Styling

-

- After conducting thorough testing, we discovered that even after performing a browser reset, the web application exhibited a distinct styling compared to the web version. This inconsistency prompted us to undertake a comprehensive overhaul of the entire CSS framework. -

-

- In our redesign, we focused on enhancing the user experience by implementing custom scrollbars that align with our overall aesthetic. Additionally, we expanded upon our existing design foundation to ensure a more cohesive and visually appealing interface. -

-

- Our efforts have resulted in a web application that is not only visually consistent across different platforms but also optimized for performance. The revamped web app is designed to be responsive, making it accessible and functional on a wide range of devices, from desktops to tablets and smartphones. This adaptability ensures that users can enjoy a seamless experience, regardless of the device they choose to use. -

-

- Overall, these improvements reflect our commitment to delivering a high-quality product that meets the diverse needs of our users while maintaining a polished and professional appearance. -

+

Web APP

+

QtWebEngine 5

+

+ We decided on going with QtWebEngine, because Qt is cross platform, and easy to use. QtWebEngine is basically a slimmed down version of Chromium that runs on the Qt Widget Framework. It looked pretty good, but the browser part is very barebones, so it broke a lot of styling. +

-

Prototype: 17.09.2024

-

- Our current prototype operates on a straightforward Python backend, which, while functional, relies heavily on our optimism that it will remain stable and not encounter any critical failures or data loss. -

-

- The existing chat system is equipped with several key features designed to enhance user interaction. Users can easily resend the AI's response, allowing for quick follow-up questions or clarifications. Additionally, the system provides the ability to edit user messages, ensuring that any mistakes can be corrected without starting the conversation anew. -

-

- Furthermore, users have the option to copy the AI's responses, facilitating easy sharing or saving of information for future reference. These features aim to create a more flexible and user-friendly experience, allowing for seamless communication and interaction with the AI. -

-

- While the current setup serves as a solid foundation, we recognize the need for further improvements and enhancements to ensure reliability and robustness as we move forward in the development process. -

-

- Theoretically there is also voice recognition but let us ignore that for now. -

+

Styling

+

+ After conducting thorough testing, we discovered that even after performing a browser reset, the web application exhibited a distinct styling compared to the web version. This inconsistency prompted us to undertake a comprehensive overhaul of the entire CSS framework. +

+

+ In our redesign, we focused on enhancing the user experience by implementing custom scrollbars that align with our overall aesthetic. Additionally, we expanded upon our existing design foundation to ensure a more cohesive and visually appealing interface. +

+

+ Our efforts have resulted in a web application that is not only visually consistent across different platforms but also optimized for performance. The revamped web app is designed to be responsive, making it accessible and functional on a wide range of devices, from desktops to tablets and smartphones. This adaptability ensures that users can enjoy a seamless experience, regardless of the device they choose to use. +

+

+ Overall, these improvements reflect our commitment to delivering a high-quality product that meets the diverse needs of our users while maintaining a polished and professional appearance. +

-

17.09.2024

-

- After transitioning to React, we made several significant changes. We renamed our History and Models components to .left-panel, which can now be hidden when not in use. Additionally, with various optimizations through React, we’ve successfully split the CSS and HTML components into more manageable parts. -

-

- We also made our first strides towards achieving a fully responsive website. With these changes, we are well on our way to completing the responsive design in the near future. -

+

Prototype: 17.09.2024

+

+ Our current prototype operates on a straightforward Python backend, which, while functional, relies heavily on our optimism that it will remain stable and not encounter any critical failures or data loss. +

+

+ The existing chat system is equipped with several key features designed to enhance user interaction. Users can easily resend the AI's response, allowing for quick follow-up questions or clarifications. Additionally, the system provides the ability to edit user messages, ensuring that any mistakes can be corrected without starting the conversation anew. +

+

+ Furthermore, users have the option to copy the AI's responses, facilitating easy sharing or saving of information for future reference. These features aim to create a more flexible and user-friendly experience, allowing for seamless communication and interaction with the AI. +

+

+ While the current setup serves as a solid foundation, we recognize the need for further improvements and enhancements to ensure reliability and robustness as we move forward in the development process. +

+

+ Theoretically there is also voice recognition but let us ignore that for now. +

-

Backend planning:

-

Task:

-

- We will develop an extensible backend that enables us to easily swap out different AI models, facilitating the creation of a versatile AI Virtual Assistant. This architecture will allow for seamless integration of new AI technologies as they become available, ensuring that our application remains up-to-date and capable of meeting diverse user needs. -

-

- The backend will also incorporate advanced features such as speech recognition, allowing users to interact with the AI Assistant through voice commands for a more natural and intuitive experience. Additionally, we will implement functionality to save chat histories, enabling users to revisit previous conversations and maintain continuity in their interactions with the AI. -

-

- For the AI model, we will utilize Microsoft Phi 3.5, which offers robust capabilities for understanding and generating human-like responses. This choice will enhance the overall performance of the Virtual Assistant, making it a valuable tool for users seeking assistance across various tasks and topics. By combining an extensible backend with powerful AI capabilities, we aim to deliver a comprehensive and user-friendly virtual assistant experience. -

+

17.09.2024

+

+ After transitioning to React, we made several significant changes. We renamed our History and Models components to .left-panel, which can now be hidden when not in use. Additionally, with various optimizations through React, we’ve successfully split the CSS and HTML components into more manageable parts. +

+

+ We also made our first strides towards achieving a fully responsive website. With these changes, we are well on our way to completing the responsive design in the near future. +

-

Design Philosophy:

-

- Our design philosophy is to create one script file per feature. This allows us to possibly reuse certain features in other projects very easily, as we can copy the individual .py files, which all work on their own, except the specific .py file for this project, which is specially tailored towards this AI chatbot. -

+

Backend planning:

+

Task:

+

+ We will develop an extensible backend that enables us to easily swap out different AI models, facilitating the creation of a versatile AI Virtual Assistant. This architecture will allow for seamless integration of new AI technologies as they become available, ensuring that our application remains up-to-date and capable of meeting diverse user needs. +

+

+ The backend will also incorporate advanced features such as speech recognition, allowing users to interact with the AI Assistant through voice commands for a more natural and intuitive experience. Additionally, we will implement functionality to save chat histories, enabling users to revisit previous conversations and maintain continuity in their interactions with the AI. +

+

+ For the AI model, we will utilize Microsoft Phi 3.5, which offers robust capabilities for understanding and generating human-like responses. This choice will enhance the overall performance of the Virtual Assistant, making it a valuable tool for users seeking assistance across various tasks and topics. By combining an extensible backend with powerful AI capabilities, we aim to deliver a comprehensive and user-friendly virtual assistant experience. +

-

UML diagrams:

-

Insert your UML diagrams here, one picture per diagram.

+

Design Philosophy:

+

+ Our design philosophy is to create one script file per feature. This allows us to possibly reuse certain features in other projects very easily, as we can copy the individual .py files, which all work on their own, except the specific .py file for this project, which is specially tailored towards this AI chatbot. +

-

First Prototype:

-

- You are able to simply select an AI Model, then type out what you want to ask. This Prototype is already ready to answer questions and detect what language you write in and give the answer in the corresponding language. -

+

UML diagrams:

+

Insert your UML diagrams here, one picture per diagram.

-

Combining back and frontend

-

Flask setup

-

- The flask setup was relatively quick. We had to adjust all of the file references, and add /static -

-
+

First Prototype:

+

+ You are able to simply select an AI Model, then type out what you want to ask. This Prototype is already ready to answer questions and detect what language you write in and give the answer in the corresponding language. +

+ +

Combining back and frontend

+

Flask setup

+

+ The flask setup was relatively quick. We had to adjust all of the file references, and add /static +

+
); }; diff --git a/app/components/Faq.tsx b/app/components/Faq.tsx index 5a274a9..757bc9b 100644 --- a/app/components/Faq.tsx +++ b/app/components/Faq.tsx @@ -12,7 +12,7 @@ const FAQ: React.FC = () => {

How does the AI assistant work?

-

The assistant uses machine learning algorithms to understand your input and provide contextually relevant answers or generate content based on the task you've described.

+

The assistant uses machine learning algorithms to understand your input and provide contextually relevant answers or generate content based on the task you've described.

@@ -32,11 +32,11 @@ const FAQ: React.FC = () => {

How can I provide feedback about the AI assistant?

-

Feedback can be provided through our feedback form, available on our website. We appreciate your input and use it to improve the AI assistant's performance.

+

Feedback can be provided through our feedback form, available on our website. We appreciate your input and use it to improve the AI assistant's performance.

-

Can I customize the AI assistant's responses?

+

Can I customize the AI assistant's responses?

Customization options are limited in the current version, but we are working on features that will allow users to tailor responses to better suit their needs.

@@ -74,10 +74,10 @@ const FAQ: React.FC = () => {

What are the system requirements to use the AI assistant?

The AI assistant can be accessed through most modern web browsers. Ensure your browser is up-to-date for the best experience.

- +

How can I access previous conversations?

-

Previous conversations can be accessed through the chat history feature available in the assistant's interface.

+

Previous conversations can be accessed through the chat history feature available in the assistant's interface.

@@ -89,12 +89,12 @@ const FAQ: React.FC = () => {

How do I update my profile or settings?

Profile and settings updates can be made through the account management section of the application. Ensure you save your changes before exiting.

- +

Can the AI assistant be used offline?

Currently, the AI assistant requires an internet connection to function. Offline capabilities are being explored for future updates.

- +

Who can I contact for technical support?

Technical support can be reached through our support contact page on the website. Our team is available to help with any technical issues you may encounter.

diff --git a/app/components/InputFrontend.tsx b/app/components/InputFrontend.tsx index 1fbd1b8..9589096 100644 --- a/app/components/InputFrontend.tsx +++ b/app/components/InputFrontend.tsx @@ -10,7 +10,7 @@ interface InputProps { } const InputFrontend = React.forwardRef( - ({ message, onSendClick, onMicClick, inputDisabled, isRecording}, ref: ForwardedRef) => { + ({ message, onSendClick, onMicClick, inputDisabled, isRecording }, ref: ForwardedRef) => { const [inputValue, setInputValue] = useState(''); useEffect(() => { @@ -49,14 +49,16 @@ const InputFrontend = React.forwardRef( onKeyDown={handleKeyDown} />
); } ); +InputFrontend.displayName = "InputFrontend"; + export default InputFrontend;