From 51fdeec00e0a24a3fc19acc4fcaeba95dc67f46a Mon Sep 17 00:00:00 2001 From: Anas Chebili Date: Wed, 11 Jun 2025 22:39:16 +0200 Subject: [PATCH 1/9] HardwareReport: add chat assistant --- HardwareReport/Chat.js | 125 ++++++++++++++++++++++++++ HardwareReport/index.html | 183 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 308 insertions(+) create mode 100644 HardwareReport/Chat.js diff --git a/HardwareReport/Chat.js b/HardwareReport/Chat.js new file mode 100644 index 00000000..1e7b1200 --- /dev/null +++ b/HardwareReport/Chat.js @@ -0,0 +1,125 @@ +import OpenAI from 'https://cdn.jsdelivr.net/npm/openai@4.85.4/+esm'; + + +//constants +const TARGET_ASSISTANT_NAME = "ArduPilot Vehicle Control via MAVLink"; + +// global variables +let openAI = null +let assistantId = null; +let currentThreadId = null; + +async function connectIfNeeded(){ + const OPENAI_API_KEY = localStorage.getItem('openai-api-key'); + if (!OPENAI_API_KEY) + throw new Error('OpenAI API key not configured.'); + if (!openAI){ + openAI = new OpenAI({apiKey: OPENAI_API_KEY, dangerouslyAllowBrowser: true}); + if (!openAI) { + throw new Error('Could not connect to open AI') + } + } + if (!assistantId){ + assistantId = await findAssistantIdByName(TARGET_ASSISTANT_NAME); + } + if (!currentThreadId){ + currentThreadId = await createThread(); + } +} + +async function createThread(){ + if (!assistantId) + throw new Error("cannot create thread before initializing assistant"); + const newThread = await openAI.beta.threads.create(); + if (!newThread) + throw new Error("something went wrong while creating thread"); + return newThread.id; +} + +async function findAssistantIdByName(name) { + if (assistantId) { + return true + } + const assistants_list = await openAI.beta.assistants.list({order: "desc", limit: 20}); + let assistant = assistants_list.data.find(a => a.name === name); + if (assistant) + return assistant.id; + else + throw new Error("could not find assistant with the specified name"); +} + +async function sendQueryToAssistant(query){ + await connectIfNeeded(); + const message = await openAI.beta.threads.messages.create(currentThreadId, { role: "user", content: query }); + if (!message) + throw new Error("Could not send message to assistant"); + const run = openAI.beta.threads.runs + .stream(currentThreadId, { assistant_id: assistantId, stream: true }) + .on('messageDelta', (delta, snapshot) => {addChatMessage(snapshot.content[0].text.value, "assistant"); + }) + if (!run) + throw new Error("Could not establish run streaming") +} + + +document.getElementById("ai-chat-bubble").addEventListener('click', ()=>toggleChat(true)) +document.getElementById("ai-chat-close-button").addEventListener('click', ()=>toggleChat(false)) +document.getElementById("ai-chat-input-area").addEventListener('submit', sendMessage) +document.getElementById('save-api-key').addEventListener('click', saveAPIKey); + +function saveAPIKey(){ + const apiKey = document.getElementById('openai-api-key').value.trim(); + if (apiKey) + localStorage.setItem('openai-api-key', apiKey); +} + + +function toggleChat(show) { + const chatWindow = document.getElementById('ai-chat-window'); + const chatBubble = document.getElementById('ai-chat-bubble'); + if (show){ + chatWindow.style.display = 'flex'; + chatBubble.style.display = 'none'; + } + else{ + chatWindow.style.display = 'none'; + chatBubble.style.display = 'flex' + } + + } + +function sendMessage(event) { + event.preventDefault(); + + const messageInput = document.getElementById('ai-chat-input'); + if (!messageInput) return; + + const messageText = messageInput.value.trim(); + if (messageText) { + addChatMessage(messageText, 'user'); + messageInput.value = ''; + addChatMessage("Thinking...", 'assistant'); + sendQueryToAssistant(messageText); + } +} + +function addChatMessage(text, sender) { + const messagesContainer = document.querySelector('#ai-chat-window .ai-chat-messages'); + + if (!messagesContainer) return; + + if (sender === "assistant") { + let last_message = messagesContainer.querySelector(`.assistant:last-of-type`); + if (last_message) { + last_message.textContent = text; + messagesContainer.scrollTop = messagesContainer.scrollHeight; + return; + } + } + + const messageElement = document.createElement('li'); + messageElement.classList.add('ai-chat-message', sender); + messageElement.textContent = text; + messagesContainer.appendChild(messageElement); + messagesContainer.scrollTop = messagesContainer.scrollHeight; +} \ No newline at end of file diff --git a/HardwareReport/index.html b/HardwareReport/index.html index 45ac8c0e..dc15eed9 100644 --- a/HardwareReport/index.html +++ b/HardwareReport/index.html @@ -31,6 +31,156 @@ div.plotly-notifier { visibility: hidden; } + + #ai-chat-widget-container { + position: fixed; + bottom: 20px; + right: 20px; + z-index: 1000; + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; + } + + #ai-chat-bubble { + width: 56px; + height: 56px; + background-color: #007AFF; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + box-shadow: 0 2px 10px rgba(0, 0, 0, 0.15); + transition: background-color 0.2s ease-in-out, transform 0.2s ease-in-out; + } + + #ai-chat-bubble:hover { + background-color: #005ecb; + transform: scale(1.05); + } + + #ai-chat-bubble svg { + width: 26px; + height: 26px; + fill: white; + } + + #ai-chat-window { + width: 360px; + height: 550px; + background-color: #ffffff; + border-radius: 12px; + box-shadow: 0 5px 25px rgba(0, 0, 0, 0.15); + display: flex; + flex-direction: column; + overflow: hidden; + } + + .ai-chat-header { + background-color: #f8f8f8; + color: #333333; + padding: 12px 16px; + display: flex; + justify-content: space-between; + align-items: center; + border-bottom: 1px solid #e0e0e0; + } + + .ai-chat-header h3 { + margin: 0; + font-size: 1em; + font-weight: 600; + } + + .ai-chat-close-button { + background: none; + border: none; + color: #8e8e93; + font-size: 22px; + cursor: pointer; + padding: 5px; + line-height: 1; + } + + .ai-chat-close-button svg { + width: 16px; + height: 16px; + fill: #8e8e93; + } + .ai-chat-close-button:hover svg { + fill: #333333; + } + + .ai-chat-messages { + flex-grow: 1; + padding: 16px; + overflow-y: auto; + background-color: #ffffff; + display: flex; + flex-direction: column; + gap: 12px; + } + + .ai-chat-message { + max-width: 85%; + padding: 10px 14px; + border-radius: 18px; + line-height: 1.45; + font-size: 0.9em; + word-wrap: break-word; + } + + .ai-chat-message.user { + background-color: #007AFF; + color: white; + align-self: flex-end; + border-bottom-right-radius: 6px; + } + + .ai-chat-message.assistant { + background-color: #f0f0f0; + color: #2c2c2e; + align-self: flex-start; + border-bottom-left-radius: 6px; + } + + .ai-chat-input-area { + display: flex; + padding: 12px; + border-top: 1px solid #e0e0e0; + background-color: #f8f8f8; + align-items: center; + } + + .ai-chat-input-area input { + flex-grow: 1; + border: 1px solid #d1d1d6; + border-radius: 20px; + padding: 10px 16px; + font-size: 0.9em; + margin-right: 8px; + background-color: #ffffff; + } + + .ai-chat-input-area input:focus { + outline: none; + border-color: #007AFF; + box-shadow: 0 0 0 2px rgba(0, 122, 255, 0.2); + } + + .ai-chat-input-area button { + background-color: #007bff; + border: none; + color: white; + padding: 10px 15px; + border-radius: 20px; + cursor: pointer; + font-size: 0.9em; + transition: background-color 0.3s ease; + } + + .ai-chat-input-area button:hover { + background-color: #0056b3; + }

ArduPilot Hardware Report

@@ -307,6 +457,38 @@

Clock drift

+
+
+ + + + +
+ +
+ + + From 322499bc9981a74e3c1d4beaf3a915c0b55aea0e Mon Sep 17 00:00:00 2001 From: Anas Chebili Date: Tue, 1 Jul 2025 11:26:06 +0200 Subject: [PATCH 2/9] HardwareReport: assitant functions with advanced features like graph extraction --- HardwareReport/Chat.js | 245 ++++++++++++++++++++++++----- HardwareReport/assistantTools.json | 44 ++++++ HardwareReport/index.html | 13 +- HardwareReport/instructions.txt | 119 ++++++++++++++ 4 files changed, 381 insertions(+), 40 deletions(-) create mode 100644 HardwareReport/assistantTools.json create mode 100644 HardwareReport/instructions.txt diff --git a/HardwareReport/Chat.js b/HardwareReport/Chat.js index 1e7b1200..68ce5b33 100644 --- a/HardwareReport/Chat.js +++ b/HardwareReport/Chat.js @@ -1,22 +1,57 @@ -import OpenAI from 'https://cdn.jsdelivr.net/npm/openai@4.85.4/+esm'; +import Openai from 'https://cdn.jsdelivr.net/npm/openai@4.85.4/+esm'; //constants -const TARGET_ASSISTANT_NAME = "ArduPilot Vehicle Control via MAVLink"; +const TARGET_ASSISTANT_NAME = "ArduPilot WebTool"; +const TARGET_ASSISTANT_MODEL = "gpt-4o"; + // global variables -let openAI = null +let openai = null; let assistantId = null; let currentThreadId = null; +let fileId; +let documentTitle = document.title; + +const globalLogs = [ + { name: "Sensor_Offset", value: Sensor_Offset }, + { name: "Temperature", value: Temperature }, + { name: "Board_Voltage", value: Board_Voltage }, + { name: "power_flags", value: power_flags }, + { name: "performance_load", value: performance_load }, + { name: "performance_mem", value: performance_mem }, + { name: "performance_time", value: performance_time }, + { name: "stack_mem", value: stack_mem }, + { name: "stack_pct", value: stack_pct }, + { name: "log_dropped", value: log_dropped }, + { name: "log_buffer", value: log_buffer }, + { name: "log_stats", value: log_stats }, + { name: "clock_drift", value: clock_drift }, + { name: "ArduPilot_GitHub_tags", value: ArduPilot_GitHub_tags }, + { name: "octokitRequest_ratelimit_reset", value: octokitRequest_ratelimit_reset }, + { name: "ins", value: ins }, + { name: "compass", value: compass }, + { name: "baro", value: baro }, + { name: "airspeed", value: airspeed }, + { name: "gps", value: gps }, + { name: "rangefinder", value: rangefinder }, + { name: "flow", value: flow }, + { name: "viso", value: viso }, + { name: "can", value: can }, + { name: "params", value: params }, + { name: "defaults", value: defaults }, + { name: "board_types", value: board_types } + ]; + async function connectIfNeeded(){ - const OPENAI_API_KEY = localStorage.getItem('openai-api-key'); - if (!OPENAI_API_KEY) - throw new Error('OpenAI API key not configured.'); - if (!openAI){ - openAI = new OpenAI({apiKey: OPENAI_API_KEY, dangerouslyAllowBrowser: true}); - if (!openAI) { - throw new Error('Could not connect to open AI') + const openai_API_KEY = localStorage.getItem('openai-api-key'); + if (!openai_API_KEY) + throw new Error('openai API key not configured.'); + if (!openai){ + openai = new Openai({apiKey: openai_API_KEY, dangerouslyAllowBrowser: true}); + if (!openai) { + throw new Error('Could not connect to open AI'); } } if (!assistantId){ @@ -25,52 +60,176 @@ async function connectIfNeeded(){ if (!currentThreadId){ currentThreadId = await createThread(); } + + if (document.title !== documentTitle) { + fileId = await uploadLogs(); + documentTitle = document.title; + } + +} + +async function uploadLogs() { + console.log(globalLogs); + + const jsonString = JSON.stringify(globalLogs); + const blob = new Blob([jsonString], {type:"application/json"}) + const file = new File([blob], "logs.json", { type: "application/json" }); + const filesList = await openai.files.list(); + filesList.data.forEach( file => file.filename == 'logs.json' && openai.files.del(file.id)) + const uploadRes = await openai.files.create({ + file, + purpose: "assistants" + }); + const fileId = uploadRes.id; + console.log("Uploaded file ID:", fileId); + await openai.beta.assistants.update(assistantId, { tools: [{type: 'file_search'}] }) + return fileId; + } async function createThread(){ if (!assistantId) throw new Error("cannot create thread before initializing assistant"); - const newThread = await openAI.beta.threads.create(); + const newThread = await openai.beta.threads.create(); if (!newThread) throw new Error("something went wrong while creating thread"); return newThread.id; } +async function createAssistant(name, instructions, model, tools){ + const assistant = await openai.beta.assistants.create({instructions,name,model,tools}); + if (!assistant) + throw new Error("error creating new assistant"); + return assistant; +} + async function findAssistantIdByName(name) { - if (assistantId) { - return true + if (assistantId) return assistantId; + const assistantsList = await openai.beta.assistants.list({order: "desc", limit: 20}); + if (!assistantsList) + throw new Error("could not retrieve the list of assistants"); + let assistant = assistantsList.data.find(a => a.name === name); + if (!assistant){ + const assistantInstructions = await loadInstructions(); + const assistantTools = await loadTools(); + assistant = await createAssistant(TARGET_ASSISTANT_NAME, assistantInstructions, TARGET_ASSISTANT_MODEL, assistantTools); } - const assistants_list = await openAI.beta.assistants.list({order: "desc", limit: 20}); - let assistant = assistants_list.data.find(a => a.name === name); - if (assistant) - return assistant.id; - else - throw new Error("could not find assistant with the specified name"); + return assistant.id; } async function sendQueryToAssistant(query){ await connectIfNeeded(); - const message = await openAI.beta.threads.messages.create(currentThreadId, { role: "user", content: query }); + const message = await openai.beta.threads.messages.create(currentThreadId, { + role: "user", + content: query, + attachments: fileId && [{ + file_id: fileId, + tools: [{ type: "file_search" }] + }] }); if (!message) throw new Error("Could not send message to assistant"); - const run = openAI.beta.threads.runs - .stream(currentThreadId, { assistant_id: assistantId, stream: true }) - .on('messageDelta', (delta, snapshot) => {addChatMessage(snapshot.content[0].text.value, "assistant"); - }) + let runId; + const run = openai.beta.threads.runs.stream(currentThreadId, {assistant_id: assistantId}); if (!run) - throw new Error("Could not establish run streaming") + throw new Error("Could not establish run streaming"); + document.getElementById('thinking-message').style.display= 'block'; + handleRunStream(run); + } +async function handleRunStream(runStream){ + for await (const event of runStream) { + switch (event.event) { + case 'thread.message.delta': + document.getElementById('thinking-message').style.display= 'none'; + addChatMessage(event.data.delta.content[0].text.value, "assistant"); + break; + case 'thread.run.requires_action': + document.getElementById('thinking-message').style.display= 'none'; + handleToolCall(event); + break; + } + } +} + +async function handleToolCall(event) { + if (!event.data.required_action.submit_tool_outputs || !event.data.required_action.submit_tool_outputs.tool_calls) + throw new Error ("passed event does not require action") + const toolCalls = event.data.required_action.submit_tool_outputs.tool_calls; + const toolOutputs = []; + for (const toolCall of toolCalls){ + const toolCallId = toolCall.id; + const toolName = toolCall.function.name; + const supportedTools = new Set(["save_all_parameters","save_changed_parameters","save_minimal_parameters"]); + let toolOutput = {tool_call_id: toolCallId}; + if (supportedTools.has(toolName)){ + window[toolName](); + toolOutput.output = "success"; + } + else { + toolOutput.output = "failure, the function that was called is not supported"; + } + toolOutputs.push(toolOutput); + } + + const run = await openai.beta.threads.runs.submitToolOutputs( + currentThreadId, + event.data.id, + { + tool_outputs: toolOutputs, + stream: true + } + ); + + if (!run) + throw new Error ("error occurred while submitting tool outputs"); + document.getElementById('thinking-message').style.display= 'block'; + handleRunStream(run); + +} + +async function loadInstructions() { + const response = await fetch('instructions.txt'); + if (!response.ok) + throw new Error('error fetching file'); + const data = await response.text(); + if (!data) + throw new Error("could not load instructions for new assistant"); + return data; +} + + + +async function loadTools(){ + const response = await fetch("assistantTools.json"); + if (!response.ok) + throw new Error("error fetching file"); + const data = response.json(); + if (!data) + throw new Error("could not load assistant tools for new assistant"); + return data; +} + +async function upgradeAssistant() { + const upgradeButton = document.getElementById('upgrade-assistant'); + upgradeButton.title = 'Upgrade in progress...' + await connectIfNeeded(); + const response = await openai.beta.assistants.del(assistantId); + assistantId=null; + if (!response) + throw new Error("error deleting assitant"); + console.log(response); + + //connecting again would automatically recreate a new assistant with no additional overhead + await connectIfNeeded(); + if (assistantId) + upgradeButton.title = 'Upgraded successfully to the newest Assistant version' +} -document.getElementById("ai-chat-bubble").addEventListener('click', ()=>toggleChat(true)) -document.getElementById("ai-chat-close-button").addEventListener('click', ()=>toggleChat(false)) -document.getElementById("ai-chat-input-area").addEventListener('submit', sendMessage) -document.getElementById('save-api-key').addEventListener('click', saveAPIKey); function saveAPIKey(){ const apiKey = document.getElementById('openai-api-key').value.trim(); - if (apiKey) - localStorage.setItem('openai-api-key', apiKey); + localStorage.setItem('openai-api-key', apiKey); } @@ -85,33 +244,29 @@ function toggleChat(show) { chatWindow.style.display = 'none'; chatBubble.style.display = 'flex' } - } function sendMessage(event) { event.preventDefault(); const messageInput = document.getElementById('ai-chat-input'); - if (!messageInput) return; const messageText = messageInput.value.trim(); if (messageText) { addChatMessage(messageText, 'user'); messageInput.value = ''; - addChatMessage("Thinking...", 'assistant'); sendQueryToAssistant(messageText); } } function addChatMessage(text, sender) { + text=text.replace(/【[\d:]+†[^】]*】/g, ''); + text = text.replace(/\*/g, ''); const messagesContainer = document.querySelector('#ai-chat-window .ai-chat-messages'); - - if (!messagesContainer) return; - if (sender === "assistant") { let last_message = messagesContainer.querySelector(`.assistant:last-of-type`); if (last_message) { - last_message.textContent = text; + last_message.textContent += text; messagesContainer.scrollTop = messagesContainer.scrollHeight; return; } @@ -122,4 +277,16 @@ function addChatMessage(text, sender) { messageElement.textContent = text; messagesContainer.appendChild(messageElement); messagesContainer.scrollTop = messagesContainer.scrollHeight; -} \ No newline at end of file +} + + +async function init(){ + document.getElementById("ai-chat-bubble").addEventListener('click', ()=>toggleChat(true)); + document.getElementById("ai-chat-close-button").addEventListener('click', ()=>toggleChat(false)); + document.getElementById("ai-chat-input-area").addEventListener('submit', sendMessage); + document.getElementById('save-api-key').addEventListener('click', saveAPIKey); + document.getElementById('openai-api-key').value=localStorage.getItem('openai-api-key'); + document.getElementById('upgrade-assistant').addEventListener('click',upgradeAssistant); +} + +init(); \ No newline at end of file diff --git a/HardwareReport/assistantTools.json b/HardwareReport/assistantTools.json new file mode 100644 index 00000000..b5b3134c --- /dev/null +++ b/HardwareReport/assistantTools.json @@ -0,0 +1,44 @@ +[ + { + "type": "function", + "function": { + "name": "save_all_parameters", + "description": "Saves all parameters", + "strict": true, + "parameters": { + "type": "object", + "properties": {}, + "additionalProperties": false, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "save_changed_parameters", + "description": "Saves changed parameters", + "strict": true, + "parameters": { + "type": "object", + "properties": {}, + "additionalProperties": false, + "required": [] + } + } + }, + { + "type": "function", + "function": { + "name": "save_minimal_parameters", + "description": "Saves minimal parameters", + "strict": true, + "parameters": { + "type": "object", + "properties": {}, + "additionalProperties": false, + "required": [] + } + } + } +] diff --git a/HardwareReport/index.html b/HardwareReport/index.html index dc15eed9..98c002df 100644 --- a/HardwareReport/index.html +++ b/HardwareReport/index.html @@ -65,7 +65,7 @@ } #ai-chat-window { - width: 360px; + width: 460px; height: 550px; background-color: #ffffff; border-radius: 12px; @@ -127,6 +127,13 @@ line-height: 1.45; font-size: 0.9em; word-wrap: break-word; + order: 0; + white-space: pre-wrap; + } + + #thinking-message { + order: 1; + display: none; } .ai-chat-message.user { @@ -467,6 +474,7 @@

Clock drift