diff --git a/Apps/Web/aiplugin/background.js b/Apps/Web/aiplugin/background.js index def9567e..e75321c3 100644 --- a/Apps/Web/aiplugin/background.js +++ b/Apps/Web/aiplugin/background.js @@ -6,6 +6,8 @@ const defaultIcon = 'zeuz.png'; var zeuz_url; var zeuz_key; var zeuz_node_id; +const _aiContentTimers = {}; // per-tab debounce for node_ai_contents +const _aiContentHashes = {}; // per-tab hashes to avoid resending unchanged contents fetch("data.json") .then(Response => Response.json()) @@ -135,7 +137,7 @@ if (navigator.userAgentData.platform.toLowerCase().includes('mac')) { } browserAppData.runtime.onMessage.addListener( function (request, sender, sendResponse) { - + if (request.action === 'toggle_from_content_script') { // allows the floating button to trigger the toggle logic toggle(sender.tab); @@ -171,36 +173,71 @@ browserAppData.runtime.onMessage.addListener( .then(text => { console.log(text); sendResponse(text); }) return true; // Will respond asynchronously. - } else if (request.apiName == 'node_ai_contents'){ - var url = `${zeuz_url}/node_ai_contents/`; - fetch(url, { - method: "POST", - headers: { - // "Content-Type": "application/json", - "X-Api-Key": zeuz_key, - }, - body: JSON.stringify({ - "dom_web": { "dom": request.dom }, - "node_id": zeuz_node_id - }), - }) - .then(response => response.json()) - .then(text => { console.log(text); sendResponse(text); }) + } else if (request.apiName == 'node_ai_contents') { + const tabId = sender.tab ? sender.tab.id : 'unknown'; + if (_aiContentTimers[tabId]) clearTimeout(_aiContentTimers[tabId]); + _aiContentTimers[tabId] = setTimeout(async () => { + delete _aiContentTimers[tabId]; + + const contentObj = { "dom": request.dom, "page_map": request.page_map, "page_map_json": request.page_map_json }; + const contentStr = JSON.stringify(contentObj); + + let hash = ''; + try { + const msgUint8 = new TextEncoder().encode(contentStr); + const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8); + const hashArray = Array.from(new Uint8Array(hashBuffer)); + hash = hashArray.map(b => b.toString(16).padStart(2, '0')).join(''); + } catch (e) { + console.error("Error generating hash", e); + hash = contentStr.length.toString(); // Fallback + } + + if (_aiContentHashes[tabId] === hash) { + console.log('node_ai_contents skipped, content unchanged for tab:', tabId, 'hash:', hash); + try { sendResponse({ status: "skipped" }); } catch (e) { } + return; + } + _aiContentHashes[tabId] = hash; + console.log('node_ai_contents sending, new hash for tab:', tabId, 'hash:', hash); + + var url = `${zeuz_url}/node_ai_contents/`; + fetch(url, { + method: "POST", + headers: { + // "Content-Type": "application/json", + "X-Api-Key": zeuz_key, + }, + body: JSON.stringify({ + "dom_web": contentObj, + "node_id": zeuz_node_id + }), + }) + .then(response => { + if (!response.ok) { + console.error("node_ai_contents failed with status:", response.status, response.statusText); + } + return response.json(); + }) + .then(text => { console.log("node_ai_contents response:", text); try { sendResponse(text); } catch (e) { } }) + .catch(e => console.error("node_ai_contents fetch error:", e)); + }, 2000); + return true; // Will respond asynchronously. } } ); // add AI Inspector to the right click menu browserAppData.runtime.onInstalled.addListener(() => { - browserAppData.contextMenus.create({ - id: "toggle-ai-inspect", - title: "Inspect with AI", - contexts: ["all"] - }); + browserAppData.contextMenus.create({ + id: "toggle-ai-inspect", + title: "Inspect with AI", + contexts: ["all"] + }); }); browserAppData.contextMenus.onClicked.addListener((info, tab) => { - if (info.menuItemId === "toggle-ai-inspect" && tab) { - toggle(tab); - } + if (info.menuItemId === "toggle-ai-inspect" && tab) { + toggle(tab); + } }); \ No newline at end of file diff --git a/Apps/Web/aiplugin/inspect.js b/Apps/Web/aiplugin/inspect.js index 0cb3a1ab..981407f8 100644 --- a/Apps/Web/aiplugin/inspect.js +++ b/Apps/Web/aiplugin/inspect.js @@ -1,10 +1,14 @@ const browserAppData = chrome || browser; setInterval(() => { + // Only run this periodic extraction on the main page, not inside iframes + // (this prevents tracking pixels from overwriting the main page DOM) + if (window.top !== window.self) return; + var html = document.createElement('html'); - html.setAttribute('zeuz','aiplugin'); + html.setAttribute('zeuz', 'aiplugin'); var myString = document.documentElement.outerHTML; html.innerHTML = myString; - + var elements = html.getElementsByTagName('head'); while (elements[0]) elements[0].parentNode.removeChild(elements[0]) @@ -20,16 +24,27 @@ setInterval(() => { var elements = html.getElementsByTagName('style'); while (elements[0]) elements[0].parentNode.removeChild(elements[0]) - + // AI model works better on indented dom, so not removing indentation. // var result = html.outerHTML.replace(/\s+/g, ' ').replace(/>\s+<'); //The following code removes non-unicode characters except newline and tab var result = html.outerHTML.replace(/[\x00-\x08\x0B-\x1F\x7F]/g, ''); + let mapData = { page_map_json: null, page_map: "" }; + if (typeof extractPageMapData === 'function') { + try { + mapData = extractPageMapData(); + } catch (e) { + console.error("Error extracting page map:", e); + } + } + browserAppData.runtime.sendMessage({ apiName: 'node_ai_contents', dom: result, + page_map: mapData.page_map, + page_map_json: mapData.page_map_json }) }, 5000); @@ -68,7 +83,7 @@ class Inspector { if (response["info"] == "success") { const modalText = 'Element data was recorded. Please Click "Add by AI"'; console.log(modalText); - + if (this.successContainer) { this.successContainer.textContent = modalText; this.successContainer.classList.add('show'); @@ -88,9 +103,9 @@ class Inspector { data: data, html: refinedHtml, }, - response => { - insert_modal_text(response, modal_id); - } + response => { + insert_modal_text(response, modal_id); + } ); } @@ -477,19 +492,19 @@ class Inspector { let value = e.target.getAttribute(name); elementText += `${name}="${value}" `; } - + this.attributesContainer.textContent = elementText.trim(); } activate() { this.createOverlayElements(); this.createSuccessMessage(); - + const style = document.createElement('style'); style.id = this.cssNode; style.textContent = '*{cursor:crosshair!important;}'; document.head.appendChild(style); - + // add listeners document.addEventListener('click', this.getData, true); this.options.inspector && (document.addEventListener('mouseover', this.draw)); @@ -506,7 +521,7 @@ class Inspector { 'zeuz-success-host' ] - for (let elemId of Remove){ + for (let elemId of Remove) { const elem = document.getElementById(elemId); elem && elem.remove(); } @@ -514,7 +529,7 @@ class Inspector { // remove listeners document.removeEventListener('click', this.getData, true); this.options && this.options.inspector && (document.removeEventListener('mouseover', this.draw)); - + // reset this.attributesHost = null; this.attributesContainer = null; diff --git a/Apps/Web/aiplugin/manifest.json b/Apps/Web/aiplugin/manifest.json index 4bf0a230..4429edae 100644 --- a/Apps/Web/aiplugin/manifest.json +++ b/Apps/Web/aiplugin/manifest.json @@ -27,6 +27,7 @@ "https://*/*" ], "js": [ + "page_map_extractor.js", "inspect.js" ] }, diff --git a/Apps/Web/aiplugin/page_map_extractor.js b/Apps/Web/aiplugin/page_map_extractor.js new file mode 100644 index 00000000..8b4b95c7 --- /dev/null +++ b/Apps/Web/aiplugin/page_map_extractor.js @@ -0,0 +1,201 @@ +function extractPageMapData() { + // ── Helpers ────────────────────────────────────────────────────────────── + const vw = window.innerWidth, vh = window.innerHeight; + + function isVisible(el) { + const s = window.getComputedStyle(el); + return s.display !== 'none' && s.visibility !== 'hidden' && parseFloat(s.opacity) >= 0.1; + } + + function inViewport(el) { + const r = el.getBoundingClientRect(); + return r.bottom > 0 && r.right > 0 && r.top < vh && r.left < vw; + } + + function norm(s, max = 120) { + return (s || '').replace(/\s+/g, ' ').trim().slice(0, max); + } + + // ── XPath generator ────────────────────────────────────────────────────── + function getXPath(el) { + if (el.id) return `//*[@id="${el.id}"]`; + const stableAttrs = ['name', 'data-testid', 'aria-label', 'placeholder']; + for (const attr of stableAttrs) { + const val = el.getAttribute(attr); + if (val) { + const tag = el.tagName.toLowerCase(); + try { + if (document.querySelectorAll(`${tag}[${attr}="${CSS.escape(val)}"]`).length === 1) + return `//${tag}[@${attr}="${val}"]`; + } catch (e) { + // Ignore escaping errors + } + } + } + if (['BUTTON', 'A'].includes(el.tagName)) { + const txt = el.innerText.trim().slice(0, 60); + if (txt) { + const tag = el.tagName.toLowerCase(); + const hits = [...document.querySelectorAll(tag)].filter(e => e.innerText.trim().startsWith(txt)); + if (hits.length === 1) return `//${tag}[normalize-space()="${txt}"]`; + } + } + function pos(e) { + const tag = e.tagName.toLowerCase(); + const sibs = [...e.parentNode.children].filter(c => c.tagName === e.tagName); + return `${tag}${sibs.length > 1 ? `[${sibs.indexOf(e) + 1}]` : ''}`; + } + const parts = []; + let cur = el; + while (cur && cur.nodeType === 1) { parts.unshift(pos(cur)); cur = cur.parentElement; } + return '/' + parts.join('/'); + } + + // ── Label resolver ──────────────────────────────────────────────────────── + function getLabel(el) { + if (el.id) { + const lbl = document.querySelector(`label[for="${el.id}"]`); + if (lbl) return lbl.innerText.trim(); + } + const wrap = el.closest('label'); + if (wrap) return wrap.innerText.replace(el.value || '', '').trim(); + const al = el.getAttribute('aria-label'); + if (al) return al.trim(); + const alby = el.getAttribute('aria-labelledby'); + if (alby) { const ref = document.getElementById(alby); if (ref) return ref.innerText.trim(); } + return null; + } + + function getNearestHeading(el) { + let cur = el.parentElement; + while (cur && cur !== document.body) { + const h = cur.querySelector('h1,h2,h3,h4,h5,h6'); + if (h) return norm(h.innerText, 80); + cur = cur.parentElement; + } + return null; + } + + // ── Interactive element selector ───────────────────────────────────────── + const ACTION_SELECTOR = [ + 'input:not([type=hidden])', 'textarea', 'select', + 'button', 'a[href]', + '[role=button]', '[role=link]', '[role=textbox]', + '[role=checkbox]', '[role=radio]', '[role=combobox]', '[role=option]' + ].join(','); + + const actionEls = new Set(document.querySelectorAll(ACTION_SELECTOR)); + + // ── Text node selector ─────────────────────────────────────────────────── + const TEXT_SELECTOR = 'h1,h2,h3,h4,h5,h6,p,li,td,th,label,span,div,[role=heading],[role=status],[role=alert],[aria-live]'; + + function getDirectText(el) { + let text = ''; + for (const node of el.childNodes) { + if (node.nodeType === Node.TEXT_NODE) text += node.textContent; + } + return text.replace(/\s+/g, ' ').trim(); + } + + function hasOwnText(el) { + const full = norm(el.innerText, 200); + if (!full || full.length < 3) return false; + if (actionEls.has(el)) return false; + const nested = [...el.querySelectorAll(ACTION_SELECTOR)]; + if (nested.length === 1 && norm(nested[0].innerText, 200) === full) return false; + return true; + } + + // ── Collect everything in DOM order ────────────────────────────────────── + const allNodes = []; + + // Pre-compute DOM order Map to avoid O(N^2) slowdown on large pages (eBay/Amazon) + const domOrderMap = new Map(); + document.querySelectorAll('*').forEach((el, idx) => domOrderMap.set(el, idx)); + + // Pass 1 — action elements + document.querySelectorAll(ACTION_SELECTOR).forEach(el => { + if (!isVisible(el)) return; + const form = el.closest('form'); + allNodes.push({ + _type: 'action', + _el: el, + _order: domOrderMap.get(el) || 0, + tag: el.tagName.toLowerCase(), + type: el.getAttribute('type') || null, + role: el.getAttribute('role') || el.tagName.toLowerCase(), + id: el.id || null, + name: el.getAttribute('name') || null, + placeholder: el.getAttribute('placeholder') || null, + label: getLabel(el) || null, + text: norm(el.innerText || el.value || '', 80) || null, + required: el.required || false, + disabled: el.disabled || false, + in_viewport: inViewport(el), + form_id: form && form.id ? form.id : null, + heading: getNearestHeading(el), + xpath: getXPath(el), + }); + }); + + // Pass 2 — text context nodes + const seenTexts = new Set(); + document.querySelectorAll(TEXT_SELECTOR).forEach(el => { + if (!isVisible(el) || !inViewport(el)) return; + if (!hasOwnText(el)) return; + const text = norm(el.innerText, 150); + if (!text || text.length < 5) return; + if (seenTexts.has(text)) return; + + let dominated = false; + for (const seen of seenTexts) { if (seen.includes(text) || text.includes(seen)) { dominated = true; break; } } + if (dominated) return; + seenTexts.add(text); + + const tag = el.tagName.toLowerCase(); + const isHeading = /^h[1-6]$/.test(tag) || el.getAttribute('role') === 'heading'; + allNodes.push({ + _type: 'text', + _el: el, + _order: domOrderMap.get(el) || 0, + kind: isHeading ? 'heading' : (el.getAttribute('role') || tag), + text, + in_viewport: true, + }); + }); + + // Sort by DOM order + allNodes.sort((a, b) => a._order - b._order); + + // Strip internal fields and assign idx + const page_map_json = allNodes.map((n, i) => { + const { _type, _el, _order, ...rest } = n; + return { idx: i, node_type: _type, ...rest }; + }); + + // Build compact text map + const lines = ["# Page Map (text context + interactive elements, in document order)\n"]; + for (const node of page_map_json) { + if (node.node_type === "text") { + const kind = (node.kind || "text").toUpperCase(); + const vp = node.in_viewport ? "👁" : "↕"; + lines.push(` ${vp} [${kind}] "${node.text}"`); + } else { + const parts = [`[${node.idx}]`, (node.role || '').toUpperCase()]; + if (node.label) parts.push(`label='${node.label}'`); + if (node.placeholder) parts.push(`placeholder='${node.placeholder}'`); + if (node.text) parts.push(`text='${node.text}'`); + if (node.type) parts.push(`type=${node.type}`); + if (node.name) parts.push(`name=${node.name}`); + if (node.required) parts.push("required"); + if (node.disabled) parts.push("disabled"); + if (node.heading) parts.push(`section='${node.heading}'`); + const vp = node.in_viewport ? "👁" : "↕"; + parts.push(vp); + lines.push(" " + parts.join(" ")); + } + } + const page_map = lines.join("\n"); + + return { page_map_json, page_map }; +}