diff --git a/apps/desktop/src/main/browser-agent/url-guard.test.ts b/apps/desktop/src/main/browser-agent/url-guard.test.ts index 8544e149b90..e50d51eb6b7 100644 --- a/apps/desktop/src/main/browser-agent/url-guard.test.ts +++ b/apps/desktop/src/main/browser-agent/url-guard.test.ts @@ -1,5 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' +// url-guard pulls in @/main/navigation, which imports electron. +vi.mock('electron', () => import('@/test/electron-mock')) + const { mockLookup } = vi.hoisted(() => ({ mockLookup: vi.fn() })) // The real resolveHostAddresses runs; only the resolver under it is mocked, so diff --git a/apps/desktop/src/main/csp.test.ts b/apps/desktop/src/main/csp.test.ts index 1319ca314fb..0bab8851199 100644 --- a/apps/desktop/src/main/csp.test.ts +++ b/apps/desktop/src/main/csp.test.ts @@ -1,4 +1,8 @@ import { beforeEach, describe, expect, it, vi } from 'vitest' + +// csp pulls in @/main/navigation, which imports electron. +vi.mock('electron', () => import('@/test/electron-mock')) + import { attachCspFallback, DEFAULT_DESKTOP_CSP } from '@/main/csp' type HeadersReceivedHandler = ( diff --git a/apps/desktop/src/main/telemetry-policy.test.ts b/apps/desktop/src/main/telemetry-policy.test.ts index 29b0cd2f6a1..c4d7ab876b0 100644 --- a/apps/desktop/src/main/telemetry-policy.test.ts +++ b/apps/desktop/src/main/telemetry-policy.test.ts @@ -1,4 +1,8 @@ -import { describe, expect, it } from 'vitest' +import { describe, expect, it, vi } from 'vitest' + +// telemetry-policy pulls in @/main/navigation, which imports electron. +vi.mock('electron', () => import('@/test/electron-mock')) + import { shouldBlockRequest } from '@/main/telemetry-policy' describe('shouldBlockRequest', () => { diff --git a/apps/sim/app/api/files/parse/route.ts b/apps/sim/app/api/files/parse/route.ts index 8047cea0f0d..a6c047ec217 100644 --- a/apps/sim/app/api/files/parse/route.ts +++ b/apps/sim/app/api/files/parse/route.ts @@ -13,6 +13,7 @@ import { checkInternalAuth } from '@/lib/auth/hybrid' import { sanitizeUrlForLog } from '@/lib/core/utils/logging' import { assertKnownSizeWithinLimit, isPayloadSizeLimitError } from '@/lib/core/utils/stream-limits' import { isSupportedFileType, parseFile } from '@/lib/file-parsers' +import { isHtmlComplexityError } from '@/lib/file-parsers/html-parser' import { isYamlComplexityError } from '@/lib/file-parsers/yaml-parser' import { isUsingCloudStorage, StorageService } from '@/lib/uploads' import { uploadExecutionFile } from '@/lib/uploads/contexts/execution' @@ -1047,6 +1048,7 @@ async function handleGenericTextBuffer( // Fail closed on a resource-exhaustion rejection instead of silently // storing the crafted document as raw text. if (isYamlComplexityError(parserError)) throw parserError + if (isHtmlComplexityError(parserError)) throw parserError logger.warn('Specialized parser failed, falling back to generic parsing:', parserError) } diff --git a/apps/sim/lib/file-parsers/html-parser.test.ts b/apps/sim/lib/file-parsers/html-parser.test.ts new file mode 100644 index 00000000000..8b7fe57bdf5 --- /dev/null +++ b/apps/sim/lib/file-parsers/html-parser.test.ts @@ -0,0 +1,99 @@ +/** + * @vitest-environment node + */ +import { rm, writeFile } from 'fs/promises' +import { tmpdir } from 'os' +import { join } from 'path' +import { describe, expect, it } from 'vitest' +import { HtmlComplexityError, HtmlParser } from '@/lib/file-parsers/html-parser' + +const parser = new HtmlParser() + +describe('HtmlParser', () => { + describe('resource limits', () => { + it('rejects a document above the input byte cap', async () => { + const sparse = Buffer.concat([ + Buffer.from('

'), + Buffer.alloc(32 * 1024 * 1024, 0x61), + Buffer.from('

'), + ]) + + await expect(parser.parseBuffer(sparse)).rejects.toThrow( + /above the maximum of 33554432 bytes/ + ) + }) + + it('rejects a tag-dense document above the markup-token cap', async () => { + const dense = Buffer.from(`${'

a

'.repeat(300_000)}`) + + const error = await parser.parseBuffer(dense).catch((e) => e) + + expect(error).toBeInstanceOf(HtmlComplexityError) + expect(error.message).toMatch(/exceeds the maximum of 500000 markup tokens/) + }) + + it('accepts a byte-heavy document whose markup stays under the token cap', async () => { + const paragraph = `

${'word '.repeat(200)}

` + const buffer = Buffer.from(`${paragraph.repeat(2000)}`) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('word') + }) + + /** + * Deep nesting overflows the stack inside cheerio's own recursive `.text()`, + * which the pre-parse caps cannot predict. It still has to be classified as + * a resource rejection so callers fail closed rather than fall back to + * storing the document as raw text. + */ + it('preserves the error type through parseFile so callers still fail closed', async () => { + const dense = `${'

a

'.repeat(300_000)}` + const path = join(tmpdir(), `html-parser-limits-${process.pid}.html`) + await writeFile(path, dense) + + try { + await expect(parser.parseFile(path)).rejects.toBeInstanceOf(HtmlComplexityError) + } finally { + await rm(path, { force: true }) + } + }) + + it('classifies a deep-nesting stack overflow as a complexity rejection', async () => { + const depth = 15_000 + const buffer = Buffer.from( + `${'
'.repeat(depth)}deep${'
'.repeat(depth)}` + ) + + await expect(parser.parseBuffer(buffer)).rejects.toThrow(HtmlComplexityError) + }) + }) + + describe('extraction', () => { + it('extracts structured text, headings, links, and metadata', async () => { + const buffer = Buffer.from( + `Doc` + + `

Title

Body text

` + + `` + + `
h
c
` + + `Example` + + `` + ) + + const result = await parser.parseBuffer(buffer) + + expect(result.metadata?.title).toBe('Doc') + expect(result.metadata?.metaDescription).toBe('About') + expect(result.content).toContain('Title') + expect(result.content).toContain('Body text') + expect(result.content).toContain('• one') + expect(result.content).toContain('| h |') + expect(result.content).toContain('Example (https://example.com)') + expect(result.content).not.toContain('alert(1)') + expect(result.metadata?.headings).toEqual([{ level: 1, text: 'Title' }]) + expect(result.metadata?.links).toEqual([{ text: 'Example', href: 'https://example.com' }]) + expect(result.metadata?.listCount).toBe(1) + expect(result.metadata?.tableCount).toBe(1) + }) + }) +}) diff --git a/apps/sim/lib/file-parsers/html-parser.ts b/apps/sim/lib/file-parsers/html-parser.ts index a8e30aa04e3..f070b276cd2 100644 --- a/apps/sim/lib/file-parsers/html-parser.ts +++ b/apps/sim/lib/file-parsers/html-parser.ts @@ -1,27 +1,94 @@ import { readFile } from 'fs/promises' import { createLogger } from '@sim/logger' +import { getErrorMessage } from '@sim/utils/errors' import * as cheerio from 'cheerio' import type { FileParseResult, FileParser } from '@/lib/file-parsers/types' import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' const logger = createLogger('HtmlParser') +/** + * `cheerio.load` retains ~530 bytes of DOM per markup token (`<`) — measured on + * cheerio 1.1.2 at 0.2M/1M/2M tokens (101/504/1008 MB), flat across all three — + * so this bounds one document's tree at roughly 256 MB. + */ +const MAX_HTML_MARKUP_TOKENS = 500_000 + +/** + * Backstop for markup sparse enough to pass the token cap: bounds the UTF-16 + * copy `buffer.toString` allocates and the text nodes the DOM keeps. + */ +const MAX_HTML_INPUT_BYTES = 32 * 1024 * 1024 + +const MARKUP_TOKEN_BYTE = 0x3c + +/** + * Raised when a document exceeds the limits above, so an input rejected on + * resource grounds is not reported as a malformed file. + */ +export class HtmlComplexityError extends Error { + constructor(message: string) { + super(message) + this.name = 'HtmlComplexityError' + } +} + +export function isHtmlComplexityError(error: unknown): error is HtmlComplexityError { + return error instanceof HtmlComplexityError +} + +function exceedsMarkupTokenLimit(buffer: Buffer): boolean { + let count = 0 + let index = buffer.indexOf(MARKUP_TOKEN_BYTE) + + while (index !== -1) { + if (++count > MAX_HTML_MARKUP_TOKENS) return true + index = buffer.indexOf(MARKUP_TOKEN_BYTE, index + 1) + } + + return false +} + +/** + * `cheerio.load` builds the entire parse5 tree before returning, so an outsized + * document has to be rejected on the buffer, before the string copy. + */ +function assertHtmlWithinLimits(buffer: Buffer): void { + if (buffer.length > MAX_HTML_INPUT_BYTES) { + throw new HtmlComplexityError( + `HTML document is ${buffer.length} bytes, above the maximum of ${MAX_HTML_INPUT_BYTES} bytes` + ) + } + + if (exceedsMarkupTokenLimit(buffer)) { + throw new HtmlComplexityError( + `HTML document exceeds the maximum of ${MAX_HTML_MARKUP_TOKENS} markup tokens` + ) + } +} + export class HtmlParser implements FileParser { async parseFile(filePath: string): Promise { + let buffer: Buffer + + /** Scoped to the read alone so `parseBuffer`'s typed rejections reach callers intact. */ try { if (!filePath) { throw new Error('No file path provided') } - const buffer = await readFile(filePath) - return this.parseBuffer(buffer) + buffer = await readFile(filePath) } catch (error) { logger.error('HTML file error:', error) - throw new Error(`Failed to parse HTML file: ${(error as Error).message}`) + throw new Error(`Failed to parse HTML file: ${getErrorMessage(error, 'Unknown error')}`) } + + return this.parseBuffer(buffer) } async parseBuffer(buffer: Buffer): Promise { + assertHtmlWithinLimits(buffer) + try { logger.info('Parsing HTML buffer, size:', buffer.length) @@ -72,8 +139,22 @@ export class HtmlParser implements FileParser { }, } } catch (error) { + /** + * Every `RangeError` reachable here is resource exhaustion the pre-parse + * caps cannot predict: a stack overflow inside cheerio's recursive + * `.text()` on deeply nested markup, or an over-long string from joining + * the extracted parts. Both must stay fail-closed rather than degrade to + * the route's raw-text fallback. + */ + if (error instanceof RangeError) { + logger.warn('HTML document exhausted parser resources:', error) + throw new HtmlComplexityError( + `HTML document could not be extracted within resource limits: ${error.message}` + ) + } + logger.error('HTML buffer parsing error:', error) - throw new Error(`Failed to parse HTML buffer: ${(error as Error).message}`) + throw new Error(`Failed to parse HTML buffer: ${getErrorMessage(error, 'Unknown error')}`) } }