diff --git a/apps/sim/lib/file-parsers/html-parser.test.ts b/apps/sim/lib/file-parsers/html-parser.test.ts index 8b7fe57bdf5..464107cc037 100644 --- a/apps/sim/lib/file-parsers/html-parser.test.ts +++ b/apps/sim/lib/file-parsers/html-parser.test.ts @@ -11,25 +11,39 @@ const parser = new HtmlParser() describe('HtmlParser', () => { describe('resource limits', () => { + /** + * Pinned by value: a 64 MB body aborts the process, so raising the cap + * toward the shared upload limit must fail here, not in production. + */ it('rejects a document above the input byte cap', async () => { - const sparse = Buffer.concat([ - Buffer.from('

'), - Buffer.alloc(32 * 1024 * 1024, 0x61), - Buffer.from('

'), - ]) - - await expect(parser.parseBuffer(sparse)).rejects.toThrow( - /above the maximum of 33554432 bytes/ - ) + const oversized = Buffer.alloc(32 * 1024 * 1024 + 1) + + const error = await parser.parseBuffer(oversized).catch((e) => e) + + expect(error).toBeInstanceOf(HtmlComplexityError) + expect(error.message).toMatch(/above the maximum of 33554432 bytes/) }) it('rejects a tag-dense document above the markup-token cap', async () => { - const dense = Buffer.from(`${'

a

'.repeat(300_000)}`) + const dense = Buffer.from(`${'

a

'.repeat(600_000)}`) const error = await parser.parseBuffer(dense).catch((e) => e) expect(error).toBeInstanceOf(HtmlComplexityError) - expect(error.message).toMatch(/exceeds the maximum of 500000 markup tokens/) + expect(error.message).toMatch(/exceeds the maximum of 1000000 markup tokens/) + }) + + /** + * A 30,000-row by 8-column export is ~540k tokens in 3.6 MB, an ordinary + * document that an earlier, tighter token cap rejected. + */ + it('accepts a realistic large table export', async () => { + const row = `${'value'.repeat(8)}` + const buffer = Buffer.from(`${row.repeat(30_000)}
`) + + const result = await parser.parseBuffer(buffer) + + expect(result.content).toContain('| value |') }) it('accepts a byte-heavy document whose markup stays under the token cap', async () => { @@ -42,13 +56,11 @@ describe('HtmlParser', () => { }) /** - * Deep nesting overflows the stack inside cheerio's own recursive `.text()`, - * which the pre-parse caps cannot predict. It still has to be classified as - * a resource rejection so callers fail closed rather than fall back to - * storing the document as raw text. + * `parseFile` must not wrap the rejection in a generic error, or the route + * stops recognising it and falls back to storing the document as raw text. */ it('preserves the error type through parseFile so callers still fail closed', async () => { - const dense = `${'

a

'.repeat(300_000)}` + const dense = `${'

a

'.repeat(600_000)}` const path = join(tmpdir(), `html-parser-limits-${process.pid}.html`) await writeFile(path, dense) @@ -59,6 +71,11 @@ describe('HtmlParser', () => { } }) + /** + * Deep nesting overflows the stack inside cheerio's own recursive `.text()`, + * which the pre-parse caps cannot predict. It still has to be classified as + * a resource rejection so callers fail closed. + */ it('classifies a deep-nesting stack overflow as a complexity rejection', async () => { const depth = 15_000 const buffer = Buffer.from( diff --git a/apps/sim/lib/file-parsers/html-parser.ts b/apps/sim/lib/file-parsers/html-parser.ts index f070b276cd2..c1532dcfbbc 100644 --- a/apps/sim/lib/file-parsers/html-parser.ts +++ b/apps/sim/lib/file-parsers/html-parser.ts @@ -8,15 +8,25 @@ import { sanitizeTextForUTF8 } from '@/lib/file-parsers/utils' const logger = createLogger('HtmlParser') /** - * `cheerio.load` retains ~530 bytes of DOM per markup token (`<`) — measured on - * cheerio 1.1.2 at 0.2M/1M/2M tokens (101/504/1008 MB), flat across all three — - * so this bounds one document's tree at roughly 256 MB. + * Bounds the DOM tree, which costs ~500 bytes per markup token (`<`) on cheerio + * 1.1.2: measured at 0.2M/1M/2M tokens as 101/504/1008 MB retained, linear + * across all three. + * + * A 50,000-row by 8-column table export is ~900k tokens in only 8.7 MB, so a + * tighter cap rejects ordinary exports; 999k tokens in a 10 MB body parses + * inside a 2 GB heap. */ -const MAX_HTML_MARKUP_TOKENS = 500_000 +const MAX_HTML_MARKUP_TOKENS = 1_000_000 /** - * Backstop for markup sparse enough to pass the token cap: bounds the UTF-16 - * copy `buffer.toString` allocates and the text nodes the DOM keeps. + * Bounds the body, which governs peak memory: extraction materialises the text + * several times over (UTF-16 buffer copy, per-node strings, the joined output). + * + * Measured against a 2 GB heap with the token cap saturated: 32 MB parses, + * 48 MB parses, 64 MB aborts the process. Deliberately NOT raised to the + * shared 100 MB document limit — that limit governs what may be uploaded, and + * a 100 MB body aborts here even with few tokens. Any change to this number + * needs the same abort test, not a consistency argument. */ const MAX_HTML_INPUT_BYTES = 32 * 1024 * 1024