Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions common/src/tools/params/tool/read-url.ts
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ export const readUrlParams = {
z.object({
url: z.string().optional(),
errorMessage: z.string(),
errorCode: z.string().optional(),
status: z.number().optional(),
}),
]),
),
Expand Down
97 changes: 97 additions & 0 deletions sdk/src/__tests__/read-url.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -122,6 +122,45 @@ describe('readUrl', () => {
expect(result.text).toContain('Bad entity: �')
})

it('extracts HTML tables into structured markdown tables', async () => {
const result = await successValue(`
<main>
<article>
<h1>API Parameters</h1>
<table>
<thead>
<tr>
<th>Parameter</th>
<th>Type</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>url</code></td>
<td>string</td>
<td>The web URL to fetch</td>
</tr>
<tr>
<td><code>max_chars</code></td>
<td>number</td>
<td>Maximum characters to return</td>
</tr>
</tbody>
</table>
</article>
</main>
`)

expect('errorMessage' in result).toBe(false)
if ('errorMessage' in result) return

expect(result.text).toContain('| Parameter | Type | Description |')
expect(result.text).toContain('| --- | --- | --- |')
expect(result.text).toContain('| url | string | The web URL to fetch |')
expect(result.text).toContain('| max_chars | number | Maximum characters to return |')
})

it('rejects non-http URLs', async () => {
const result = await readUrl({
url: 'file:///etc/passwd',
Expand All @@ -133,6 +172,7 @@ describe('readUrl', () => {
expect(result[0].value).toEqual({
url: 'file:///etc/passwd',
errorMessage: 'Only http:// and https:// URLs are supported',
errorCode: 'INVALID_URL',
})
})

Expand Down Expand Up @@ -267,6 +307,7 @@ describe('readUrl SSRF protection', () => {
url: 'http://169.254.169.254/latest/meta-data/',
errorMessage:
'Refusing to fetch private or reserved address: 169.254.169.254',
errorCode: 'BLOCKED_ADDRESS',
})
})

Expand All @@ -284,6 +325,7 @@ describe('readUrl SSRF protection', () => {
url: 'http://intranet.example.com/secrets',
errorMessage:
'Host "intranet.example.com" resolves to a private or reserved address (10.0.0.5)',
errorCode: 'BLOCKED_ADDRESS',
})
})

Expand Down Expand Up @@ -326,6 +368,7 @@ describe('readUrl SSRF protection', () => {
url: 'https://public.example.com/start',
errorMessage:
'Refusing to fetch private or reserved address: 169.254.169.254',
errorCode: 'BLOCKED_ADDRESS',
})
})

Expand All @@ -349,6 +392,7 @@ describe('readUrl SSRF protection', () => {
expect(result[0].value).toEqual({
url: 'https://example.com/loop',
errorMessage: 'Too many redirects (>5)',
errorCode: 'TOO_MANY_REDIRECTS',
})
})

Expand All @@ -367,6 +411,59 @@ describe('readUrl SSRF protection', () => {
expect(result[0].value).toEqual({
url: 'https://example.com/start',
errorMessage: 'Invalid redirect location: http://',
errorCode: 'INVALID_URL',
})
})

it('returns HTTP_ERROR and status code for 404 and 429 responses', async () => {
const notFoundResult = await readUrl({
url: 'https://example.com/not-found',
fetch: async () =>
new Response('Not Found', {
status: 404,
statusText: 'Not Found',
}),
})

expect(notFoundResult[0].value).toEqual({
url: 'https://example.com/not-found',
errorMessage: 'Failed to fetch URL: 404 Not Found',
errorCode: 'HTTP_ERROR',
status: 404,
})

const rateLimitedResult = await readUrl({
url: 'https://example.com/api',
fetch: async () =>
new Response('Too Many Requests', {
status: 429,
statusText: 'Too Many Requests',
}),
})

expect(rateLimitedResult[0].value).toEqual({
url: 'https://example.com/api',
errorMessage: 'Failed to fetch URL: 429 Too Many Requests',
errorCode: 'HTTP_ERROR',
status: 429,
})
})

it('returns UNSUPPORTED_CONTENT_TYPE for binary payloads', async () => {
const binaryResult = await readUrl({
url: 'https://example.com/image.png',
fetch: async () =>
new Response(new Uint8Array([0x89, 0x50, 0x4e, 0x47]), {
status: 200,
headers: { 'content-type': 'image/png' },
}),
})

expect(binaryResult[0].value).toEqual({
url: 'https://example.com/image.png',
errorMessage: 'Unsupported content type: image/png',
errorCode: 'UNSUPPORTED_CONTENT_TYPE',
status: 200,
})
})
})
112 changes: 102 additions & 10 deletions sdk/src/tools/read-url.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,37 @@ type FetchLike = (
init?: RequestInit,
) => Promise<Response>

export type ReadUrlErrorCode =
| 'INVALID_URL'
| 'BLOCKED_ADDRESS'
| 'HTTP_ERROR'
| 'TOO_MANY_REDIRECTS'
| 'UNSUPPORTED_CONTENT_TYPE'
| 'RESPONSE_TOO_LARGE'
| 'TIMEOUT'
| 'ABORTED'
| 'NO_READABLE_TEXT'
| 'FETCH_ERROR'

function errorResult(
url: string | undefined,
errorMessage: string,
options?: {
errorCode?: ReadUrlErrorCode
status?: number
},
): ReadUrlOutput {
return [{ type: 'json', value: { ...(url ? { url } : {}), errorMessage } }]
return [
{
type: 'json',
value: {
...(url ? { url } : {}),
errorMessage,
...(options?.errorCode ? { errorCode: options.errorCode } : {}),
...(options?.status !== undefined ? { status: options.status } : {}),
},
},
]
}

function getHeader(headers: Headers, name: string): string | undefined {
Expand Down Expand Up @@ -174,6 +200,41 @@ function extractMetaContent(html: string, name: string): string | undefined {
return undefined
}

function formatHtmlTables(html: string): string {
return html.replace(
/<table\b[^>]*>([\s\S]*?)<\/table>/gi,
(_, tableContent: string) => {
const rows = Array.from(
tableContent.matchAll(/<tr\b[^>]*>([\s\S]*?)<\/tr>/gi),
(m) => m[1],
)
if (rows.length === 0) return tableContent

const formattedRows: string[] = []
let isFirstRow = true

for (const row of rows) {
const cells = Array.from(
row.matchAll(/<(th|td)\b[^>]*>([\s\S]*?)<\/\1>/gi),
(m) => normalizeText(decodeHtmlEntities(stripTags(m[2]))),
)
if (cells.length === 0) continue

formattedRows.push(`| ${cells.join(' | ')} |`)

if (isFirstRow) {
isFirstRow = false
formattedRows.push(`| ${cells.map(() => '---').join(' | ')} |`)
}
}

return formattedRows.length > 0
? `\n\n${formattedRows.join('\n')}\n\n`
: tableContent
},
)
}

function extractHtml(html: string): {
title?: string
description?: string
Expand Down Expand Up @@ -206,6 +267,7 @@ function extractHtml(html: string): {
}

readable = selectReadableHtml(readable)
readable = formatHtmlTables(readable)

readable = readable
.replace(/<br\s*\/?>/gi, '\n')
Expand Down Expand Up @@ -345,14 +407,16 @@ export async function readUrl({
signal?: AbortSignal
}): Promise<ReadUrlOutput> {
if (signal?.aborted) {
return errorResult(url, 'Cancelled: the run was aborted by the user.')
return errorResult(url, 'Cancelled: the run was aborted by the user.', {
errorCode: 'ABORTED',
})
}

let parsedUrl: URL
try {
parsedUrl = new URL(url)
} catch {
return errorResult(url, 'Invalid URL')
return errorResult(url, 'Invalid URL', { errorCode: 'INVALID_URL' })
}

const controller = new AbortController()
Expand All @@ -375,10 +439,14 @@ export async function readUrl({
// are still blocked.
await assertUrlAllowed(currentUrl, { lookupHost, resolveDns })
} catch (error) {
return errorResult(
url,
error instanceof Error ? error.message : 'Blocked URL',
)
const errorMsg =
error instanceof Error ? error.message : 'Blocked URL'
const isInvalid =
error instanceof Error &&
error.message.includes('Only http:// and https:// URLs are supported')
return errorResult(url, errorMsg, {
errorCode: isInvalid ? 'INVALID_URL' : 'BLOCKED_ADDRESS',
})
}

response = await fetchImpl(currentUrl.toString(), {
Expand All @@ -401,19 +469,26 @@ export async function readUrl({
break
}
if (redirects >= MAX_REDIRECTS) {
return errorResult(url, `Too many redirects (>${MAX_REDIRECTS})`)
return errorResult(url, `Too many redirects (>${MAX_REDIRECTS})`, {
errorCode: 'TOO_MANY_REDIRECTS',
})
}
try {
currentUrl = new URL(location, currentUrl)
} catch {
return errorResult(url, `Invalid redirect location: ${location}`)
return errorResult(
url,
`Invalid redirect location: ${location}`,
{ errorCode: 'INVALID_URL' },
)
}
}

if (!response.ok) {
return errorResult(
url,
`Failed to fetch URL: ${response.status} ${response.statusText}`,
{ errorCode: 'HTTP_ERROR', status: response.status },
)
}

Expand All @@ -422,6 +497,7 @@ export async function readUrl({
return errorResult(
url,
`Unsupported content type: ${contentType || 'unknown'}`,
{ errorCode: 'UNSUPPORTED_CONTENT_TYPE', status: response.status },
)
}

Expand All @@ -430,7 +506,10 @@ export async function readUrl({
const truncated = truncateText(extracted.text, max_chars)

if (!truncated.text) {
return errorResult(url, 'No readable text found at URL')
return errorResult(url, 'No readable text found at URL', {
errorCode: 'NO_READABLE_TEXT',
status: response.status,
})
}

return [
Expand All @@ -452,6 +531,10 @@ export async function readUrl({
]
} catch (error) {
const isAbort = error instanceof Error && error.name === 'AbortError'
const isTooLarge =
error instanceof Error &&
(error.message.includes('Response is too large') ||
error.message.includes('Response exceeded'))
return errorResult(
url,
isAbort
Expand All @@ -461,6 +544,15 @@ export async function readUrl({
: error instanceof Error
? error.message
: 'Unknown error',
{
errorCode: isAbort
? signal?.aborted
? 'ABORTED'
: 'TIMEOUT'
: isTooLarge
? 'RESPONSE_TOO_LARGE'
: 'FETCH_ERROR',
},
)
} finally {
clearTimeout(timeout)
Expand Down
Loading