From d4a031e8e1cbb9f93a356f70c901c039036672e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Hanu=C5=A1?= Date: Thu, 2 Jul 2026 00:55:21 +0200 Subject: [PATCH] feat(api): add --verbose, --paginate, --no-auth flags to apify api `apify api` is the escape-hatch REST tool but agents debugging with it currently have no way to see the resolved outbound URL, no shortcut for walking a paginated endpoint, and no way to hit auth-less resources through the same command. This adds three additive, independent flags: - --verbose (-v): logs "-> METHOD URL" to stderr before the fetch and "<- STATUS (content-type)" after, so callers can see exactly what the CLI sent and what came back. - --paginate: for endpoints returning { data: { items, total, offset, limit } }, automatically advances offset until exhausted and emits all items as a single JSON array on stdout. Pair with --paginate-max N to cap. Fails loudly if the response is not paginated (never silently truncates). - --no-auth: skips the Authorization header. Accepts absolute URLs verbatim so the same escape hatch can reach public resources like raw.githubusercontent.com (e.g. actor-templates manifest) without requiring a login. None of the flags change existing behavior when omitted; all three compose. Refs internal findings F31 + EF8. Co-Authored-By: Claude Opus 4.7 --- src/commands/api.ts | 206 +++++++++++++++++++++++++++++++++++++++----- 1 file changed, 186 insertions(+), 20 deletions(-) diff --git a/src/commands/api.ts b/src/commands/api.ts index 25ed0ad7d..67a6ba9bb 100644 --- a/src/commands/api.ts +++ b/src/commands/api.ts @@ -179,6 +179,23 @@ export class ApiCommand extends ApifyCommand { description: 'Print a reference for an endpoint (methods, summary, path params).', command: 'apify api --describe actor-runs/{runId}', }, + { + description: 'Log the resolved outbound method + URL (and response status) to stderr.', + command: 'apify api users/me --verbose', + }, + { + description: 'Follow pagination automatically and print all items as a single JSON array.', + command: 'apify api acts --paginate', + }, + { + description: 'Follow pagination but cap at 200 items.', + command: 'apify api acts --paginate --paginate-max 200', + }, + { + description: 'Hit an unauthenticated URL through the same escape hatch.', + command: + 'apify api --no-auth https://raw.githubusercontent.com/apify/actor-templates/master/templates/manifest.json', + }, ]; static override docsUrl = 'https://docs.apify.com/api/v2'; @@ -243,6 +260,31 @@ export class ApiCommand extends ApifyCommand { required: false, exclusive: ['list-endpoints', 'search'], }), + verbose: Flags.boolean({ + char: 'v', + description: + 'Print the resolved outbound request (method + URL) to stderr before the fetch, and the ' + + 'HTTP status + Content-Type after. Useful for debugging what the CLI actually sends.', + default: false, + }), + paginate: Flags.boolean({ + description: + 'For endpoints returning { data: { items, total, offset, limit } }, automatically follow ' + + 'pagination by advancing offset until all items are fetched, then emit the collected items ' + + 'as a single JSON array on stdout. Combine with --paginate-max to cap the number of items.', + default: false, + }), + 'paginate-max': Flags.integer({ + description: 'When used with --paginate, stop after collecting this many items. Ignored without --paginate.', + required: false, + }), + 'no-auth': Flags.boolean({ + description: + 'Skip the Authorization header. Useful for hitting public / auth-less endpoints via the ' + + 'same escape hatch. When the endpoint is an absolute URL (e.g. https://raw.githubusercontent.com/...), ' + + 'the request is sent verbatim; otherwise the endpoint is resolved against the Apify API base URL.', + default: false, + }), }; async run() { @@ -310,26 +352,49 @@ export class ApiCommand extends ApifyCommand { } } - const apifyClient = await getLoggedClientOrThrow(); - const token = apifyClient.token!; - - // apifyClient.baseUrl already ends in "/v2" - const endpoint = normalizePath(endpointArg); + const { noAuth, verbose, paginate, paginateMax } = this.flags; - let url = `${apifyClient.baseUrl}/${endpoint}`; + // Resolve the base URL and token. When --no-auth is set we don't require a login; + // we also allow the endpoint to be an absolute URL so agents can hit auth-less + // resources (e.g. raw GitHub) through the same escape hatch. + let token: string | undefined; + let baseUrl = `${(process.env.APIFY_CLIENT_BASE_URL || 'https://api.apify.com').replace(/\/$/, '')}/v2`; - if (queryString) { - const separator = url.includes('?') ? '&' : '?'; - url = `${url}${separator}${queryString}`; + if (!noAuth) { + const apifyClient = await getLoggedClientOrThrow(); + token = apifyClient.token!; + // apifyClient.baseUrl already ends in "/v2" + baseUrl = apifyClient.baseUrl; } + const endpointIsAbsolute = /^https?:\/\//i.test(endpointArg); + const endpointForSuggestions = endpointIsAbsolute ? endpointArg : normalizePath(endpointArg); + + const buildUrl = (extraQuery?: string): string => { + let url = endpointIsAbsolute ? endpointArg! : `${baseUrl}/${normalizePath(endpointArg!)}`; + + const parts: string[] = []; + if (queryString) parts.push(queryString); + if (extraQuery) parts.push(extraQuery); + + if (parts.length > 0) { + const separator = url.includes('?') ? '&' : '?'; + url = `${url}${separator}${parts.join('&')}`; + } + + return url; + }; + // Build headers. Custom headers overwrite defaults case-insensitively so // callers can override e.g. Content-Type without creating duplicate entries. const headers: Record = { ...APIFY_CLIENT_DEFAULT_HEADERS, - Authorization: `Bearer ${token}`, }; + if (token) { + headers.Authorization = `Bearer ${token}`; + } + if (this.flags.body) { headers['Content-Type'] = 'application/json'; } @@ -343,16 +408,31 @@ export class ApiCommand extends ApifyCommand { headers[key] = value; } - // Make the request - const response = await fetch(url, { - method, - headers, - body: this.flags.body || undefined, - }); + const doFetch = async (url: string): Promise<{ response: Response; responseText: string }> => { + if (verbose) { + simpleLog({ message: chalk.gray(`→ ${method} ${url}`), stdout: false }); + } + + const response = await fetch(url, { + method, + headers, + body: this.flags.body || undefined, + }); - const responseText = await response.text(); + const responseText = await response.text(); - if (!response.ok) { + if (verbose) { + const contentType = response.headers.get('content-type') || ''; + simpleLog({ + message: chalk.gray(`← ${response.status} ${response.statusText}${contentType ? ` (${contentType})` : ''}`), + stdout: false, + }); + } + + return { response, responseText }; + }; + + const handleErrorResponse = async (response: Response, responseText: string): Promise => { process.exitCode = CommandExitCodes.RunFailed; // Print status to stderr but JSON response bodies to stdout so that @@ -368,10 +448,96 @@ export class ApiCommand extends ApifyCommand { } } - if (response.status === 404) { - await this.print404Suggestions(endpoint); + if (response.status === 404 && !endpointIsAbsolute) { + await this.print404Suggestions(endpointForSuggestions); + } + }; + + // --paginate: only meaningful for GET requests on endpoints that return + // { data: { items, total, offset, limit } }. Any other shape falls back to + // a single response and we warn on stderr so callers aren't silently misled. + if (paginate) { + if (method !== 'GET') { + throw new Error('--paginate can only be used with GET requests.'); + } + + const collected: unknown[] = []; + let offset = 0; + let limit: number | undefined; + let total: number | undefined; + + // eslint-disable-next-line no-constant-condition + while (true) { + const extra = new URLSearchParams(); + extra.set('offset', String(offset)); + if (limit !== undefined) extra.set('limit', String(limit)); + const url = buildUrl(extra.toString()); + + const { response, responseText } = await doFetch(url); + + if (!response.ok) { + await handleErrorResponse(response, responseText); + return; + } + + let parsed: unknown; + try { + parsed = JSON.parse(responseText); + } catch { + throw new Error('--paginate requires a JSON response body, but got non-JSON content.'); + } + + const data = (parsed as { data?: unknown } | null)?.data as + | { items?: unknown[]; total?: number; offset?: number; limit?: number; count?: number } + | undefined; + + if (!data || !Array.isArray(data.items)) { + throw new Error( + '--paginate expected a { data: { items: [...] } } response shape from the endpoint. ' + + 'This endpoint does not appear to be paginated — retry without --paginate.', + ); + } + + const { items } = data; + const pageCount = items.length; + + for (const item of items) { + if (paginateMax !== undefined && collected.length >= paginateMax) break; + collected.push(item); + } + + if (typeof data.total === 'number') total = data.total; + if (typeof data.limit === 'number' && limit === undefined) limit = data.limit; + + const nextOffset = offset + (pageCount || (limit ?? 0)); + + const reachedCap = paginateMax !== undefined && collected.length >= paginateMax; + const exhausted = total !== undefined ? nextOffset >= total : pageCount === 0; + + if (reachedCap || exhausted || pageCount === 0) break; + + offset = nextOffset; + } + + // Emit collected items as one JSON array. This matches the intent + // documented in the flag description ("emit all items as one JSON array"). + simpleLog({ message: JSON.stringify(collected, null, 2), stdout: true }); + if (verbose) { + simpleLog({ + message: chalk.gray( + ` collected ${collected.length}${total !== undefined ? ` of ${total}` : ''} item(s) across pagination`, + ), + stdout: false, + }); } + return; + } + // Single-request path. + const { response, responseText } = await doFetch(buildUrl()); + + if (!response.ok) { + await handleErrorResponse(response, responseText); return; }