diff --git a/README.md b/README.md index 3d72d06b83..9a036867ca 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ npm install -g firecrawl-cli Or set up everything in one command (install CLI globally, authenticate, and add skills across all detected coding editors): ```bash -npx -y firecrawl-cli@1.16.2 init -y --browser +npx -y firecrawl-cli@1.19.3 init -y --browser ``` - `-y` runs setup non-interactively @@ -50,6 +50,29 @@ To install the Firecrawl MCP server into your editors (Cursor, Claude Code, VS C firecrawl setup mcp ``` +To make Firecrawl the default web provider for supported AI agents: + +```bash +firecrawl setup defaults +``` + +This disables native web fetch/search where supported so agents route web work +through Firecrawl. When run interactively, it asks harness by harness (Claude +Code, Codex) so you can choose exactly which to change. Use `-y` to skip the +picker and apply to all, or `--agent` to target one: + +```bash +firecrawl setup defaults --agent codex # only Codex +firecrawl setup defaults -y # all harnesses, no prompts +``` + +To undo those config changes (also interactive, harness by harness): + +```bash +firecrawl setup defaults --undo # pick which to restore +firecrawl setup defaults --undo --agent claude +``` + ## Quick Start Just run a command - the CLI will prompt you to authenticate if needed: @@ -652,7 +675,7 @@ firecrawl --status ``` ``` - 🔥 firecrawl cli v1.16.2 + 🔥 firecrawl cli v1.19.3 ● Authenticated via stored credentials Concurrency: 0/100 jobs (parallel scrape limit) diff --git a/package.json b/package.json index 2e2cc67339..8afe602165 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "firecrawl-cli", - "version": "1.19.2", + "version": "1.19.3", "description": "Command-line interface for Firecrawl. Scrape, crawl, and extract data from any website directly from your terminal.", "main": "dist/index.js", "bin": { diff --git a/skills/firecrawl-cli/rules/install.md b/skills/firecrawl-cli/rules/install.md index ca9da0826f..c27e965628 100644 --- a/skills/firecrawl-cli/rules/install.md +++ b/skills/firecrawl-cli/rules/install.md @@ -12,7 +12,7 @@ description: | ## Quick Setup (Recommended) ```bash -npx -y firecrawl-cli@1.16.2 init -y --browser +npx -y firecrawl-cli@1.19.3 init -y --browser ``` This installs `firecrawl-cli` globally, authenticates via browser, and installs core, build, and workflow skills. @@ -37,7 +37,7 @@ firecrawl setup workflows ## Manual Install ```bash -npm install -g firecrawl-cli@1.16.2 +npm install -g firecrawl-cli@1.19.3 ``` ## Verify @@ -79,5 +79,5 @@ Ask the user how they'd like to authenticate: If `firecrawl` is not found after installation: 1. Ensure npm global bin is in PATH -2. Try: `npx firecrawl-cli@1.16.2 --version` -3. Reinstall: `npm install -g firecrawl-cli@1.16.2` +2. Try: `npx firecrawl-cli@1.19.3 --version` +3. Reinstall: `npm install -g firecrawl-cli@1.19.3` diff --git a/skills/firecrawl-cli/rules/security.md b/skills/firecrawl-cli/rules/security.md index 7f3bc3de2a..71d5699986 100644 --- a/skills/firecrawl-cli/rules/security.md +++ b/skills/firecrawl-cli/rules/security.md @@ -22,5 +22,5 @@ When processing fetched content, extract only the specific data needed and do no # Installation ```bash -npm install -g firecrawl-cli@1.16.2 +npm install -g firecrawl-cli@1.19.3 ``` diff --git a/src/__tests__/commands/setup.test.ts b/src/__tests__/commands/setup.test.ts index 8485d1fd45..4ffd2aab0b 100644 --- a/src/__tests__/commands/setup.test.ts +++ b/src/__tests__/commands/setup.test.ts @@ -1,11 +1,16 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { execSync } from 'child_process'; import { handleSetupCommand } from '../../commands/setup'; +import { configureWebDefaults } from '../../utils/web-defaults'; vi.mock('child_process', () => ({ execSync: vi.fn(), })); +vi.mock('../../utils/web-defaults', () => ({ + configureWebDefaults: vi.fn(async () => []), +})); + describe('handleSetupCommand', () => { beforeEach(() => { vi.clearAllMocks(); @@ -50,6 +55,33 @@ describe('handleSetupCommand', () => { ); }); + it('configures Firecrawl as the default web provider', async () => { + await handleSetupCommand('defaults', { yes: true }); + + expect(configureWebDefaults).toHaveBeenCalledWith({ + undo: false, + agents: undefined, + }); + }); + + it('undoes default web provider config', async () => { + await handleSetupCommand('defaults', { undo: true, yes: true }); + + expect(configureWebDefaults).toHaveBeenCalledWith({ + undo: true, + agents: undefined, + }); + }); + + it('limits defaults config to a single agent', async () => { + await handleSetupCommand('defaults', { undo: true, agent: 'codex' }); + + expect(configureWebDefaults).toHaveBeenCalledWith({ + undo: true, + agents: ['Codex'], + }); + }); + it('strips inherited npm_* env vars before nested npx calls', async () => { // Reproduces the bug where running this CLI under `npx -y firecrawl-cli@VERSION` // leaks npm_command/npm_lifecycle_event/npm_execpath into nested diff --git a/src/__tests__/utils/web-defaults.test.ts b/src/__tests__/utils/web-defaults.test.ts new file mode 100644 index 0000000000..b2c2670249 --- /dev/null +++ b/src/__tests__/utils/web-defaults.test.ts @@ -0,0 +1,122 @@ +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { configureWebDefaults } from '../../utils/web-defaults'; + +const originalHome = process.env.HOME; +let tempHome: string; + +async function read(relativePath: string): Promise { + return fs.readFile(path.join(tempHome, relativePath), 'utf8'); +} + +async function write(relativePath: string, content: string): Promise { + const filePath = path.join(tempHome, relativePath); + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, 'utf8'); +} + +describe('configureWebDefaults', () => { + beforeEach(async () => { + tempHome = await fs.mkdtemp(path.join(os.tmpdir(), 'firecrawl-web-')); + process.env.HOME = tempHome; + }); + + afterEach(async () => { + if (originalHome === undefined) delete process.env.HOME; + else process.env.HOME = originalHome; + await fs.rm(tempHome, { recursive: true, force: true }); + }); + + it('disables native Claude Code and Codex web tools', async () => { + const results = await configureWebDefaults(); + + expect(results.map((result) => result.changed)).toEqual([true, true]); + expect(JSON.parse(await read('.claude/settings.json'))).toEqual({ + permissions: { + deny: ['WebSearch', 'WebFetch'], + }, + }); + expect(await read('.codex/config.toml')).toBe('web_search = "disabled"\n'); + }); + + it('preserves existing Claude permissions and Codex config while disabling web', async () => { + await write( + '.claude/settings.json', + JSON.stringify({ + permissions: { + allow: ['Read'], + deny: ['Bash(rm *)', 'WebSearch'], + }, + }) + ); + await write( + '.codex/config.toml', + 'model = "gpt-5"\nweb_search = "cached"\n' + ); + + await configureWebDefaults(); + + expect(JSON.parse(await read('.claude/settings.json'))).toEqual({ + permissions: { + allow: ['Read'], + deny: ['Bash(rm *)', 'WebSearch', 'WebFetch'], + }, + }); + expect(await read('.codex/config.toml')).toBe( + 'model = "gpt-5"\nweb_search = "disabled"\n' + ); + }); + + it('undoes only the native web defaults', async () => { + await write( + '.claude/settings.json', + JSON.stringify({ + permissions: { + deny: ['Bash(rm *)', 'WebSearch', 'WebFetch'], + }, + }) + ); + await write( + '.codex/config.toml', + 'model = "gpt-5"\nweb_search = "disabled"\n' + ); + + const results = await configureWebDefaults({ undo: true }); + + expect(results.map((result) => result.changed)).toEqual([true, true]); + expect(JSON.parse(await read('.claude/settings.json'))).toEqual({ + permissions: { + deny: ['Bash(rm *)'], + }, + }); + expect(await read('.codex/config.toml')).toBe('model = "gpt-5"\n'); + }); + + it('writes Codex web_search at the root before TOML tables', async () => { + await write( + '.codex/config.toml', + 'model = "gpt-5"\n\n[mcp_servers.firecrawl]\ncommand = "npx"\n' + ); + + await configureWebDefaults(); + + expect(await read('.codex/config.toml')).toBe( + 'model = "gpt-5"\n\nweb_search = "disabled"\n[mcp_servers.firecrawl]\ncommand = "npx"\n' + ); + }); + + it('does not undo table-local Codex web_search settings', async () => { + await write( + '.codex/config.toml', + 'model = "gpt-5"\n\n[profiles.research]\nweb_search = "disabled"\n' + ); + + await configureWebDefaults({ undo: true }); + + expect(await read('.codex/config.toml')).toBe( + 'model = "gpt-5"\n\n[profiles.research]\nweb_search = "disabled"\n' + ); + }); +}); diff --git a/src/commands/init.ts b/src/commands/init.ts index 1bac7fc6e4..f3f55bc159 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -219,6 +219,9 @@ function printNextSteps(skillCount: number | null): void { console.log( ` ${arrow} ${dim}Add MCP: ${reset} ${bold}firecrawl setup mcp${reset}` ); + console.log( + ` ${arrow} ${dim}Default web:${reset} ${bold}firecrawl setup defaults${reset}` + ); console.log( ` ${arrow} ${dim}All commands:${reset} ${bold}firecrawl --help${reset}` ); diff --git a/src/commands/setup.ts b/src/commands/setup.ts index 8176aae18f..06fc2e7c46 100644 --- a/src/commands/setup.ts +++ b/src/commands/setup.ts @@ -4,6 +4,7 @@ */ import { execSync } from 'child_process'; +import readline from 'readline'; import { getApiKey } from '../utils/config'; import { buildSkillsInstallArgs, @@ -12,12 +13,20 @@ import { WORKFLOW_SKILL_REPOS, } from './skills-install'; import { hasNpx, installSkillsNative } from './skills-native'; +import { + configureWebDefaults, + WEB_AGENTS, + type WebAgent, +} from '../utils/web-defaults'; -export type SetupSubcommand = 'skills' | 'workflows' | 'mcp'; +export type SetupSubcommand = 'skills' | 'workflows' | 'mcp' | 'defaults'; export interface SetupOptions { global?: boolean; agent?: string; + undo?: boolean; + /** Skip the interactive harness picker and apply to all agents. */ + yes?: boolean; } /** @@ -37,6 +46,9 @@ export async function handleSetupCommand( case 'mcp': await installMcp(options); break; + case 'defaults': + await handleMakeDefaultCommand(options); + break; default: console.error(`Unknown setup subcommand: ${subcommand}`); console.log('\nAvailable subcommands:'); @@ -49,10 +61,106 @@ export async function handleSetupCommand( console.log( ' mcp Install firecrawl MCP server into editors (Cursor, Claude Code, VS Code, etc.)' ); + console.log( + ' defaults Make Firecrawl the default web provider (use --undo to restore native web tools)' + ); process.exit(1); } } +/** Map a user-supplied --agent value to a known web agent. */ +function resolveWebAgent(agent: string): WebAgent | null { + const normalized = agent.trim().toLowerCase(); + if (normalized === 'claude' || normalized === 'claude code') { + return 'Claude Code'; + } + if (normalized === 'codex') return 'Codex'; + return null; +} + +function promptInput(question: string): Promise { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + return new Promise((resolve) => { + rl.question(question, (answer: string) => { + rl.close(); + resolve(answer.trim()); + }); + }); +} + +/** + * Interactively ask which harnesses to apply the change to, one by one. + * Returns the selected agents, or null if the user aborted. + */ +async function pickWebAgents(undo: boolean): Promise { + const verb = undo + ? 'Re-enable native web tools for' + : 'Disable native web tools for'; + console.log( + undo + ? 'Choose which harnesses to restore native web tools for:' + : 'Choose which harnesses to route through Firecrawl:' + ); + console.log(''); + + const selected: WebAgent[] = []; + for (const agent of WEB_AGENTS) { + const answer = ( + await promptInput(` ${verb} ${agent}? [Y/n] `) + ).toLowerCase(); + if (answer === '' || answer === 'y' || answer === 'yes') { + selected.push(agent); + } + } + console.log(''); + return selected; +} + +export async function handleMakeDefaultCommand( + options: SetupOptions = {} +): Promise { + const undo = Boolean(options.undo); + let agents: readonly WebAgent[] | undefined; + + if (options.agent) { + const resolved = resolveWebAgent(options.agent); + if (!resolved) { + console.error( + `Unknown agent "${options.agent}" for setup defaults. Use "claude" or "codex".` + ); + process.exit(1); + } + agents = [resolved]; + } else if (!options.yes && process.stdin.isTTY) { + const picked = await pickWebAgents(undo); + if (!picked || picked.length === 0) { + console.log('No harnesses selected. Nothing changed.'); + return; + } + agents = picked; + } + + const results = await configureWebDefaults({ undo, agents }); + + for (const result of results) { + const prefix = result.skipped ? '!' : result.changed ? '✓' : '•'; + console.log(`${prefix} ${result.message}`); + console.log(` ${result.path}`); + } + + console.log(''); + if (undo) { + console.log('Native web tools restored where supported.'); + } else { + console.log( + 'Firecrawl is now the default web provider for supported AI agents.' + ); + } +} + async function installSkills( options: SetupOptions, repos: readonly string[] diff --git a/src/index.ts b/src/index.ts index 1927862e21..44801d7339 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1809,11 +1809,25 @@ program program .command('setup') .description( - 'Set up individual firecrawl integrations (skills, workflows, mcp)' + 'Set up individual firecrawl integrations (skills, workflows, mcp, defaults)' + ) + .argument( + '', + 'What to set up: "skills", "workflows", "mcp", or "defaults"' ) - .argument('', 'What to set up: "skills", "workflows", or "mcp"') .option('-g, --global', 'Install globally (user-level)') - .option('-a, --agent ', 'Install to a specific agent') + .option( + '-a, --agent ', + 'Limit to a specific agent (for "defaults": "claude" or "codex")' + ) + .option( + '-y, --yes', + 'For "defaults": skip the interactive harness picker and apply to all' + ) + .option( + '--undo', + 'Undo setup defaults by re-enabling native web tools where supported' + ) .action(async (subcommand: SetupSubcommand, options) => { await handleSetupCommand(subcommand, options); }); diff --git a/src/utils/web-defaults.ts b/src/utils/web-defaults.ts new file mode 100644 index 0000000000..87734ed2d8 --- /dev/null +++ b/src/utils/web-defaults.ts @@ -0,0 +1,211 @@ +import { promises as fs } from 'fs'; +import os from 'os'; +import path from 'path'; + +const CLAUDE_DENY_TOOLS = ['WebSearch', 'WebFetch'] as const; +const CODEX_WEB_SEARCH_DISABLED = 'web_search = "disabled"'; + +export type WebAgent = 'Claude Code' | 'Codex'; + +export const WEB_AGENTS: readonly WebAgent[] = ['Claude Code', 'Codex']; + +export interface WebDefaultsOptions { + undo?: boolean; + /** Limit configuration to these agents. Defaults to all agents. */ + agents?: readonly WebAgent[]; +} + +export interface WebDefaultResult { + agent: WebAgent; + path: string; + changed: boolean; + skipped?: boolean; + message: string; +} + +async function readText(filePath: string): Promise { + try { + return await fs.readFile(filePath, 'utf8'); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; + throw error; + } +} + +async function writeText(filePath: string, content: string): Promise { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content, 'utf8'); +} + +function removeJsonComments(content: string): string { + return content + .replace(/\/\*[\s\S]*?\*\//g, '') + .replace(/(^|[^:\\])\/\/.*$/gm, '$1'); +} + +async function configureClaudeDefaults( + undo: boolean +): Promise { + const filePath = path.join(os.homedir(), '.claude', 'settings.json'); + const existing = await readText(filePath); + let config: Record = {}; + + if (existing && existing.trim()) { + try { + config = JSON.parse(removeJsonComments(existing)); + } catch { + return { + agent: 'Claude Code', + path: filePath, + changed: false, + skipped: true, + message: + 'Skipped Claude Code settings because settings.json is not valid JSON', + }; + } + } + + const permissions = + config.permissions && typeof config.permissions === 'object' + ? (config.permissions as Record) + : {}; + const deny = Array.isArray(permissions.deny) ? [...permissions.deny] : []; + + let nextDeny: unknown[]; + const denyTools = new Set(CLAUDE_DENY_TOOLS); + if (undo) { + nextDeny = deny.filter( + (tool) => typeof tool !== 'string' || !denyTools.has(tool) + ); + } else { + const existing = new Set( + deny.filter((tool): tool is string => typeof tool === 'string') + ); + nextDeny = [...deny]; + for (const tool of CLAUDE_DENY_TOOLS) { + if (!existing.has(tool)) nextDeny.push(tool); + } + } + + const changed = JSON.stringify(deny) !== JSON.stringify(nextDeny); + + if (!changed) { + return { + agent: 'Claude Code', + path: filePath, + changed: false, + message: undo + ? 'Claude Code native WebSearch/WebFetch were already enabled' + : 'Claude Code already denies native WebSearch/WebFetch', + }; + } + + const nextPermissions = { ...permissions }; + if (nextDeny.length > 0) { + nextPermissions.deny = nextDeny; + } else { + delete nextPermissions.deny; + } + + const nextConfig = { ...config }; + if (Object.keys(nextPermissions).length > 0) { + nextConfig.permissions = nextPermissions; + } else { + delete nextConfig.permissions; + } + + await writeText(filePath, `${JSON.stringify(nextConfig, null, 2)}\n`); + + return { + agent: 'Claude Code', + path: filePath, + changed: true, + message: undo + ? 'Enabled Claude Code native WebSearch/WebFetch' + : 'Disabled Claude Code native WebSearch/WebFetch', + }; +} + +function setCodexWebSearchDisabled(content: string): { + content: string; + changed: boolean; +} { + if (content.trim().length === 0) { + return { content: `${CODEX_WEB_SEARCH_DISABLED}\n`, changed: true }; + } + + const lines = content.split(/\r?\n/); + const firstTableIndex = lines.findIndex((line) => /^\s*\[/.test(line)); + const rootEnd = firstTableIndex === -1 ? lines.length : firstTableIndex; + + for (let index = 0; index < rootEnd; index += 1) { + if (/^\s*web_search\s*=/.test(lines[index])) { + if (lines[index] === CODEX_WEB_SEARCH_DISABLED) { + return { content, changed: false }; + } + lines[index] = CODEX_WEB_SEARCH_DISABLED; + return { content: lines.join('\n'), changed: true }; + } + } + + lines.splice(rootEnd, 0, CODEX_WEB_SEARCH_DISABLED); + return { content: lines.join('\n'), changed: true }; +} + +function removeCodexWebSearchDisabled(content: string): { + content: string; + changed: boolean; +} { + const lines = content.split(/\r?\n/); + const firstTableIndex = lines.findIndex((line) => /^\s*\[/.test(line)); + const rootEnd = firstTableIndex === -1 ? lines.length : firstTableIndex; + const nextLines = lines.filter((line, index) => { + if (index >= rootEnd) return true; + return !/^web_search\s*=\s*["']disabled["']\s*(#.*)?$/.test(line.trim()); + }); + const next = nextLines.join('\n').replace(/\n{3,}/g, '\n\n'); + return { content: next, changed: next !== content }; +} + +async function configureCodexDefaults( + undo: boolean +): Promise { + const filePath = path.join(os.homedir(), '.codex', 'config.toml'); + const existing = (await readText(filePath)) ?? ''; + const result = undo + ? removeCodexWebSearchDisabled(existing) + : setCodexWebSearchDisabled(existing); + + if (!result.changed) { + return { + agent: 'Codex', + path: filePath, + changed: false, + message: undo + ? 'Codex native web search was already enabled' + : 'Codex native web search was already disabled', + }; + } + + await writeText(filePath, result.content); + + return { + agent: 'Codex', + path: filePath, + changed: true, + message: undo + ? 'Enabled Codex native web search' + : 'Disabled Codex native web search', + }; +} + +export async function configureWebDefaults( + options: WebDefaultsOptions = {} +): Promise { + const undo = Boolean(options.undo); + const selected = new Set(options.agents ?? WEB_AGENTS); + const tasks: Promise[] = []; + if (selected.has('Claude Code')) tasks.push(configureClaudeDefaults(undo)); + if (selected.has('Codex')) tasks.push(configureCodexDefaults(undo)); + return Promise.all(tasks); +}