From 0896ab313d75edcdf97631c690ac005d93855c66 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 00:40:43 +0800 Subject: [PATCH 01/16] chore(release): 1.5.2 --- extension/manifest.json | 2 +- extension/package-lock.json | 4 ++-- extension/package.json | 2 +- package-lock.json | 4 ++-- package.json | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extension/manifest.json b/extension/manifest.json index b042a678..7bd3aa86 100644 --- a/extension/manifest.json +++ b/extension/manifest.json @@ -1,7 +1,7 @@ { "manifest_version": 3, "name": "OpenCLI", - "version": "1.5.1", + "version": "1.5.2", "description": "Browser automation bridge for the OpenCLI CLI tool. Executes commands in isolated Chrome windows via a local daemon.", "permissions": [ "debugger", diff --git a/extension/package-lock.json b/extension/package-lock.json index 7fc5b82c..ed0c08cd 100644 --- a/extension/package-lock.json +++ b/extension/package-lock.json @@ -1,12 +1,12 @@ { "name": "opencli-extension", - "version": "1.5.1", + "version": "1.5.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "opencli-extension", - "version": "1.5.1", + "version": "1.5.2", "devDependencies": { "@types/chrome": "^0.0.287", "typescript": "^5.7.0", diff --git a/extension/package.json b/extension/package.json index c2fc974e..31d854b4 100644 --- a/extension/package.json +++ b/extension/package.json @@ -1,6 +1,6 @@ { "name": "opencli-extension", - "version": "1.5.1", + "version": "1.5.2", "private": true, "type": "module", "scripts": { diff --git a/package-lock.json b/package-lock.json index e6b2d88b..f11b3ed1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@jackwener/opencli", - "version": "1.5.1", + "version": "1.5.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@jackwener/opencli", - "version": "1.5.1", + "version": "1.5.2", "hasInstallScript": true, "license": "Apache-2.0", "dependencies": { diff --git a/package.json b/package.json index d51082b2..26c337f0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@jackwener/opencli", - "version": "1.5.1", + "version": "1.5.2", "publishConfig": { "access": "public" }, From b80ca80e5eba5acb06a121be0c0c27a0c5c46395 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 00:45:36 +0800 Subject: [PATCH 02/16] test(e2e): stabilize output format checks --- tests/e2e/output-formats.test.ts | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/tests/e2e/output-formats.test.ts b/tests/e2e/output-formats.test.ts index 2f622e1b..638863dd 100644 --- a/tests/e2e/output-formats.test.ts +++ b/tests/e2e/output-formats.test.ts @@ -1,6 +1,7 @@ /** * E2E tests for output format rendering. - * Uses hackernews (public, fast) as a stable data source. + * Uses the built-in list command so renderer coverage does not depend on + * external network availability. */ import { describe, it, expect } from 'vitest'; @@ -10,19 +11,22 @@ const FORMATS = ['json', 'yaml', 'csv', 'md'] as const; describe('output formats E2E', () => { for (const fmt of FORMATS) { - it(`hackernews top -f ${fmt} produces valid output`, async () => { - const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '2', '-f', fmt]); + it(`list -f ${fmt} produces valid output`, async () => { + const { stdout, code } = await runCli(['list', '-f', fmt]); expect(code).toBe(0); expect(stdout.trim().length).toBeGreaterThan(0); if (fmt === 'json') { const data = parseJsonOutput(stdout); expect(Array.isArray(data)).toBe(true); - expect(data.length).toBe(2); + expect(data.length).toBeGreaterThan(50); + expect(data[0]).toHaveProperty('command'); + expect(data[0]).toHaveProperty('site'); } if (fmt === 'yaml') { - expect(stdout).toContain('title:'); + expect(stdout).toContain('command:'); + expect(stdout).toContain('site:'); } if (fmt === 'csv') { @@ -33,16 +37,8 @@ describe('output formats E2E', () => { if (fmt === 'md') { // Markdown table should have pipe characters - expect(stdout).toContain('|'); + expect(stdout).toContain('| command |'); } }, 30_000); } - - it('list -f csv produces valid csv', async () => { - const { stdout, code } = await runCli(['list', '-f', 'csv']); - expect(code).toBe(0); - const lines = stdout.trim().split('\n'); - // Header + many data lines - expect(lines.length).toBeGreaterThan(50); - }); }); From 72138bf0f67c4f1e33496511fe7a3d137f23571a Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:45:06 +0800 Subject: [PATCH 03/16] docs: add perf smart-wait design spec (waitForCapture + selector wait + backoff) --- .../2026-03-28-perf-smart-wait-design.md | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 docs/superpowers/specs/2026-03-28-perf-smart-wait-design.md diff --git a/docs/superpowers/specs/2026-03-28-perf-smart-wait-design.md b/docs/superpowers/specs/2026-03-28-perf-smart-wait-design.md new file mode 100644 index 00000000..9c2e0955 --- /dev/null +++ b/docs/superpowers/specs/2026-03-28-perf-smart-wait-design.md @@ -0,0 +1,170 @@ +# Performance: Smart Wait & INTERCEPT Fix + +**Date**: 2026-03-28 +**Status**: Approved + +## Problem + +Three distinct performance/correctness issues: + +1. **INTERCEPT strategy semantic bug**: After `installInterceptor()` + `goto()`, adapters call `wait(N)` — which now uses `waitForDomStableJs` and returns early when the DOM settles. But DOM-settle != network capture. The API response may arrive *after* DOM is stable, causing `getInterceptedRequests()` to return an empty array. + +2. **Blind `wait(N)` in adapters**: ~30 high-traffic adapters (Twitter family, Medium, Substack, etc.) call `wait(5)` waiting for React/Vue to hydrate. These should wait for a specific DOM element to appear, not a fixed cap. + +3. **Daemon cold-start polling**: Fixed 300ms poll loop means ~600ms before first successful `isExtensionConnected()` check, even though the daemon is typically ready in 500–800ms. + +## Design + +### Layer 1 — `waitForCapture()` (correctness fix + perf) + +Add `waitForCapture(timeout?: number): Promise` to `IPage`. + +Polls `window.__opencli_xhr.length > 0` every 100ms inside the browser tab. Resolves as soon as ≥1 capture arrives; rejects after `timeout` seconds. + +```typescript +// dom-helpers.ts +export function waitForCaptureJs(maxMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${maxMs}; + const check = () => { + if ((window.__opencli_xhr || []).length > 0) return resolve('captured'); + if (Date.now() > deadline) return reject(new Error('No capture within ${maxMs / 1000}s')); + setTimeout(check, 100); + }; + check(); + }) + `; +} +``` + +`page.ts` and `cdp.ts` implement `waitForCapture()` by calling `waitForCaptureJs`. + +**All INTERCEPT adapters** replace `wait(N)` → `waitForCapture(N+2)` (slightly longer timeout as safety margin). + +`stepIntercept` in `pipeline/steps/intercept.ts` replaces its internal `wait(timeout)` with `waitForCapture(timeout)`. + +**Expected gain**: 36kr hot/search: 6s → ~1–2s. Twitter search/followers: 5–8s → ~1–3s. + +### Layer 2 — `wait({ selector })` (semantic precision) + +Extend `WaitOptions` with `selector?: string`. + +Add `waitForSelectorJs(selector, timeoutMs)` to `dom-helpers.ts` — polls `document.querySelector(selector)` every 100ms, resolves on first match, rejects on timeout. + +```typescript +// types.ts +export interface WaitOptions { + text?: string; + selector?: string; // NEW + time?: number; + timeout?: number; +} +``` + +```typescript +// dom-helpers.ts +export function waitForSelectorJs(selector: string, timeoutMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${timeoutMs}; + const check = () => { + if (document.querySelector(${JSON.stringify(selector)})) return resolve('found'); + if (Date.now() > deadline) return reject(new Error('Selector not found: ' + ${JSON.stringify(selector)})); + setTimeout(check, 100); + }; + check(); + }) + `; +} +``` + +`page.ts` and `cdp.ts` handle `selector` branch in `wait()`. + +**High-impact adapter changes**: + +| Adapter | Old | New | +|---------|-----|-----| +| `twitter/*` (15 adapters) | `wait(5)` | `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `twitter/reply.ts` | `wait(5)` | `wait({ selector: '[data-testid="tweetTextarea_0"]', timeout: 8 })` | +| `medium/utils.ts` | `wait(5)` + inline 3s setTimeout | `wait({ selector: 'article', timeout: 8 })` + remove inline sleep | +| `substack/utils.ts` | `wait(5)` × 2 | `wait({ selector: 'article', timeout: 8 })` | +| `bloomberg/news.ts` | `wait(5)` | `wait({ selector: 'article', timeout: 6 })` | +| `sinablog/utils.ts` | `wait(5)` | `wait({ selector: 'article, .article', timeout: 6 })` | +| `producthunt` (already covered by layer 1) | — | — | + +**Expected gain**: Twitter commands: 5s → ~0.5–2s. Medium: 8s → ~1–3s. + +### Layer 3 — Daemon exponential backoff (cold-start) + +Replace fixed 300ms poll in `_ensureDaemon()` (`browser/mcp.ts`) with exponential backoff: + +```typescript +// before +while (Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 300)); + if (await isExtensionConnected()) return; +} + +// after +const backoffs = [50, 100, 200, 400, 800, 1500, 3000]; +let i = 0; +while (Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, backoffs[Math.min(i++, backoffs.length - 1)])); + if (await isExtensionConnected()) return; +} +``` + +**Expected gain**: First cold-start check succeeds at ~150ms instead of ~600ms. + +## Files Changed + +### New / Modified (framework) +- `src/types.ts` — `WaitOptions.selector`, `IPage.waitForCapture()` +- `src/browser/dom-helpers.ts` — `waitForCaptureJs()`, `waitForSelectorJs()` +- `src/browser/page.ts` — `waitForCapture()`, `wait()` selector branch +- `src/browser/cdp.ts` — `waitForCapture()`, `wait()` selector branch +- `src/browser/mcp.ts` — exponential backoff in `_ensureDaemon()` +- `src/pipeline/steps/intercept.ts` — use `waitForCapture()` + +### Modified (adapters — Layer 1, INTERCEPT) +- `src/clis/36kr/hot.ts` +- `src/clis/36kr/search.ts` +- `src/clis/twitter/search.ts` +- `src/clis/twitter/followers.ts` +- `src/clis/twitter/following.ts` +- `src/clis/producthunt/hot.ts` +- `src/clis/producthunt/browse.ts` + +### Modified (adapters — Layer 2, selector) +- `src/clis/twitter/reply.ts` +- `src/clis/twitter/follow.ts` +- `src/clis/twitter/unfollow.ts` +- `src/clis/twitter/like.ts` +- `src/clis/twitter/bookmark.ts` +- `src/clis/twitter/unbookmark.ts` +- `src/clis/twitter/block.ts` +- `src/clis/twitter/unblock.ts` +- `src/clis/twitter/hide-reply.ts` +- `src/clis/twitter/notifications.ts` +- `src/clis/twitter/profile.ts` +- `src/clis/twitter/thread.ts` +- `src/clis/twitter/timeline.ts` +- `src/clis/twitter/delete.ts` +- `src/clis/twitter/reply-dm.ts` +- `src/clis/medium/utils.ts` +- `src/clis/substack/utils.ts` +- `src/clis/bloomberg/news.ts` +- `src/clis/sinablog/utils.ts` + +## Delivery Order + +1. Layer 1 (`waitForCapture`) — correctness fix, highest ROI +2. Layer 3 (backoff) — 3-line change, zero risk +3. Layer 2 (`wait({ selector })`) — largest adapter surface, can be done per-site + +## Testing + +- Unit tests: `waitForCaptureJs`, `waitForSelectorJs` exported and tested in `dom-helpers.test.ts` (if exists) or new test file +- Adapter tests: existing tests must continue to pass (mock `page.wait` / `page.waitForCapture`) +- Run: `npx vitest run --project unit --project adapter` From 9e4fc0ad3bb6c24e190d2d6e188eca752be4d584 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:51:29 +0800 Subject: [PATCH 04/16] docs: add perf smart-wait implementation plan --- .../plans/2026-03-28-perf-smart-wait.md | 1143 +++++++++++++++++ 1 file changed, 1143 insertions(+) create mode 100644 docs/superpowers/plans/2026-03-28-perf-smart-wait.md diff --git a/docs/superpowers/plans/2026-03-28-perf-smart-wait.md b/docs/superpowers/plans/2026-03-28-perf-smart-wait.md new file mode 100644 index 00000000..4986f4ec --- /dev/null +++ b/docs/superpowers/plans/2026-03-28-perf-smart-wait.md @@ -0,0 +1,1143 @@ +# Performance: Smart Wait & INTERCEPT Fix — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fix the INTERCEPT strategy correctness bug, add `wait({ selector })` for event-driven waits, and speed up daemon cold-start — eliminating up to 8s of unnecessary fixed sleeps per command. + +**Architecture:** Three independent layers applied in order: (1) add `waitForCaptureJs` + `waitForSelectorJs` to `dom-helpers.ts` and expose via `IPage`, (2) update `page.ts`/`cdp.ts` implementations, (3) update adapters from the inside out — framework first, then adapters. + +**Tech Stack:** TypeScript, Vitest (unit + adapter projects), Node.js, browser JS (eval'd strings) + +--- + +## File Map + +| File | Change | +|------|--------| +| `src/browser/dom-helpers.ts` | Add `waitForCaptureJs()`, `waitForSelectorJs()` | +| `src/browser/dom-helpers.test.ts` | **New** — unit tests for new helpers | +| `src/types.ts` | Add `selector?` to `WaitOptions`; add `waitForCapture()` to `IPage` | +| `src/browser/page.ts` | Implement `waitForCapture()`, add `selector` branch to `wait()` | +| `src/browser/cdp.ts` | Implement `waitForCapture()`, add `selector` branch to `wait()` | +| `src/pipeline/steps/intercept.ts` | Use `page.installInterceptor()` + `page.waitForCapture()` + `page.getInterceptedRequests()` | +| `src/browser/mcp.ts` | Exponential backoff in `_ensureDaemon()` | +| `src/clis/36kr/hot.ts` | `wait(6)` → `waitForCapture(10)` | +| `src/clis/36kr/search.ts` | `wait(6)` → `waitForCapture(10)` | +| `src/clis/twitter/search.ts` | `wait(5)` → `waitForCapture(8)` (already INTERCEPT) | +| `src/clis/twitter/followers.ts` | `wait(5)` → `waitForCapture(8)` (already INTERCEPT) | +| `src/clis/twitter/following.ts` | `wait(5)` → `waitForCapture(8)` (already INTERCEPT) | +| `src/clis/twitter/notifications.ts` | `wait(3)` → selector + `wait(5)` → `waitForCapture(8)` | +| `src/clis/producthunt/hot.ts` | `wait(5)` → `waitForCapture(8)` | +| `src/clis/producthunt/browse.ts` | `wait(5)` → `waitForCapture(8)` | +| `src/clis/twitter/reply.ts` | `wait(5)` → `wait({ selector: '[data-testid="tweetTextarea_0"]', timeout: 8 })` | +| `src/clis/twitter/follow.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/unfollow.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/like.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/bookmark.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/unbookmark.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/block.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/unblock.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/hide-reply.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/profile.ts` | `wait(5)` + `wait(3)` → selector variants | +| `src/clis/twitter/thread.ts` | `wait(3)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 })` | +| `src/clis/twitter/timeline.ts` | `wait(3)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 })` | +| `src/clis/twitter/delete.ts` | `wait(5)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })` | +| `src/clis/twitter/reply-dm.ts` | `wait(5)` + `wait(3)` → selector variants | +| `src/clis/medium/utils.ts` | `wait(5)` → selector; remove inline `setTimeout(3000)` | +| `src/clis/substack/utils.ts` | `wait(5)` × 2 → selector; remove inline `setTimeout(3000)` × 2 | +| `src/clis/bloomberg/news.ts` | `wait(5)` → `wait({ selector: '#__NEXT_DATA__', timeout: 8 })`; `wait(4)` → `wait({ selector: '#__NEXT_DATA__', timeout: 5 })` | +| `src/clis/sinablog/utils.ts` | `wait(3/5)` → selector; remove inline polling loop | + +--- + +## Task 1: Add `waitForCaptureJs` and `waitForSelectorJs` to dom-helpers.ts + +**Files:** +- Modify: `src/browser/dom-helpers.ts` +- Create: `src/browser/dom-helpers.test.ts` + +- [ ] **Step 1: Add two new exported functions at the end of `src/browser/dom-helpers.ts`** + +```typescript +/** + * Generate JS to wait until window.__opencli_xhr has ≥1 captured response. + * Polls every 100ms. Resolves 'captured' on success; rejects after maxMs. + * Used after installInterceptor() + goto() instead of a fixed sleep. + */ +export function waitForCaptureJs(maxMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${maxMs}; + const check = () => { + if ((window.__opencli_xhr || []).length > 0) return resolve('captured'); + if (Date.now() > deadline) return reject(new Error('No network capture within ${maxMs / 1000}s')); + setTimeout(check, 100); + }; + check(); + }) + `; +} + +/** + * Generate JS to wait until document.querySelector(selector) returns a match. + * Polls every 100ms. Resolves 'found' on success; rejects after timeoutMs. + */ +export function waitForSelectorJs(selector: string, timeoutMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${timeoutMs}; + const check = () => { + if (document.querySelector(${JSON.stringify(selector)})) return resolve('found'); + if (Date.now() > deadline) return reject(new Error('Selector not found: ' + ${JSON.stringify(selector)})); + setTimeout(check, 100); + }; + check(); + }) + `; +} +``` + +- [ ] **Step 2: Create `src/browser/dom-helpers.test.ts` with failing tests** + +```typescript +import { describe, it, expect } from 'vitest'; +import { waitForCaptureJs, waitForSelectorJs } from './dom-helpers.js'; + +describe('waitForCaptureJs', () => { + it('returns a non-empty string', () => { + const code = waitForCaptureJs(1000); + expect(typeof code).toBe('string'); + expect(code.length).toBeGreaterThan(0); + expect(code).toContain('__opencli_xhr'); + expect(code).toContain('resolve'); + expect(code).toContain('reject'); + }); + + it('resolves "captured" when __opencli_xhr is populated before deadline', async () => { + const g = globalThis as any; + g.__opencli_xhr = []; + const code = waitForCaptureJs(1000); + const promise = eval(code) as Promise; + g.__opencli_xhr.push({ data: 'test' }); + await expect(promise).resolves.toBe('captured'); + delete g.__opencli_xhr; + }); + + it('rejects when __opencli_xhr stays empty past deadline', async () => { + const g = globalThis as any; + g.__opencli_xhr = []; + const code = waitForCaptureJs(50); // 50ms timeout + const promise = eval(code) as Promise; + await expect(promise).rejects.toThrow('No network capture within 0.05s'); + delete g.__opencli_xhr; + }); + + it('resolves immediately when __opencli_xhr already has data', async () => { + const g = globalThis as any; + g.__opencli_xhr = [{ data: 'already here' }]; + const code = waitForCaptureJs(1000); + await expect(eval(code) as Promise).resolves.toBe('captured'); + delete g.__opencli_xhr; + }); +}); + +describe('waitForSelectorJs', () => { + it('returns a non-empty string', () => { + const code = waitForSelectorJs('#app', 1000); + expect(typeof code).toBe('string'); + expect(code).toContain('#app'); + expect(code).toContain('querySelector'); + }); + + it('rejects when document.querySelector returns null within timeout', async () => { + const g = globalThis as any; + g.document = { querySelector: (_: string) => null }; + const code = waitForSelectorJs('#missing', 50); + await expect(eval(code) as Promise).rejects.toThrow('Selector not found: #missing'); + delete g.document; + }); + + it('resolves "found" when document.querySelector returns an element', async () => { + const g = globalThis as any; + const fakeEl = { tagName: 'DIV' }; + g.document = { querySelector: (_: string) => fakeEl }; + const code = waitForSelectorJs('[data-testid="primaryColumn"]', 1000); + await expect(eval(code) as Promise).resolves.toBe('found'); + delete g.document; + }); +}); +``` + +- [ ] **Step 3: Run tests to verify they fail (functions not yet exported)** + +```bash +cd /Users/jakevin/code/opencli +npx vitest run --project unit src/browser/dom-helpers.test.ts +``` +Expected: Tests for `waitForCaptureJs` pass (function exists), tests for `waitForSelectorJs` fail (not yet added). + +- [ ] **Step 4: Run tests again after Step 1 to verify all pass** + +```bash +npx vitest run --project unit src/browser/dom-helpers.test.ts +``` +Expected: All 7 tests PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/browser/dom-helpers.ts src/browser/dom-helpers.test.ts +git commit -m "feat(perf): add waitForCaptureJs and waitForSelectorJs to dom-helpers" +``` + +--- + +## Task 2: Extend `IPage` interface and `WaitOptions` in `types.ts` + +**Files:** +- Modify: `src/types.ts` + +- [ ] **Step 1: Add `selector` to `WaitOptions` and `waitForCapture` to `IPage`** + +In `src/types.ts`, find `WaitOptions` and add `selector?`: + +```typescript +export interface WaitOptions { + text?: string; + selector?: string; // wait until document.querySelector(selector) matches + time?: number; + timeout?: number; +} +``` + +In the same file, find `IPage` and add `waitForCapture` after `getInterceptedRequests`: + +```typescript + installInterceptor(pattern: string): Promise; + getInterceptedRequests(): Promise; + waitForCapture(timeout?: number): Promise; +``` + +- [ ] **Step 2: Run unit tests to confirm no type errors** + +```bash +npx vitest run --project unit +``` +Expected: All existing unit tests PASS (no adapter tests broken since IPage is extended, not changed). + +- [ ] **Step 3: Commit** + +```bash +git add src/types.ts +git commit -m "feat(perf): extend WaitOptions with selector, add waitForCapture to IPage" +``` + +--- + +## Task 3: Implement `waitForCapture()` and `wait({ selector })` in `page.ts` + +**Files:** +- Modify: `src/browser/page.ts` + +- [ ] **Step 1: Add `waitForCaptureJs` and `waitForSelectorJs` to the imports at the top of `page.ts`** + +Find the existing import from `./dom-helpers.js`: + +```typescript +import { + clickJs, + typeTextJs, + pressKeyJs, + waitForTextJs, + scrollJs, + autoScrollJs, + networkRequestsJs, + waitForDomStableJs, +} from './dom-helpers.js'; +``` + +Replace with: + +```typescript +import { + clickJs, + typeTextJs, + pressKeyJs, + waitForTextJs, + waitForCaptureJs, + waitForSelectorJs, + scrollJs, + autoScrollJs, + networkRequestsJs, + waitForDomStableJs, +} from './dom-helpers.js'; +``` + +- [ ] **Step 2: Add `selector` branch to the existing `wait()` method in `page.ts`** + +Find the current `wait()` implementation and add the `selector` branch before the `text` branch: + +```typescript + async wait(options: number | WaitOptions): Promise { + if (typeof options === 'number') { + if (options >= 1) { + try { + const maxMs = options * 1000; + await sendCommand('exec', { + code: waitForDomStableJs(maxMs, Math.min(500, maxMs)), + ...this._cmdOpts(), + }); + return; + } catch { + // Fallback: fixed sleep (e.g. if page has no DOM yet) + } + } + await new Promise(resolve => setTimeout(resolve, options * 1000)); + return; + } + if (typeof options.time === 'number') { + await new Promise(resolve => setTimeout(resolve, options.time! * 1000)); + return; + } + if (options.selector) { + const timeout = (options.timeout ?? 10) * 1000; + const code = waitForSelectorJs(options.selector, timeout); + await sendCommand('exec', { code, ...this._cmdOpts() }); + return; + } + if (options.text) { + const timeout = (options.timeout ?? 30) * 1000; + const code = waitForTextJs(options.text, timeout); + await sendCommand('exec', { code, ...this._cmdOpts() }); + } + } +``` + +- [ ] **Step 3: Add `waitForCapture()` method to `page.ts`, just after `getInterceptedRequests()`** + +Find `getInterceptedRequests()` at the end of the `Page` class and add after it: + +```typescript + async waitForCapture(timeout: number = 10): Promise { + const maxMs = timeout * 1000; + await sendCommand('exec', { + code: waitForCaptureJs(maxMs), + ...this._cmdOpts(), + }); + } +``` + +- [ ] **Step 4: Run unit tests** + +```bash +npx vitest run --project unit +``` +Expected: All PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/browser/page.ts +git commit -m "feat(perf): implement waitForCapture() and wait({ selector }) in Page" +``` + +--- + +## Task 4: Implement `waitForCapture()` and `wait({ selector })` in `cdp.ts` + +**Files:** +- Modify: `src/browser/cdp.ts` + +- [ ] **Step 1: Add `waitForCaptureJs` and `waitForSelectorJs` to the imports in `cdp.ts`** + +Find the existing import from `./dom-helpers.js` in `cdp.ts`: + +```typescript +import { + clickJs, + typeTextJs, + pressKeyJs, + waitForTextJs, + scrollJs, + autoScrollJs, + networkRequestsJs, +} from './dom-helpers.js'; +``` + +Replace with: + +```typescript +import { + clickJs, + typeTextJs, + pressKeyJs, + waitForTextJs, + waitForCaptureJs, + waitForSelectorJs, + scrollJs, + autoScrollJs, + networkRequestsJs, +} from './dom-helpers.js'; +``` + +- [ ] **Step 2: Add `selector` branch to `wait()` in `cdp.ts`** + +Find the current `wait()` in `cdp.ts` and replace it entirely: + +```typescript + async wait(options: number | WaitOptions): Promise { + if (typeof options === 'number') { + await new Promise((resolve) => setTimeout(resolve, options * 1000)); + return; + } + if (typeof options.time === 'number') { + const waitTime = options.time; + await new Promise((resolve) => setTimeout(resolve, waitTime * 1000)); + return; + } + if (options.selector) { + const timeout = (options.timeout ?? 10) * 1000; + await this.evaluate(waitForSelectorJs(options.selector, timeout)); + return; + } + if (options.text) { + const timeout = (options.timeout ?? 30) * 1000; + await this.evaluate(waitForTextJs(options.text, timeout)); + } + } +``` + +- [ ] **Step 3: Add `waitForCapture()` to `cdp.ts`, just after `getInterceptedRequests()`** + +Find `getInterceptedRequests()` at the end of the `CDPPage` class and add after it: + +```typescript + async waitForCapture(timeout: number = 10): Promise { + const maxMs = timeout * 1000; + await this.evaluate(waitForCaptureJs(maxMs)); + } +``` + +- [ ] **Step 4: Run unit tests** + +```bash +npx vitest run --project unit +``` +Expected: All PASS. + +- [ ] **Step 5: Commit** + +```bash +git add src/browser/cdp.ts +git commit -m "feat(perf): implement waitForCapture() and wait({ selector }) in CDPPage" +``` + +--- + +## Task 5: Update `stepIntercept` to use unified IPage methods + +**Files:** +- Modify: `src/pipeline/steps/intercept.ts` + +The current `stepIntercept` uses `generateInterceptorJs`/`generateReadInterceptedJs` directly, writing to `__opencli_intercepted`. We unify this to use `page.installInterceptor()` (→ `__opencli_xhr`) + `page.waitForCapture()` + `page.getInterceptedRequests()`. + +- [ ] **Step 1: Rewrite `src/pipeline/steps/intercept.ts`** + +```typescript +/** + * Pipeline step: intercept — declarative XHR interception. + */ + +import type { IPage } from '../../types.js'; +import { render, normalizeEvaluateSource } from '../template.js'; + +export async function stepIntercept(page: IPage | null, params: any, data: any, args: Record): Promise { + const cfg = typeof params === 'object' ? params : {}; + const trigger = cfg.trigger ?? ''; + const capturePattern = cfg.capture ?? ''; + const timeout = cfg.timeout ?? 8; + const selectPath = cfg.select ?? null; + + if (!capturePattern) return data; + + // Step 1: Install fetch/XHR interceptor BEFORE trigger + await page!.installInterceptor(capturePattern); + + // Step 2: Execute the trigger action + if (trigger.startsWith('navigate:')) { + const url = render(trigger.slice('navigate:'.length), { args, data }); + await page!.goto(String(url)); + } else if (trigger.startsWith('evaluate:')) { + const js = trigger.slice('evaluate:'.length); + await page!.evaluate(normalizeEvaluateSource(render(js, { args, data }) as string)); + } else if (trigger.startsWith('click:')) { + const ref = render(trigger.slice('click:'.length), { args, data }); + await page!.click(String(ref).replace(/^@/, '')); + } else if (trigger === 'scroll') { + await page!.scroll('down'); + } + + // Step 3: Wait for network capture instead of fixed sleep + await page!.waitForCapture(timeout); + + // Step 4: Retrieve captured data + const matchingResponses = await page!.getInterceptedRequests(); + + // Step 5: Select from response if specified + let result = matchingResponses.length === 1 ? matchingResponses[0] : + matchingResponses.length > 1 ? matchingResponses : data; + + if (selectPath && result) { + let current = result; + for (const part of String(selectPath).split('.')) { + if (current && typeof current === 'object' && !Array.isArray(current)) { + current = current[part]; + } else break; + } + result = current ?? result; + } + + return result; +} +``` + +- [ ] **Step 2: Run unit + adapter tests** + +```bash +npx vitest run --project unit --project adapter +``` +Expected: All PASS. + +- [ ] **Step 3: Commit** + +```bash +git add src/pipeline/steps/intercept.ts +git commit -m "perf(intercept): use installInterceptor+waitForCapture in stepIntercept pipeline step" +``` + +--- + +## Task 6: Fix INTERCEPT adapters (Layer 1) + +**Files:** `36kr/hot.ts`, `36kr/search.ts`, `twitter/search.ts`, `twitter/followers.ts`, `twitter/following.ts`, `twitter/notifications.ts`, `producthunt/hot.ts`, `producthunt/browse.ts` + +- [ ] **Step 1: Fix `src/clis/36kr/hot.ts`** + +Find: +```typescript + await page.installInterceptor('36kr.com/api'); + await page.goto(url); + await page.wait(6); +``` +Replace with: +```typescript + await page.installInterceptor('36kr.com/api'); + await page.goto(url); + await page.waitForCapture(10); +``` + +- [ ] **Step 2: Fix `src/clis/36kr/search.ts`** + +Find: +```typescript + await page.installInterceptor('36kr.com/api'); + await page.goto(`https://www.36kr.com/search/articles/${query}`); + await page.wait(6); +``` +Replace with: +```typescript + await page.installInterceptor('36kr.com/api'); + await page.goto(`https://www.36kr.com/search/articles/${query}`); + await page.waitForCapture(10); +``` + +- [ ] **Step 3: Fix `src/clis/twitter/search.ts`** + +Find the two lines that contain `await page.wait(5)` in the `navigateToSearch` helper: +```typescript + await page.wait(5); +``` +(there are two of them: one after `pushState`, one in the retry). Replace both with: +```typescript + await page.waitForCapture(8); +``` + +- [ ] **Step 4: Fix `src/clis/twitter/followers.ts`** + +Find: +```typescript + await page.wait(5); + + // 4. Scroll to trigger pagination API calls +``` +Replace with: +```typescript + await page.waitForCapture(8); + + // 4. Scroll to trigger pagination API calls +``` + +Also find the earlier `wait(5)` after going to profile and `wait(3)` after going to home — those are UI waits (not INTERCEPT), replace with selector: +```typescript + // After page.goto('https://x.com/home'): + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 5 }); + // After page.goto(`https://x.com/${targetUser}`): + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 }); +``` + +- [ ] **Step 5: Fix `src/clis/twitter/following.ts`** + +Same pattern as `followers.ts`. Find and apply identically: +- `wait(5)` after `goto('https://x.com/home')` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 5 })` +- `wait(3)` after `goto(\`https://x.com/${targetUser}\`)` → `wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 })` +- `wait(5)` after SPA click that triggers INTERCEPT → `waitForCapture(8)` + +- [ ] **Step 6: Fix `src/clis/twitter/notifications.ts`** + +Find: +```typescript + await page.goto('https://x.com/home'); + await page.wait(3); +``` +Replace with: +```typescript + await page.goto('https://x.com/home'); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 5 }); +``` + +Find: +```typescript + await page.wait(5); + + // Verify SPA navigation succeeded +``` +Replace with: +```typescript + await page.waitForCapture(8); + + // Verify SPA navigation succeeded +``` + +- [ ] **Step 7: Fix `src/clis/producthunt/hot.ts`** + +Find: +```typescript + await page.installInterceptor( +``` +Look at the full pattern and replace the subsequent `wait(5)` with `waitForCapture(8)`. + +- [ ] **Step 8: Fix `src/clis/producthunt/browse.ts`** + +Same as `hot.ts` — replace `wait(5)` after `installInterceptor` + `goto` with `waitForCapture(8)`. + +- [ ] **Step 9: Run adapter tests** + +```bash +npx vitest run --project adapter +``` +Expected: All PASS (adapter tests mock `page.wait` and `page.waitForCapture`; existing mocks will need `waitForCapture: vi.fn()` if not already present). + +If any adapter test file lacks `waitForCapture` mock, add `waitForCapture: vi.fn().mockResolvedValue(undefined)` to its mock page object. + +- [ ] **Step 10: Commit** + +```bash +git add src/clis/36kr/hot.ts src/clis/36kr/search.ts \ + src/clis/twitter/search.ts src/clis/twitter/followers.ts \ + src/clis/twitter/following.ts src/clis/twitter/notifications.ts \ + src/clis/producthunt/hot.ts src/clis/producthunt/browse.ts +git commit -m "perf(intercept): replace wait(N) with waitForCapture() in all INTERCEPT adapters" +``` + +--- + +## Task 7: Daemon exponential backoff (Layer 3) + +**Files:** +- Modify: `src/browser/mcp.ts` + +- [ ] **Step 1: Replace fixed 300ms poll loop in `_ensureDaemon()`** + +In `src/browser/mcp.ts`, find: + +```typescript + // Wait for daemon to be ready AND extension to connect + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + await new Promise(resolve => setTimeout(resolve, 300)); + if (await isExtensionConnected()) return; + } +``` + +Replace with: + +```typescript + // Wait for daemon to be ready AND extension to connect. + // Exponential backoff: daemon typically ready in 500–800ms, + // so first check at 50ms then 100ms gets a fast result without hammering. + const deadline = Date.now() + timeoutMs; + const backoffs = [50, 100, 200, 400, 800, 1500, 3000]; + let backoffIdx = 0; + while (Date.now() < deadline) { + const delay = backoffs[Math.min(backoffIdx++, backoffs.length - 1)]; + await new Promise(resolve => setTimeout(resolve, delay)); + if (await isExtensionConnected()) return; + } +``` + +- [ ] **Step 2: Run unit tests** + +```bash +npx vitest run --project unit +``` +Expected: All PASS. + +- [ ] **Step 3: Commit** + +```bash +git add src/browser/mcp.ts +git commit -m "perf(daemon): exponential backoff for cold-start extension polling" +``` + +--- + +## Task 8: Fix Twitter UI adapters with `wait({ selector })` (Layer 2, part 1) + +**Files:** 13 adapters in `src/clis/twitter/` + +For all adapters below, the pattern is identical: `await page.goto(url)` followed by `await page.wait(5)` waiting for React to hydrate. Replace `wait(5)` with `wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 })`. + +- [ ] **Step 1: Fix `src/clis/twitter/reply.ts`** + +Find: +```typescript + await page.goto(kwargs.url); + await page.wait(5); // Wait for the react application to hydrate +``` +Replace with: +```typescript + await page.goto(kwargs.url); + await page.wait({ selector: '[data-testid="tweetTextarea_0"]', timeout: 8 }); +``` +(reply.ts uses the reply textarea directly — more precise than primaryColumn) + +- [ ] **Step 2: Fix `src/clis/twitter/follow.ts`** + +Find: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 3: Fix `src/clis/twitter/unfollow.ts`** + +Find: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 4: Fix `src/clis/twitter/like.ts`** + +Find: +```typescript + await page.goto(kwargs.url); + await page.wait(5); // Wait for tweet to load completely +``` +Replace with: +```typescript + await page.goto(kwargs.url); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 5: Fix `src/clis/twitter/bookmark.ts`** + +Find: +```typescript + await page.goto(kwargs.url); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(kwargs.url); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 6: Fix `src/clis/twitter/unbookmark.ts`** + +Find: +```typescript + await page.goto(kwargs.url); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(kwargs.url); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 7: Fix `src/clis/twitter/block.ts`** + +Find: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 8: Fix `src/clis/twitter/unblock.ts`** + +Find: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 9: Fix `src/clis/twitter/hide-reply.ts`** + +Find: +```typescript + await page.goto(kwargs.url); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(kwargs.url); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 10: Fix `src/clis/twitter/delete.ts`** + +Find: +```typescript + await page.goto(kwargs.url); + await page.wait(5); // Wait for tweet to load completely +``` +Replace with: +```typescript + await page.goto(kwargs.url); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +- [ ] **Step 11: Fix `src/clis/twitter/profile.ts`** + +There are two wait calls: + +Find (detecting logged-in user): +```typescript + await page.goto('https://x.com/home'); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto('https://x.com/home'); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 6 }); +``` + +Find (after going to profile): +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait(3); +``` +Replace with: +```typescript + await page.goto(`https://x.com/${username}`); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 }); +``` + +- [ ] **Step 12: Fix `src/clis/twitter/thread.ts`** + +Find: +```typescript + await page.goto('https://x.com'); + await page.wait(3); +``` +Replace with: +```typescript + await page.goto('https://x.com'); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 }); +``` + +- [ ] **Step 13: Fix `src/clis/twitter/timeline.ts`** + +Find: +```typescript + await page.goto('https://x.com'); + await page.wait(3); +``` +Replace with: +```typescript + await page.goto('https://x.com'); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 }); +``` + +- [ ] **Step 14: Fix `src/clis/twitter/reply-dm.ts`** + +Find: +```typescript + await page.goto('https://x.com/messages'); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto('https://x.com/messages'); + await page.wait({ selector: '[data-testid="DMDrawer"], [data-testid="primaryColumn"]', timeout: 6 }); +``` + +Find the second wait in `reply-dm.ts`: +```typescript + await page.goto(convUrl); + await page.wait(3); +``` +Replace with: +```typescript + await page.goto(convUrl); + await page.wait({ selector: '[data-testid="primaryColumn"]', timeout: 4 }); +``` + +- [ ] **Step 15: Run adapter tests** + +```bash +npx vitest run --project adapter +``` +Expected: All PASS. + +- [ ] **Step 16: Commit** + +```bash +git add src/clis/twitter/reply.ts src/clis/twitter/follow.ts src/clis/twitter/unfollow.ts \ + src/clis/twitter/like.ts src/clis/twitter/bookmark.ts src/clis/twitter/unbookmark.ts \ + src/clis/twitter/block.ts src/clis/twitter/unblock.ts src/clis/twitter/hide-reply.ts \ + src/clis/twitter/delete.ts src/clis/twitter/profile.ts src/clis/twitter/thread.ts \ + src/clis/twitter/timeline.ts src/clis/twitter/reply-dm.ts +git commit -m "perf(twitter): replace wait(N) with wait({ selector }) for React hydration waits" +``` + +--- + +## Task 9: Fix medium, substack, bloomberg, sinablog (Layer 2, part 2) + +**Files:** `medium/utils.ts`, `substack/utils.ts`, `bloomberg/news.ts`, `sinablog/utils.ts` + +The pattern for medium/substack: outer `wait(5)` + inner `setTimeout(3000)` in `evaluate()`. Fix: replace outer with `wait({ selector: 'article', timeout: 8 })`, remove inner setTimeout, and let the evaluate run synchronously. + +- [ ] **Step 1: Fix `src/clis/medium/utils.ts`** + +Find the `loadMediumPosts` function. Replace: +```typescript + await page.goto(url); + await page.wait(5); + const data = await page.evaluate(` + (async () => { + await new Promise((resolve) => setTimeout(resolve, 3000)); +``` +With: +```typescript + await page.goto(url); + await page.wait({ selector: 'article', timeout: 8 }); + const data = await page.evaluate(` + (() => { +``` +Also remove the closing `})()` (async) and replace with `()()` (sync). The full evaluate becomes a sync IIFE since the inner sleep is removed. + +**Complete replacement** — find the entire evaluate block starting with `(async () => {` and ending with `})()`: + +The evaluate body starting line is: +```typescript + const data = await page.evaluate(` + (async () => { + await new Promise((resolve) => setTimeout(resolve, 3000)); + + const limit = ${Math.max(1, Math.min(limit, 50))}; +``` +Replace `(async () => {` with `(() => {` and remove the `await new Promise((resolve) => setTimeout(resolve, 3000));` line (and the blank line after it). Change `})()` closing to `})()`. Remove `async` from the arrow function signature. + +- [ ] **Step 2: Fix `src/clis/substack/utils.ts` — `loadSubstackFeed`** + +Find: +```typescript + await page.goto(url); + await page.wait(5); + const data = await page.evaluate(` + (async () => { + await new Promise((resolve) => setTimeout(resolve, 3000)); +``` +Replace with: +```typescript + await page.goto(url); + await page.wait({ selector: 'article, [class*="post"]', timeout: 8 }); + const data = await page.evaluate(` + (() => { +``` +And remove the `await new Promise((resolve) => setTimeout(resolve, 3000));` line. Change `(async () => {` to `(() => {`. + +- [ ] **Step 3: Fix `src/clis/substack/utils.ts` — `loadSubstackArchive`** + +Same fix as Step 2 but for `loadSubstackArchive`: +```typescript + await page.goto(`${baseUrl}/archive`); + await page.wait(5); + const data = await page.evaluate(` + (async () => { + await new Promise((resolve) => setTimeout(resolve, 3000)); +``` +Replace with: +```typescript + await page.goto(`${baseUrl}/archive`); + await page.wait({ selector: 'a[href*="/p/"]', timeout: 8 }); + const data = await page.evaluate(` + (() => { +``` +Remove inner setTimeout line. Change async to sync. + +- [ ] **Step 4: Fix `src/clis/bloomberg/news.ts`** + +Find: +```typescript + await page.goto(url); + await page.wait(5); +``` +Replace with: +```typescript + await page.goto(url); + await page.wait({ selector: '#__NEXT_DATA__, article', timeout: 8 }); +``` + +Find the retry wait: +```typescript + if (result?.errorCode === 'NO_NEXT_DATA' || result?.errorCode === 'NO_STORY') { + await page.wait(4); + result = await loadStory(); + } +``` +Replace with: +```typescript + if (result?.errorCode === 'NO_NEXT_DATA' || result?.errorCode === 'NO_STORY') { + await page.wait({ selector: '#__NEXT_DATA__', timeout: 5 }); + result = await loadStory(); + } +``` + +- [ ] **Step 5: Fix `src/clis/sinablog/utils.ts`** + +`sinablog` has three functions to fix. + +**`loadSinaBlogHot` and `loadSinaBlogUser`** — find their `wait(3)` calls followed by inline `setTimeout(1500)` loops: + +```typescript + await page.goto(url); + await page.wait(3); + const data = await page.evaluate(` + (async () => { + await new Promise((resolve) => setTimeout(resolve, 1500)); +``` +Replace with: +```typescript + await page.goto(url); + await page.wait({ selector: '.article-list, .blog-article, article', timeout: 6 }); + const data = await page.evaluate(` + (() => { +``` +Remove the inner setTimeout line. Change `async` arrow to sync. + +**`loadSinaBlogSearch`** — find: +```typescript + await page.goto(buildSinaBlogSearchUrl(keyword)); + await page.wait(5); + const data = await page.evaluate(` + (async () => { + const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + for (let i = 0; i < 20; i += 1) { + if (document.querySelector('.result-item')) break; + await sleep(500); + } +``` +Replace with: +```typescript + await page.goto(buildSinaBlogSearchUrl(keyword)); + await page.wait({ selector: '.result-item', timeout: 8 }); + const data = await page.evaluate(` + (() => { +``` +Remove the `sleep` helper definition and the polling loop (they're replaced by the outer `wait({ selector })`). Change `async` to sync. + +- [ ] **Step 6: Run full unit + adapter tests** + +```bash +npx vitest run --project unit --project adapter +``` +Expected: All PASS. + +- [ ] **Step 7: Commit** + +```bash +git add src/clis/medium/utils.ts src/clis/substack/utils.ts \ + src/clis/bloomberg/news.ts src/clis/sinablog/utils.ts +git commit -m "perf(adapters): replace wait(N)+inline-sleep with wait({ selector }) in medium/substack/bloomberg/sinablog" +``` + +--- + +## Task 10: Final verification and PR + +- [ ] **Step 1: Run full test suite** + +```bash +npx vitest run --project unit --project adapter +``` +Expected: All tests PASS with no regressions. + +- [ ] **Step 2: TypeScript compile check** + +```bash +npx tsc --noEmit +``` +Expected: No errors. + +- [ ] **Step 3: Push and create PR** + +```bash +git push -u origin HEAD +gh pr create \ + --title "perf: smart wait — waitForCapture, wait({ selector }), daemon backoff" \ + --body "$(cat <<'EOF' +## Summary + +Three layered performance + correctness improvements: + +- **Layer 1 — `waitForCapture()`**: Fixes a correctness bug in INTERCEPT adapters where `wait(N)` (now DOM-stable-aware) could return before network captures arrive. Adds `waitForCapture(timeout)` to `IPage` — polls `window.__opencli_xhr` at 100ms intervals, resolves as soon as ≥1 capture exists. Applied to 36kr, twitter/search, followers, following, notifications, producthunt. +- **Layer 2 — `wait({ selector })`**: Extends `WaitOptions` with `selector?: string`. Adds `waitForSelectorJs()` to dom-helpers. Applied to 14 Twitter adapters (replacing `wait(5)` "React hydration" waits with precise element checks) and medium/substack/bloomberg/sinablog (removing duplicate inner `setTimeout` inside `evaluate()`). +- **Layer 3 — daemon backoff**: Replaces fixed 300ms poll with exponential backoff (50→100→200→400→800ms) in `_ensureDaemon()`. Cold-start first-success at ~150ms vs ~600ms. + +## Expected gains +- 36kr hot/search: 6s → ~1–2s +- Twitter INTERCEPT commands: 5–8s → ~1–3s +- Twitter UI commands: 5s → ~0.5–2s +- Medium/Substack: 8s → ~1–3s +- Daemon cold-start: ~600ms → ~150ms + +## Test plan +- [ ] `npx vitest run --project unit --project adapter` — all pass +- [ ] `npx tsc --noEmit` — no type errors +EOF +)" +``` From cac1dee86f670d3c048f89a3664502c3b6a1fb67 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:53:39 +0800 Subject: [PATCH 05/16] feat(perf): add waitForCaptureJs and waitForSelectorJs to dom-helpers --- src/browser/dom-helpers.test.ts | 72 +++++++++++++++++++++++++++++++++ src/browser/dom-helpers.ts | 37 +++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 src/browser/dom-helpers.test.ts diff --git a/src/browser/dom-helpers.test.ts b/src/browser/dom-helpers.test.ts new file mode 100644 index 00000000..0ad47725 --- /dev/null +++ b/src/browser/dom-helpers.test.ts @@ -0,0 +1,72 @@ +import { describe, it, expect } from 'vitest'; +import { waitForCaptureJs, waitForSelectorJs } from './dom-helpers.js'; + +describe('waitForCaptureJs', () => { + it('returns a non-empty string', () => { + const code = waitForCaptureJs(1000); + expect(typeof code).toBe('string'); + expect(code.length).toBeGreaterThan(0); + expect(code).toContain('__opencli_xhr'); + expect(code).toContain('resolve'); + expect(code).toContain('reject'); + }); + + it('resolves "captured" when __opencli_xhr is populated before deadline', async () => { + const g = globalThis as any; + g.__opencli_xhr = []; + g.window = g; // stub window for Node eval + const code = waitForCaptureJs(1000); + const promise = eval(code) as Promise; + g.__opencli_xhr.push({ data: 'test' }); + await expect(promise).resolves.toBe('captured'); + delete g.__opencli_xhr; + delete g.window; + }); + + it('rejects when __opencli_xhr stays empty past deadline', async () => { + const g = globalThis as any; + g.__opencli_xhr = []; + g.window = g; + const code = waitForCaptureJs(50); // 50ms timeout + const promise = eval(code) as Promise; + await expect(promise).rejects.toThrow('No network capture within 0.05s'); + delete g.__opencli_xhr; + delete g.window; + }); + + it('resolves immediately when __opencli_xhr already has data', async () => { + const g = globalThis as any; + g.__opencli_xhr = [{ data: 'already here' }]; + g.window = g; + const code = waitForCaptureJs(1000); + await expect(eval(code) as Promise).resolves.toBe('captured'); + delete g.__opencli_xhr; + delete g.window; + }); +}); + +describe('waitForSelectorJs', () => { + it('returns a non-empty string', () => { + const code = waitForSelectorJs('#app', 1000); + expect(typeof code).toBe('string'); + expect(code).toContain('#app'); + expect(code).toContain('querySelector'); + }); + + it('rejects when document.querySelector returns null within timeout', async () => { + const g = globalThis as any; + g.document = { querySelector: (_: string) => null }; + const code = waitForSelectorJs('#missing', 50); + await expect(eval(code) as Promise).rejects.toThrow('Selector not found: #missing'); + delete g.document; + }); + + it('resolves "found" when document.querySelector returns an element', async () => { + const g = globalThis as any; + const fakeEl = { tagName: 'DIV' }; + g.document = { querySelector: (_: string) => fakeEl }; + const code = waitForSelectorJs('[data-testid="primaryColumn"]', 1000); + await expect(eval(code) as Promise).resolves.toBe('found'); + delete g.document; + }); +}); diff --git a/src/browser/dom-helpers.ts b/src/browser/dom-helpers.ts index bcc9b413..204444c6 100644 --- a/src/browser/dom-helpers.ts +++ b/src/browser/dom-helpers.ts @@ -179,3 +179,40 @@ export function waitForDomStableJs(maxMs: number, quietMs: number): string { }) `; } + +/** + * Generate JS to wait until window.__opencli_xhr has ≥1 captured response. + * Polls every 100ms. Resolves 'captured' on success; rejects after maxMs. + * Used after installInterceptor() + goto() instead of a fixed sleep. + */ +export function waitForCaptureJs(maxMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${maxMs}; + const check = () => { + if ((window.__opencli_xhr || []).length > 0) return resolve('captured'); + if (Date.now() > deadline) return reject(new Error('No network capture within ${maxMs / 1000}s')); + setTimeout(check, 100); + }; + check(); + }) + `; +} + +/** + * Generate JS to wait until document.querySelector(selector) returns a match. + * Polls every 100ms. Resolves 'found' on success; rejects after timeoutMs. + */ +export function waitForSelectorJs(selector: string, timeoutMs: number): string { + return ` + new Promise((resolve, reject) => { + const deadline = Date.now() + ${timeoutMs}; + const check = () => { + if (document.querySelector(${JSON.stringify(selector)})) return resolve('found'); + if (Date.now() > deadline) return reject(new Error('Selector not found: ' + ${JSON.stringify(selector)})); + setTimeout(check, 100); + }; + check(); + }) + `; +} From 265d43131e6e446cf2c97123233d033dabc69c0a Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:54:35 +0800 Subject: [PATCH 06/16] feat(perf): extend WaitOptions with selector, add waitForCapture to IPage --- src/types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/types.ts b/src/types.ts index d043e49e..43ceb983 100644 --- a/src/types.ts +++ b/src/types.ts @@ -26,6 +26,7 @@ export interface SnapshotOptions { export interface WaitOptions { text?: string; + selector?: string; // wait until document.querySelector(selector) matches time?: number; timeout?: number; } @@ -64,6 +65,7 @@ export interface IPage { autoScroll(options?: { times?: number; delayMs?: number }): Promise; installInterceptor(pattern: string): Promise; getInterceptedRequests(): Promise; + waitForCapture(timeout?: number): Promise; screenshot(options?: ScreenshotOptions): Promise; closeWindow?(): Promise; /** Returns the current page URL, or null if unavailable. */ From 49e99f683300ab102422d5de77847bbc244bdede Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:55:42 +0800 Subject: [PATCH 07/16] feat(perf): implement waitForCapture() and wait({ selector }) in Page --- src/browser/page.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/browser/page.ts b/src/browser/page.ts index 92286394..a1bd2264 100644 --- a/src/browser/page.ts +++ b/src/browser/page.ts @@ -22,6 +22,8 @@ import { typeTextJs, pressKeyJs, waitForTextJs, + waitForCaptureJs, + waitForSelectorJs, scrollJs, autoScrollJs, networkRequestsJs, @@ -236,6 +238,12 @@ export class Page implements IPage { await new Promise(resolve => setTimeout(resolve, options.time! * 1000)); return; } + if (options.selector) { + const timeout = (options.timeout ?? 10) * 1000; + const code = waitForSelectorJs(options.selector, timeout); + await sendCommand('exec', { code, ...this._cmdOpts() }); + return; + } if (options.text) { const timeout = (options.timeout ?? 30) * 1000; const code = waitForTextJs(options.text, timeout); @@ -330,6 +338,14 @@ export class Page implements IPage { const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr')); return Array.isArray(result) ? result : []; } + + async waitForCapture(timeout: number = 10): Promise { + const maxMs = timeout * 1000; + await sendCommand('exec', { + code: waitForCaptureJs(maxMs), + ...this._cmdOpts(), + }); + } } // (End of file) From 42ab87ba6ae16d4983896aa87fa6bf0774f82b05 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:58:26 +0800 Subject: [PATCH 08/16] feat(perf): implement waitForCapture() and wait({ selector }) in CDPPage --- src/browser/cdp.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index 7ebead3d..018d5a92 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -25,6 +25,8 @@ import { autoScrollJs, networkRequestsJs, waitForDomStableJs, + waitForCaptureJs, + waitForSelectorJs, } from './dom-helpers.js'; import { isRecord, saveBase64ToFile } from '../utils.js'; @@ -255,6 +257,11 @@ class CDPPage implements IPage { await new Promise((resolve) => setTimeout(resolve, waitTime * 1000)); return; } + if (options.selector) { + const timeout = (options.timeout ?? 10) * 1000; + await this.evaluate(waitForSelectorJs(options.selector, timeout)); + return; + } if (options.text) { const timeout = (options.timeout ?? 30) * 1000; await this.evaluate(waitForTextJs(options.text, timeout)); @@ -326,6 +333,11 @@ class CDPPage implements IPage { const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr')); return Array.isArray(result) ? result : []; } + + async waitForCapture(timeout: number = 10): Promise { + const maxMs = timeout * 1000; + await this.evaluate(waitForCaptureJs(maxMs)); + } } function isCookie(value: unknown): value is BrowserCookie { From 7393b1b8060f4ced6b3099bb49cdb742576c790c Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 01:59:19 +0800 Subject: [PATCH 09/16] feat(perf): stepIntercept uses installInterceptor+waitForCapture+getInterceptedRequests --- src/pipeline/steps/intercept.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/pipeline/steps/intercept.ts b/src/pipeline/steps/intercept.ts index 26b60d9e..442d6c28 100644 --- a/src/pipeline/steps/intercept.ts +++ b/src/pipeline/steps/intercept.ts @@ -4,7 +4,6 @@ import type { IPage } from '../../types.js'; import { render, normalizeEvaluateSource } from '../template.js'; -import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js'; export async function stepIntercept(page: IPage | null, params: any, data: any, args: Record): Promise { const cfg = typeof params === 'object' ? params : {}; @@ -16,7 +15,7 @@ export async function stepIntercept(page: IPage | null, params: any, data: any, if (!capturePattern) return data; // Step 1: Inject fetch/XHR interceptor BEFORE trigger - await page!.evaluate(generateInterceptorJs(JSON.stringify(capturePattern))); + await page!.installInterceptor(capturePattern); // Step 2: Execute the trigger action if (trigger.startsWith('navigate:')) { @@ -32,11 +31,11 @@ export async function stepIntercept(page: IPage | null, params: any, data: any, await page!.scroll('down'); } - // Step 3: Wait a bit for network requests to fire - await page!.wait(Math.min(timeout, 3)); + // Step 3: Wait for network capture (event-driven, not fixed sleep) + await page!.waitForCapture(timeout); // Step 4: Retrieve captured data - const matchingResponses = await page!.evaluate(generateReadInterceptedJs()); + const matchingResponses = await page!.getInterceptedRequests(); // Step 5: Select from response if specified let result = matchingResponses.length === 1 ? matchingResponses[0] : From 295000f9b676d2f3363681b9866d0f39bbbf040c Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:00:39 +0800 Subject: [PATCH 10/16] fix(perf): replace wait(N) with waitForCapture(N) in 7 INTERCEPT adapters --- src/clis/36kr/hot.ts | 2 +- src/clis/36kr/search.ts | 2 +- src/clis/producthunt/browse.ts | 2 +- src/clis/producthunt/hot.ts | 2 +- src/clis/twitter/followers.ts | 2 +- src/clis/twitter/following.ts | 2 +- src/clis/twitter/notifications.ts | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/clis/36kr/hot.ts b/src/clis/36kr/hot.ts index 3caa8aaa..2c462818 100644 --- a/src/clis/36kr/hot.ts +++ b/src/clis/36kr/hot.ts @@ -60,7 +60,7 @@ cli({ await page.installInterceptor('36kr.com/api'); await page.goto(url); - await page.wait(6); + await page.waitForCapture(6); // Scrape rendered article links from DOM (deduplicated) const domItems: any = await page.evaluate(` diff --git a/src/clis/36kr/search.ts b/src/clis/36kr/search.ts index c508cd71..971b9c9a 100644 --- a/src/clis/36kr/search.ts +++ b/src/clis/36kr/search.ts @@ -24,7 +24,7 @@ cli({ await page.installInterceptor('36kr.com/api'); await page.goto(`https://www.36kr.com/search/articles/${query}`); - await page.wait(6); + await page.waitForCapture(6); const domItems: any = await page.evaluate(` (() => { diff --git a/src/clis/producthunt/browse.ts b/src/clis/producthunt/browse.ts index 27fdeaa1..9fa7f7bb 100644 --- a/src/clis/producthunt/browse.ts +++ b/src/clis/producthunt/browse.ts @@ -32,7 +32,7 @@ cli({ await page.installInterceptor('producthunt.com'); await page.goto(`https://www.producthunt.com/categories/${slug}`); - await page.wait(5); + await page.waitForCapture(5); const domItems: any = await page.evaluate(` (() => { diff --git a/src/clis/producthunt/hot.ts b/src/clis/producthunt/hot.ts index ab05b807..bc6d39d6 100644 --- a/src/clis/producthunt/hot.ts +++ b/src/clis/producthunt/hot.ts @@ -23,7 +23,7 @@ cli({ await page.installInterceptor('producthunt.com'); await page.goto('https://www.producthunt.com'); - await page.wait(5); + await page.waitForCapture(5); const domItems: any = await page.evaluate(` (() => { diff --git a/src/clis/twitter/followers.ts b/src/clis/twitter/followers.ts index 365a5b51..84fa7cb4 100644 --- a/src/clis/twitter/followers.ts +++ b/src/clis/twitter/followers.ts @@ -58,7 +58,7 @@ cli({ if (!clicked) { throw new SelectorError('Twitter followers link', 'Twitter may have changed the layout.'); } - await page.wait(5); + await page.waitForCapture(5); // 4. Scroll to trigger pagination API calls await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 }); diff --git a/src/clis/twitter/following.ts b/src/clis/twitter/following.ts index 5169331a..8c2c17af 100644 --- a/src/clis/twitter/following.ts +++ b/src/clis/twitter/following.ts @@ -51,7 +51,7 @@ cli({ if (!clicked) { throw new SelectorError('Twitter following link', 'Twitter may have changed the layout.'); } - await page.wait(5); + await page.waitForCapture(5); // 4. Scroll to trigger pagination API calls await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 }); diff --git a/src/clis/twitter/notifications.ts b/src/clis/twitter/notifications.ts index 3ca66d82..30c3e2ae 100644 --- a/src/clis/twitter/notifications.ts +++ b/src/clis/twitter/notifications.ts @@ -25,7 +25,7 @@ cli({ window.history.pushState({}, '', '/notifications'); window.dispatchEvent(new PopStateEvent('popstate', { state: {} })); }`); - await page.wait(5); + await page.waitForCapture(5); // Verify SPA navigation succeeded const currentUrl = await page.evaluate('() => window.location.pathname'); From feb0c239ee457cdf0016786cac19d6d10039d608 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:01:13 +0800 Subject: [PATCH 11/16] feat(perf): daemon cold-start uses exponential backoff [50..3000ms] --- src/browser/mcp.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/browser/mcp.ts b/src/browser/mcp.ts index d817f5d7..47810632 100644 --- a/src/browser/mcp.ts +++ b/src/browser/mcp.ts @@ -96,10 +96,11 @@ export class BrowserBridge implements IBrowserFactory { }); this._daemonProc.unref(); - // Wait for daemon to be ready AND extension to connect + // Wait for daemon to be ready AND extension to connect (exponential backoff) + const backoffs = [50, 100, 200, 400, 800, 1500, 3000]; const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - await new Promise(resolve => setTimeout(resolve, 300)); + for (let i = 0; Date.now() < deadline; i++) { + await new Promise(resolve => setTimeout(resolve, backoffs[Math.min(i, backoffs.length - 1)])); if (await isExtensionConnected()) return; } From 05f231b61435161fd93fbfbc1ef6a2cce38ea3db Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:03:09 +0800 Subject: [PATCH 12/16] fix(perf): replace wait(5) with wait({ selector }) in 15 Twitter UI adapters --- src/clis/twitter/block.ts | 2 +- src/clis/twitter/bookmark.ts | 2 +- src/clis/twitter/delete.ts | 2 +- src/clis/twitter/follow.ts | 2 +- src/clis/twitter/followers.ts | 2 +- src/clis/twitter/following.ts | 2 +- src/clis/twitter/hide-reply.ts | 2 +- src/clis/twitter/like.ts | 2 +- src/clis/twitter/profile.ts | 2 +- src/clis/twitter/reply-dm.ts | 2 +- src/clis/twitter/reply.ts | 2 +- src/clis/twitter/search.ts | 2 +- src/clis/twitter/unblock.ts | 2 +- src/clis/twitter/unbookmark.ts | 2 +- src/clis/twitter/unfollow.ts | 2 +- 15 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/clis/twitter/block.ts b/src/clis/twitter/block.ts index d3b8243a..857c03ee 100644 --- a/src/clis/twitter/block.ts +++ b/src/clis/twitter/block.ts @@ -18,7 +18,7 @@ cli({ const username = kwargs.username.replace(/^@/, ''); await page.goto(`https://x.com/${username}`); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/bookmark.ts b/src/clis/twitter/bookmark.ts index f1c2af2b..b718a381 100644 --- a/src/clis/twitter/bookmark.ts +++ b/src/clis/twitter/bookmark.ts @@ -17,7 +17,7 @@ cli({ if (!page) throw new CommandExecutionError('Browser session required for twitter bookmark'); await page.goto(kwargs.url); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/delete.ts b/src/clis/twitter/delete.ts index 15a2817f..97823cc2 100644 --- a/src/clis/twitter/delete.ts +++ b/src/clis/twitter/delete.ts @@ -17,7 +17,7 @@ cli({ if (!page) throw new CommandExecutionError('Browser session required for twitter delete'); await page.goto(kwargs.url); - await page.wait(5); // Wait for tweet to load completely + await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Wait for tweet to load completely const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/follow.ts b/src/clis/twitter/follow.ts index a7c714db..9c3b2551 100644 --- a/src/clis/twitter/follow.ts +++ b/src/clis/twitter/follow.ts @@ -18,7 +18,7 @@ cli({ const username = kwargs.username.replace(/^@/, ''); await page.goto(`https://x.com/${username}`); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/followers.ts b/src/clis/twitter/followers.ts index 84fa7cb4..fd57ab46 100644 --- a/src/clis/twitter/followers.ts +++ b/src/clis/twitter/followers.ts @@ -19,7 +19,7 @@ cli({ // If no user is specified, figure out the logged-in user's handle if (!targetUser) { await page.goto('https://x.com/home'); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const href = await page.evaluate(`() => { const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); diff --git a/src/clis/twitter/following.ts b/src/clis/twitter/following.ts index 8c2c17af..dce077e4 100644 --- a/src/clis/twitter/following.ts +++ b/src/clis/twitter/following.ts @@ -19,7 +19,7 @@ cli({ // If no user is specified, figure out the logged-in user's handle if (!targetUser) { await page.goto('https://x.com/home'); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const href = await page.evaluate(`() => { const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); diff --git a/src/clis/twitter/hide-reply.ts b/src/clis/twitter/hide-reply.ts index 98a7d0e0..ce96ab4e 100644 --- a/src/clis/twitter/hide-reply.ts +++ b/src/clis/twitter/hide-reply.ts @@ -17,7 +17,7 @@ cli({ if (!page) throw new CommandExecutionError('Browser session required for twitter hide-reply'); await page.goto(kwargs.url); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/like.ts b/src/clis/twitter/like.ts index bac90b86..91e4217d 100644 --- a/src/clis/twitter/like.ts +++ b/src/clis/twitter/like.ts @@ -17,7 +17,7 @@ cli({ if (!page) throw new CommandExecutionError('Browser session required for twitter like'); await page.goto(kwargs.url); - await page.wait(5); // Wait for tweet to load completely + await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Wait for tweet to load completely const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/profile.ts b/src/clis/twitter/profile.ts index 81a1d32e..d7cbc47a 100644 --- a/src/clis/twitter/profile.ts +++ b/src/clis/twitter/profile.ts @@ -21,7 +21,7 @@ cli({ // If no username, detect the logged-in user if (!username) { await page.goto('https://x.com/home'); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const href = await page.evaluate(`() => { const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]'); return link ? link.getAttribute('href') : null; diff --git a/src/clis/twitter/reply-dm.ts b/src/clis/twitter/reply-dm.ts index e2ffb6a1..a7df0acb 100644 --- a/src/clis/twitter/reply-dm.ts +++ b/src/clis/twitter/reply-dm.ts @@ -27,7 +27,7 @@ cli({ // Step 1: Navigate to messages to get conversation list await page.goto('https://x.com/messages'); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); // Step 2: Collect conversations with scroll-to-load const needed = maxSend + 10; // extra buffer for skips diff --git a/src/clis/twitter/reply.ts b/src/clis/twitter/reply.ts index 4ab045e2..0fd1e595 100644 --- a/src/clis/twitter/reply.ts +++ b/src/clis/twitter/reply.ts @@ -19,7 +19,7 @@ cli({ // 1. Navigate to the tweet page await page.goto(kwargs.url); - await page.wait(5); // Wait for the react application to hydrate + await page.wait({ selector: '[data-testid="primaryColumn"]' }); // 2. Automate typing the reply and clicking reply const result = await page.evaluate(`(async () => { diff --git a/src/clis/twitter/search.ts b/src/clis/twitter/search.ts index e11fd9da..5496524a 100644 --- a/src/clis/twitter/search.ts +++ b/src/clis/twitter/search.ts @@ -20,7 +20,7 @@ async function navigateToSearch(page: Pick, query: s window.dispatchEvent(new PopStateEvent('popstate', { state: {} })); })() `); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); lastPath = String(await page.evaluate('() => window.location.pathname') || ''); if (lastPath.startsWith('/search')) { diff --git a/src/clis/twitter/unblock.ts b/src/clis/twitter/unblock.ts index 2aa16f96..1ceb4712 100644 --- a/src/clis/twitter/unblock.ts +++ b/src/clis/twitter/unblock.ts @@ -18,7 +18,7 @@ cli({ const username = kwargs.username.replace(/^@/, ''); await page.goto(`https://x.com/${username}`); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/unbookmark.ts b/src/clis/twitter/unbookmark.ts index 3df63aa5..847400cf 100644 --- a/src/clis/twitter/unbookmark.ts +++ b/src/clis/twitter/unbookmark.ts @@ -17,7 +17,7 @@ cli({ if (!page) throw new CommandExecutionError('Browser session required for twitter unbookmark'); await page.goto(kwargs.url); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { diff --git a/src/clis/twitter/unfollow.ts b/src/clis/twitter/unfollow.ts index 5c7ec4e8..9e9e5587 100644 --- a/src/clis/twitter/unfollow.ts +++ b/src/clis/twitter/unfollow.ts @@ -18,7 +18,7 @@ cli({ const username = kwargs.username.replace(/^@/, ''); await page.goto(`https://x.com/${username}`); - await page.wait(5); + await page.wait({ selector: '[data-testid="primaryColumn"]' }); const result = await page.evaluate(`(async () => { try { From 7b0e18922ac27fd503a91f63dbdb49308cde5a88 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:04:58 +0800 Subject: [PATCH 13/16] fix(perf): replace wait(N) with wait({ selector }) in medium/substack/bloomberg/sinablog --- src/clis/bloomberg/news.ts | 2 +- src/clis/medium/utils.ts | 2 +- src/clis/sinablog/utils.ts | 8 ++++---- src/clis/substack/utils.ts | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/clis/bloomberg/news.ts b/src/clis/bloomberg/news.ts index 3ce84a46..0a16a71d 100644 --- a/src/clis/bloomberg/news.ts +++ b/src/clis/bloomberg/news.ts @@ -23,7 +23,7 @@ cli({ // Navigate and wait for the page to hydrate before extracting story data. await page.goto(url); - await page.wait(5); + await page.wait({ selector: 'article', timeout: 5 }); const loadStory = async () => page.evaluate(`(() => { const isRobot = /Are you a robot/i.test(document.title) diff --git a/src/clis/medium/utils.ts b/src/clis/medium/utils.ts index 9f6fcc80..b4326749 100644 --- a/src/clis/medium/utils.ts +++ b/src/clis/medium/utils.ts @@ -16,7 +16,7 @@ export function buildMediumUserUrl(username: string): string { export async function loadMediumPosts(page: IPage, url: string, limit: number): Promise { if (!page) throw new CommandExecutionError('Browser session required for medium posts'); await page.goto(url); - await page.wait(5); + await page.wait({ selector: 'article', timeout: 5 }); const data = await page.evaluate(` (async () => { await new Promise((resolve) => setTimeout(resolve, 3000)); diff --git a/src/clis/sinablog/utils.ts b/src/clis/sinablog/utils.ts index 3e2b4f54..eb873ce1 100644 --- a/src/clis/sinablog/utils.ts +++ b/src/clis/sinablog/utils.ts @@ -14,7 +14,7 @@ export function buildSinaBlogUserUrl(uid: string): string { export async function loadSinaBlogArticle(page: IPage, url: string): Promise { await page.goto(url); - await page.wait(3); + await page.wait({ selector: 'h1', timeout: 3 }); return page.evaluate(` (async () => { await new Promise((resolve) => setTimeout(resolve, 1500)); @@ -52,7 +52,7 @@ export async function loadSinaBlogArticle(page: IPage, url: string): Promise { const safeLimit = clampLimit(limit); await page.goto('https://blog.sina.com.cn/'); - await page.wait(3); + await page.wait({ selector: 'h1', timeout: 3 }); const data = await page.evaluate(` (async () => { await new Promise((resolve) => setTimeout(resolve, 1500)); @@ -122,7 +122,7 @@ export async function loadSinaBlogHot(page: IPage, limit: number): Promise { const safeLimit = clampLimit(limit); await page.goto(buildSinaBlogSearchUrl(keyword)); - await page.wait(5); + await page.wait({ selector: '.result-item', timeout: 5 }); const data = await page.evaluate(` (async () => { const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); @@ -159,7 +159,7 @@ export async function loadSinaBlogSearch(page: IPage, keyword: string, limit: nu export async function loadSinaBlogUser(page: IPage, uid: string, limit: number): Promise { const safeLimit = clampLimit(limit); await page.goto(buildSinaBlogUserUrl(uid)); - await page.wait(3); + await page.wait({ selector: 'h1', timeout: 3 }); const data = await page.evaluate(` (async () => { await new Promise((resolve) => setTimeout(resolve, 1000)); diff --git a/src/clis/substack/utils.ts b/src/clis/substack/utils.ts index c8dfd24c..0b0b958d 100644 --- a/src/clis/substack/utils.ts +++ b/src/clis/substack/utils.ts @@ -10,7 +10,7 @@ export function buildSubstackBrowseUrl(category?: string): string { export async function loadSubstackFeed(page: IPage, url: string, limit: number): Promise { if (!page) throw new CommandExecutionError('Browser session required for substack feed'); await page.goto(url); - await page.wait(5); + await page.wait({ selector: 'article', timeout: 5 }); const data = await page.evaluate(` (async () => { await new Promise((resolve) => setTimeout(resolve, 3000)); @@ -79,7 +79,7 @@ export async function loadSubstackFeed(page: IPage, url: string, limit: number): export async function loadSubstackArchive(page: IPage, baseUrl: string, limit: number): Promise { if (!page) throw new CommandExecutionError('Browser session required for substack archive'); await page.goto(`${baseUrl}/archive`); - await page.wait(5); + await page.wait({ selector: 'article', timeout: 5 }); const data = await page.evaluate(` (async () => { await new Promise((resolve) => setTimeout(resolve, 3000)); From b7277567919b3a2bc16ca316cb5c6219208511ea Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:06:44 +0800 Subject: [PATCH 14/16] fix(types): add waitForCapture to IPage mock helpers in tests --- src/clis/xiaohongshu/comments.test.ts | 1 + src/clis/xiaohongshu/creator-note-detail.test.ts | 1 + src/clis/xiaohongshu/creator-notes.test.ts | 1 + src/clis/xiaohongshu/publish.test.ts | 1 + src/clis/xiaohongshu/search.test.ts | 1 + src/pipeline/executor.test.ts | 1 + src/pipeline/steps/download.test.ts | 1 + 7 files changed, 7 insertions(+) diff --git a/src/clis/xiaohongshu/comments.test.ts b/src/clis/xiaohongshu/comments.test.ts index a60c42a0..9650025e 100644 --- a/src/clis/xiaohongshu/comments.test.ts +++ b/src/clis/xiaohongshu/comments.test.ts @@ -26,6 +26,7 @@ function createPageMock(evaluateResult: any): IPage { getInterceptedRequests: vi.fn().mockResolvedValue([]), getCookies: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), }; } diff --git a/src/clis/xiaohongshu/creator-note-detail.test.ts b/src/clis/xiaohongshu/creator-note-detail.test.ts index e40ae0db..af566242 100644 --- a/src/clis/xiaohongshu/creator-note-detail.test.ts +++ b/src/clis/xiaohongshu/creator-note-detail.test.ts @@ -33,6 +33,7 @@ function createPageMock(evaluateResult: any): IPage { getInterceptedRequests: vi.fn().mockResolvedValue([]), getCookies: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), }; } diff --git a/src/clis/xiaohongshu/creator-notes.test.ts b/src/clis/xiaohongshu/creator-notes.test.ts index bc0fe145..c2698c91 100644 --- a/src/clis/xiaohongshu/creator-notes.test.ts +++ b/src/clis/xiaohongshu/creator-notes.test.ts @@ -37,6 +37,7 @@ function createPageMock(evaluateResult: any, interceptedRequests: any[] = []): I getInterceptedRequests, getCookies: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), }; } diff --git a/src/clis/xiaohongshu/publish.test.ts b/src/clis/xiaohongshu/publish.test.ts index 9fcc0326..959961fb 100644 --- a/src/clis/xiaohongshu/publish.test.ts +++ b/src/clis/xiaohongshu/publish.test.ts @@ -36,6 +36,7 @@ function createPageMock(evaluateResults: any[]): IPage { getInterceptedRequests: vi.fn().mockResolvedValue([]), getCookies: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), }; } diff --git a/src/clis/xiaohongshu/search.test.ts b/src/clis/xiaohongshu/search.test.ts index 5771ae4c..afa82fbf 100644 --- a/src/clis/xiaohongshu/search.test.ts +++ b/src/clis/xiaohongshu/search.test.ts @@ -31,6 +31,7 @@ function createPageMock(evaluateResults: any[]): IPage { getInterceptedRequests: vi.fn().mockResolvedValue([]), getCookies: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), }; } diff --git a/src/pipeline/executor.test.ts b/src/pipeline/executor.test.ts index 2bc373ad..ca7ad555 100644 --- a/src/pipeline/executor.test.ts +++ b/src/pipeline/executor.test.ts @@ -31,6 +31,7 @@ function createMockPage(overrides: Partial = {}): IPage { installInterceptor: vi.fn(), getInterceptedRequests: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), ...overrides, }; } diff --git a/src/pipeline/steps/download.test.ts b/src/pipeline/steps/download.test.ts index 8604e3f5..bf177eb5 100644 --- a/src/pipeline/steps/download.test.ts +++ b/src/pipeline/steps/download.test.ts @@ -44,6 +44,7 @@ function createMockPage(getCookies: IPage['getCookies']): IPage { installInterceptor: vi.fn(), getInterceptedRequests: vi.fn().mockResolvedValue([]), screenshot: vi.fn().mockResolvedValue(''), + waitForCapture: vi.fn().mockResolvedValue(undefined), }; } From e9915498f44e80dabd93745d0ead6c045021d139 Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:15:07 +0800 Subject: [PATCH 15/16] docs: simplify README to 50-line overview with docs link Remove redundant command table (already in docs/adapters/index.md). Keep badges, quick-start, and star history only. --- README.md | 383 +++--------------------------------------------------- 1 file changed, 21 insertions(+), 362 deletions(-) diff --git a/README.md b/README.md index 48dfd290..66d4b0bb 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # OpenCLI -> **Make any website, Electron App, or Local Tool your CLI.** +> **Make any website, Electron App, or Local Tool your CLI.** > Zero risk · Reuse Chrome login · AI-powered discovery · Universal CLI Hub [![中文文档](https://img.shields.io/badge/docs-%E4%B8%AD%E6%96%87-0F766E?style=flat-square)](./README.zh-CN.md) @@ -8,384 +8,43 @@ [![Node.js Version](https://img.shields.io/node/v/@jackwener/opencli?style=flat-square)](https://nodejs.org) [![License](https://img.shields.io/npm/l/@jackwener/opencli?style=flat-square)](./LICENSE) -A CLI tool that turns **any website**, **Electron app**, or **local CLI tool** into a command-line interface — Bilibili, Zhihu, 小红书, Twitter/X, Reddit, YouTube, Antigravity, `gh`, `docker`, and [many more](#built-in-commands) — powered by browser session reuse and AI-native discovery. +A CLI tool that turns **any website**, **Electron app**, or **local CLI tool** into a command-line interface — powered by browser session reuse and AI-native discovery. -**Built for AI Agents**: Simply configure an instruction in your global `AGENT.md` or `.cursorrules` guiding the AI to execute `opencli list` via Bash to discover available tools. Register your favorite local CLIs (`opencli register mycli`), and the AI will automatically learn how to invoke all your tools perfectly! - -**CLI All Electron Apps! The Most Powerful Update Has Arrived!** -Turn ANY Electron application into a CLI tool! Recombine, script, and extend applications like Antigravity Ultra seamlessly. Now AI can control itself natively. Unlimited possibilities await! - ---- - -## Highlights - -- **CLI All Electron** — CLI-ify apps like Antigravity Ultra! Now AI can control itself natively using cc/openclaw! +- **Zero LLM cost** — No tokens consumed at runtime. - **Account-safe** — Reuses Chrome's logged-in state; your credentials never leave the browser. -- **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies. -- **External CLI Hub** — Discover, auto-install, and passthrough commands to any external CLI (gh, obsidian, docker, etc). Zero setup. -- **Self-healing setup** — `opencli doctor` diagnoses and auto-starts the daemon, extension, and live browser connectivity. -- **Dynamic Loader** — Simply drop `.ts` or `.yaml` adapters into the `clis/` folder for auto-registration. -- **Dual-Engine Architecture** — Supports both YAML declarative data pipelines and robust browser runtime TypeScript injections. - -## Why opencli? - -There are many great browser automation tools. Here's when opencli is the right choice: - -| Your need | Best tool | Why | -|-----------|-----------|-----| -| Scheduled data extraction from specific sites | **opencli** | Pre-built adapters, deterministic JSON, zero LLM cost | -| AI agent needs reliable site operations | **opencli** | Hundreds of commands, structured output, fast deterministic response | -| Explore an unknown website ad-hoc | Browser-Use, Stagehand | LLM-driven general browsing for one-off tasks | -| Large-scale web crawling | Crawl4AI, Scrapy | Purpose-built for throughput and scale | -| Control desktop Electron apps from terminal | **opencli** | CDP + AppleScript — the only CLI tool that does this | - -**What makes opencli different:** - -- **Zero LLM cost** — No tokens consumed at runtime. Run 10,000 times and pay nothing. - **Deterministic** — Same command, same output schema, every time. Pipeable, scriptable, CI-friendly. -- **Broad coverage** — 50+ sites across global and Chinese platforms (Bilibili, Zhihu, Xiaohongshu, Reddit, HackerNews, and more), plus desktop Electron apps via CDP. - -> For a detailed comparison with Browser-Use, Crawl4AI, Firecrawl, and others, see the [Comparison Guide](./docs/comparison.md). - -## Prerequisites - -- **Node.js**: >= 20.0.0 (or **Bun** >= 1.0 — see [Runtime Support](#runtime-support) below) -- **Chrome** running **and logged into the target site** (e.g. bilibili.com, zhihu.com, xiaohongshu.com). - -> **⚠️ Important**: Browser commands reuse your Chrome login session. You must be logged into the target website in Chrome before running commands. If you get empty data or errors, check your login status first. - -### Runtime Support - -OpenCLI works with both **Node.js** (≥ 20) and **Bun** (≥ 1.0). All commands and adapters are runtime-agnostic. - -```bash -# Development with Bun (faster startup) -npm run dev:bun - -# Run the built CLI with Bun -npm run start:bun - -# Run unit tests under Bun -npm run test:bun - -# Run E2E tests with Bun as the runtime -OPENCLI_TEST_RUNTIME=bun npm run test:e2e -``` - -Use `opencli doctor` to check your current runtime — it displays the active engine (e.g. `node v22.13.0` or `bun 1.1.42`). - -OpenCLI connects to your browser through a lightweight **Browser Bridge** Chrome Extension + micro-daemon (zero config, auto-start). - -### Browser Bridge Extension Setup - -You can install the extension via either method: - -**Method 1: Download Pre-built Release (Recommended)** -1. Go to the GitHub [Releases page](https://github.com/jackwener/opencli/releases) and download the latest `opencli-extension.zip`. -2. Unzip the file and open `chrome://extensions`, enable **Developer mode** (top-right toggle). -3. Click **Load unpacked** and select the unzipped folder. - -**Method 2: Load Source (For Developers)** -1. Open `chrome://extensions` and enable **Developer mode**. -2. Click **Load unpacked** and select the `extension/` directory from this repository. - -That's it! The daemon auto-starts when you run any browser command. No tokens, no manual configuration. - -> **Tip**: Use `opencli doctor` for ongoing diagnosis: -> ```bash -> opencli doctor # Check extension + daemon connectivity -> ``` +- **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies. +- **65+ built-in adapters** — Global and Chinese platforms, plus desktop Electron apps via CDP. ## Quick Start -### Install via npm (recommended) - ```bash npm install -g @jackwener/opencli +opencli doctor # Check setup +opencli list # See all commands +opencli hackernews top --limit 5 # Public API, no browser needed +opencli bilibili hot -f json # Browser command, JSON output ``` -Then use directly: - -```bash -opencli list # See all commands -opencli list -f yaml # List commands as YAML -opencli hackernews top --limit 5 # Public API, no browser -opencli bilibili hot --limit 5 # Browser command -opencli zhihu hot -f json # JSON output -opencli zhihu hot -f yaml # YAML output -``` - -### Install from source (for developers) - -```bash -git clone git@github.com:jackwener/opencli.git -cd opencli -npm install -npm run build -npm link # Link binary globally -opencli list # Now you can use it anywhere! -``` - -### Update - -```bash -npm install -g @jackwener/opencli@latest -``` - -## Built-in Commands - -Run `opencli list` for the live registry. - -| Site | Commands | Mode | -|------|----------|------| -| **twitter** | `trending` `bookmarks` `profile` `search` `timeline` `thread` `following` `followers` `notifications` `post` `reply` `delete` `like` `article` `follow` `unfollow` `bookmark` `unbookmark` `download` `accept` `reply-dm` `block` `unblock` `hide-reply` | Browser | -| **reddit** | `hot` `frontpage` `popular` `search` `subreddit` `read` `user` `user-posts` `user-comments` `upvote` `save` `comment` `subscribe` `saved` `upvoted` | Browser | -| **cursor** | `status` `send` `read` `new` `dump` `composer` `model` `extract-code` `ask` `screenshot` `history` `export` | Desktop | -| **bilibili** | `hot` `search` `me` `favorite` `history` `feed` `subtitle` `dynamic` `ranking` `following` `user-videos` `download` | Browser | -| **codex** | `status` `send` `read` `new` `dump` `extract-diff` `model` `ask` `screenshot` `history` `export` | Desktop | -| **chatwise** | `status` `new` `send` `read` `ask` `model` `history` `export` `screenshot` | Desktop | -| **doubao** | `status` `new` `send` `read` `ask` | Browser | -| **doubao-app** | `status` `new` `send` `read` `ask` `screenshot` `dump` | Desktop | -| **notion** | `status` `search` `read` `new` `write` `sidebar` `favorites` `export` | Desktop | -| **discord-app** | `status` `send` `read` `channels` `servers` `search` `members` | Desktop | -| **v2ex** | `hot` `latest` `topic` `node` `user` `member` `replies` `nodes` `daily` `me` `notifications` | Public / Browser | -| **xueqiu** | `feed` `hot-stock` `hot` `search` `stock` `watchlist` `earnings-date` `fund-holdings` `fund-snapshot` | Browser | -| **antigravity** | `status` `send` `read` `new` `dump` `extract-code` `model` `watch` | Desktop | -| **chatgpt** | `status` `new` `send` `read` `ask` `model` | Desktop | -| **xiaohongshu** | `search` `notifications` `feed` `user` `download` `publish` `creator-notes` `creator-note-detail` `creator-notes-summary` `creator-profile` `creator-stats` | Browser | -| **apple-podcasts** | `search` `episodes` `top` | Public | -| **xiaoyuzhou** | `podcast` `podcast-episodes` `episode` | Public | -| **zhihu** | `hot` `search` `question` `download` | Browser | -| **weixin** | `download` | Browser | -| **youtube** | `search` `video` `transcript` | Browser | -| **boss** | `search` `detail` `recommend` `joblist` `greet` `batchgreet` `send` `chatlist` `chatmsg` `invite` `mark` `exchange` `resume` `stats` | Browser | -| **coupang** | `search` `add-to-cart` | Browser | -| **bbc** | `news` | Public | -| **bloomberg** | `main` `markets` `economics` `industries` `tech` `politics` `businessweek` `opinions` `feeds` `news` | Public / Browser | -| **ctrip** | `search` | Browser | -| **devto** | `top` `tag` `user` | Public | -| **dictionary** | `search` `synonyms` `examples` | Public | -| **arxiv** | `search` `paper` | Public | -| **paperreview** | `submit` `review` `feedback` | Public | -| **wikipedia** | `search` `summary` `random` `trending` | Public | -| **hackernews** | `top` `new` `best` `ask` `show` `jobs` `search` `user` | Public | -| **jd** | `item` | Browser | -| **linkedin** | `search` `timeline` | Browser | -| **reuters** | `search` | Browser | -| **smzdm** | `search` | Browser | -| **web** | `read` | Browser | -| **weibo** | `hot` `search` | Browser | -| **yahoo-finance** | `quote` | Browser | -| **sinafinance** | `news` | 🌐 Public | -| **barchart** | `quote` `options` `greeks` `flow` | Browser | -| **chaoxing** | `assignments` `exams` | Browser | -| **grok** | `ask` | Browser | -| **hf** | `top` | Public | -| **jike** | `feed` `search` `create` `like` `comment` `repost` `notifications` `post` `topic` `user` | Browser | -| **jimeng** | `generate` `history` | Browser | -| **yollomi** | `generate` `video` `edit` `upload` `models` `remove-bg` `upscale` `face-swap` `restore` `try-on` `background` `object-remover` | Browser | -| **linux-do** | `feed` `categories` `tags` `search` `topic` `user-topics` `user-posts` | Browser | -| **stackoverflow** | `hot` `search` `bounties` `unanswered` | Public | -| **steam** | `top-sellers` | Public | -| **weread** | `shelf` `search` `book` `highlights` `notes` `notebooks` `ranking` | Browser | -| **douban** | `search` `top250` `subject` `photos` `download` `marks` `reviews` `movie-hot` `book-hot` | Browser | -| **facebook** | `feed` `profile` `search` `friends` `groups` `events` `notifications` `memories` `add-friend` `join-group` | Browser | -| **google** | `news` `search` `suggest` `trends` | Public | -| **36kr** | `news` `hot` `search` `article` | Public / Browser | -| **imdb** | `search` `title` `top` `trending` `person` `reviews` | Public | -| **producthunt** | `posts` `today` `hot` `browse` | Public / Browser | -| **instagram** | `explore` `profile` `search` `user` `followers` `following` `follow` `unfollow` `like` `unlike` `comment` `save` `unsave` `saved` | Browser | -| **lobsters** | `hot` `newest` `active` `tag` | Public | -| **medium** | `feed` `search` `user` | Browser | -| **sinablog** | `hot` `search` `article` `user` | Browser | -| **substack** | `feed` `search` `publication` | Browser | -| **pixiv** | `ranking` `search` `user` `illusts` `detail` `download` | Browser | -| **tiktok** | `explore` `search` `profile` `user` `following` `follow` `unfollow` `like` `unlike` `comment` `save` `unsave` `live` `notifications` `friends` | Browser | - - -### External CLI Hub - -OpenCLI acts as a universal hub for your existing command-line tools. It provides unified discovery, automatic installation, and pure passthrough execution. - -| External CLI | Description | Commands Example | -|--------------|-------------|------------------| -| **gh** | GitHub CLI | `opencli gh pr list --limit 5` | -| **obsidian** | Obsidian vault management | `opencli obsidian search query="AI"` | -| **docker** | Docker command-line interface | `opencli docker ps` | -| **readwise** | Readwise & Reader CLI | `opencli readwise login` | -| **gws** | Google Workspace CLI — Docs, Sheets, Drive, Gmail, Calendar | `opencli gws docs list` | - -**Zero Configuration**: OpenCLI purely passes your inputs to the underlying binary via standard I/O streams. The external CLI works exactly as it naturally would, maintaining its standard output formats. - -**Auto-Installation**: If you run `opencli gh ...` and `gh` is not installed on your system, OpenCLI will automatically try to install it using your system's package manager (e.g., `brew install gh`) before seamlessly re-running the command. - -**Register Your Own**: -Add any local CLI to your OpenCLI registry so AI agents can automatically discover it via the `opencli list` command. -```bash -opencli register mycli -``` - -### Desktop App Adapters - -Each desktop adapter has its own detailed documentation with commands reference, setup guide, and examples: - -If you want to add support for a new Electron desktop app, start with [docs/guide/electron-app-cli.md](./docs/guide/electron-app-cli.md) and the deeper [Electron guide](./docs/advanced/electron.md). - -| App | Description | Doc | -|-----|-------------|-----| -| **Cursor** | Control Cursor IDE — Composer, chat, code extraction | [Doc](./docs/adapters/desktop/cursor.md) | -| **Codex** | Drive OpenAI Codex CLI agent headlessly | [Doc](./docs/adapters/desktop/codex.md) | -| **Antigravity** | Control Antigravity Ultra from terminal | [Doc](./docs/adapters/desktop/antigravity.md) | -| **ChatGPT** | Automate ChatGPT macOS desktop app | [Doc](./docs/adapters/desktop/chatgpt.md) | -| **ChatWise** | Multi-LLM client (GPT-4, Claude, Gemini) | [Doc](./docs/adapters/desktop/chatwise.md) | -| **Notion** | Search, read, write Notion pages | [Doc](./docs/adapters/desktop/notion.md) | -| **Discord** | Discord Desktop — messages, channels, servers | [Doc](./docs/adapters/desktop/discord.md) | -| **Doubao** | Control Doubao AI desktop app via CDP | [Doc](./docs/adapters/desktop/doubao-app.md) | - -## Download Support - -OpenCLI supports downloading images, videos, and articles from supported platforms. - -### Supported Platforms - -| Platform | Content Types | Notes | -|----------|---------------|-------| -| **xiaohongshu** | Images, Videos | Downloads all media from a note | -| **bilibili** | Videos | Requires `yt-dlp` installed | -| **twitter** | Images, Videos | Downloads from user media tab or single tweet | -| **douban** | Images | Downloads poster / still image lists from movie subjects | -| **pixiv** | Images | Downloads original-quality illustrations, supports multi-page works | -| **zhihu** | Articles (Markdown) | Exports articles with optional image download | -| **weixin** | Articles (Markdown) | Exports WeChat Official Account articles | - -### Prerequisites - -For video downloads from streaming platforms, you need to install `yt-dlp`: - -```bash -# Install yt-dlp -pip install yt-dlp -# or -brew install yt-dlp -``` - -### Usage Examples +**[→ Full documentation](./docs/index.md)** -```bash -# Download images/videos from Xiaohongshu note -opencli xiaohongshu download abc123 --output ./xhs - -# Download Bilibili video (requires yt-dlp) -opencli bilibili download BV1xxx --output ./bilibili -opencli bilibili download BV1xxx --quality 1080p # Specify quality - -# Download Twitter media from user -opencli twitter download elonmusk --limit 20 --output ./twitter - -# Download single tweet media -opencli twitter download --tweet-url "https://x.com/user/status/123" --output ./twitter - -# Download Douban posters / stills -opencli douban download 30382501 --output ./douban - -# Export Zhihu article to Markdown -opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu - -# Export with local images -opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --download-images - -# Export WeChat article to Markdown -opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin -``` - - - -## Output Formats - -All built-in commands support `--format` / `-f` with `table`, `json`, `yaml`, `md`, and `csv`. -The `list` command supports the same format options, and keeps `--json` for backward compatibility. - -```bash -opencli list -f yaml # Command registry as YAML -opencli bilibili hot -f table # Default: rich terminal table -opencli bilibili hot -f json # JSON (pipe to jq or LLMs) -opencli bilibili hot -f yaml # YAML (human-readable structured output) -opencli bilibili hot -f md # Markdown -opencli bilibili hot -f csv # CSV -opencli bilibili hot -v # Verbose: show pipeline debug steps -``` - -## Plugins - -Extend OpenCLI with community-contributed adapters. Plugins use the same YAML/TS format as built-in commands and are automatically discovered at startup. - -```bash -opencli plugin install github:user/opencli-plugin-my-tool # Install -opencli plugin list # List installed -opencli plugin update my-tool # Update to latest -opencli plugin update --all # Update all installed plugins -opencli plugin uninstall my-tool # Remove -``` - -`opencli plugin list` also shows the tracked short commit hash when a plugin version is recorded in `~/.opencli/plugins.lock.json`. - -| Plugin | Type | Description | -|--------|------|-------------| -| [opencli-plugin-github-trending](https://github.com/ByteYue/opencli-plugin-github-trending) | YAML | GitHub Trending repositories | -| [opencli-plugin-hot-digest](https://github.com/ByteYue/opencli-plugin-hot-digest) | TS | Multi-platform trending aggregator | -| [opencli-plugin-juejin](https://github.com/Astro-Han/opencli-plugin-juejin) | YAML | 稀土掘金 (Juejin) hot articles | - -See [Plugins Guide](./docs/guide/plugins.md) for creating your own plugin. - -## For AI Agents (Developer Guide) - -If you are an AI assistant tasked with creating a new command adapter for `opencli`, please follow the AI Agent workflow below: - -> **Quick mode**: To generate a single command for a specific page URL, see [CLI-ONESHOT.md](./CLI-ONESHOT.md) — just a URL + one-line goal, 4 steps done. - -> **Full mode**: Before writing any adapter code, read [CLI-EXPLORER.md](./CLI-EXPLORER.md). It contains the complete browser exploration workflow, the 5-tier authentication strategy decision tree, and debugging guide. - -```bash -# 1. Deep Explore — discover APIs, infer capabilities, detect framework -opencli explore https://example.com --site mysite - -# 2. Synthesize — generate YAML adapters from explore artifacts -opencli synthesize mysite - -# 3. Generate — one-shot: explore → synthesize → register -opencli generate https://example.com --goal "hot" - -# 4. Strategy Cascade — auto-probe: PUBLIC → COOKIE → HEADER -opencli cascade https://api.example.com/data -``` - -Explore outputs to `.opencli/explore//` (manifest.json, endpoints.json, capabilities.json, auth.json). - -## Testing - -See **[TESTING.md](./TESTING.md)** for how to run and write tests. - -## Troubleshooting - -- **"Extension not connected"** - - Ensure the opencli Browser Bridge extension is installed and **enabled** in `chrome://extensions`. -- **"attach failed: Cannot access a chrome-extension:// URL"** - - Another Chrome extension (e.g. youmind, New Tab Override, or AI assistant extensions) may be interfering. Try **disabling other extensions** temporarily, then retry. -- **Empty data returns or 'Unauthorized' error** - - Your login session in Chrome might have expired. Open a normal Chrome tab, navigate to the target site, and log in or refresh the page. -- **Node API errors** - - Make sure you are using Node.js >= 20. Some dependencies require modern Node APIs. -- **Daemon issues** - - Check daemon status: `curl localhost:19825/status` - - View extension logs: `curl localhost:19825/logs` +## Documentation +| Topic | Link | +|-------|------| +| Installation & Setup | [docs/guide/installation.md](./docs/guide/installation.md) | +| Getting Started | [docs/guide/getting-started.md](./docs/guide/getting-started.md) | +| Built-in Commands | [docs/adapters/index.md](./docs/adapters/index.md) | +| Desktop App Adapters | [docs/adapters/desktop](./docs/adapters/desktop) | +| Plugins | [docs/guide/plugins.md](./docs/guide/plugins.md) | +| Electron App Guide | [docs/guide/electron-app-cli.md](./docs/guide/electron-app-cli.md) | +| Troubleshooting | [docs/guide/troubleshooting.md](./docs/guide/troubleshooting.md) | +| Comparison with other tools | [docs/comparison.md](./docs/comparison.md) | ## Star History [![Star History Chart](https://api.star-history.com/svg?repos=jackwener/opencli&type=Date)](https://star-history.com/#jackwener/opencli&Date) - - ## License [Apache-2.0](./LICENSE) From 75973288c2a645f321195e2e78f950d0ad6d425c Mon Sep 17 00:00:00 2001 From: jackwener Date: Sat, 28 Mar 2026 02:15:14 +0800 Subject: [PATCH 16/16] fix(perf): CDPPage smart wait + MutationObserver selector wait - CDPPage.wait(N>=1) now uses waitForDomStableJs instead of fixed sleep, matching Page.wait() behavior and saving unnecessary idle time - waitForSelectorJs switches from 100ms polling to MutationObserver, resolving instantly when the target element appears in the DOM - Update dom-helpers tests to stub MutationObserver for Node eval context --- src/browser/cdp.ts | 9 +++++++ src/browser/dom-helpers.test.ts | 46 ++++++++++++++++++++++++++------- src/browser/dom-helpers.ts | 23 +++++++++++------ 3 files changed, 61 insertions(+), 17 deletions(-) diff --git a/src/browser/cdp.ts b/src/browser/cdp.ts index 018d5a92..72bcf87f 100644 --- a/src/browser/cdp.ts +++ b/src/browser/cdp.ts @@ -249,6 +249,15 @@ class CDPPage implements IPage { async wait(options: number | WaitOptions): Promise { if (typeof options === 'number') { + if (options >= 1) { + try { + const maxMs = options * 1000; + await this.evaluate(waitForDomStableJs(maxMs, Math.min(500, maxMs))); + return; + } catch { + // Fallback: fixed sleep + } + } await new Promise((resolve) => setTimeout(resolve, options * 1000)); return; } diff --git a/src/browser/dom-helpers.test.ts b/src/browser/dom-helpers.test.ts index 0ad47725..44bd29db 100644 --- a/src/browser/dom-helpers.test.ts +++ b/src/browser/dom-helpers.test.ts @@ -51,17 +51,10 @@ describe('waitForSelectorJs', () => { expect(typeof code).toBe('string'); expect(code).toContain('#app'); expect(code).toContain('querySelector'); + expect(code).toContain('MutationObserver'); }); - it('rejects when document.querySelector returns null within timeout', async () => { - const g = globalThis as any; - g.document = { querySelector: (_: string) => null }; - const code = waitForSelectorJs('#missing', 50); - await expect(eval(code) as Promise).rejects.toThrow('Selector not found: #missing'); - delete g.document; - }); - - it('resolves "found" when document.querySelector returns an element', async () => { + it('resolves "found" immediately when selector already present', async () => { const g = globalThis as any; const fakeEl = { tagName: 'DIV' }; g.document = { querySelector: (_: string) => fakeEl }; @@ -69,4 +62,39 @@ describe('waitForSelectorJs', () => { await expect(eval(code) as Promise).resolves.toBe('found'); delete g.document; }); + + it('resolves "found" when selector appears after DOM mutation', async () => { + const g = globalThis as any; + let mutationCallback!: () => void; + g.MutationObserver = class { + constructor(cb: () => void) { mutationCallback = cb; } + observe() {} + disconnect() {} + }; + let calls = 0; + g.document = { + querySelector: (_: string) => (calls++ > 0 ? { tagName: 'DIV' } : null), + body: {}, + }; + const code = waitForSelectorJs('#app', 1000); + const promise = eval(code) as Promise; + mutationCallback(); // simulate DOM mutation + await expect(promise).resolves.toBe('found'); + delete g.document; + delete g.MutationObserver; + }); + + it('rejects when selector never appears within timeout', async () => { + const g = globalThis as any; + g.MutationObserver = class { + constructor(_cb: () => void) {} + observe() {} + disconnect() {} + }; + g.document = { querySelector: (_: string) => null, body: {} }; + const code = waitForSelectorJs('#missing', 50); + await expect(eval(code) as Promise).rejects.toThrow('Selector not found: #missing'); + delete g.document; + delete g.MutationObserver; + }); }); diff --git a/src/browser/dom-helpers.ts b/src/browser/dom-helpers.ts index 204444c6..de2fbfa0 100644 --- a/src/browser/dom-helpers.ts +++ b/src/browser/dom-helpers.ts @@ -201,18 +201,25 @@ export function waitForCaptureJs(maxMs: number): string { /** * Generate JS to wait until document.querySelector(selector) returns a match. - * Polls every 100ms. Resolves 'found' on success; rejects after timeoutMs. + * Uses MutationObserver for near-instant resolution; falls back to reject after timeoutMs. */ export function waitForSelectorJs(selector: string, timeoutMs: number): string { return ` new Promise((resolve, reject) => { - const deadline = Date.now() + ${timeoutMs}; - const check = () => { - if (document.querySelector(${JSON.stringify(selector)})) return resolve('found'); - if (Date.now() > deadline) return reject(new Error('Selector not found: ' + ${JSON.stringify(selector)})); - setTimeout(check, 100); - }; - check(); + const sel = ${JSON.stringify(selector)}; + if (document.querySelector(sel)) return resolve('found'); + const cap = setTimeout(() => { + obs.disconnect(); + reject(new Error('Selector not found: ' + sel)); + }, ${timeoutMs}); + const obs = new MutationObserver(() => { + if (document.querySelector(sel)) { + clearTimeout(cap); + obs.disconnect(); + resolve('found'); + } + }); + obs.observe(document.body || document.documentElement, { childList: true, subtree: true }); }) `; }