diff --git a/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts b/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts new file mode 100644 index 0000000000..069fb2310f --- /dev/null +++ b/packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; +import { buildTelemetryJoinKeys } from "./feedback.js"; + +describe("buildTelemetryJoinKeys", () => { + it("emits fid + tid and omits renders when the ring is empty", () => { + const keys = buildTelemetryJoinKeys({ + feedbackId: "feedback-uuid", + anonymousId: "install-uuid", + }); + expect(keys).toBe("fid=feedback-uuid tid=install-uuid"); + }); + + it("appends recent render ids newest-last with a ! marking failed renders", () => { + const keys = buildTelemetryJoinKeys({ + feedbackId: "f", + anonymousId: "t", + recentRenders: [ + { id: "render-a", at: "2026-07-21T00:00:00Z", ok: true }, + { id: "render-b", at: "2026-07-21T01:00:00Z", ok: false }, + ], + }); + expect(keys).toBe("fid=f tid=t renders=render-a,render-b!"); + }); + + it("stays within the backend env cap for a full ring of uuid render ids", () => { + const uuid = "01234567-89ab-cdef-0123-456789abcdef"; + const keys = buildTelemetryJoinKeys({ + feedbackId: uuid, + anonymousId: uuid, + recentRenders: Array.from({ length: 5 }, (_, i) => ({ + id: uuid, + at: "2026-07-21T00:00:00Z", + ok: i % 2 === 0, + })), + }); + // submitFeedback caps env at 500 chars; the doctor summary consumes ~100. + expect(keys.length).toBeLessThan(400); + }); +}); diff --git a/packages/cli/src/commands/feedback.ts b/packages/cli/src/commands/feedback.ts index ebae8619bc..f3be656f0f 100644 --- a/packages/cli/src/commands/feedback.ts +++ b/packages/cli/src/commands/feedback.ts @@ -1,4 +1,5 @@ import { failCommand } from "../utils/commandResult.js"; +import { randomUUID } from "node:crypto"; import { resolve } from "node:path"; import { defineCommand } from "citty"; import * as clack from "@clack/prompts"; @@ -7,6 +8,7 @@ import type { Example } from "./_examples.js"; import { trackRenderFeedback } from "../telemetry/events.js"; import { shouldTrack, flush } from "../telemetry/client.js"; import { getDoctorSummary } from "../telemetry/feedback.js"; +import { readConfig, type RecentRenderRecord } from "../telemetry/config.js"; import { publishProjectArchive } from "../utils/publishProject.js"; import { submitFeedback } from "../utils/submitFeedback.js"; import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js"; @@ -28,6 +30,27 @@ function normalizeComment(raw?: string): string | undefined { return raw || undefined; } +/** + * Compact PostHog join keys appended to the environment string that rides + * along with the forwarded report (and therefore lands verbatim in the wild + * feedback channel): `fid` = this submission's PostHog `survey sent` + * `feedback_id`; `tid` = the install's telemetry distinct_id; `renders` = + * recent `render_job_id`s (newest last, `!` suffix = the render failed). + * Together they turn a wild report into an exact telemetry lookup instead of + * a hardware-fingerprint hunt. + */ +export function buildTelemetryJoinKeys(input: { + feedbackId: string; + anonymousId: string; + recentRenders?: RecentRenderRecord[]; +}): string { + const parts = [`fid=${input.feedbackId}`, `tid=${input.anonymousId}`]; + if (input.recentRenders?.length) { + parts.push(`renders=${input.recentRenders.map((r) => `${r.id}${r.ok ? "" : "!"}`).join(",")}`); + } + return parts.join(" "); +} + function printIssueConsent(dir: string): void { console.log(); console.log( @@ -170,6 +193,18 @@ export default defineCommand({ const comment = normalizeComment(args.comment); const doctorSummary = await getDoctorSummary(); + // Join keys tying this report to the install's PostHog rows — see + // buildTelemetryJoinKeys. Appended to the env string so they surface in + // the forwarded report; mirrored as structured props on the PostHog event. + const feedbackId = randomUUID(); + const config = readConfig(); + const joinKeys = buildTelemetryJoinKeys({ + feedbackId, + anonymousId: config.anonymousId, + recentRenders: config.recentRenders, + }); + const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys; + // Soft-warn (never blocks) when the comment for a non-clean report is // missing the mandated reproduction-packet markers. Prints before the // submission ack so the reporter sees the nudge while their run is fresh. @@ -177,13 +212,19 @@ export default defineCommand({ // The standalone command runs separately from `render`, so it has no real // elapsed time to report. Omit it rather than recording a fake duration. - trackRenderFeedback({ rating, comment, doctorSummary }); + trackRenderFeedback({ + rating, + comment, + doctorSummary, + feedbackId, + recentRenderIds: config.recentRenders?.map((r) => r.id), + }); await flush(); // Ack first so the user isn't kept waiting on the best-effort forward (which // is bounded to a few seconds and never surfaces an error either way). console.log(c.dim("Thanks for the feedback!")); - await submitFeedback({ rating, comment, cliVersion: VERSION, env: doctorSummary }); + await submitFeedback({ rating, comment, cliVersion: VERSION, env: envWithJoinKeys }); if (args["file-issue"] === true) { await fileGithubIssue({ diff --git a/packages/cli/src/commands/render.ts b/packages/cli/src/commands/render.ts index a53cbfbc68..9b3d067e4c 100644 --- a/packages/cli/src/commands/render.ts +++ b/packages/cli/src/commands/render.ts @@ -61,7 +61,12 @@ import { trackRenderObservation, } from "../telemetry/events.js"; import { maybePromptRenderFeedback } from "../telemetry/feedback.js"; -import { readConfigFresh, writeConfig, type HyperframesConfig } from "../telemetry/config.js"; +import { + readConfigFresh, + recordRecentRender, + writeConfig, + type HyperframesConfig, +} from "../telemetry/config.js"; import { shouldTrack } from "../telemetry/client.js"; import { renderJobObservabilityTelemetryPayload } from "../telemetry/renderObservability.js"; import { bytesToMb } from "../telemetry/system.js"; @@ -1339,6 +1344,9 @@ function handleRenderError( ...renderJobObservabilityTelemetryPayload(job), ...getMemorySnapshot(), }); + // Failed renders join the recent-renders ring too — a bug report filed via + // `hyperframes feedback` is MOST likely to be about a failed render. + if (job?.id) recordRecentRender(job.id, false); if (options.throwOnError) { throw new Error(message); } @@ -1376,6 +1384,9 @@ function trackRenderMetrics( options: RenderOptions, docker: boolean, ): void { + // Successful render → recent-renders ring, so a later `hyperframes + // feedback` can attach this render's telemetry id to the report. + recordRecentRender(job.id, true); const perf = job.perfSummary; const compositionDurationMs = perf ? Math.round(perf.compositionDurationSeconds * 1000) @@ -1393,6 +1404,13 @@ function trackRenderMetrics( fps: fpsToNumber(options.fps), quality: options.quality, workers: options.workers ?? perf?.workers, + workersBoundBy: perf?.workerSizing?.boundBy, + workersCpuBased: perf?.workerSizing?.cpuBasedWorkers, + workersMemoryBased: perf?.workerSizing?.memoryBasedWorkers, + workersHeapBased: perf?.workerSizing?.heapBasedWorkers, + workersFrameBased: perf?.workerSizing?.frameBasedWorkers, + workersHeapLimitMb: perf?.workerSizing?.heapLimitMb, + workersExceedHeapAdvisory: perf?.workerSizing?.exceedsHeapAdvisory, docker, gpu: options.gpu, authoringSkill: options.authoringSkill, diff --git a/packages/cli/src/telemetry/config.ts b/packages/cli/src/telemetry/config.ts index 98054a71b6..cd41a4d50c 100644 --- a/packages/cli/src/telemetry/config.ts +++ b/packages/cli/src/telemetry/config.ts @@ -85,6 +85,39 @@ export interface HyperframesConfig { * `DE_PARALLEL_ROUTER_TRIAL_MAX_RENDERS` in `render.ts`. */ deParallelRouterTrialRenderCount?: number; + /** + * Ring of the last few local renders (newest last). `hyperframes feedback` + * attaches these ids — which are the `render_job_id` / + * `observability_render_job_id` on this install's PostHog events — to the + * feedback it submits, so a wild bug report can be joined to the exact + * telemetry rows of the renders it describes. + */ + recentRenders?: RecentRenderRecord[]; +} + +/** One entry in {@link HyperframesConfig.recentRenders}. */ +export interface RecentRenderRecord { + /** The render job id (`RenderJob.id` — the telemetry `render_job_id`). */ + id: string; + /** ISO timestamp of when the render finished. */ + at: string; + /** Whether the render completed successfully. */ + ok: boolean; +} + +/** Ring size for {@link HyperframesConfig.recentRenders}. */ +const MAX_RECENT_RENDERS = 5; + +/** + * Append a finished render to the recent-renders ring (newest last, capped). + * Fresh read-modify-write like the trial counters — narrows, but does not + * eliminate, lost updates against a concurrent CLI process. + */ +export function recordRecentRender(id: string, ok: boolean): void { + const config = readConfigFresh(); + const ring = [...(config.recentRenders ?? []), { id, at: new Date().toISOString(), ok }]; + config.recentRenders = ring.slice(-MAX_RECENT_RENDERS); + writeConfig(config); } const DEFAULT_CONFIG: HyperframesConfig = { @@ -142,6 +175,18 @@ export function readConfig(): HyperframesConfig { typeof parsed.deParallelRouterTrialRenderCount === "number" ? parsed.deParallelRouterTrialRenderCount : undefined, + recentRenders: Array.isArray(parsed.recentRenders) + ? parsed.recentRenders + .filter( + (r): r is RecentRenderRecord => + typeof r === "object" && + r !== null && + typeof (r as RecentRenderRecord).id === "string" && + typeof (r as RecentRenderRecord).at === "string" && + typeof (r as RecentRenderRecord).ok === "boolean", + ) + .slice(-MAX_RECENT_RENDERS) + : undefined, }; cachedConfig = config; diff --git a/packages/cli/src/telemetry/events.test.ts b/packages/cli/src/telemetry/events.test.ts index 9ee0b8c040..5dd6c6253e 100644 --- a/packages/cli/src/telemetry/events.test.ts +++ b/packages/cli/src/telemetry/events.test.ts @@ -186,6 +186,59 @@ describe("render telemetry events", () => { expect(flush).toHaveBeenCalledTimes(2); }); + // The enforcement decision for the advisory heap budget reads these fleet + // props (see computeWorkerSizing) — a silent drop in the summary→event hop + // would invalidate that decision without anyone noticing. + it("carries every worker-sizing provenance prop on render_complete", () => { + trackRenderComplete({ + durationMs: 1000, + fps: 30, + quality: "high", + docker: false, + gpu: false, + workers: 6, + workersBoundBy: "max_workers", + workersCpuBased: 16, + workersMemoryBased: 8, + workersHeapBased: 4, + workersFrameBased: 24, + workersHeapLimitMb: 4096, + workersExceedHeapAdvisory: true, + }); + + expect(trackEvent).toHaveBeenCalledWith( + "render_complete", + expect.objectContaining({ + workers: 6, + workers_bound_by: "max_workers", + workers_cpu_based: 16, + workers_memory_based: 8, + workers_heap_based: 4, + workers_frame_based: 24, + workers_heap_limit_mb: 4096, + workers_exceed_heap_advisory: true, + }), + undefined, + ); + }); + + it("ties feedback to its report and recent renders via feedback_id + recent_render_ids", () => { + trackRenderFeedback({ + rating: 3, + comment: "hook scene blank", + feedbackId: "feedback-uuid", + recentRenderIds: ["render-a", "render-b"], + }); + + expect(trackEvent).toHaveBeenCalledWith( + "survey sent", + expect.objectContaining({ + feedback_id: "feedback-uuid", + recent_render_ids: "render-a,render-b", + }), + ); + }); + it("redacts paths and URL query strings from render error messages", () => { trackRenderError({ fps: 30, diff --git a/packages/cli/src/telemetry/events.ts b/packages/cli/src/telemetry/events.ts index 9e100ae3ee..23085eb342 100644 --- a/packages/cli/src/telemetry/events.ts +++ b/packages/cli/src/telemetry/events.ts @@ -149,6 +149,17 @@ export function trackRenderComplete( /** Authoring workflow skill that drove this render (e.g. "product-launch-video"). */ authoringSkill?: string; workers?: number; + // Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers + // "why N workers?" fleet-wide, and validates the advisory per-worker heap + // budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap + // machine — see computeWorkerSizing in @hyperframes/engine). + workersBoundBy?: string; + workersCpuBased?: number; + workersMemoryBased?: number; + workersHeapBased?: number; + workersFrameBased?: number; + workersHeapLimitMb?: number; + workersExceedHeapAdvisory?: boolean; docker: boolean; gpu: boolean; // Static-frame dedup outcome (opt-out HF_STATIC_DEDUP=false). Undefined on @@ -245,6 +256,13 @@ export function trackRenderComplete( quality: props.quality, authoring_skill: props.authoringSkill, workers: props.workers, + workers_bound_by: props.workersBoundBy, + workers_cpu_based: props.workersCpuBased, + workers_memory_based: props.workersMemoryBased, + workers_heap_based: props.workersHeapBased, + workers_frame_based: props.workersFrameBased, + workers_heap_limit_mb: props.workersHeapLimitMb, + workers_exceed_heap_advisory: props.workersExceedHeapAdvisory, docker: props.docker, gpu: props.gpu, static_dedup_enabled: props.staticDedupEnabled, @@ -609,6 +627,14 @@ export function trackRenderFeedback(props: { renderDurationMs?: number; comment?: string; doctorSummary?: string; + /** + * Join key shared with the forwarded feedback report (Slack/backend): the + * same uuid rides in the report's env string as `fid=…`, so a wild report + * resolves to exactly one PostHog "survey sent" event and vice versa. + */ + feedbackId?: string; + /** render_job_id values of this install's recent renders (newest last). */ + recentRenderIds?: string[]; }): void { trackEvent("survey sent", { $survey_id: "render_satisfaction", @@ -617,6 +643,11 @@ export function trackRenderFeedback(props: { ...(props.comment ? { $survey_response_2: props.comment } : {}), ...(props.renderDurationMs !== undefined ? { render_duration_ms: props.renderDurationMs } : {}), ...(props.doctorSummary ? { doctor_summary: props.doctorSummary } : {}), + ...(props.feedbackId ? { feedback_id: props.feedbackId } : {}), + // Comma-joined: EventProperties values are scalars only. + ...(props.recentRenderIds?.length + ? { recent_render_ids: props.recentRenderIds.join(",") } + : {}), }); } diff --git a/packages/engine/src/index.ts b/packages/engine/src/index.ts index c90bab6f6c..62ed54ab22 100644 --- a/packages/engine/src/index.ts +++ b/packages/engine/src/index.ts @@ -208,6 +208,7 @@ export type { // ── Parallel rendering ───────────────────────────────────────────────────────── export { calculateOptimalWorkers, + computeWorkerSizing, distributeFrames, distributeFramesInterleaved, executeParallelCapture, @@ -215,6 +216,8 @@ export { getSystemResources, type WorkerTask, type WorkerResult, + type WorkerSizing, + type WorkerSizingBound, type ParallelProgress, } from "./services/parallelCoordinator.js"; diff --git a/packages/engine/src/services/parallelCoordinator.test.ts b/packages/engine/src/services/parallelCoordinator.test.ts index 91d294bad8..31f7b59410 100644 --- a/packages/engine/src/services/parallelCoordinator.test.ts +++ b/packages/engine/src/services/parallelCoordinator.test.ts @@ -1,6 +1,7 @@ import { describe, it, expect } from "vitest"; import { calculateOptimalWorkers, + computeWorkerSizing, distributeFrames, expectedFramesForTask, flagSilentWorkerExits, @@ -93,6 +94,50 @@ describe("calculateOptimalWorkers", () => { }); }); +describe("computeWorkerSizing", () => { + it("matches calculateOptimalWorkers and reports every constraint", () => { + const config = { concurrency: "auto" as const }; + const sizing = computeWorkerSizing(900, undefined, config); + expect(sizing.workers).toBe(calculateOptimalWorkers(900, undefined, config)); + expect(sizing.cpuBasedWorkers).toBeGreaterThanOrEqual(1); + expect(sizing.memoryBasedWorkers).toBeGreaterThanOrEqual(1); + expect(sizing.heapBasedWorkers).toBeGreaterThanOrEqual(1); + expect(sizing.frameBasedWorkers).toBe(30); // 900 / MIN_FRAMES_PER_WORKER(30) + expect(sizing.heapLimitMb).toBeGreaterThan(0); + expect(sizing.totalMemoryMb).toBeGreaterThan(0); + expect(sizing.cpuCount).toBeGreaterThan(0); + }); + + it("labels explicit requests and clamps them like the legacy wrapper", () => { + const sizing = computeWorkerSizing(900, 25, { concurrency: "auto" }); + expect(sizing.workers).toBe(24); + expect(sizing.boundBy).toBe("explicit"); + }); + + it("labels tiny renders as too_few_frames", () => { + const sizing = computeWorkerSizing(30, undefined, { concurrency: "auto" }); + expect(sizing.workers).toBe(1); + expect(sizing.boundBy).toBe("too_few_frames"); + }); + + it("labels the contention cap when high capture cost binds", () => { + const sizing = computeWorkerSizing(180, undefined, { + concurrency: 6, + coresPerWorker: 100, + minParallelFrames: 120, + largeRenderThreshold: 1000, + captureCostMultiplier: 4, + }); + expect(sizing.workers).toBe(1); + expect(sizing.boundBy).toBe("contention"); + }); + + it("flags exceedsHeapAdvisory exactly when workers exceed the heap budget", () => { + const sizing = computeWorkerSizing(900, 24, { concurrency: "auto" }); + expect(sizing.exceedsHeapAdvisory).toBe(sizing.workers > sizing.heapBasedWorkers); + }); +}); + describe("shouldDisableBrowserPoolForParallelWorker", () => { const linuxHeadlessWorker = { parallel: true, diff --git a/packages/engine/src/services/parallelCoordinator.ts b/packages/engine/src/services/parallelCoordinator.ts index d6c028e4a3..113c2cf9f9 100644 --- a/packages/engine/src/services/parallelCoordinator.ts +++ b/packages/engine/src/services/parallelCoordinator.ts @@ -9,6 +9,7 @@ import { cpus, freemem } from "os"; import { existsSync, mkdirSync, readdirSync } from "fs"; import { copyFile, rename } from "fs/promises"; import { join } from "path"; +import { getHeapStatistics } from "v8"; import { createCaptureSession, @@ -109,7 +110,22 @@ type WorkerBrowserPoolDecision = { headlessShellPath?: string; }; -const MEMORY_PER_WORKER_MB = 256; +// System-memory budget per parallel worker. Each worker is a full Chrome +// process (SwiftShader compositor + raster threads) plus its share of parent- +// process frame buffering — the old 256MB figure was ~6× under a measured +// Chrome-under-capture RSS and let memory-constrained hosts overcommit (wild +// 16GB black-slab report; 0.7.66 heap-OOM report on 24GB with 6 auto workers). +const MEMORY_PER_WORKER_MB = 1536; +// Parent-process V8 heap the coordinator itself needs regardless of worker +// count (compile artifacts, puppeteer sessions, encoder state). +const HEAP_RESERVED_MB = 1024; +// Parent-process V8 heap consumed per worker (protocol buffers + in-flight +// frame buffers). Derived from the field OOM: 6 workers exhausted a ~4GB +// default heap ⇒ >~500MB/worker + base. ponytail: advisory-only until the +// workers_heap_* telemetry added alongside this constant validates the figure +// — enforcing a guessed budget could silently cut worker counts fleet-wide. +// TODO(PRINFRA-341): decide enforcement after ~2 weeks of fleet soak. +const HEAP_PER_WORKER_MB = 640; const MIN_WORKERS = 1; const MAX_WORKER_DIAGNOSTIC_LINES = 8; // Hard ceiling on explicit `--workers N` requests. Above this, the cost of @@ -230,13 +246,95 @@ export function formatWorkerFailure(result: WorkerResult): string { return `${base}; diagnostics: ${diagnostics}`; } -export function calculateOptimalWorkers( +/** Which constraint produced the final auto-sized worker count. */ +export type WorkerSizingBound = + | "explicit" + | "too_few_frames" + | "cpu" + | "memory" + | "frames" + | "max_workers" + | "min_parallel_floor" + | "contention"; + +/** + * Full provenance of a worker-sizing decision. Threaded into render + * observability/telemetry so fleet data can answer "why N workers?" and + * "would the heap budget have prevented this OOM?" without a repro. + */ +export interface WorkerSizing { + workers: number; + boundBy: WorkerSizingBound; + cpuBasedWorkers: number; + memoryBasedWorkers: number; + frameBasedWorkers: number; + effectiveMaxWorkers: number; + /** + * ADVISORY, not enforced (see HEAP_PER_WORKER_MB): how many workers the + * parent process's V8 heap could feed. Compare against `workers` in + * telemetry to validate the budget before enforcement. + */ + heapBasedWorkers: number; + /** V8 `heap_size_limit` for the parent process, MB. */ + heapLimitMb: number; + totalMemoryMb: number; + cpuCount: number; + captureCostMultiplier: number; + /** true when the chosen count exceeds the advisory heap budget. */ + exceedsHeapAdvisory: boolean; +} + +/** + * Compute the auto worker count together with the full sizing provenance. + * `calculateOptimalWorkers` is the thin legacy wrapper returning `.workers`. + */ +// The branch count is the decision provenance itself — each if records WHICH +// constraint bound, which is the entire point of the function. +// fallow-ignore-next-line complexity +export function computeWorkerSizing( totalFrames: number, requested?: number, config?: WorkerSizingConfig, -): number { +): WorkerSizing { + const cpuCount = cpus().length; + // Use total memory instead of free memory — macOS reports misleadingly low + // freemem() because it aggressively caches files in "inactive" memory that + // is immediately reclaimable. + const totalMemoryMb = getSystemTotalMb(); + const heapLimitMb = Math.round(getHeapStatistics().heap_size_limit / (1024 * 1024)); + const heapBasedWorkers = Math.max( + 1, + Math.floor((heapLimitMb - HEAP_RESERVED_MB) / HEAP_PER_WORKER_MB), + ); + const cpuBasedWorkers = Math.max(1, cpuCount - 2); + const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMb * 0.5) / MEMORY_PER_WORKER_MB)); + const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER); + const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1); + + const base = { + cpuBasedWorkers, + memoryBasedWorkers, + frameBasedWorkers, + heapBasedWorkers, + heapLimitMb, + totalMemoryMb, + cpuCount, + captureCostMultiplier, + }; + const finish = (workers: number, boundBy: WorkerSizingBound, effectiveMaxWorkers: number) => ({ + workers, + boundBy, + effectiveMaxWorkers, + ...base, + exceedsHeapAdvisory: workers > heapBasedWorkers, + }); + if (requested !== undefined) { - return Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested)); + return finish( + Math.max(MIN_WORKERS, Math.min(ABSOLUTE_MAX_WORKERS, requested)), + "explicit", + ABSOLUTE_MAX_WORKERS, + ); } // Resolve effective values: config overrides → DEFAULT_CONFIG fallback. @@ -251,24 +349,22 @@ export function calculateOptimalWorkers( const effectiveMinParallelFrames = config?.minParallelFrames ?? DEFAULT_CONFIG.minParallelFrames; const effectiveLargeRenderThreshold = config?.largeRenderThreshold ?? DEFAULT_CONFIG.largeRenderThreshold; - const captureCostMultiplier = Math.max(1, config?.captureCostMultiplier ?? 1); - - if (totalFrames < MIN_FRAMES_PER_WORKER * 2) return 1; - const cpuCount = cpus().length; - const cpuBasedWorkers = Math.max(1, cpuCount - 2); - - // Use total memory instead of free memory — macOS reports misleadingly low - // freemem() because it aggressively caches files in "inactive" memory that - // is immediately reclaimable. - const totalMemoryMB = getSystemTotalMb(); - const memoryBasedWorkers = Math.max(1, Math.floor((totalMemoryMB * 0.5) / MEMORY_PER_WORKER_MB)); - - const frameBasedWorkers = Math.floor(totalFrames / MIN_FRAMES_PER_WORKER); + if (totalFrames < MIN_FRAMES_PER_WORKER * 2) { + return finish(1, "too_few_frames", effectiveMaxWorkers); + } const optimal = Math.min(cpuBasedWorkers, memoryBasedWorkers, frameBasedWorkers); + const optimalBound: WorkerSizingBound = + optimal === cpuBasedWorkers ? "cpu" : optimal === memoryBasedWorkers ? "memory" : "frames"; const minWorkersForJob = totalFrames >= effectiveMinParallelFrames ? 2 : MIN_WORKERS; let finalWorkers = Math.max(minWorkersForJob, Math.min(effectiveMaxWorkers, optimal)); + let boundBy: WorkerSizingBound = + finalWorkers === optimal + ? optimalBound + : finalWorkers === effectiveMaxWorkers && effectiveMaxWorkers < optimal + ? "max_workers" + : "min_parallel_floor"; // Adaptive scaling: cap workers for large or expensive renders to prevent // CPU contention. Each Chrome process (with SwiftShader) is CPU-heavy; too @@ -285,10 +381,19 @@ export function calculateOptimalWorkers( const cpuScaledMax = Math.max(MIN_WORKERS, Math.floor(cpuCount / weightedCoresPerWorker)); if (finalWorkers > cpuScaledMax) { finalWorkers = cpuScaledMax; + boundBy = "contention"; } } - return finalWorkers; + return finish(finalWorkers, boundBy, effectiveMaxWorkers); +} + +export function calculateOptimalWorkers( + totalFrames: number, + requested?: number, + config?: WorkerSizingConfig, +): number { + return computeWorkerSizing(totalFrames, requested, config).workers; } export function distributeFrames( diff --git a/packages/producer/src/services/render/captureCost.heapAdvisory.test.ts b/packages/producer/src/services/render/captureCost.heapAdvisory.test.ts new file mode 100644 index 0000000000..39d2fdce1b --- /dev/null +++ b/packages/producer/src/services/render/captureCost.heapAdvisory.test.ts @@ -0,0 +1,42 @@ +import { describe, expect, it } from "vitest"; +import type { WorkerSizing } from "@hyperframes/engine"; +import { buildHeapAdvisoryWarning } from "./captureCost.js"; + +function sizing(overrides: Partial): WorkerSizing { + return { + workers: 6, + boundBy: "max_workers", + cpuBasedWorkers: 16, + memoryBasedWorkers: 8, + frameBasedWorkers: 24, + effectiveMaxWorkers: 6, + heapBasedWorkers: 4, + heapLimitMb: 4096, + totalMemoryMb: 24576, + cpuCount: 18, + captureCostMultiplier: 1, + exceedsHeapAdvisory: true, + ...overrides, + }; +} + +describe("buildHeapAdvisoryWarning", () => { + it("names the chosen count, heap limit, safe count, and both remediation knobs", () => { + const message = buildHeapAdvisoryWarning(sizing({}), undefined); + expect(message).toContain("6 capture workers"); + expect(message).toContain("limit 4096MB"); + expect(message).toContain("~4"); + expect(message).toContain("NODE_OPTIONS=--max-old-space-size=8192"); + expect(message).toContain("--workers 4"); + }); + + it("stays silent for explicit --workers requests (operator's own call)", () => { + expect(buildHeapAdvisoryWarning(sizing({}), 6)).toBeUndefined(); + }); + + it("stays silent when the chosen count fits the heap budget", () => { + expect( + buildHeapAdvisoryWarning(sizing({ exceedsHeapAdvisory: false }), undefined), + ).toBeUndefined(); + }); +}); diff --git a/packages/producer/src/services/render/captureCost.ts b/packages/producer/src/services/render/captureCost.ts index 3a12d790ba..db00538d92 100644 --- a/packages/producer/src/services/render/captureCost.ts +++ b/packages/producer/src/services/render/captureCost.ts @@ -16,9 +16,11 @@ import { type CaptureOptions, type CaptureSession, type EngineConfig, + type WorkerSizing, calculateOptimalWorkers, captureFrameToBuffer, closeCaptureSession, + computeWorkerSizing, createCaptureSession, initializeSession, } from "@hyperframes/engine"; @@ -106,6 +108,36 @@ function combineCaptureCostEstimates( }; } +/** + * Advisory heap warning (field OOM, 0.7.66 on 24GB: auto-picked 6 workers + * blew Node's default ~4GB heap). Returns the warning message, or undefined + * when it should not fire: + * + * - Auto-sized renders only (`requestedWorkers === undefined`) — the field + * failure was auto sizing, and an explicit `--workers N` is the operator's + * own call. + * - Not enforced as a cap yet — the per-worker budget constant is derived + * from one field report; the `workers_heap_*` telemetry emitted with the + * sizing decides whether to enforce (see the TODO on HEAP_PER_WORKER_MB in + * @hyperframes/engine's parallelCoordinator). The message gives the + * operator the actionable knobs today. + * + * Pure so the message shape + firing condition are unit-testable with a + * synthetic `WorkerSizing` (the real one depends on the host's heap). + */ +export function buildHeapAdvisoryWarning( + sizing: WorkerSizing, + requestedWorkers: number | undefined, +): string | undefined { + if (requestedWorkers !== undefined || !sizing.exceedsHeapAdvisory) return undefined; + return ( + `[Render] ${sizing.workers} capture workers may exceed this process's V8 heap ` + + `(limit ${sizing.heapLimitMb}MB supports ~${sizing.heapBasedWorkers}). If the render ` + + `dies with "JavaScript heap out of memory", raise the heap ` + + `(NODE_OPTIONS=--max-old-space-size=8192) or pass --workers ${sizing.heapBasedWorkers}.` + ); +} + export function resolveRenderWorkerCount( totalFrames: number, requestedWorkers: number | undefined, @@ -113,6 +145,8 @@ export function resolveRenderWorkerCount( compiled: Pick, log: ProducerLogger = defaultLogger, measuredCaptureCost?: CaptureCostEstimate, + /** Sink for the sizing provenance so the orchestrator can thread it into telemetry. */ + onSizing?: (sizing: WorkerSizing) => void, ): number { // TODO(htmlInCanvas): workaround — Chrome's experimental drawElementImage // API (CanvasDrawElement) is non-deterministic across concurrent browser @@ -144,10 +178,16 @@ export function resolveRenderWorkerCount( estimateCaptureCostMultiplier(compiled), measuredCaptureCost, ); - const workerCount = calculateOptimalWorkers(totalFrames, requestedWorkers, { + const sizing = computeWorkerSizing(totalFrames, requestedWorkers, { ...cfg, captureCostMultiplier: captureCost.multiplier, }); + // The heap advisory is deliberately NOT logged here: this function's + // "no unexpected warns" unit-test contract would become dependent on the + // test machine's actual V8 heap limit. The orchestrator's onSizing consumer + // emits it via buildHeapAdvisoryWarning. + onSizing?.(sizing); + const workerCount = sizing.workers; if (requestedWorkers !== undefined || captureCost.multiplier <= 1) { return workerCount; diff --git a/packages/producer/src/services/render/perfSummary.ts b/packages/producer/src/services/render/perfSummary.ts index 1b0b790dd3..2e288edc4c 100644 --- a/packages/producer/src/services/render/perfSummary.ts +++ b/packages/producer/src/services/render/perfSummary.ts @@ -4,7 +4,7 @@ */ import { fpsToNumber } from "@hyperframes/core"; -import type { CapturePerfSummary, SubTimelineWaitOutcome } from "@hyperframes/engine"; +import type { CapturePerfSummary, SubTimelineWaitOutcome, WorkerSizing } from "@hyperframes/engine"; import type { CaptureCalibrationSample, CaptureCostEstimate } from "./captureCost.js"; import type { CaptureAttemptSummary, @@ -181,6 +181,8 @@ function aggregateBeginFrameReuse( export function buildRenderPerfSummary(input: { job: RenderJob; workerCount: number; + /** Auto-sizing provenance; undefined when a pin (htmlInCanvas / low-memory) short-circuited sizing. */ + workerSizing?: WorkerSizing; enableChunkedEncode: boolean; chunkedEncodeSize: number; compositionDurationSeconds: number; @@ -217,6 +219,7 @@ export function buildRenderPerfSummary(input: { fps: fpsToNumber(input.job.config.fps), quality: input.job.config.quality, workers: input.workerCount, + workerSizing: input.workerSizing, chunkedEncode: input.enableChunkedEncode, chunkSizeFrames: input.enableChunkedEncode ? input.chunkedEncodeSize : null, compositionDurationSeconds: input.compositionDurationSeconds, diff --git a/packages/producer/src/services/renderOrchestrator.ts b/packages/producer/src/services/renderOrchestrator.ts index bcb6efcbf5..f0673fda9e 100644 --- a/packages/producer/src/services/renderOrchestrator.ts +++ b/packages/producer/src/services/renderOrchestrator.ts @@ -76,6 +76,7 @@ import { type CapturePerfSummary, type CaptureWarning, type SubTimelineWaitOutcome, + type WorkerSizing, resolveBrowserGpuMode, resolveHeadlessShellPath, applyConcreteGpuScreenshotClamp, @@ -124,6 +125,7 @@ import { resolveVideoCaptureBeyondViewport } from "./render/captureBeyondViewpor import { type CaptureCalibrationSample, type CaptureCostEstimate, + buildHeapAdvisoryWarning, resolveRenderWorkerCount, runCaptureCalibration, } from "./render/captureCost.js"; @@ -373,6 +375,14 @@ export interface RenderPerfSummary { fps: number; quality: string; workers: number; + /** + * Provenance of the auto worker-sizing decision (undefined when the + * htmlInCanvas / low-memory pins short-circuited sizing). `boundBy` names + * the binding constraint; the heap fields are the advisory budget being + * validated by fleet telemetry before enforcement — see + * `computeWorkerSizing` in @hyperframes/engine. + */ + workerSizing?: WorkerSizing; chunkedEncode: boolean; chunkSizeFrames: number | null; compositionDurationSeconds: number; @@ -1703,6 +1713,7 @@ async function executeRenderPipeline(input: { // "routed" = the parallel router fired and held; "reverted" = fired but // the self-verify retry rolled back; undefined = never fired. let deParallelRouter: "routed" | "reverted" | undefined; + let workerSizing: WorkerSizing | undefined; execution.defer("rollback staged artifact", () => artifactTransaction.rollback()); execution.defer("close file server", () => { @@ -2475,6 +2486,16 @@ async function executeRenderPipeline(input: { compiled, log, captureCalibration?.estimate, + (sizing) => { + workerSizing = sizing; + const heapAdvisory = buildHeapAdvisoryWarning(sizing, job.config.workers); + if (heapAdvisory) { + log.warn(heapAdvisory, { + heapLimitMb: sizing.heapLimitMb, + heapBasedWorkers: sizing.heapBasedWorkers, + }); + } + }, ); // DE priority inversion — see shouldPreferSingleWorkerDrawElement for the // policy and benchmark rationale (eligibility resolved above, before @@ -3281,6 +3302,7 @@ async function executeRenderPipeline(input: { const perfSummary = buildRenderPerfSummary({ job, workerCount, + workerSizing, enableChunkedEncode, chunkedEncodeSize, compositionDurationSeconds: composition.duration,