Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions packages/cli/src/commands/feedback.telemetryJoinKeys.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import { buildTelemetryJoinKeys } from "./feedback.js";

describe("buildTelemetryJoinKeys", () => {
it("emits fid + tid and omits renders when the ring is empty", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "feedback-uuid",
anonymousId: "install-uuid",
});
expect(keys).toBe("fid=feedback-uuid tid=install-uuid");
});

it("appends recent render ids newest-last with a ! marking failed renders", () => {
const keys = buildTelemetryJoinKeys({
feedbackId: "f",
anonymousId: "t",
recentRenders: [
{ id: "render-a", at: "2026-07-21T00:00:00Z", ok: true },
{ id: "render-b", at: "2026-07-21T01:00:00Z", ok: false },
],
});
expect(keys).toBe("fid=f tid=t renders=render-a,render-b!");
});

it("stays within the backend env cap for a full ring of uuid render ids", () => {
const uuid = "01234567-89ab-cdef-0123-456789abcdef";
const keys = buildTelemetryJoinKeys({
feedbackId: uuid,
anonymousId: uuid,
recentRenders: Array.from({ length: 5 }, (_, i) => ({
id: uuid,
at: "2026-07-21T00:00:00Z",
ok: i % 2 === 0,
})),
});
// submitFeedback caps env at 500 chars; the doctor summary consumes ~100.
expect(keys.length).toBeLessThan(400);
});
});
45 changes: 43 additions & 2 deletions packages/cli/src/commands/feedback.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import { failCommand } from "../utils/commandResult.js";
import { randomUUID } from "node:crypto";
import { resolve } from "node:path";
import { defineCommand } from "citty";
import * as clack from "@clack/prompts";
Expand All @@ -7,6 +8,7 @@ import type { Example } from "./_examples.js";
import { trackRenderFeedback } from "../telemetry/events.js";
import { shouldTrack, flush } from "../telemetry/client.js";
import { getDoctorSummary } from "../telemetry/feedback.js";
import { readConfig, type RecentRenderRecord } from "../telemetry/config.js";
import { publishProjectArchive } from "../utils/publishProject.js";
import { submitFeedback } from "../utils/submitFeedback.js";
import { buildIssueUrl, HYPERFRAMES_REPO_URL } from "../utils/feedbackIssue.js";
Expand All @@ -28,6 +30,27 @@ function normalizeComment(raw?: string): string | undefined {
return raw || undefined;
}

/**
* Compact PostHog join keys appended to the environment string that rides
* along with the forwarded report (and therefore lands verbatim in the wild
* feedback channel): `fid` = this submission's PostHog `survey sent`
* `feedback_id`; `tid` = the install's telemetry distinct_id; `renders` =
* recent `render_job_id`s (newest last, `!` suffix = the render failed).
* Together they turn a wild report into an exact telemetry lookup instead of
* a hardware-fingerprint hunt.
*/
export function buildTelemetryJoinKeys(input: {
feedbackId: string;
anonymousId: string;
recentRenders?: RecentRenderRecord[];
}): string {
const parts = [`fid=${input.feedbackId}`, `tid=${input.anonymousId}`];
if (input.recentRenders?.length) {
parts.push(`renders=${input.recentRenders.map((r) => `${r.id}${r.ok ? "" : "!"}`).join(",")}`);
}
return parts.join(" ");
}

function printIssueConsent(dir: string): void {
console.log();
console.log(
Expand Down Expand Up @@ -170,20 +193,38 @@ export default defineCommand({
const comment = normalizeComment(args.comment);
const doctorSummary = await getDoctorSummary();

// Join keys tying this report to the install's PostHog rows — see
// buildTelemetryJoinKeys. Appended to the env string so they surface in
// the forwarded report; mirrored as structured props on the PostHog event.
const feedbackId = randomUUID();
const config = readConfig();
const joinKeys = buildTelemetryJoinKeys({
feedbackId,
anonymousId: config.anonymousId,
recentRenders: config.recentRenders,
});
const envWithJoinKeys = doctorSummary ? `${doctorSummary} ${joinKeys}` : joinKeys;

// Soft-warn (never blocks) when the comment for a non-clean report is
// missing the mandated reproduction-packet markers. Prints before the
// submission ack so the reporter sees the nudge while their run is fresh.
printFeedbackLintWarnings({ rating, comment });

// The standalone command runs separately from `render`, so it has no real
// elapsed time to report. Omit it rather than recording a fake duration.
trackRenderFeedback({ rating, comment, doctorSummary });
trackRenderFeedback({
rating,
comment,
doctorSummary,
feedbackId,
recentRenderIds: config.recentRenders?.map((r) => r.id),
});

await flush();
// Ack first so the user isn't kept waiting on the best-effort forward (which
// is bounded to a few seconds and never surfaces an error either way).
console.log(c.dim("Thanks for the feedback!"));
await submitFeedback({ rating, comment, cliVersion: VERSION, env: doctorSummary });
await submitFeedback({ rating, comment, cliVersion: VERSION, env: envWithJoinKeys });

if (args["file-issue"] === true) {
await fileGithubIssue({
Expand Down
20 changes: 19 additions & 1 deletion packages/cli/src/commands/render.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,12 @@ import {
trackRenderObservation,
} from "../telemetry/events.js";
import { maybePromptRenderFeedback } from "../telemetry/feedback.js";
import { readConfigFresh, writeConfig, type HyperframesConfig } from "../telemetry/config.js";
import {
readConfigFresh,
recordRecentRender,
writeConfig,
type HyperframesConfig,
} from "../telemetry/config.js";
import { shouldTrack } from "../telemetry/client.js";
import { renderJobObservabilityTelemetryPayload } from "../telemetry/renderObservability.js";
import { bytesToMb } from "../telemetry/system.js";
Expand Down Expand Up @@ -1339,6 +1344,9 @@ function handleRenderError(
...renderJobObservabilityTelemetryPayload(job),
...getMemorySnapshot(),
});
// Failed renders join the recent-renders ring too — a bug report filed via
// `hyperframes feedback` is MOST likely to be about a failed render.
if (job?.id) recordRecentRender(job.id, false);
if (options.throwOnError) {
throw new Error(message);
}
Expand Down Expand Up @@ -1376,6 +1384,9 @@ function trackRenderMetrics(
options: RenderOptions,
docker: boolean,
): void {
// Successful render → recent-renders ring, so a later `hyperframes
// feedback` can attach this render's telemetry id to the report.
recordRecentRender(job.id, true);
const perf = job.perfSummary;
const compositionDurationMs = perf
? Math.round(perf.compositionDurationSeconds * 1000)
Expand All @@ -1393,6 +1404,13 @@ function trackRenderMetrics(
fps: fpsToNumber(options.fps),
quality: options.quality,
workers: options.workers ?? perf?.workers,
workersBoundBy: perf?.workerSizing?.boundBy,
workersCpuBased: perf?.workerSizing?.cpuBasedWorkers,
workersMemoryBased: perf?.workerSizing?.memoryBasedWorkers,
workersHeapBased: perf?.workerSizing?.heapBasedWorkers,
workersFrameBased: perf?.workerSizing?.frameBasedWorkers,
workersHeapLimitMb: perf?.workerSizing?.heapLimitMb,
workersExceedHeapAdvisory: perf?.workerSizing?.exceedsHeapAdvisory,
docker,
gpu: options.gpu,
authoringSkill: options.authoringSkill,
Expand Down
45 changes: 45 additions & 0 deletions packages/cli/src/telemetry/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,39 @@ export interface HyperframesConfig {
* `DE_PARALLEL_ROUTER_TRIAL_MAX_RENDERS` in `render.ts`.
*/
deParallelRouterTrialRenderCount?: number;
/**
* Ring of the last few local renders (newest last). `hyperframes feedback`
* attaches these ids — which are the `render_job_id` /
* `observability_render_job_id` on this install's PostHog events — to the
* feedback it submits, so a wild bug report can be joined to the exact
* telemetry rows of the renders it describes.
*/
recentRenders?: RecentRenderRecord[];
}

/** One entry in {@link HyperframesConfig.recentRenders}. */
export interface RecentRenderRecord {
/** The render job id (`RenderJob.id` — the telemetry `render_job_id`). */
id: string;
/** ISO timestamp of when the render finished. */
at: string;
/** Whether the render completed successfully. */
ok: boolean;
}

/** Ring size for {@link HyperframesConfig.recentRenders}. */
const MAX_RECENT_RENDERS = 5;

/**
* Append a finished render to the recent-renders ring (newest last, capped).
* Fresh read-modify-write like the trial counters — narrows, but does not
* eliminate, lost updates against a concurrent CLI process.
*/
export function recordRecentRender(id: string, ok: boolean): void {

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Nit — read-modify-write lost-update on concurrent CLIs. The comment above (L112) acknowledges this: 'narrows, but does not eliminate, lost updates against a concurrent CLI process.' In practice the impact is small — a single render's id may be missing from the ring if two hyperframes render invocations write within the same read-modify window. Feedback still works because the join keys carry other ids in the ring plus feedback_id/tid. Fine as-is; noting for reader awareness in case a future user reports 'my last render isn't in the join keys.' — Rames D Jusso

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed, leaving as-is — same read-modify-write posture as the existing trial counters, and a dropped ring entry degrades to a slightly less complete join (fid/tid still resolve the install).

const config = readConfigFresh();
const ring = [...(config.recentRenders ?? []), { id, at: new Date().toISOString(), ok }];
config.recentRenders = ring.slice(-MAX_RECENT_RENDERS);
writeConfig(config);
}

const DEFAULT_CONFIG: HyperframesConfig = {
Expand Down Expand Up @@ -142,6 +175,18 @@ export function readConfig(): HyperframesConfig {
typeof parsed.deParallelRouterTrialRenderCount === "number"
? parsed.deParallelRouterTrialRenderCount
: undefined,
recentRenders: Array.isArray(parsed.recentRenders)
? parsed.recentRenders
.filter(
(r): r is RecentRenderRecord =>
typeof r === "object" &&
r !== null &&
typeof (r as RecentRenderRecord).id === "string" &&
typeof (r as RecentRenderRecord).at === "string" &&
typeof (r as RecentRenderRecord).ok === "boolean",
)
.slice(-MAX_RECENT_RENDERS)
: undefined,
};

cachedConfig = config;
Expand Down
53 changes: 53 additions & 0 deletions packages/cli/src/telemetry/events.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,59 @@ describe("render telemetry events", () => {
expect(flush).toHaveBeenCalledTimes(2);
});

// The enforcement decision for the advisory heap budget reads these fleet
// props (see computeWorkerSizing) — a silent drop in the summary→event hop
// would invalidate that decision without anyone noticing.
it("carries every worker-sizing provenance prop on render_complete", () => {
trackRenderComplete({
durationMs: 1000,
fps: 30,
quality: "high",
docker: false,
gpu: false,
workers: 6,
workersBoundBy: "max_workers",
workersCpuBased: 16,
workersMemoryBased: 8,
workersHeapBased: 4,
workersFrameBased: 24,
workersHeapLimitMb: 4096,
workersExceedHeapAdvisory: true,
});

expect(trackEvent).toHaveBeenCalledWith(
"render_complete",
expect.objectContaining({
workers: 6,
workers_bound_by: "max_workers",
workers_cpu_based: 16,
workers_memory_based: 8,
workers_heap_based: 4,
workers_frame_based: 24,
workers_heap_limit_mb: 4096,
workers_exceed_heap_advisory: true,
}),
undefined,
);
});

it("ties feedback to its report and recent renders via feedback_id + recent_render_ids", () => {
trackRenderFeedback({
rating: 3,
comment: "hook scene blank",
feedbackId: "feedback-uuid",
recentRenderIds: ["render-a", "render-b"],
});

expect(trackEvent).toHaveBeenCalledWith(
"survey sent",
expect.objectContaining({
feedback_id: "feedback-uuid",
recent_render_ids: "render-a,render-b",
}),
);
});

it("redacts paths and URL query strings from render error messages", () => {
trackRenderError({
fps: 30,
Expand Down
31 changes: 31 additions & 0 deletions packages/cli/src/telemetry/events.ts
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,17 @@ export function trackRenderComplete(
/** Authoring workflow skill that drove this render (e.g. "product-launch-video"). */
authoringSkill?: string;
workers?: number;
// Worker auto-sizing provenance (RenderPerfSummary.workerSizing). Answers
// "why N workers?" fleet-wide, and validates the advisory per-worker heap
// budget before it's enforced (field OOM: 6 auto workers on a 24GB/4GB-heap
// machine — see computeWorkerSizing in @hyperframes/engine).
workersBoundBy?: string;
workersCpuBased?: number;
workersMemoryBased?: number;
workersHeapBased?: number;
workersFrameBased?: number;
workersHeapLimitMb?: number;
workersExceedHeapAdvisory?: boolean;
docker: boolean;
gpu: boolean;
// Static-frame dedup outcome (opt-out HF_STATIC_DEDUP=false). Undefined on
Expand Down Expand Up @@ -245,6 +256,13 @@ export function trackRenderComplete(
quality: props.quality,
authoring_skill: props.authoringSkill,
workers: props.workers,
workers_bound_by: props.workersBoundBy,
workers_cpu_based: props.workersCpuBased,
workers_memory_based: props.workersMemoryBased,
workers_heap_based: props.workersHeapBased,

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟠 Concern — telemetry prop pass-through not test-locked. Fields workers_bound_by, workers_cpu_based, workers_memory_based, workers_heap_based, workers_frame_based, workers_heap_limit_mb, workers_exceed_heap_advisory land here from the summary → event body wiring, but nothing asserts they actually reach the emitted PostHog properties. Since the PR body's central thesis is 'fleet data validates the 640MB budget before enforcement,' a silent drop of these fields (e.g. a future rename that misses one hop) would silently invalidate the enforcement decision. Suggest a trackRenderComplete test with all seven fields set on the props → assert they appear on the mocked PostHog capture call payload. — Rames D Jusso

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done in c6462a0events.test.ts now asserts all seven workers_* props reach the emitted render_complete payload, plus a second test locking the survey sent join keys (feedback_id, comma-joined recent_render_ids).

workers_frame_based: props.workersFrameBased,
workers_heap_limit_mb: props.workersHeapLimitMb,
workers_exceed_heap_advisory: props.workersExceedHeapAdvisory,
docker: props.docker,
gpu: props.gpu,
static_dedup_enabled: props.staticDedupEnabled,
Expand Down Expand Up @@ -609,6 +627,14 @@ export function trackRenderFeedback(props: {
renderDurationMs?: number;
comment?: string;
doctorSummary?: string;
/**
* Join key shared with the forwarded feedback report (Slack/backend): the
* same uuid rides in the report's env string as `fid=…`, so a wild report
* resolves to exactly one PostHog "survey sent" event and vice versa.
*/
feedbackId?: string;
/** render_job_id values of this install's recent renders (newest last). */
recentRenderIds?: string[];
}): void {
trackEvent("survey sent", {
$survey_id: "render_satisfaction",
Expand All @@ -617,6 +643,11 @@ export function trackRenderFeedback(props: {
...(props.comment ? { $survey_response_2: props.comment } : {}),
...(props.renderDurationMs !== undefined ? { render_duration_ms: props.renderDurationMs } : {}),
...(props.doctorSummary ? { doctor_summary: props.doctorSummary } : {}),
...(props.feedbackId ? { feedback_id: props.feedbackId } : {}),
// Comma-joined: EventProperties values are scalars only.
...(props.recentRenderIds?.length
? { recent_render_ids: props.recentRenderIds.join(",") }
: {}),
});
}

Expand Down
3 changes: 3 additions & 0 deletions packages/engine/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -208,13 +208,16 @@ export type {
// ── Parallel rendering ─────────────────────────────────────────────────────────
export {
calculateOptimalWorkers,
computeWorkerSizing,
distributeFrames,
distributeFramesInterleaved,
executeParallelCapture,
mergeWorkerFrames,
getSystemResources,
type WorkerTask,
type WorkerResult,
type WorkerSizing,
type WorkerSizingBound,
type ParallelProgress,
} from "./services/parallelCoordinator.js";

Expand Down
Loading
Loading