Skip to content

perf(studio): virtualize timeline thumbnail media #2718

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
miguel-heygen wants to merge 1 commit into
base: codex/studio-timeline-g-decoder-resources-v2
Choose a base branch
from
Draft

perf(studio): virtualize timeline thumbnail media #2718

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
143 changes: 74 additions & 69 deletions packages/studio/src/player/components/CompositionThumbnail.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { memo, useCallback, useState, useRef } from "react";
import { useMountEffect } from "../../hooks/useMountEffect";
import { memo, useCallback, useMemo, useRef, useState } from "react";
import { useThumbnailLease } from "../../hooks/useThumbnailLease";
import { createThumbnailKey, type ThumbnailPriority } from "../lib/thumbnailScheduler";
import { TIMELINE_VIEWPORT_BUDGETS } from "../lib/timelineViewportBudgets";

interface CompositionThumbnailProps {
previewUrl: string;
Expand All @@ -11,6 +13,10 @@ interface CompositionThumbnailProps {
duration?: number;
width?: number;
height?: number;
projectId?: string;
sessionEpoch?: number;
priority?: ThumbnailPriority;
rich?: boolean;
}

const CLIP_HEIGHT = 66;
Expand All @@ -34,9 +40,8 @@ export function buildCompositionThumbnailUrl({
const thumbnailBase = previewUrl
.replace("/preview/comp/", "/thumbnail/")
.replace(/\/preview$/, "/thumbnail/index.html");
const midTime = seekTime + duration / 2;
const thumbnailUrl = new URL(thumbnailBase, origin);
thumbnailUrl.searchParams.set("t", midTime.toFixed(2));
thumbnailUrl.searchParams.set("t", (seekTime + duration / 2).toFixed(2));
thumbnailUrl.searchParams.set("v", THUMBNAIL_URL_VERSION);
if (selector) {
thumbnailUrl.searchParams.set("selector", selector);
Expand All @@ -47,6 +52,23 @@ export function buildCompositionThumbnailUrl({
return thumbnailUrl.toString();
}

async function loadCompositionImage(url: string, signal: AbortSignal) {
const response = await fetch(url, { signal });
if (!response.ok) throw new Error(`Composition thumbnail failed (${response.status})`);
const blob = await response.blob();
if (signal.aborted) throw new DOMException("Aborted", "AbortError");
const objectUrl = URL.createObjectURL(blob);
return {
value: { kind: "image" as const, url: objectUrl, aspect: 16 / 9 },
weight:
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalWidth *
TIMELINE_VIEWPORT_BUDGETS.posterMaxPhysicalHeight *
4,
dispose: () => URL.revokeObjectURL(objectUrl),
};
}

/** Server-rendered composition poster, deduplicated and budgeted by project/session. */
export const CompositionThumbnail = memo(function CompositionThumbnail({
previewUrl,
label,
Expand All @@ -55,31 +77,12 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
selectorIndex,
seekTime = 2,
duration = 5,
projectId = previewUrl,
sessionEpoch = 0,
priority = "visible",
}: CompositionThumbnailProps) {
const [containerWidth, setContainerWidth] = useState(0);
const [loaded, setLoaded] = useState(false);
const [aspect, setAspect] = useState(16 / 9);
const roRef = useRef<ResizeObserver | null>(null);

const setContainerRef = useCallback((el: HTMLDivElement | null) => {
roRef.current?.disconnect();
if (!el) return;

const measured = el.parentElement?.clientWidth || el.clientWidth;
// fallow-ignore-next-line code-duplication
setContainerWidth(measured);

const target = el.parentElement || el;
roRef.current = new ResizeObserver(([entry]) => {
setContainerWidth(entry.contentRect.width);
});
roRef.current.observe(target);
}, []);

useMountEffect(() => () => {
roRef.current?.disconnect();
});

const observerRef = useRef<ResizeObserver | null>(null);
const url = buildCompositionThumbnailUrl({
previewUrl,
seekTime,
Expand All @@ -88,57 +91,59 @@ export const CompositionThumbnail = memo(function CompositionThumbnail({
selectorIndex,
origin: window.location.origin,
});
const frameW = Math.max(48, Math.round(CLIP_HEIGHT * aspect));
const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameW)) : 1;
const request = useMemo(
() => ({
key: createThumbnailKey({ kind: "composition", url }),
projectId,
sessionEpoch,
kind: "composition" as const,
priority,
rich: true,
load: (signal: AbortSignal) => loadCompositionImage(url, signal),
}),
[priority, projectId, sessionEpoch, url],
);
const snapshot = useThumbnailLease(request);
const value =
snapshot.status === "ready" && snapshot.value.kind === "image" ? snapshot.value : null;
const frameWidth = Math.max(48, Math.round(CLIP_HEIGHT * (value?.aspect ?? 16 / 9)));
const frameCount = containerWidth > 0 ? Math.max(1, Math.ceil(containerWidth / frameWidth)) : 1;

const setContainerRef = useCallback((element: HTMLDivElement | null) => {
observerRef.current?.disconnect();
if (!element) return;
const target = element.parentElement ?? element;
setContainerWidth(target.clientWidth);
observerRef.current = new ResizeObserver(([entry]) =>
setContainerWidth(entry.contentRect.width),
);
observerRef.current.observe(target);
}, []);

return (
<div ref={setContainerRef} className="absolute inset-0 overflow-hidden">
<img
src={url}
alt=""
draggable={false}
loading="eager"
onLoad={(e) => {
const img = e.currentTarget;
if (img.naturalWidth > 0 && img.naturalHeight > 0) {
setAspect(img.naturalWidth / img.naturalHeight);
}
setLoaded(true);
}}
className="hidden"
/>

{loaded && (
<div
className="absolute inset-0 flex"
style={{ animation: "hf-thumb-fade 200ms ease-out", mixBlendMode: "lighten" }}
>
{Array.from({ length: frameCount }).map((_, i) => (
<div
key={i}
className="relative h-full flex-shrink-0 overflow-hidden"
style={{ width: frameW }}
>
<img
src={url}
alt=""
draggable={false}
className="absolute inset-0 h-full w-full object-cover"
style={{ opacity: 0.7 }}
/>
</div>
{value && (
<div className="absolute inset-0 flex">
{Array.from({ length: frameCount }, (_, index) => (
<img
key={index}
src={value.url}
alt=""
draggable={false}
className="h-full flex-shrink-0 object-cover opacity-70"
style={{ width: frameWidth }}
/>
))}
</div>
)}

{snapshot.status === "loading" && (
<div className="absolute inset-0 animate-pulse bg-white/[0.035]" />
)}
{label && (
<div className="absolute left-3 top-0 bottom-0 flex items-center" style={{ zIndex: 10 }}>
<div className="absolute inset-y-0 left-3 z-10 flex items-center">
<span
className="block max-w-full truncate text-[10px] font-semibold leading-none"
style={{
color: labelColor,
textShadow: loaded ? "0 1px 4px rgba(0,0,0,0.9), 0 0 8px rgba(0,0,0,0.6)" : "none",
}}
style={{ color: labelColor }}
>
{label}
</span>
Expand Down
40 changes: 26 additions & 14 deletions packages/studio/src/player/components/ImageThumbnail.test.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import React, { act } from "react";
import { createRoot, type Root } from "react-dom/client";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { thumbnailScheduler } from "../lib/thumbnailScheduler";
import { ImageThumbnail } from "./ImageThumbnail";

Object.defineProperty(globalThis, "IS_REACT_ACT_ENVIRONMENT", {
Expand Down Expand Up @@ -68,6 +69,7 @@ beforeEach(() => {
afterEach(() => {
act(() => root?.unmount());
root = null;
thumbnailScheduler.clear();
globalThis.IntersectionObserver = originalIO;
globalThis.ResizeObserver = originalRO;
globalThis.Image = originalImage;
Expand All @@ -82,6 +84,10 @@ function render(props: { imageSrc: string; label?: string; labelColor?: string }
imageSrc={props.imageSrc}
label={props.label ?? ""}
labelColor={props.labelColor ?? "#fff"}
projectId="p"
sessionEpoch={1}
priority="visible"
rich={false}
/>,
);
});
Expand All @@ -93,6 +99,13 @@ function lastProbe(): MockImage {
return probe!;
}

async function resolveProbe(update: (probe: MockImage) => void): Promise<void> {
await act(async () => {
update(lastProbe());
await new Promise((resolve) => setTimeout(resolve, 0));
});
}

/** Assert at least one tile rendered and the first tile serves `expectedSrc`. */
function expectFirstTileSrc(expectedSrc: string): void {
const imgs = [...host.querySelectorAll("img")];
Expand All @@ -107,15 +120,15 @@ describe("ImageThumbnail", () => {
expect(host.querySelectorAll("img").length).toBe(0);
});

it("probes the resolved src and renders repeated object-cover tiles on load", () => {
it("probes the resolved src and renders repeated object-cover tiles on load", async () => {
render({ imageSrc: "/api/projects/p/preview/assets/pic.png" });
const probe = lastProbe();
expect(probe.src).toBe("/api/projects/p/preview/assets/pic.png");

act(() => {
probe.naturalWidth = 1920;
probe.naturalHeight = 1080;
probe.onload?.();
await resolveProbe((current) => {
current.naturalWidth = 1920;
current.naturalHeight = 1080;
current.onload?.();
});

const imgs = [...host.querySelectorAll("img")];
Expand All @@ -127,18 +140,18 @@ describe("ImageThumbnail", () => {
expect(host.querySelector(".animate-pulse")).toBeNull();
});

it("drops the shimmer and renders no tiles when a raster image fails to load", () => {
it("drops the shimmer and renders no tiles when a raster image fails to load", async () => {
render({ imageSrc: "/api/projects/p/preview/assets/missing.png" });
act(() => lastProbe().onerror?.());
await resolveProbe((probe) => probe.onerror?.());
expect(host.querySelectorAll("img").length).toBe(0);
expect(host.querySelector(".animate-pulse")).toBeNull();
});

it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", () => {
it("renders tiles at 16:9 when an SVG has no intrinsic dimensions (naturalWidth=0)", async () => {
render({ imageSrc: "/api/projects/p/preview/assets/logo.svg" });
const probe = lastProbe();

act(() => {
await resolveProbe(() => {
// naturalWidth stays 0 — SVG with no width/height attribute
probe.onload?.();
});
Expand All @@ -147,21 +160,20 @@ describe("ImageThumbnail", () => {
expect(host.querySelector(".animate-pulse")).toBeNull();
});

it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", () => {
it("renders SVG tiles at 16:9 fallback even when the probe fires onerror", async () => {
// Some browser/sandbox environments fire onerror for SVGs even though the
// <img> element itself can render the file — we must not blank the strip.
render({ imageSrc: "/api/projects/p/preview/assets/icon.svg" });

act(() => lastProbe().onerror?.());
await resolveProbe((probe) => probe.onerror?.());

expectFirstTileSrc("/api/projects/p/preview/assets/icon.svg");
expect(host.querySelector(".animate-pulse")).toBeNull();
});

it("renders the label above the strip when provided", () => {
it("renders the label above the strip when provided", async () => {
render({ imageSrc: "/x.png", label: "hero", labelColor: "#abc" });
act(() => {
const probe = lastProbe();
await resolveProbe((probe) => {
probe.naturalWidth = 100;
probe.naturalHeight = 100;
probe.onload?.();
Expand Down
Loading
Loading