diff --git a/package.json b/package.json index f908e5a4..348ffa37 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "vite-plugin-dts": "^4.5.4" }, "dependencies": { - "@shotstack/schemas": "1.14.1", + "@shotstack/schemas": "1.16.0", "@shotstack/shotstack-canvas": "^2.10.2", "howler": "^2.2.4", "mediabunny": "^1.11.2", diff --git a/readme.md b/readme.md index 224e9f61..3f6d4cb0 100644 --- a/readme.md +++ b/readme.md @@ -183,6 +183,7 @@ Available event names: | Playback | `playback:play`, `playback:pause` | | Timeline | `timeline:updated`, `timeline:backgroundChanged`, `timeline:resized` | | Clip lifecycle | `clip:added`, `clip:selected`, `clip:updated`, `clip:deleted`, `clip:restored`, `clip:copied`, `clip:loadFailed`, `clip:unresolved` | +| Asset generation | `clip:generationStarted`, `clip:generationCompleted`, `clip:generationFailed` | | Selection | `selection:cleared` | | Edit state | `edit:changed`, `edit:undo`, `edit:redo` | | Track | `track:added`, `track:removed` | @@ -190,6 +191,33 @@ Available event names: | Output | `output:resized`, `output:resolutionChanged`, `output:aspectRatioChanged`, `output:fpsChanged`, `output:formatChanged`, `output:destinationsChanged` | | Merge fields | `mergefield:changed` | +### Generating assets from prompts + +An image, video or audio asset can carry a `prompt` instead of a `src`. Those clips +render as a placeholder until something fills the `src` in. Register a generator and +the editor offers a generate action on the placeholder and in the toolbar: + +```typescript +edit.registerAssetGenerator(async ({ clipId, asset, signal }) => { + const url = await myBackend.generate(asset, { signal }); + return { url }; +}); +``` + +The SDK writes the returned URL to the clip, so the change is undoable and autosaves +like any other edit. It tracks whether a clip is generating or has failed, and renders +those states; a rejection's message is shown as-is next to a retry action. Everything +else — which models exist, what they cost, what an error means — stays with the host. + +```typescript +edit.hasAssetGenerator(); // false until a handler is registered +edit.generateClipAsset(clipId); // what the generate action calls +edit.getClipGenerationState(clipId); // { status: "generating" | "failed", error? } +``` + +Generation state is transient: it is never saved to the edit, never part of undo, and +gone on reload. Deleting a clip mid-generation aborts its request via the `signal`. + ### Canvas `Canvas` renders the current edit. diff --git a/smoketest.html b/smoketest.html new file mode 100644 index 00000000..8848e1fb --- /dev/null +++ b/smoketest.html @@ -0,0 +1,57 @@ + + + + + + + + Shotstack Studio — generation smoketest + + + +
+
+ + + + diff --git a/src/components/canvas/players/ai-generation-binding.ts b/src/components/canvas/players/ai-generation-binding.ts new file mode 100644 index 00000000..186f33dc --- /dev/null +++ b/src/components/canvas/players/ai-generation-binding.ts @@ -0,0 +1,49 @@ +import type { Edit } from "@core/edit-session"; +import { EditEvent } from "@core/events/edit-events"; + +import type { AiPendingOverlay } from "./ai-pending-overlay"; + +export function aiGenerateHandler(edit: Edit, clipId: string | null): (() => void) | undefined { + if (!clipId || !edit.hasAssetGenerator()) return undefined; + return () => { + edit.generateClipAsset(clipId).catch(() => { + // Failures surface as clip state; nothing to do here. + }); + }; +} + +/** Hosts may register a generator after mount. Cheap to call every frame: only acts on a flip. */ +export function createGenerateActionSync(edit: Edit, overlay: AiPendingOverlay, clipId: string | null): () => void { + let last: boolean | null = null; + return () => { + const canGenerate = edit.hasAssetGenerator(); + if (canGenerate === last) return; + last = canGenerate; + overlay.setOnGenerate(aiGenerateHandler(edit, clipId)); + }; +} + +/** Mirrors generation state onto the overlay; returns its unsubscribe. */ +export function bindGenerationState(edit: Edit, clipId: string | null, overlay: AiPendingOverlay): () => void { + if (!clipId) return () => {}; + + const onStarted = ({ clipId: id }: { clipId: string }): void => { + if (id === clipId) overlay.setGenerating(true); + }; + const onCompleted = ({ clipId: id }: { clipId: string }): void => { + if (id === clipId) overlay.setGenerating(false); + }; + const onFailed = ({ clipId: id, error }: { clipId: string; error: string }): void => { + if (id === clipId) overlay.setFailed(error); + }; + + edit.events.on(EditEvent.ClipGenerationStarted, onStarted); + edit.events.on(EditEvent.ClipGenerationCompleted, onCompleted); + edit.events.on(EditEvent.ClipGenerationFailed, onFailed); + + return () => { + edit.events.off(EditEvent.ClipGenerationStarted, onStarted); + edit.events.off(EditEvent.ClipGenerationCompleted, onCompleted); + edit.events.off(EditEvent.ClipGenerationFailed, onFailed); + }; +} diff --git a/src/components/canvas/players/ai-pending-overlay.ts b/src/components/canvas/players/ai-pending-overlay.ts index 3f667235..06b17e69 100644 --- a/src/components/canvas/players/ai-pending-overlay.ts +++ b/src/components/canvas/players/ai-pending-overlay.ts @@ -13,6 +13,8 @@ export interface AiPendingOverlayOptions { assetNumber?: number; prompt?: string; assetType?: string; + /** Supplied only when the host can generate; omitting it hides the action. */ + onGenerate?: () => void; } /** @@ -197,6 +199,8 @@ export class AiPendingOverlay { private time = 0; private rafId: number | null = null; private lastTime: number | null = null; + private generating = false; + private failure: string | null = null; constructor(private options: AiPendingOverlayOptions) { this.container = new pixi.Container(); @@ -221,6 +225,26 @@ export class AiPendingOverlay { this.rebuild(); } + setGenerating(generating: boolean): void { + if (generating === this.generating) return; + this.generating = generating; + if (generating) this.failure = null; + this.rebuild(); + } + + setOnGenerate(onGenerate: (() => void) | undefined): void { + if (Boolean(onGenerate) === Boolean(this.options.onGenerate)) return; + this.options.onGenerate = onGenerate; + this.rebuild(); + } + + setFailed(message: string | null): void { + if (message === this.failure) return; + this.failure = message; + if (message !== null) this.generating = false; + this.rebuild(); + } + dispose(): void { this.stopAnimation(); this.container.destroy({ children: true }); @@ -375,7 +399,7 @@ export class AiPendingOverlay { // Prompt text (if provided) if (prompt) { - const truncated = truncatePrompt(prompt, 60); + const truncated = this.failure ?? truncatePrompt(prompt, 60); const promptText = new pixi.Text({ text: truncated, style: { @@ -394,6 +418,38 @@ export class AiPendingOverlay { badge.addChild(promptText); } + if (this.options.onGenerate && this.options.mode === "panel") { + badge.addChild(this.buildGenerateButton(prompt ? BADGE_SIZE + 90 : BADGE_SIZE + 45)); + } + this.container.addChild(badge); } + + private buildGenerateButton(y: number): pixi.Container { + const label = this.generating ? "Generating…" : (this.failure && "Retry") || "Generate"; + const button = new pixi.Container(); + button.position.set(BADGE_SIZE / 2, y); + + const text = new pixi.Text({ + text: label, + style: { fontFamily: "Arial", fontSize: 15, fontWeight: "bold", fill: "#ffffff" } + }); + text.anchor.set(0.5, 0.5); + + const paddingX = 18; + const paddingY = 9; + const bg = new pixi.Graphics(); + bg.roundRect(-text.width / 2 - paddingX, -text.height / 2 - paddingY, text.width + paddingX * 2, text.height + paddingY * 2, 999); + bg.fill({ color: this.generating ? "#3f3f46" : "#7C3AED", alpha: this.generating ? 0.7 : 0.95 }); + + button.addChild(bg, text); + + if (!this.generating) { + button.eventMode = "static"; + button.cursor = "pointer"; + button.on("pointertap", () => this.options.onGenerate?.()); + } + + return button; + } } diff --git a/src/components/canvas/players/image-to-video-player.ts b/src/components/canvas/players/image-to-video-player.ts index d377f13f..f4f9d2c6 100644 --- a/src/components/canvas/players/image-to-video-player.ts +++ b/src/components/canvas/players/image-to-video-player.ts @@ -4,6 +4,7 @@ import { type Size } from "@layouts/geometry"; import { type ResolvedClip } from "@schemas"; import * as pixi from "pixi.js"; +import { aiGenerateHandler, bindGenerationState, createGenerateActionSync } from "./ai-generation-binding"; import { AiPendingOverlay } from "./ai-pending-overlay"; import { createPlaceholderGraphic } from "./placeholder-graphic"; import { Player, PlayerType } from "./player"; @@ -13,6 +14,8 @@ export class ImageToVideoPlayer extends Player { private texture: pixi.Texture | null = null; private placeholder: pixi.Graphics | null = null; private aiOverlay: AiPendingOverlay | null = null; + private unbindGeneration: (() => void) | null = null; + private syncGenerateAction: (() => void) | null = null; constructor(edit: Edit, clipConfiguration: ResolvedClip) { super(edit, clipConfiguration, PlayerType.ImageToVideo); @@ -32,10 +35,10 @@ export class ImageToVideoPlayer extends Player { const prompt = isAiAsset(asset) ? asset.prompt || "" : ""; const assetType = isAiAsset(asset) ? asset.type : "image-to-video"; - // Legacy image-to-video carries its input image in src; the unified - // video asset carries it in seed (src holds the generated output) - const { src, seed } = asset as { src?: string; seed?: string }; - const inputImage = seed ?? src; + // Legacy image-to-video carries its input image in src; the unified video + // asset carries it in options.inputSrc (src holds the generated output). + const { type, src, options } = asset as { type?: string; src?: string; options?: { inputSrc?: string } }; + const inputImage = type === "image-to-video" ? src : options?.inputSrc; const loaded = inputImage ? await this.tryLoadTexture(inputImage) : false; if (!loaded) { @@ -50,8 +53,11 @@ export class ImageToVideoPlayer extends Player { height: displaySize.height, assetNumber: assetNumber ?? undefined, prompt, - assetType + assetType, + onGenerate: aiGenerateHandler(this.edit, this.clipId ?? null) }); + this.syncGenerateAction = createGenerateActionSync(this.edit, this.aiOverlay, this.clipId ?? null); + this.unbindGeneration = bindGenerationState(this.edit, this.clipId ?? null, this.aiOverlay); this.contentContainer.addChild(this.aiOverlay.getContainer()); this.configureKeyframes(); @@ -62,6 +68,7 @@ export class ImageToVideoPlayer extends Player { const displaySize = this.getDisplaySize(); this.aiOverlay?.resize(displaySize.width, displaySize.height); + this.syncGenerateAction?.(); const overlayContainer = this.aiOverlay?.getContainer(); if (overlayContainer) { @@ -106,6 +113,9 @@ export class ImageToVideoPlayer extends Player { this.placeholder?.destroy(); this.placeholder = null; + this.unbindGeneration?.(); + this.unbindGeneration = null; + this.syncGenerateAction = null; this.aiOverlay?.dispose(); this.aiOverlay = null; diff --git a/src/components/canvas/players/text-to-image-player.ts b/src/components/canvas/players/text-to-image-player.ts index 410123b1..0b21d035 100644 --- a/src/components/canvas/players/text-to-image-player.ts +++ b/src/components/canvas/players/text-to-image-player.ts @@ -3,11 +3,14 @@ import { computeAiAssetNumber, isAiAsset } from "@core/shared/ai-asset-utils"; import { type Size } from "@layouts/geometry"; import type { ResolvedClip } from "@schemas"; +import { aiGenerateHandler, bindGenerationState, createGenerateActionSync } from "./ai-generation-binding"; import { AiPendingOverlay } from "./ai-pending-overlay"; import { Player, PlayerType } from "./player"; export class TextToImagePlayer extends Player { private aiOverlay: AiPendingOverlay | null = null; + private unbindGeneration: (() => void) | null = null; + private syncGenerateAction: (() => void) | null = null; private lastPrompt = ""; constructor(edit: Edit, clipConfiguration: ResolvedClip) { @@ -35,8 +38,11 @@ export class TextToImagePlayer extends Player { height, assetNumber: assetNumber ?? undefined, prompt, - assetType + assetType, + onGenerate: aiGenerateHandler(this.edit, this.clipId ?? null) }); + this.syncGenerateAction = createGenerateActionSync(this.edit, this.aiOverlay, this.clipId ?? null); + this.unbindGeneration = bindGenerationState(this.edit, this.clipId ?? null, this.aiOverlay); this.contentContainer.addChild(this.aiOverlay.getContainer()); this.configureKeyframes(); @@ -46,6 +52,7 @@ export class TextToImagePlayer extends Player { super.update(deltaTime, elapsed); const { width, height } = this.getSize(); this.aiOverlay?.resize(width, height); + this.syncGenerateAction?.(); // Sync prompt text only when it actually changes (e.g. via toolbar editing) const { asset } = this.clipConfiguration; @@ -65,6 +72,9 @@ export class TextToImagePlayer extends Player { } public override dispose(): void { + this.unbindGeneration?.(); + this.unbindGeneration = null; + this.syncGenerateAction = null; this.aiOverlay?.dispose(); this.aiOverlay = null; super.dispose(); diff --git a/src/components/timeline/components/clip/clip-component.ts b/src/components/timeline/components/clip/clip-component.ts index 4ae197b9..6eb136aa 100644 --- a/src/components/timeline/components/clip/clip-component.ts +++ b/src/components/timeline/components/clip/clip-component.ts @@ -17,6 +17,8 @@ export interface ClipComponentOptions { getRenderer: (type: string) => ClipRenderer | undefined; /** Get error state for a clip (if asset failed to load) */ getClipError?: (trackIndex: number, clipIndex: number) => { error: string; assetType: string } | null; + /** Get generation state for a clip (while an AI asset is being generated) */ + getClipGenerationState?: (clipId: string) => { status: "generating" | "failed"; error?: string } | undefined; /** Reference to attached luma (if this clip has a mask) */ attachedLuma?: LumaRef; /** Callback when mask badge is clicked - passes the CONTENT clip indices */ @@ -192,6 +194,9 @@ export class ClipComponent { // Update error state (show if asset failed to load) this.updateErrorState(); + // Update generation state (show while an AI asset is generating) + this.updateGenerationState(); + // Apply custom renderer if available const renderer = this.options.getRenderer(assetType); if (renderer) { @@ -224,6 +229,21 @@ export class ClipComponent { } } + /** Generation is transient: a failure shows the same treatment as a load error. */ + private updateGenerationState(): void { + const { clipId } = this.element.dataset; + const state = clipId ? this.options.getClipGenerationState?.(clipId) : undefined; + + this.element.classList.toggle("ss-clip--generating", state?.status === "generating"); + + if (state?.status === "failed") { + this.element.classList.add("ss-clip--error"); + this.element.title = state.error ?? "Generation failed"; + } else if (this.element.title && !this.currentError) { + this.element.title = ""; + } + } + /** Show/hide error state based on clip error */ private updateErrorState(): void { const error = this.currentState ? this.options.getClipError?.(this.currentState.trackIndex, this.currentState.clipIndex) : null; diff --git a/src/components/timeline/components/track/track-component.ts b/src/components/timeline/components/track/track-component.ts index 0794bb8a..6112ef72 100644 --- a/src/components/timeline/components/track/track-component.ts +++ b/src/components/timeline/components/track/track-component.ts @@ -8,6 +8,8 @@ export interface TrackComponentOptions { getClipRenderer: (type: string) => ClipRenderer | undefined; /** Get error state for a clip (if asset failed to load) */ getClipError?: (trackIndex: number, clipIndex: number) => { error: string; assetType: string } | null; + /** Get generation state for a clip (while an AI asset is being generated) */ + getClipGenerationState?: (clipId: string) => { status: "generating" | "failed"; error?: string } | undefined; /** Check if content clip has an attached luma (pure function) */ hasAttachedLuma?: (trackIndex: number, clipIndex: number) => boolean; /** Find attached luma for a content clip via timing match (pure function) */ @@ -152,6 +154,7 @@ export class TrackComponent { onSelect: this.options.onClipSelect, getRenderer: this.options.getClipRenderer, getClipError: this.options.getClipError, + getClipGenerationState: this.options.getClipGenerationState, aiAssetNumbers: this.options.aiAssetNumbers }); this.clipComponents.set(clipState.id, clipComponent); @@ -180,6 +183,7 @@ export class TrackComponent { onSelect: this.options.onClipSelect, getRenderer: this.options.getClipRenderer, getClipError: this.options.getClipError, + getClipGenerationState: this.options.getClipGenerationState, attachedLuma: attachedLuma ?? undefined, onMaskClick: this.options.onMaskClick, onMenuClick: this.options.onMenuClick, diff --git a/src/components/timeline/components/track/track-list.ts b/src/components/timeline/components/track/track-list.ts index edee3e5f..816f3156 100644 --- a/src/components/timeline/components/track/track-list.ts +++ b/src/components/timeline/components/track/track-list.ts @@ -9,6 +9,8 @@ export interface TrackListOptions { getClipRenderer: (type: string) => ClipRenderer | undefined; /** Get error state for a clip (if asset failed to load) */ getClipError?: (trackIndex: number, clipIndex: number) => { error: string; assetType: string } | null; + /** Get generation state for a clip (while an AI asset is being generated) */ + getClipGenerationState?: (clipId: string) => { status: "generating" | "failed"; error?: string } | undefined; /** Check if content clip has an attached luma */ hasAttachedLuma?: (trackIndex: number, clipIndex: number) => boolean; /** Find attached luma for a content clip via timing match */ @@ -85,6 +87,7 @@ export class TrackListComponent { onClipSelect: this.options.onClipSelect, getClipRenderer: this.options.getClipRenderer, getClipError: this.options.getClipError, + getClipGenerationState: this.options.getClipGenerationState, hasAttachedLuma: this.options.hasAttachedLuma, findAttachedLuma: this.options.findAttachedLuma, onMaskClick: this.options.onMaskClick, diff --git a/src/components/timeline/timeline.ts b/src/components/timeline/timeline.ts index b606a842..3c1741c4 100644 --- a/src/components/timeline/timeline.ts +++ b/src/components/timeline/timeline.ts @@ -65,6 +65,7 @@ export class Timeline { private readonly handlePlaybackPause: () => void; private readonly handleClipSelected: () => void; private readonly handleClipLoadFailed: () => void; + private readonly handleClipGeneration: () => void; private readonly handleClipUpdated: () => void; private readonly handleClipFocusChanged: () => void; private readonly handleRulerMouseMove: (e: MouseEvent) => void; @@ -115,6 +116,7 @@ export class Timeline { }; this.handleClipSelected = () => this.requestRender(); this.handleClipLoadFailed = () => this.requestRender(); + this.handleClipGeneration = () => this.requestRender(); this.handleClipUpdated = () => this.requestRender(); this.handleClipFocusChanged = () => this.requestRender(); this.handleRulerMouseMove = (e: MouseEvent) => { @@ -257,6 +259,9 @@ export class Timeline { // Listen for clip load failures (to show error badge on timeline) this.edit.events.on(EditEvent.ClipLoadFailed, this.handleClipLoadFailed); + this.edit.events.on(EditEvent.ClipGenerationStarted, this.handleClipGeneration); + this.edit.events.on(EditEvent.ClipGenerationCompleted, this.handleClipGeneration); + this.edit.events.on(EditEvent.ClipGenerationFailed, this.handleClipGeneration); // Listen for focus changes (source popup hover-to-highlight) const internal = this.edit.getInternalEvents(); @@ -276,6 +281,9 @@ export class Timeline { this.edit.events.off(EditEvent.ClipSelected, this.handleClipSelected); this.edit.events.off(EditEvent.ClipUpdated, this.handleClipUpdated); this.edit.events.off(EditEvent.ClipLoadFailed, this.handleClipLoadFailed); + this.edit.events.off(EditEvent.ClipGenerationStarted, this.handleClipGeneration); + this.edit.events.off(EditEvent.ClipGenerationCompleted, this.handleClipGeneration); + this.edit.events.off(EditEvent.ClipGenerationFailed, this.handleClipGeneration); const internal = this.edit.getInternalEvents(); internal.off(InternalEvent.ClipFocused, this.handleClipFocusChanged); @@ -394,6 +402,7 @@ export class Timeline { }, getClipRenderer: type => this.clipRenderers.get(type), getClipError: (trackIndex, clipIndex) => this.edit.getClipError(trackIndex, clipIndex), + getClipGenerationState: clipId => this.edit.getClipGenerationState(clipId), hasAttachedLuma: (trackIndex, clipIndex) => this.stateManager.hasAttachedLuma(trackIndex, clipIndex), findAttachedLuma: (trackIndex, clipIndex) => this.stateManager.findAttachedLuma(trackIndex, clipIndex), onMaskClick: (contentTrackIndex, contentClipIndex) => { diff --git a/src/core/edit-session.ts b/src/core/edit-session.ts index cccb4160..fe0de0b4 100644 --- a/src/core/edit-session.ts +++ b/src/core/edit-session.ts @@ -57,6 +57,7 @@ import * as pixi from "pixi.js"; import { CommandQueue } from "./commands/command-queue"; import { CommandNoop, type EditCommand, type CommandContext, type CommandResult } from "./commands/types"; import { EditDocument } from "./edit-document"; +import { AssetGenerator, type AssetGeneratorHandler, type ClipGenerationState } from "./generation/asset-generator"; import { PlayerReconciler } from "./player-reconciler"; import { resolve as resolveDocument, resolveClip as resolveClipById, type SingleClipContext } from "./resolver"; import { InvalidAssetUrlError, extractClipUrls, extractTrackUrls } from "./url-validation"; @@ -102,6 +103,7 @@ export class Edit { private timingManager!: TimingManager; private lumaMaskController: LumaMaskController; private playerReconciler: PlayerReconciler; + private assetGenerator: AssetGenerator; private outputSettings!: OutputSettingsManager; private selectionManager!: SelectionManager; /** @internal */ @@ -151,6 +153,17 @@ export class Edit { this.internalEvents ); this.playerReconciler = new PlayerReconciler(this); + this.assetGenerator = new AssetGenerator({ + getClipAsset: clipId => this.getClipById(clipId)?.asset as Record | undefined, + applyGeneratedSrc: async (clipId, url) => { + const asset = this.getClipById(clipId)?.asset; + if (!asset) return; + await this.updateClipById(clipId, { asset: { ...asset, src: url } } as Partial); + }, + emitStarted: clipId => this.internalEvents.emit(EditEvent.ClipGenerationStarted, { clipId }), + emitCompleted: clipId => this.internalEvents.emit(EditEvent.ClipGenerationCompleted, { clipId }), + emitFailed: (clipId, error) => this.internalEvents.emit(EditEvent.ClipGenerationFailed, { clipId, error }) + }); this.mergeFieldService = new MergeFieldService(this.internalEvents); this.outputSettings = new OutputSettingsManager(this); this.selectionManager = new SelectionManager(this); @@ -263,6 +276,7 @@ export class Edit { /** @internal */ public dispose(): void { this.clearClips(); + this.assetGenerator.abortAll(); this.lumaMaskController.dispose(); this.playerReconciler.dispose(); @@ -410,6 +424,41 @@ export class Edit { return clip ? structuredClone(clip) : null; } + /** + * Register the handler that turns a prompt-bearing clip into a generated asset. + * + * The SDK owns the pending, generating and failed states and writes the returned + * URL back to the clip; the host owns how generation happens and what a failure + * message says. Without a handler, no generate affordance is shown. + */ + public registerAssetGenerator(handler: AssetGeneratorHandler): void { + this.assetGenerator.register(handler); + } + + /** Whether a generator handler has been registered. */ + public hasAssetGenerator(): boolean { + return this.assetGenerator.hasHandler(); + } + + /** + * Generate the asset for a prompt-bearing clip and write the result to it. + * + * Resolves once the clip is updated or the attempt has failed; inspect + * `getClipGenerationState` or the `clip:generation*` events for the outcome. + * A second call while one is in flight for the same clip is ignored. + */ + public generateClipAsset(clipId: string): Promise { + return this.assetGenerator.generate(clipId); + } + + /** + * Transient generation state for a clip, or undefined when idle. Not part of the + * edit: never saved, never undone, cleared on reload. + */ + public getClipGenerationState(clipId: string): ClipGenerationState | undefined { + return this.assetGenerator.getState(clipId); + } + /** * Look up the (trackIndex, clipIndex) position of a clip by its stable ID. */ @@ -937,6 +986,11 @@ export class Edit { const clipToDelete = track[clipIdx]; if (!clipToDelete) return CommandNoop(`No clip at track ${trackIdx}, index ${clipIdx}`); + // Every deletion funnels through here, so in-flight generation is dropped + // whichever way the clip goes (toolbar, keyboard, or by id). + const deletedClipId = this.document.getClipId(trackIdx, clipIdx); + if (deletedClipId) this.assetGenerator.abort(deletedClipId); + // Check if this is a content clip (not a luma) const isContentClip = clipToDelete.playerType !== PlayerType.Luma; diff --git a/src/core/events/edit-events.ts b/src/core/events/edit-events.ts index fb6d2658..89228e2b 100644 --- a/src/core/events/edit-events.ts +++ b/src/core/events/edit-events.ts @@ -71,6 +71,9 @@ export const EditEvent = { ClipCaptureCompleted: "clip:captureCompleted", ClipCaptureFailed: "clip:captureFailed", ClipUnresolved: "clip:unresolved", + ClipGenerationStarted: "clip:generationStarted", + ClipGenerationCompleted: "clip:generationCompleted", + ClipGenerationFailed: "clip:generationFailed", // Selection SelectionCleared: "selection:cleared", @@ -161,6 +164,9 @@ export type EditEventMap = { [EditEvent.ClipCaptureCompleted]: { clipId: string | null; assetType: string; frameCount: number }; [EditEvent.ClipCaptureFailed]: { clipId: string | null; assetType: string; error: string; fallback: string }; [EditEvent.ClipUnresolved]: ClipLocation & { assetType: string; clipId: string }; + [EditEvent.ClipGenerationStarted]: { clipId: string }; + [EditEvent.ClipGenerationCompleted]: { clipId: string }; + [EditEvent.ClipGenerationFailed]: { clipId: string; error: string }; // Selection [EditEvent.SelectionCleared]: void; diff --git a/src/core/generation/asset-generator.ts b/src/core/generation/asset-generator.ts new file mode 100644 index 00000000..d03159de --- /dev/null +++ b/src/core/generation/asset-generator.ts @@ -0,0 +1,101 @@ +import { isAiAsset } from "@core/shared/ai-asset-utils"; + +/** Passed to the host handler for one generation. */ +export interface AssetGenerationRequest { + clipId: string; + /** Snapshot of the clip's asset when generation started. */ + asset: Record; + /** Aborted when the clip is deleted or the edit is disposed. */ + signal: AbortSignal; +} + +/** Resolves with the URL of the generated asset. */ +export type AssetGeneratorHandler = (request: AssetGenerationRequest) => Promise<{ url: string }>; + +export interface ClipGenerationState { + status: "generating" | "failed"; + /** Host-supplied message, shown as-is. */ + error?: string; +} + +export interface AssetGeneratorDeps { + getClipAsset: (clipId: string) => Record | undefined; + applyGeneratedSrc: (clipId: string, url: string) => Promise; + emitStarted: (clipId: string) => void; + emitCompleted: (clipId: string) => void; + emitFailed: (clipId: string, error: string) => void; +} + +/** + * Owns the generation lifecycle: one host handler, one in-flight request per + * clip, and the transient state the UI renders. State lives here rather than in + * the document so it is never autosaved, undone, or restored on reload. + */ +export class AssetGenerator { + private handler?: AssetGeneratorHandler; + private readonly states = new Map(); + private readonly controllers = new Map(); + + constructor(private readonly deps: AssetGeneratorDeps) {} + + public register(handler: AssetGeneratorHandler): void { + this.handler = handler; + } + + public hasHandler(): boolean { + return this.handler !== undefined; + } + + public getState(clipId: string): ClipGenerationState | undefined { + return this.states.get(clipId); + } + + public async generate(clipId: string): Promise { + if (!this.handler) { + console.warn("generateClipAsset: no asset generator registered"); + return; + } + if (this.states.get(clipId)?.status === "generating") return; + + const asset = this.deps.getClipAsset(clipId); + if (!asset || !isAiAsset(asset)) { + console.warn(`generateClipAsset: clip ${clipId} has no generatable asset`); + return; + } + + const controller = new AbortController(); + this.controllers.set(clipId, controller); + this.states.set(clipId, { status: "generating" }); + this.deps.emitStarted(clipId); + + try { + const { url } = await this.handler({ + clipId, + asset: structuredClone(asset) as Record, + signal: controller.signal + }); + if (controller.signal.aborted) return; + this.states.delete(clipId); + await this.deps.applyGeneratedSrc(clipId, url); + this.deps.emitCompleted(clipId); + } catch (error) { + if (controller.signal.aborted) return; + const message = error instanceof Error ? error.message : String(error); + this.states.set(clipId, { status: "failed", error: message }); + this.deps.emitFailed(clipId, message); + } finally { + this.controllers.delete(clipId); + } + } + + public abort(clipId: string): void { + this.controllers.get(clipId)?.abort(); + this.controllers.delete(clipId); + this.states.delete(clipId); + } + + public abortAll(): void { + for (const clipId of [...this.controllers.keys()]) this.abort(clipId); + this.states.clear(); + } +} diff --git a/src/core/merge/merge-field-service.ts b/src/core/merge/merge-field-service.ts index 445681f7..1734088f 100644 --- a/src/core/merge/merge-field-service.ts +++ b/src/core/merge/merge-field-service.ts @@ -91,8 +91,11 @@ export class MergeFieldService { resolveToNumber(input: string): number | null { if (!this.isMergeFieldTemplate(input)) return null; - const resolved = this.resolve(input); - const num = parseFloat(resolved); + // Whole-string match only: parseFloat takes a numeric prefix, so a + // resolved "03 image of a cat" would collapse to the number 3. + const resolved = this.resolve(input).trim(); + if (resolved === "") return null; + const num = Number(resolved); return Number.isFinite(num) ? num : null; } diff --git a/src/core/resolver.ts b/src/core/resolver.ts index 1df2e621..83f72599 100644 --- a/src/core/resolver.ts +++ b/src/core/resolver.ts @@ -69,7 +69,7 @@ interface ClipLocation { * - Tries numeric conversion first (for timing, scale, offset, etc.) * - Falls back to string resolution (for text content) */ -const STRING_ONLY_KEYS = new Set(["text", "src"]); +const STRING_ONLY_KEYS = new Set(["text", "src", "prompt"]); function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldService): InternalClip { function processValue(value: unknown, key?: string): unknown { @@ -81,7 +81,7 @@ function resolveMergeFieldsInClip(clip: InternalClip, mergeFields: MergeFieldSer return num !== null ? num : mergeFields.resolve(value); } if (Array.isArray(value)) { - return value.map((item) => processValue(item, key)); + return value.map(item => processValue(item, key)); } if (value !== null && typeof value === "object") { const result: Record = {}; diff --git a/src/core/ui/media-toolbar.ts b/src/core/ui/media-toolbar.ts index 09cc1bb1..d579af9b 100644 --- a/src/core/ui/media-toolbar.ts +++ b/src/core/ui/media-toolbar.ts @@ -11,6 +11,7 @@ import { } from "@core/animations/opacity-keyframes"; import type { Edit } from "@core/edit-session"; import { EditEvent } from "@core/events/edit-events"; +import { isAiAsset, isPendingAiAsset } from "@core/shared/ai-asset-utils"; import { hasKeyframedVisualProperty } from "@core/shared/clip-utils"; import { validateAssetUrl } from "@core/shared/utils"; import { ShotstackEdit } from "@core/shotstack-edit"; @@ -66,6 +67,7 @@ type MediaAssetType = "video" | "image" | "audio" | "text-to-image" | "image-to- const VISUAL_ASSET_TYPES: ReadonlySet = new Set(["video", "image", "text-to-image", "image-to-video"]); /** Asset types that have volume controls */ +const PROMPT_DEBOUNCE_MS = 300; const VOLUME_ASSET_TYPES: ReadonlySet = new Set(["video", "audio", "text-to-speech"]); /** Asset types that have audio fade controls (audio-only types) */ @@ -158,6 +160,14 @@ export class MediaToolbar extends BaseToolbar { private volumeDisplayInput: HTMLInputElement | null = null; private volumeSection: HTMLDivElement | null = null; private visualSection: HTMLDivElement | null = null; + private aiSection: HTMLDivElement | null = null; + private aiDivider: HTMLDivElement | null = null; + private promptTextarea: HTMLTextAreaElement | null = null; + private promptPopup: HTMLDivElement | null = null; + private generateBtn: HTMLButtonElement | null = null; + private generateError: HTMLElement | null = null; + private promptDebounceTimer: ReturnType | null = null; + private generationUnsubscribers: (() => void)[] = []; private audioSection: HTMLDivElement | null = null; private speedSection: HTMLDivElement | null = null; private speedSlider: HTMLInputElement | null = null; @@ -209,6 +219,27 @@ export class MediaToolbar extends BaseToolbar {
+ + + +
@@ -417,6 +448,12 @@ export class MediaToolbar extends BaseToolbar { this.volumeDisplayInput = this.container.querySelector("[data-volume-display]"); this.volumeSection = this.container.querySelector("[data-volume-section]"); this.visualSection = this.container.querySelector("[data-visual-section]"); + this.aiSection = this.container.querySelector("[data-ai-section]"); + this.aiDivider = this.container.querySelector("[data-ai-divider]"); + this.promptTextarea = this.container.querySelector("[data-prompt-textarea]"); + this.promptPopup = this.container.querySelector("[data-popup='prompt']"); + this.generateBtn = this.container.querySelector("[data-action='generate']"); + this.generateError = this.container.querySelector("[data-generate-error]"); this.audioSection = this.container.querySelector("[data-audio-section]"); this.speedSection = this.container.querySelector("[data-speed-section]"); this.speedSlider = this.container.querySelector("[data-speed-slider]"); @@ -533,6 +570,31 @@ export class MediaToolbar extends BaseToolbar { this.abortController = new AbortController(); const { signal } = this.abortController; + // Generation controls + this.container?.querySelector("[data-action='prompt']")?.addEventListener( + "click", + e => { + e.stopPropagation(); + this.togglePopup(this.promptPopup); + }, + { signal } + ); + this.promptTextarea?.addEventListener("input", () => this.handlePromptInput(), { signal }); + this.generateBtn?.addEventListener( + "click", + e => { + e.stopPropagation(); + const clipId = this.getSelectedClipId(); + if (clipId) { + this.edit.generateClipAsset(clipId).catch(() => { + // Failures surface as clip state. + }); + } + }, + { signal } + ); + this.subscribeToGeneration(); + // Toggle popups this.fitBtn?.addEventListener( "click", @@ -785,6 +847,101 @@ export class MediaToolbar extends BaseToolbar { this.speedBtn?.classList.remove("active"); } + private subscribeToGeneration(): void { + // mount() can run more than once on an instance; never stack listeners. + if (this.generationUnsubscribers.length > 0) return; + + const names = [EditEvent.ClipGenerationStarted, EditEvent.ClipGenerationCompleted, EditEvent.ClipGenerationFailed] as const; + + for (const name of names) { + const handler = (payload: { clipId: string }): void => { + if (payload.clipId === this.getSelectedClipId()) this.updateGenerationUI(); + }; + this.edit.events.on(name, handler); + this.generationUnsubscribers.push(() => this.edit.events.off(name, handler)); + } + } + + private handlePromptInput(): void { + if (this.promptDebounceTimer) clearTimeout(this.promptDebounceTimer); + + this.promptDebounceTimer = setTimeout(() => { + const rawText = this.promptTextarea?.value ?? ""; + const shotstackEdit = this.getShotstackEdit(); + const resolvedText = shotstackEdit?.mergeFields.resolve(rawText) ?? rawText; + + const document = this.edit.getDocument(); + const clipId = this.getSelectedClipId(); + + if (clipId && document) { + if (shotstackEdit?.mergeFields.isMergeFieldTemplate(rawText)) { + document.setClipBinding(clipId, "asset.prompt", { placeholder: rawText, resolvedValue: resolvedText }); + } else { + document.removeClipBinding(clipId, "asset.prompt"); + } + } + + const clip = this.edit.getResolvedClip(this.selectedTrackIdx, this.selectedClipIdx); + if (clip) { + this.edit.updateClip(this.selectedTrackIdx, this.selectedClipIdx, { + asset: { ...clip.asset, prompt: resolvedText } + } as never); + } + this.updatePromptButtonText(rawText); + }, PROMPT_DEBOUNCE_MS); + } + + private updatePromptButtonText(text: string): void { + const label = this.container?.querySelector("[data-prompt-text]"); + if (label) label.textContent = text.trim() ? text.trim().slice(0, 20) : "Prompt"; + } + + /** Prompt controls appear only for prompt-bearing assets; the action only with a generator. */ + private updateGenerationUI(): void { + const clip = this.edit.getResolvedClip(this.selectedTrackIdx, this.selectedClipIdx); + const asset = clip?.asset; + const isGenerative = isAiAsset(asset); + + this.aiSection?.classList.toggle("hidden", !isGenerative); + this.aiDivider?.classList.toggle("hidden", !isGenerative); + if (!isGenerative || !this.generateBtn) return; + + if (this.promptTextarea) { + const document = this.edit.getDocument(); + const clipId = this.getSelectedClipId(); + const binding = clipId ? document?.getClipBinding(clipId, "asset.prompt") : undefined; + const text = binding?.placeholder ?? asset.prompt ?? ""; + this.promptTextarea.value = text; + this.updatePromptButtonText(text); + } + + if (!this.edit.hasAssetGenerator()) { + this.generateBtn.hidden = true; + return; + } + this.generateBtn.hidden = false; + + const clipId = this.getSelectedClipId(); + const state = clipId ? this.edit.getClipGenerationState(clipId) : undefined; + const generating = state?.status === "generating"; + const label = this.generateBtn.querySelector("[data-generate-label]"); + + this.generateBtn.disabled = generating; + this.generateBtn.classList.toggle("is-generating", generating); + + if (label) { + if (generating) label.textContent = "Generating…"; + else if (state?.status === "failed") label.textContent = "Retry"; + else label.textContent = isPendingAiAsset(asset) ? "Generate" : "Regenerate"; + } + + if (this.generateError) { + const message = state?.status === "failed" ? (state.error ?? "Generation failed") : ""; + this.generateError.textContent = message; + this.generateError.hidden = message === ""; + } + } + protected override getPopupList(): (HTMLElement | null)[] { return [ this.fitPopup, @@ -795,7 +952,8 @@ export class MediaToolbar extends BaseToolbar { this.effectPopup, this.advancedPopup, this.audioFadePopup, - this.speedPopup + this.speedPopup, + this.promptPopup ]; } @@ -875,6 +1033,8 @@ export class MediaToolbar extends BaseToolbar { this.advancedBtn.parentElement.classList.toggle("hidden", !VISUAL_ASSET_TYPES.has(this.assetType) || !this.showMergeFields); } + this.updateGenerationUI(); + // Sync merge field label bound states if (this.showMergeFields && this.mergeFieldManager?.hasLabels) { this.mergeFieldManager.sync(); @@ -1591,6 +1751,11 @@ export class MediaToolbar extends BaseToolbar { this.abortController?.abort(); this.abortController = null; + if (this.promptDebounceTimer) clearTimeout(this.promptDebounceTimer); + this.promptDebounceTimer = null; + for (const off of this.generationUnsubscribers) off(); + this.generationUnsubscribers = []; + // Clear any in-progress drag sessions this.dragManager.clear(); if (this.playbackPauseListener) { diff --git a/src/index.ts b/src/index.ts index 95edc633..c528c085 100644 --- a/src/index.ts +++ b/src/index.ts @@ -9,6 +9,7 @@ export { UIController } from "@core/ui/ui-controller"; export { WebGLUnsupportedError } from "@core/webgl-support"; export type { UIControllerOptions, ToolbarButtonConfig } from "@core/ui/ui-controller"; +export type { AssetGenerationRequest, AssetGeneratorHandler, ClipGenerationState } from "@core/generation/asset-generator"; export type { EditConfig } from "@core/schemas"; export type { CommandResult } from "@core/commands/types"; diff --git a/src/smoketest.ts b/src/smoketest.ts new file mode 100644 index 00000000..e8643f79 --- /dev/null +++ b/src/smoketest.ts @@ -0,0 +1,112 @@ +import { type Edit as EditSchema } from "@schemas"; +import { Timeline } from "@timeline/index"; + +import template from "./templates/generation-smoketest.json"; + +import { Edit, Canvas, Controls, UIController } from "./index"; + +/** + * Manual smoketest for the asset generation lifecycle. Run `npm run dev` and open + * /smoketest.html. The generator is fake: it returns placeholder media so every + * state can be exercised without spending credits. + */ + +type Outcome = "success" | "failure" | "slow"; + +const PLACEHOLDER = { + image: "https://shotstack-assets.s3.amazonaws.com/images/waterfall.jpeg", + video: "https://shotstack-assets.s3.amazonaws.com/footage/city-timelapse.mp4", + audio: "https://shotstack-assets.s3.amazonaws.com/music/unminus/lit.mp3" +} as const; + +let outcome: Outcome = "success"; +let delayMs = 1500; + +const wait = (ms: number, signal: AbortSignal): Promise => + new Promise((resolve, reject) => { + const timer = setTimeout(resolve, ms); + signal.addEventListener("abort", () => { + clearTimeout(timer); + reject(new Error("aborted")); + }); + }); + +function buildControls(edit: Edit): void { + const panel = document.createElement("div"); + panel.className = "smoketest-panel"; + panel.innerHTML = ` + Fake generator + + + + + +
+ `; + document.body.appendChild(panel); + + panel.querySelectorAll("input[name=outcome]").forEach(input => { + input.addEventListener("change", () => { + if (input.checked) outcome = input.value as Outcome; + }); + }); + panel.querySelector("#delay")?.addEventListener("change", event => { + delayMs = Number((event.target as HTMLInputElement).value); + }); + + // Registering is one-way in the API, so this only demonstrates the + // no-generator state on a fresh load. + panel.querySelector("#toggle-generator")?.addEventListener("click", () => { + // eslint-disable-next-line no-alert -- dev harness only + alert("Reload with ?nogen to see the editor without a registered generator."); + }); + + const log = panel.querySelector("#smoketest-log") as HTMLDivElement; + const append = (line: string): void => { + log.textContent = `${line}\n${log.textContent ?? ""}`.split("\n").slice(0, 8).join("\n"); + }; + + edit.events.on("clip:generationStarted", ({ clipId }) => append(`started ${clipId.slice(0, 8)}`)); + edit.events.on("clip:generationCompleted", ({ clipId }) => append(`done ${clipId.slice(0, 8)}`)); + edit.events.on("clip:generationFailed", ({ clipId, error }) => append(`failed ${clipId.slice(0, 8)} — ${error}`)); +} + +async function main(): Promise { + const edit = new Edit(template as EditSchema); + const canvas = new Canvas(edit); + const ui = UIController.create(edit, canvas); + + await canvas.load(); + await edit.load(); + + const timeline = new Timeline(edit, document.querySelector("[data-shotstack-timeline]") as HTMLElement); + await timeline.load(); + + const controls = new Controls(edit); + await controls.load(); + + // Registered after load() on purpose: overlays must pick the generator up late. + if (!new URLSearchParams(window.location.search).has("nogen")) { + edit.registerAssetGenerator(async ({ clipId, asset, signal }) => { + const kind = (asset as { type?: string }).type ?? "image"; + // eslint-disable-next-line no-console -- dev harness only + console.log("[smoketest] generate", clipId, asset); + + await wait(outcome === "slow" ? 10_000 : delayMs, signal); + + if (outcome === "failure") throw new Error("Not enough credits"); + + const url = PLACEHOLDER[kind as keyof typeof PLACEHOLDER] ?? PLACEHOLDER.image; + return { url: `${url}?generated=${Date.now()}` }; + }); + } + + buildControls(edit); + if (!ui) throw new Error("UI controller failed to initialise"); + (window as unknown as { edit: Edit }).edit = edit; +} + +main().catch(error => { + // eslint-disable-next-line no-console -- dev harness only + console.error("Smoketest failed to start:", error); +}); diff --git a/src/styles/timeline/timeline.css b/src/styles/timeline/timeline.css index dda04582..0098b8be 100644 --- a/src/styles/timeline/timeline.css +++ b/src/styles/timeline/timeline.css @@ -1156,3 +1156,22 @@ transform: translateX(2px); } } + +/* Generating state — a quiet pulse while an AI asset is being produced */ +.ss-clip.ss-clip--generating { + animation: ss-clip-generating-pulse 1.6s ease-in-out infinite; +} + +.ss-clip.ss-clip--generating .ss-clip-icon { + opacity: 0.9; +} + +@keyframes ss-clip-generating-pulse { + 0%, + 100% { + opacity: 1; + } + 50% { + opacity: 0.62; + } +} diff --git a/src/styles/ui/media-toolbar.css b/src/styles/ui/media-toolbar.css index 15e4cc5d..57b6a803 100644 --- a/src/styles/ui/media-toolbar.css +++ b/src/styles/ui/media-toolbar.css @@ -1092,3 +1092,78 @@ .ss-audio-fade-btn.active .ss-audio-fade-label { color: rgba(255, 255, 255, 0.9); } + +/* Generation controls — only rendered for prompt-bearing assets */ +.ss-media-toolbar-ai { + display: flex; + align-items: center; + gap: 2px; +} + +.ss-media-toolbar-ai.hidden { + display: none; +} + +.ss-ai-prompt-btn { + max-width: 170px; +} + +.ss-ai-prompt-btn .ss-ai-prompt-text { + max-width: 110px; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.ss-ai-prompt-popup { + width: 320px; + padding: 10px; +} + +.ss-ai-prompt-textarea { + width: 100%; + box-sizing: border-box; + resize: vertical; + background: rgba(0, 0, 0, 0.3); + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 6px; + color: #fafafa; + font-family: inherit; + font-size: 12px; + line-height: 1.5; + padding: 8px; +} + +.ss-ai-prompt-textarea:focus { + outline: none; + border-color: rgba(255, 255, 255, 0.25); +} + +.ss-ai-prompt-hint { + margin-top: 6px; + font-size: 11px; + color: rgba(255, 255, 255, 0.45); +} + +.ss-ai-generate-btn[hidden] { + display: none; +} + +.ss-ai-generate-btn.is-generating { + opacity: 0.65; + cursor: default; +} + +.ss-ai-error { + max-width: 180px; + padding: 0 6px; + font-size: 11px; + color: #fca5a5; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.ss-ai-error[hidden] { + display: none; +} diff --git a/src/templates/generation-smoketest.json b/src/templates/generation-smoketest.json new file mode 100644 index 00000000..b39ccaf2 --- /dev/null +++ b/src/templates/generation-smoketest.json @@ -0,0 +1,158 @@ +{ + "timeline": { + "background": "#111111", + "tracks": [ + { + "clips": [ + { + "asset": { + "type": "image", + "prompt": "01 image pending, bare prompt" + }, + "start": 0, + "length": 3 + }, + { + "asset": { + "type": "image", + "prompt": "02 image pending, model and options", + "model": "nano-banana-2", + "options": { "resolution": "2K", "aspectRatio": "16:9" } + }, + "start": 3, + "length": 3 + }, + { + "asset": { + "type": "image", + "prompt": "03 image pending, merge field {{ SUBJECT }}" + }, + "start": 6, + "length": 3 + }, + { + "asset": { + "type": "image", + "prompt": "04 image realised, regenerate replaces this", + "src": "https://shotstack-assets.s3.amazonaws.com/images/wave-barrel.jpg" + }, + "start": 9, + "length": 3, + "fit": "crop" + }, + { + "asset": { + "type": "image", + "src": "https://shotstack-assets.s3.amazonaws.com/images/earth.jpg" + }, + "start": 12, + "length": 3, + "fit": "crop" + }, + { + "asset": { + "type": "video", + "prompt": "06 video pending, text to video, no input image" + }, + "start": 15, + "length": 3 + }, + { + "asset": { + "type": "video", + "prompt": "07 video pending, image to video over a start frame", + "model": "seedance-2.0", + "options": { "inputSrc": "https://shotstack-assets.s3.amazonaws.com/images/dog1.jpg" } + }, + "start": 18, + "length": 3 + }, + { + "asset": { + "type": "video", + "prompt": "08 video realised, regenerate replaces this", + "src": "https://shotstack-assets.s3.amazonaws.com/footage/skateboarder.mp4" + }, + "start": 21, + "length": 3, + "fit": "crop" + }, + { + "asset": { + "type": "video", + "src": "https://shotstack-assets.s3.amazonaws.com/footage/beach.mp4" + }, + "start": 24, + "length": 3, + "fit": "crop" + }, + { + "asset": { + "type": "text-to-image", + "prompt": "10 legacy text-to-image" + }, + "start": 27, + "length": 3, + "width": 512, + "height": 512 + }, + { + "asset": { + "type": "image-to-video", + "src": "https://shotstack-assets.s3.amazonaws.com/images/waterfall.jpeg", + "prompt": "11 legacy image-to-video" + }, + "start": 30, + "length": 3 + } + ] + }, + { + "clips": [ + { + "asset": { + "type": "audio", + "prompt": "12 audio pending, bare prompt and default voice" + }, + "start": 0, + "length": 6 + }, + { + "asset": { + "type": "audio", + "prompt": "13 audio pending, explicit voice option", + "model": "polly-neural", + "options": { "voice": "Matthew" } + }, + "start": 6, + "length": 6 + }, + { + "asset": { + "type": "audio", + "prompt": "14 audio realised, regenerate replaces this", + "src": "https://shotstack-assets.s3.amazonaws.com/music/unminus/lit.mp3" + }, + "start": 12, + "length": 6 + }, + { + "asset": { + "type": "text-to-speech", + "text": "15 legacy text to speech", + "voice": "Matthew" + }, + "start": 18, + "length": 6 + } + ] + } + ] + }, + "merge": [{ "find": "SUBJECT", "replace": "a red apple" }], + "output": { + "format": "mp4", + "fps": 25, + "size": { "width": 1280, "height": 720 } + } +} diff --git a/src/templates/prompt-assets.json b/src/templates/prompt-assets.json index 30ad5350..fb83e26b 100644 --- a/src/templates/prompt-assets.json +++ b/src/templates/prompt-assets.json @@ -18,7 +18,10 @@ "asset": { "type": "video", "prompt": "Slowly zoom out and orbit left around the trees", - "seed": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg" + "model": "seedance-2.0", + "options": { + "inputSrc": "https://shotstack-assets.s3.amazonaws.com/images/woods1.jpg" + } }, "start": 5, "length": 5, @@ -81,7 +84,9 @@ "asset": { "type": "audio", "prompt": "Welcome to the unified prompt asset demo.", - "voice": "Matthew" + "options": { + "voice": "Matthew" + } }, "start": 10, "length": 10 diff --git a/tests/ai-asset-utils.test.ts b/tests/ai-asset-utils.test.ts index b25d689a..e0c96a62 100644 --- a/tests/ai-asset-utils.test.ts +++ b/tests/ai-asset-utils.test.ts @@ -11,7 +11,7 @@ describe("ai-asset-utils", () => { it("accepts prompt-bearing media assets", () => { expect(isAiAsset({ type: "image", prompt: "a cat" })).toBe(true); - expect(isAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true); + expect(isAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true); expect(isAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true); }); @@ -34,7 +34,7 @@ describe("ai-asset-utils", () => { describe("isPendingAiAsset", () => { it("is pending while a prompt-bearing media asset has no src", () => { expect(isPendingAiAsset({ type: "image", prompt: "a cat" })).toBe(true); - expect(isPendingAiAsset({ type: "video", prompt: "waves", seed: "https://cdn/seed.png" })).toBe(true); + expect(isPendingAiAsset({ type: "video", prompt: "waves", options: { inputSrc: "https://cdn/start.png" } })).toBe(true); expect(isPendingAiAsset({ type: "audio", prompt: "calm piano" })).toBe(true); }); diff --git a/tests/asset-generator.test.ts b/tests/asset-generator.test.ts new file mode 100644 index 00000000..234e68bb --- /dev/null +++ b/tests/asset-generator.test.ts @@ -0,0 +1,168 @@ +import { AssetGenerator, type AssetGeneratorDeps } from "@core/generation/asset-generator"; + +const PROMPT_ASSET = { type: "image", prompt: "a red apple" }; + +function makeDeps(overrides: Partial = {}) { + const started: string[] = []; + const completed: string[] = []; + const failed: { clipId: string; error: string }[] = []; + const applied: { clipId: string; url: string }[] = []; + + const deps: AssetGeneratorDeps = { + getClipAsset: () => ({ ...PROMPT_ASSET }), + applyGeneratedSrc: async (clipId, url) => { + applied.push({ clipId, url }); + }, + emitStarted: clipId => started.push(clipId), + emitCompleted: clipId => completed.push(clipId), + emitFailed: (clipId, error) => failed.push({ clipId, error }), + ...overrides + }; + + return { deps, started, completed, failed, applied }; +} + +describe("AssetGenerator", () => { + it("writes the generated src back and reports completion", async () => { + const { deps, started, completed, applied } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/out.png" })); + + await generator.generate("clip-1"); + + expect(applied).toEqual([{ clipId: "clip-1", url: "https://cdn/out.png" }]); + expect(started).toEqual(["clip-1"]); + expect(completed).toEqual(["clip-1"]); + expect(generator.getState("clip-1")).toBeUndefined(); + }); + + it("keeps the host's message on failure and writes nothing", async () => { + const { deps, failed, applied } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register(async () => { + throw new Error("Not enough credits"); + }); + + await generator.generate("clip-1"); + + expect(applied).toEqual([]); + expect(generator.getState("clip-1")).toEqual({ status: "failed", error: "Not enough credits" }); + expect(failed).toEqual([{ clipId: "clip-1", error: "Not enough credits" }]); + }); + + it("clears the failed state when retried", async () => { + const { deps } = makeDeps(); + const generator = new AssetGenerator(deps); + let attempt = 0; + generator.register(async () => { + attempt += 1; + if (attempt === 1) throw new Error("boom"); + return { url: "https://cdn/out.png" }; + }); + + await generator.generate("clip-1"); + expect(generator.getState("clip-1")?.status).toBe("failed"); + + await generator.generate("clip-1"); + expect(generator.getState("clip-1")).toBeUndefined(); + expect(attempt).toBe(2); + }); + + it("ignores a second request while one is in flight for the same clip", async () => { + const { deps } = makeDeps(); + const generator = new AssetGenerator(deps); + let calls = 0; + let release: (() => void) | undefined; + generator.register(async () => { + calls += 1; + await new Promise(resolve => { + release = resolve; + }); + return { url: "https://cdn/out.png" }; + }); + + const first = generator.generate("clip-1"); + await generator.generate("clip-1"); + expect(calls).toBe(1); + + release?.(); + await first; + }); + + it("generates different clips concurrently", async () => { + const { deps, completed } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register(async ({ clipId }) => ({ url: `https://cdn/${clipId}.png` })); + + await Promise.all([generator.generate("clip-1"), generator.generate("clip-2")]); + + expect(completed.sort()).toEqual(["clip-1", "clip-2"]); + }); + + it("does nothing without a registered handler", async () => { + const { deps, started } = makeDeps(); + const generator = new AssetGenerator(deps); + + await generator.generate("clip-1"); + + expect(started).toEqual([]); + expect(generator.getState("clip-1")).toBeUndefined(); + }); + + it("refuses a clip whose asset is not generatable", async () => { + const { deps, started } = makeDeps({ getClipAsset: () => ({ type: "image", src: "https://cdn/a.png" }) }); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/out.png" })); + + await generator.generate("clip-1"); + + expect(started).toEqual([]); + }); + + it("accepts a realised asset so it can be regenerated", async () => { + const { deps, applied } = makeDeps({ + getClipAsset: () => ({ type: "image", prompt: "a red apple", src: "https://cdn/old.png" }) + }); + const generator = new AssetGenerator(deps); + generator.register(async () => ({ url: "https://cdn/new.png" })); + + await generator.generate("clip-1"); + + expect(applied).toEqual([{ clipId: "clip-1", url: "https://cdn/new.png" }]); + }); + + it("leaves no failed state when aborted mid-flight", async () => { + const { deps, failed, applied } = makeDeps(); + const generator = new AssetGenerator(deps); + generator.register( + ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => reject(new Error("aborted"))); + }) + ); + + const pending = generator.generate("clip-1"); + generator.abort("clip-1"); + await pending; + + expect(generator.getState("clip-1")).toBeUndefined(); + expect(failed).toEqual([]); + expect(applied).toEqual([]); + }); + + it("passes a snapshot rather than the live asset", async () => { + const live = { type: "image", prompt: "a red apple" }; + const { deps } = makeDeps({ getClipAsset: () => live }); + const generator = new AssetGenerator(deps); + let received: Record | undefined; + generator.register(async ({ asset }) => { + received = asset; + return { url: "https://cdn/out.png" }; + }); + + await generator.generate("clip-1"); + + expect(received).toEqual(live); + expect(received).not.toBe(live); + }); +}); diff --git a/tests/edit-clip-operations.test.ts b/tests/edit-clip-operations.test.ts index f93ae50e..a15727b8 100644 --- a/tests/edit-clip-operations.test.ts +++ b/tests/edit-clip-operations.test.ts @@ -571,6 +571,31 @@ describe("Edit Clip Operations", () => { expect((await edit.deleteClipById(id as string)).status).toBe("success"); }); + it("aborts in-flight generation when the clip is deleted by position", async () => { + await edit.addClip(0, { asset: { type: "image", prompt: "a red apple" }, start: 0, length: 5 } as never); + let aborted = false; + edit.registerAssetGenerator( + ({ signal }) => + new Promise((_resolve, reject) => { + signal.addEventListener("abort", () => { + aborted = true; + reject(new Error("aborted")); + }); + }) + ); + const doc = (edit as unknown as { document: { getClipId(t: number, c: number): string | null } }).document; + const id = doc.getClipId(0, 1) as string; + + const pending = edit.generateClipAsset(id); + expect(edit.getClipGenerationState(id)?.status).toBe("generating"); + + await edit.deleteClip(0, 1); + await pending; + + expect(aborted).toBe(true); + expect(edit.getClipGenerationState(id)).toBeUndefined(); + }); + it("updateClip resolves success and noop by position", async () => { expect((await edit.updateClip(0, 0, { fit: "contain" })).status).toBe("success"); expect(await edit.updateClip(0, 99, {})).toMatchObject({ status: "noop" }); diff --git a/tests/media-toolbar.test.ts b/tests/media-toolbar.test.ts index 4a4ce819..ef12c874 100644 --- a/tests/media-toolbar.test.ts +++ b/tests/media-toolbar.test.ts @@ -57,6 +57,10 @@ function createMockEditSession() { getClipId: jest.fn().mockReturnValue("clip-1"), getResolvedClip: jest.fn(), getDocumentClip: jest.fn(), + getDocument: jest.fn(), + hasAssetGenerator: jest.fn().mockReturnValue(false), + getClipGenerationState: jest.fn(), + generateClipAsset: jest.fn().mockResolvedValue(undefined), updateClip: jest.fn(), updateClipInDocument: jest.fn(), resolveClip: jest.fn(), @@ -103,6 +107,10 @@ function createMergeFieldMockEditSession() { getResolvedClip: jest.fn(), getResolvedClipById: jest.fn(), getDocumentClip: jest.fn(), + getDocument: jest.fn(), + hasAssetGenerator: jest.fn().mockReturnValue(false), + getClipGenerationState: jest.fn(), + generateClipAsset: jest.fn().mockResolvedValue(undefined), updateClip: jest.fn(), updateClipInDocument: jest.fn(), resolveClip: jest.fn(), diff --git a/tests/merge-field-numeric-resolution.test.ts b/tests/merge-field-numeric-resolution.test.ts new file mode 100644 index 00000000..48c2366d --- /dev/null +++ b/tests/merge-field-numeric-resolution.test.ts @@ -0,0 +1,32 @@ +import { EventEmitter } from "../src/core/events/event-emitter"; +import { MergeFieldService } from "../src/core/merge/merge-field-service"; + +import type { EditEventMap } from "../src/core/events/edit-events"; + +function serviceWith(fields: Record): MergeFieldService { + const service = new MergeFieldService(new EventEmitter()); + Object.entries(fields).forEach(([name, defaultValue]) => service.register({ name, defaultValue }, { silent: true })); + return service; +} + +describe("MergeFieldService.resolveToNumber", () => { + it("converts a template that resolves to a bare number", () => { + expect(serviceWith({ WIDTH: "1920" }).resolveToNumber("{{ WIDTH }}")).toBe(1920); + expect(serviceWith({ SCALE: " 2.5 " }).resolveToNumber("{{ SCALE }}")).toBe(2.5); + }); + + it("returns null when the resolved text merely starts with digits", () => { + const service = serviceWith({ SUBJECT: "a red apple" }); + + expect(service.resolveToNumber("03 image pending, merge field {{ SUBJECT }}")).toBeNull(); + expect(service.resolveToNumber("{{ SUBJECT }} 42")).toBeNull(); + }); + + it("returns null for text that resolves to an empty string", () => { + expect(serviceWith({ EMPTY: "" }).resolveToNumber("{{ EMPTY }}")).toBeNull(); + }); + + it("returns null for a non-template input", () => { + expect(serviceWith({ WIDTH: "1920" }).resolveToNumber("1920")).toBeNull(); + }); +}); diff --git a/tests/schema.test.ts b/tests/schema.test.ts index f0a1075b..083f1e6d 100644 --- a/tests/schema.test.ts +++ b/tests/schema.test.ts @@ -400,15 +400,29 @@ describe("API-accepted templates parse at load", () => { expect(result.success).toBe(true); }); - it("accepts prompt-driven audio assets with voice and newscaster", () => { + it("accepts prompt-driven audio assets with model-scoped options", () => { const result = ClipSchema.safeParse({ - asset: { type: "audio", prompt: "Read the news intro", voice: "Joanna", newscaster: true }, + asset: { + type: "audio", + prompt: "Read the news intro", + model: "polly-neural", + options: { voice: "Joanna", newscaster: true } + }, start: 0, length: 5 }); expect(result.success).toBe(true); }); + it("rejects generation settings at the asset root", () => { + const result = ClipSchema.safeParse({ + asset: { type: "audio", prompt: "Read the news intro", voice: "Joanna" }, + start: 0, + length: 5 + }); + expect(result.success).toBe(false); + }); + it("accepts a destination without an options object", () => { const result = EditSchema.safeParse({ timeline: { diff --git a/tests/toolbar-delete-button.test.ts b/tests/toolbar-delete-button.test.ts index d869fdc8..5efd39d3 100644 --- a/tests/toolbar-delete-button.test.ts +++ b/tests/toolbar-delete-button.test.ts @@ -194,12 +194,21 @@ describe("Toolbar delete button", () => { // Simulating a re-mount path: call mount() again on the same toolbar. toolbar.mount(document.body); - [EditEvent.ClipAdded, EditEvent.ClipDeleted, EditEvent.ClipRestored, EditEvent.PlaybackPause, EditEvent.EditChanged].forEach(event => { + [ + EditEvent.ClipAdded, + EditEvent.ClipDeleted, + EditEvent.ClipRestored, + EditEvent.PlaybackPause, + EditEvent.EditChanged, + EditEvent.ClipGenerationStarted, + EditEvent.ClipGenerationCompleted, + EditEvent.ClipGenerationFailed + ].forEach(event => { expect(mockEdit.events.on.mock.calls.filter(([name]) => name === event)).toHaveLength(1); }); - // Any listener beyond those five would survive a re-mount unremoved. - expect(mockEdit.events.on.mock.calls).toHaveLength(5); + // Any listener beyond those eight would survive a re-mount unremoved. + expect(mockEdit.events.on.mock.calls).toHaveLength(8); }); });