diff --git a/apps/desktop/src-tauri/src/export.rs b/apps/desktop/src-tauri/src/export.rs index 0eba1e546a..3ba63d6268 100644 --- a/apps/desktop/src-tauri/src/export.rs +++ b/apps/desktop/src-tauri/src/export.rs @@ -293,6 +293,7 @@ pub async fn generate_export_preview( .iter() .map(|s| RenderSegment { cursor: s.cursor.clone(), + keyboard: s.keyboard.clone(), decoders: s.decoders.clone(), }) .collect(); diff --git a/apps/desktop/src-tauri/src/import.rs b/apps/desktop/src-tauri/src/import.rs index c37b701bb9..2af5408a85 100644 --- a/apps/desktop/src-tauri/src/import.rs +++ b/apps/desktop/src-tauri/src/import.rs @@ -506,6 +506,7 @@ pub async fn start_video_import(app: AppHandle, source_path: PathBuf) -> Result< mic: None, system_audio: None, cursor: None, + keyboard: None, }], cursors: Cursors::default(), status: Some(StudioRecordingStatus::InProgress), @@ -599,6 +600,7 @@ pub async fn start_video_import(app: AppHandle, source_path: PathBuf) -> Result< mic: None, system_audio, cursor: None, + keyboard: None, }], cursors: Cursors::default(), status: Some(StudioRecordingStatus::Complete), diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 7497f352dc..7d82995608 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -2030,6 +2030,76 @@ async fn generate_zoom_segments_from_clicks( Ok(zoom_segments) } +fn load_keyboard_events_from_display_path( + display_path: &relative_path::RelativePathBuf, + meta: &cap_project::RecordingMeta, +) -> cap_project::keyboard::KeyboardEvents { + let Some(dir) = display_path.parent() else { + return Default::default(); + }; + let kb_path = dir.join("keyboard.json"); + let full = meta.path(&kb_path); + if full.exists() { + cap_project::keyboard::KeyboardEvents::load_from_file(&full).unwrap_or_default() + } else { + Default::default() + } +} + +#[tauri::command] +#[specta::specta] +#[instrument(skip(editor_instance))] +async fn generate_keyboard_segments( + editor_instance: WindowEditorInstance, + grouping_threshold_ms: f64, + display_duration_ms: f64, + show_modifiers: bool, + show_special_keys: bool, +) -> Result, String> { + let meta = editor_instance.meta(); + + let RecordingMetaInner::Studio(studio_meta) = &meta.inner else { + return Ok(vec![]); + }; + + let segments = match studio_meta.as_ref() { + StudioRecordingMeta::MultipleSegments { inner, .. } => &inner.segments, + StudioRecordingMeta::SingleSegment { segment } => { + let events = load_keyboard_events_from_display_path(&segment.display.path, meta); + return Ok(cap_project::keyboard::group_key_events( + &events, + grouping_threshold_ms, + display_duration_ms, + show_modifiers, + show_special_keys, + )); + } + }; + + let mut all_events = cap_project::keyboard::KeyboardEvents { presses: vec![] }; + + for segment in segments { + let events = segment.keyboard_events(meta); + all_events.presses.extend(events.presses); + } + + all_events.presses.sort_by(|a, b| { + a.time_ms + .partial_cmp(&b.time_ms) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + let grouped = cap_project::keyboard::group_key_events( + &all_events, + grouping_threshold_ms, + display_duration_ms, + show_modifiers, + show_special_keys, + ); + + Ok(grouped) +} + #[tauri::command] #[specta::specta] #[instrument] @@ -3019,6 +3089,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) { set_project_config, update_project_config_in_memory, generate_zoom_segments_from_clicks, + generate_keyboard_segments, permissions::open_permission_settings, permissions::do_permissions_check, permissions::request_permission, diff --git a/apps/desktop/src-tauri/src/recording.rs b/apps/desktop/src-tauri/src/recording.rs index d65bce27fc..959aba7684 100644 --- a/apps/desktop/src-tauri/src/recording.rs +++ b/apps/desktop/src-tauri/src/recording.rs @@ -2370,6 +2370,7 @@ fn project_config_from_recording( scene_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), + keyboard_segments: Vec::new(), }); config diff --git a/apps/desktop/src/routes/editor/ConfigSidebar.tsx b/apps/desktop/src/routes/editor/ConfigSidebar.tsx index 8765313716..c5814a702c 100644 --- a/apps/desktop/src/routes/editor/ConfigSidebar.tsx +++ b/apps/desktop/src/routes/editor/ConfigSidebar.tsx @@ -65,7 +65,12 @@ import IconLucideTimer from "~icons/lucide/timer"; import IconLucideType from "~icons/lucide/type"; import IconLucideWind from "~icons/lucide/wind"; import { CaptionsTab } from "./CaptionsTab"; -import { type CornerRoundingType, useEditorContext } from "./context"; +import { + type CornerRoundingType, + type KeyboardTrackSegment, + useEditorContext, +} from "./context"; +import { KeyboardTab } from "./KeyboardTab"; import { evaluateMask, type MaskKind, type MaskSegment } from "./masks"; import { DEFAULT_GRADIENT_FROM, @@ -79,6 +84,7 @@ import { ComingSoonTooltip, EditorButton, Field, + Input, MenuItem, MenuItemList, PopperContent, @@ -370,7 +376,8 @@ export function ConfigSidebar() { | "audio" | "cursor" | "hotkeys" - | "captions", + | "captions" + | "keyboard", }); let scrollRef!: HTMLDivElement; @@ -403,6 +410,10 @@ export function ConfigSidebar() { id: "captions" as const, icon: IconCapMessageBubble, }, + { + id: "keyboard" as const, + icon: IconLucideKeyboard, + }, // { id: "hotkeys" as const, icon: IconCapHotkeys }, ].filter(Boolean)} > @@ -819,6 +830,12 @@ export function ConfigSidebar() { > + + +
)} + { + const kbSelection = selection(); + if (kbSelection.type !== "keyboard") return; + + const segments = kbSelection.indices + .map((idx) => ({ + segment: project.timeline?.keyboardSegments?.[idx], + index: idx, + })) + .filter( + ( + s, + ): s is { + index: number; + segment: KeyboardTrackSegment; + } => s.segment !== undefined, + ); + + if (segments.length === 0) { + setEditorState("timeline", "selection", null); + return; + } + return { selection: kbSelection, segments }; + })()} + > + {(value) => ( +
+
+
+ + setEditorState("timeline", "selection", null) + } + leftIcon={} + > + Done + + + {value().segments.length} keyboard{" "} + {value().segments.length === 1 + ? "segment" + : "segments"}{" "} + selected + +
+ + projectActions.deleteKeyboardSegments( + value().segments.map((s) => s.index), + ) + } + leftIcon={} + > + Delete + +
+ + {(item) => ( + + )} + +
+ )} + )} @@ -3729,4 +3815,95 @@ function hexToRgb(hex: string): [number, number, number, number] | null { return [...rgb, 255]; } +function KeyboardSegmentConfig(props: { + segment: KeyboardTrackSegment; + segmentIndex: number; +}) { + const { project, setProject } = useEditorContext(); + + const getFadeDuration = () => { + const settings = project?.keyboard?.settings; + return settings?.fadeDurationSecs ?? 0.15; + }; + + return ( +
+ + + setProject( + "timeline", + "keyboardSegments", + props.segmentIndex, + "displayText", + e.target.value, + ) + } + /> + + + + setProject( + "timeline", + "keyboardSegments", + props.segmentIndex, + "start", + Number.parseFloat(e.target.value), + ) + } + /> + + + + setProject( + "timeline", + "keyboardSegments", + props.segmentIndex, + "end", + Number.parseFloat(e.target.value), + ) + } + /> + + + + setProject( + "timeline", + "keyboardSegments", + props.segmentIndex, + "fadeDurationOverride", + v[0] / 100, + ) + } + minValue={0} + maxValue={50} + step={1} + /> + + {( + (props.segment.fadeDurationOverride ?? getFadeDuration()) * 1000 + ).toFixed(0)} + ms + + +
+ ); +} + const CHECKERED_BUTTON_BACKGROUND = `url("data:image/svg+xml,%3Csvg width='16' height='16' xmlns='http://www.w3.org/2000/svg'%3E%3Crect width='8' height='8' fill='%23a0a0a0'/%3E%3Crect x='8' y='8' width='8' height='8' fill='%23a0a0a0'/%3E%3C/svg%3E")`; diff --git a/apps/desktop/src/routes/editor/KeyboardTab.tsx b/apps/desktop/src/routes/editor/KeyboardTab.tsx new file mode 100644 index 0000000000..75062ccb2c --- /dev/null +++ b/apps/desktop/src/routes/editor/KeyboardTab.tsx @@ -0,0 +1,449 @@ +import { Button } from "@cap/ui-solid"; +import { Select as KSelect } from "@kobalte/core/select"; +import { cx } from "cva"; +import { createEffect, createMemo, createSignal, on, Show } from "solid-js"; +import toast from "solid-toast"; +import { Toggle } from "~/components/Toggle"; +import { + defaultKeyboardSettings, + type KeyboardSettings, +} from "~/store/keyboard"; +import { commands } from "~/utils/tauri"; +import IconCapChevronDown from "~icons/cap/chevron-down"; +import { useEditorContext } from "./context"; +import { + Field, + Input, + MenuItem, + MenuItemList, + PopperContent, + Slider, + Subfield, + topSlideAnimateClasses, +} from "./ui"; + +const FONT_FAMILY_OPTIONS = [ + { label: "Monospace", value: "System Monospace" }, + { label: "Sans-Serif", value: "System Sans-Serif" }, + { label: "Serif", value: "System Serif" }, +]; + +const FONT_WEIGHT_OPTIONS = [ + { label: "Normal", value: 400 }, + { label: "Medium", value: 500 }, + { label: "Bold", value: 700 }, +]; + +export function KeyboardTab() { + const { project, setProject, editorState, setEditorState } = + useEditorContext(); + + const getSetting = ( + key: K, + ): NonNullable => { + const value = project?.keyboard?.settings?.[key]; + return value ?? defaultKeyboardSettings[key]; + }; + + const updateSetting = ( + key: K, + value: KeyboardSettings[K], + ) => { + if (!project?.keyboard) { + setProject("keyboard", { + settings: { ...defaultKeyboardSettings, [key]: value }, + }); + return; + } + setProject("keyboard", "settings", key, value); + }; + + const hasKeyboardSegments = createMemo( + () => (project.timeline?.keyboardSegments?.length ?? 0) > 0, + ); + + const [isGenerating, setIsGenerating] = createSignal(false); + + const generateSegments = async () => { + setIsGenerating(true); + try { + const segments = await commands.generateKeyboardSegments( + getSetting("groupingThresholdMs"), + getSetting("displayDurationSecs") * 1000, + getSetting("showModifiers"), + getSetting("showSpecialKeys"), + ); + + if (segments.length > 0) { + setProject("timeline", "keyboardSegments", segments); + setEditorState("timeline", "tracks", "keyboard", true); + } + } catch (e) { + console.error("Failed to generate keyboard segments:", e); + toast.error( + `Failed to generate keyboard segments: ${e instanceof Error ? e.message : String(e)}`, + ); + } finally { + setIsGenerating(false); + } + }; + + const groupingParams = createMemo(() => ({ + threshold: getSetting("groupingThresholdMs"), + display: getSetting("displayDurationSecs"), + modifiers: getSetting("showModifiers"), + special: getSetting("showSpecialKeys"), + })); + + createEffect( + on( + groupingParams, + () => { + if (hasKeyboardSegments()) { + generateSegments(); + } + }, + { defer: true }, + ), + ); + + const selectedSegment = createMemo(() => { + const selection = editorState.timeline.selection; + if (selection?.type !== "keyboard" || selection.indices.length !== 1) + return null; + return project.timeline?.keyboardSegments?.[selection.indices[0]] ?? null; + }); + + const selectedIndex = createMemo(() => { + const selection = editorState.timeline.selection; + if (selection?.type !== "keyboard" || selection.indices.length !== 1) + return -1; + return selection.indices[0]; + }); + + return ( + }> +
+ + updateSetting("enabled", checked)} + /> + + +
+ }> +
+ + o.value === getSetting("font"), + ) ?? FONT_FAMILY_OPTIONS[0] + } + onChange={(value) => { + if (!value) return; + updateSetting("font", value.value); + }} + itemComponent={(selectItemProps) => ( + + as={KSelect.Item} + item={selectItemProps.item} + > + + {selectItemProps.item.rawValue.label} + + + )} + > + + class="truncate"> + {(state) => state.selectedOption()?.label ?? "Monospace"} + + + + + + + + as={KSelect.Content} + class={cx(topSlideAnimateClasses, "z-50")} + > + + class="overflow-y-auto max-h-40" + as={KSelect.Listbox} + /> + + + + + + + o.value === getSetting("fontWeight"), + ) ?? FONT_WEIGHT_OPTIONS[1] + } + onChange={(value) => { + if (!value) return; + updateSetting("fontWeight", value.value); + }} + itemComponent={(selectItemProps) => ( + + as={KSelect.Item} + item={selectItemProps.item} + > + + {selectItemProps.item.rawValue.label} + + + )} + > + + class="truncate"> + {(state) => state.selectedOption()?.label ?? "Medium"} + + + + + + + + as={KSelect.Content} + class={cx(topSlideAnimateClasses, "z-50")} + > + + class="overflow-y-auto max-h-40" + as={KSelect.Listbox} + /> + + + + + +
+ Size + updateSetting("size", v[0])} + minValue={12} + maxValue={72} + step={1} + /> +
+ +
+ Background Opacity + updateSetting("backgroundOpacity", v[0])} + minValue={0} + maxValue={100} + step={1} + /> +
+
+ + + }> +
+
+ Fade Duration + + updateSetting("fadeDurationSecs", v[0] / 100) + } + minValue={0} + maxValue={50} + step={1} + /> + + {(getSetting("fadeDurationSecs") * 1000).toFixed(0)}ms + +
+ +
+ Display Duration + + updateSetting("displayDurationSecs", v[0] / 100) + } + minValue={0} + maxValue={300} + step={5} + /> + + {(getSetting("displayDurationSecs") * 1000).toFixed(0)}ms + +
+ +
+ Grouping Threshold + updateSetting("groupingThresholdMs", v[0])} + minValue={50} + maxValue={1000} + step={10} + /> + + {getSetting("groupingThresholdMs").toFixed(0)}ms + +
+
+ + + }> +
+
+
+ Show Modifier Keys + + updateSetting("showModifiers", checked) + } + /> +
+
+ +
+
+ Show Special Keys + + updateSetting("showSpecialKeys", checked) + } + /> +
+
+
+ + +
+ +
+ + + {(seg) => ( + }> +
+ + + setProject( + "timeline", + "keyboardSegments", + selectedIndex(), + "start", + Number.parseFloat(e.target.value), + ) + } + /> + + + + setProject( + "timeline", + "keyboardSegments", + selectedIndex(), + "end", + Number.parseFloat(e.target.value), + ) + } + /> + + + + setProject( + "timeline", + "keyboardSegments", + selectedIndex(), + "displayText", + e.target.value, + ) + } + /> + + + + setProject( + "timeline", + "keyboardSegments", + selectedIndex(), + "fadeDurationOverride", + v[0] / 100, + ) + } + minValue={0} + maxValue={50} + step={1} + /> + +
+ + )} + + + +
+

No keyboard segments yet.

+

+ Click "Generate Keyboard Segments" to create segments from + recorded keyboard presses. +

+
+ +
+
+ + ); +} diff --git a/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx b/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx new file mode 100644 index 0000000000..67a390d90b --- /dev/null +++ b/apps/desktop/src/routes/editor/Timeline/KeyboardTrack.tsx @@ -0,0 +1,272 @@ +import { createEventListenerMap } from "@solid-primitives/event-listener"; +import { cx } from "cva"; +import { createMemo, createRoot, For } from "solid-js"; + +import { MIN_KEYBOARD_SEGMENT_SECS } from "~/store/keyboard"; +import { useEditorContext } from "../context"; +import { useTimelineContext } from "./context"; +import { SegmentContent, SegmentHandle, SegmentRoot, TrackRoot } from "./Track"; + +export type KeyboardSegmentDragState = + | { type: "idle" } + | { type: "movePending" } + | { type: "moving" }; + +const MIN_SEGMENT_PIXELS = 30; +const KEYBOARD_FALLBACK_DISPLAY = "\u2328"; + +export function KeyboardTrack(props: { + onDragStateChanged: (v: KeyboardSegmentDragState) => void; + handleUpdatePlayhead: (e: MouseEvent) => void; +}) { + const { + project, + setProject, + editorState, + setEditorState, + totalDuration, + projectHistory, + projectActions, + } = useEditorContext(); + const { secsPerPixel } = useTimelineContext(); + + const minDuration = () => + Math.max(MIN_KEYBOARD_SEGMENT_SECS, secsPerPixel() * MIN_SEGMENT_PIXELS); + + const keyboardSegments = () => project.timeline?.keyboardSegments ?? []; + + const neighborBounds = (index: number) => { + const segments = keyboardSegments(); + return { + prevEnd: segments[index - 1]?.end ?? 0, + nextStart: segments[index + 1]?.start ?? totalDuration(), + }; + }; + + function createMouseDownDrag( + segmentIndex: () => number, + setup: () => T, + update: (e: MouseEvent, value: T, initialMouseX: number) => void, + ) { + return (downEvent: MouseEvent) => { + if (editorState.timeline.interactMode !== "seek") return; + downEvent.stopPropagation(); + const initial = setup(); + let moved = false; + let initialMouseX: number | null = null; + + const resumeHistory = projectHistory.pause(); + props.onDragStateChanged({ type: "movePending" }); + + function finish(e: MouseEvent) { + resumeHistory(); + if (!moved) { + e.stopPropagation(); + const index = segmentIndex(); + const isMultiSelect = e.ctrlKey || e.metaKey; + + if (isMultiSelect) { + const currentSelection = editorState.timeline.selection; + if (currentSelection?.type === "keyboard") { + const base = currentSelection.indices; + const exists = base.includes(index); + const next = exists + ? base.filter((i) => i !== index) + : [...base, index]; + setEditorState( + "timeline", + "selection", + next.length > 0 ? { type: "keyboard", indices: next } : null, + ); + } else { + setEditorState("timeline", "selection", { + type: "keyboard", + indices: [index], + }); + } + } else { + setEditorState("timeline", "selection", { + type: "keyboard", + indices: [index], + }); + } + props.handleUpdatePlayhead(e); + } + props.onDragStateChanged({ type: "idle" }); + } + + function handleUpdate(event: MouseEvent) { + if (Math.abs(event.clientX - downEvent.clientX) > 2) { + if (!moved) { + moved = true; + initialMouseX = event.clientX; + props.onDragStateChanged({ type: "moving" }); + } + } + if (initialMouseX === null) return; + update(event, initial, initialMouseX); + } + + createRoot((dispose) => { + createEventListenerMap(window, { + mousemove: (e) => handleUpdate(e), + mouseup: (e) => { + handleUpdate(e); + finish(e); + dispose(); + }, + }); + }); + }; + } + + return ( + + setEditorState("timeline", "hoveredTrack", "keyboard") + } + onMouseLeave={() => setEditorState("timeline", "hoveredTrack", null)} + > + +
No keyboard events
+
+ Record keyboard presses or generate from recording +
+
+ } + > + {(segment, i) => { + const isSelected = createMemo(() => { + const selection = editorState.timeline.selection; + if (!selection || selection.type !== "keyboard") return false; + return selection.indices.includes(i()); + }); + + const segmentDuration = () => segment.end - segment.start; + + return ( + { + e.stopPropagation(); + if (editorState.timeline.interactMode === "split") { + const rect = e.currentTarget.getBoundingClientRect(); + const fraction = (e.clientX - rect.left) / rect.width; + const splitTime = fraction * segmentDuration(); + projectActions.splitKeyboardSegment(i(), splitTime); + } + }} + > + { + const bounds = neighborBounds(i()); + const start = segment.start; + const minValue = bounds.prevEnd; + const maxValue = Math.max( + minValue, + Math.min( + segment.end - minDuration(), + bounds.nextStart - minDuration(), + ), + ); + return { start, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const delta = (e.clientX - initialMouseX) * secsPerPixel(); + const next = Math.max( + value.minValue, + Math.min(value.maxValue, value.start + delta), + ); + setProject( + "timeline", + "keyboardSegments", + i(), + "start", + next, + ); + }, + )} + /> + { + const original = { ...segment }; + const bounds = neighborBounds(i()); + const minDelta = bounds.prevEnd - original.start; + const maxDelta = bounds.nextStart - original.end; + return { original, minDelta, maxDelta }; + }, + (e, value, initialMouseX) => { + const delta = (e.clientX - initialMouseX) * secsPerPixel(); + const lowerBound = Math.min(value.minDelta, value.maxDelta); + const upperBound = Math.max(value.minDelta, value.maxDelta); + const clampedDelta = Math.min( + upperBound, + Math.max(lowerBound, delta), + ); + setProject("timeline", "keyboardSegments", i(), { + ...value.original, + start: value.original.start + clampedDelta, + end: value.original.end + clampedDelta, + }); + }, + )} + > +
+
+ + {segment.displayText || KEYBOARD_FALLBACK_DISPLAY} + +
+
+ + { + const bounds = neighborBounds(i()); + const end = segment.end; + const minValue = segment.start + minDuration(); + const maxValue = Math.max(minValue, bounds.nextStart); + return { end, minValue, maxValue }; + }, + (e, value, initialMouseX) => { + const delta = (e.clientX - initialMouseX) * secsPerPixel(); + const next = Math.max( + value.minValue, + Math.min(value.maxValue, value.end + delta), + ); + setProject( + "timeline", + "keyboardSegments", + i(), + "end", + next, + ); + }, + )} + /> + + ); + }} + + + ); +} diff --git a/apps/desktop/src/routes/editor/Timeline/index.tsx b/apps/desktop/src/routes/editor/Timeline/index.tsx index 8aacea955f..17ff01b9b5 100644 --- a/apps/desktop/src/routes/editor/Timeline/index.tsx +++ b/apps/desktop/src/routes/editor/Timeline/index.tsx @@ -20,6 +20,7 @@ import { FPS, type TimelineTrackType, useEditorContext } from "../context"; import { formatTime } from "../utils"; import { ClipTrack } from "./ClipTrack"; import { TimelineContextProvider, useTimelineContext } from "./context"; +import { type KeyboardSegmentDragState, KeyboardTrack } from "./KeyboardTrack"; import { type MaskSegmentDragState, MaskTrack } from "./MaskTrack"; import { type SceneSegmentDragState, SceneTrack } from "./SceneTrack"; import { type TextSegmentDragState, TextTrack } from "./TextTrack"; @@ -36,6 +37,8 @@ const trackIcons: Record = { mask: , zoom: , scene: , + caption: , + keyboard: , }; type TrackDefinition = { @@ -52,6 +55,12 @@ const trackDefinitions: TrackDefinition[] = [ icon: trackIcons.clip, locked: true, }, + { + type: "keyboard", + label: "Keyboard", + icon: trackIcons.keyboard, + locked: false, + }, { type: "text", label: "Text", @@ -112,7 +121,9 @@ export function Timeline() { ? trackState().mask : definition.type === "text" ? trackState().text - : true, + : definition.type === "keyboard" + ? trackState().keyboard + : true, available: definition.type === "scene" ? sceneAvailable() : true, })); const sceneTrackVisible = () => trackState().scene && sceneAvailable(); @@ -120,6 +131,7 @@ export function Timeline() { 2 + (trackState().text ? 1 : 0) + (trackState().mask ? 1 : 0) + + (trackState().keyboard ? 1 : 0) + (sceneTrackVisible() ? 1 : 0); const trackHeight = () => (visibleTrackCount() > 2 ? "3rem" : "3.25rem"); @@ -142,6 +154,14 @@ export function Timeline() { if (!next && editorState.timeline.selection?.type === "mask") { setEditorState("timeline", "selection", null); } + return; + } + + if (type === "keyboard") { + setEditorState("timeline", "tracks", "keyboard", next); + if (!next && editorState.timeline.selection?.type === "keyboard") { + setEditorState("timeline", "selection", null); + } } } @@ -160,6 +180,7 @@ export function Timeline() { sceneSegments: [], maskSegments: [], textSegments: [], + keyboardSegments: [], }); resume(); } @@ -201,11 +222,13 @@ export function Timeline() { sceneSegments: [], maskSegments: [], textSegments: [], + keyboardSegments: [], }; project.timeline.sceneSegments ??= []; project.timeline.maskSegments ??= []; project.timeline.textSegments ??= []; project.timeline.zoomSegments ??= []; + project.timeline.keyboardSegments ??= []; }), ); } @@ -214,6 +237,7 @@ export function Timeline() { let sceneSegmentDragState = { type: "idle" } as SceneSegmentDragState; let maskSegmentDragState = { type: "idle" } as MaskSegmentDragState; let textSegmentDragState = { type: "idle" } as TextSegmentDragState; + let keyboardSegmentDragState = { type: "idle" } as KeyboardSegmentDragState; let pendingZoomDelta = 0; let pendingZoomOrigin: number | null = null; @@ -272,7 +296,8 @@ export function Timeline() { zoomSegmentDragState.type !== "moving" && sceneSegmentDragState.type !== "moving" && maskSegmentDragState.type !== "moving" && - textSegmentDragState.type !== "moving" + textSegmentDragState.type !== "moving" && + keyboardSegmentDragState.type !== "moving" ) { // Guard against missing bounds and clamp computed time to [0, totalDuration()] if (left == null) return; @@ -326,6 +351,8 @@ export function Timeline() { projectActions.deleteMaskSegments(selection.indices); } else if (selection.type === "text") { projectActions.deleteTextSegments(selection.indices); + } else if (selection.type === "keyboard") { + projectActions.deleteKeyboardSegments(selection.indices); } else if (selection.type === "clip") { // Delete all selected clips in reverse order [...selection.indices] @@ -535,6 +562,16 @@ export function Timeline() { handleUpdatePlayhead={handleUpdatePlayhead} /> + + + { + keyboardSegmentDragState = v; + }} + handleUpdatePlayhead={handleUpdatePlayhead} + /> + + = T & { roundingType: CornerRoundingType }; +export type KeyboardTrackSegment = { + id: string; + start: number; + end: number; + displayText: string; + keys?: Array<{ key: string; timeOffsetMs: number }>; + fadeDurationOverride?: number | null; + positionOverride?: string | null; + colorOverride?: string | null; + backgroundColorOverride?: string | null; + fontSizeOverride?: number | null; +}; + type EditorTimelineConfiguration = Omit< TimelineConfiguration, "sceneSegments" | "maskSegments" @@ -111,6 +132,7 @@ type EditorTimelineConfiguration = Omit< sceneSegments?: SceneSegment[]; maskSegments: MaskSegment[]; textSegments: TextSegment[]; + keyboardSegments: KeyboardTrackSegment[]; }; export type EditorProjectConfiguration = Omit< @@ -155,6 +177,12 @@ export function normalizeProject( textSegments?: TextSegment[]; } ).textSegments ?? [], + keyboardSegments: + ( + config.timeline as TimelineConfiguration & { + keyboardSegments?: KeyboardTrackSegment[]; + } + ).keyboardSegments ?? [], } : undefined; @@ -179,6 +207,7 @@ export function serializeProjectConfiguration( ...project.timeline, maskSegments: project.timeline.maskSegments ?? [], textSegments: project.timeline.textSegments ?? [], + keyboardSegments: project.timeline.keyboardSegments ?? [], } : project.timeline; @@ -417,6 +446,77 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( setEditorState("timeline", "selection", null); }); }, + splitKeyboardSegment: (index: number, time: number) => { + setProject( + "timeline", + "keyboardSegments", + produce((segments) => { + const segment = segments?.[index]; + if (!segment) return; + + const duration = segment.end - segment.start; + const remaining = duration - time; + if ( + time < MIN_KEYBOARD_SEGMENT_SECS || + remaining < MIN_KEYBOARD_SEGMENT_SECS + ) + return; + + const timeMs = time * 1000; + const charIndex = segment.keys + ? segment.keys.findIndex((k) => k.timeOffsetMs >= timeMs) + : -1; + const chars = [...segment.displayText]; + + const firstDisplayText = + charIndex >= 0 + ? chars.slice(0, charIndex).join("") + : segment.displayText; + const secondDisplayText = + charIndex >= 0 ? chars.slice(charIndex).join("") : ""; + const firstKeys = + charIndex >= 0 && segment.keys + ? segment.keys.slice(0, charIndex) + : (segment.keys ?? []); + const secondKeys = + charIndex >= 0 && segment.keys + ? segment.keys + .slice(charIndex) + .map((k) => ({ ...k, timeOffsetMs: k.timeOffsetMs - timeMs })) + : []; + + segments.splice(index + 1, 0, { + ...segment, + id: crypto.randomUUID(), + start: segment.start + time, + end: segment.end, + displayText: secondDisplayText, + keys: secondKeys, + }); + segments[index].end = segment.start + time; + segments[index].displayText = firstDisplayText; + segments[index].keys = firstKeys; + }), + ); + }, + deleteKeyboardSegments: (segmentIndices: number[]) => { + batch(() => { + setProject( + "timeline", + "keyboardSegments", + produce((segments) => { + if (!segments) return; + const sorted = [...new Set(segmentIndices)] + .filter( + (i) => Number.isInteger(i) && i >= 0 && i < segments.length, + ) + .sort((a, b) => b - a); + for (const i of sorted) segments.splice(i, 1); + }), + ); + setEditorState("timeline", "selection", null); + }); + }, setClipSegmentTimescale: (index: number, timescale: number) => { setProject( produce((project) => { @@ -460,6 +560,11 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( textSegment.end += diff(textSegment.end); } + for (const keyboardSegment of timeline.keyboardSegments) { + keyboardSegment.start += diff(keyboardSegment.start); + keyboardSegment.end += diff(keyboardSegment.end); + } + segment.timescale = timescale; }), ); @@ -633,6 +738,8 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( (project.timeline?.maskSegments?.length ?? 0) > 0; const initialTextTrackEnabled = (project.timeline?.textSegments?.length ?? 0) > 0; + const initialKeyboardTrackEnabled = + (project.timeline?.keyboardSegments?.length ?? 0) > 0; const [editorState, setEditorState] = createStore({ previewTime: null as number | null, @@ -652,7 +759,9 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( | { type: "clip"; indices: number[] } | { type: "scene"; indices: number[] } | { type: "mask"; indices: number[] } - | { type: "text"; indices: number[] }, + | { type: "text"; indices: number[] } + | { type: "caption"; indices: number[] } + | { type: "keyboard"; indices: number[] }, transform: { // visible seconds zoom: zoomOutLimit(), @@ -695,6 +804,7 @@ export const [EditorContextProvider, useEditorContext] = createContextProvider( scene: true, mask: initialMaskTrackEnabled, text: initialTextTrackEnabled, + keyboard: initialKeyboardTrackEnabled, }, hoveredTrack: null as null | TimelineTrackType, }, diff --git a/apps/desktop/src/store/keyboard.ts b/apps/desktop/src/store/keyboard.ts new file mode 100644 index 0000000000..7caded8ab3 --- /dev/null +++ b/apps/desktop/src/store/keyboard.ts @@ -0,0 +1,35 @@ +export const MIN_KEYBOARD_SEGMENT_SECS = 0.3; + +export type KeyboardPosition = "above-captions" | "top" | "bottom"; + +export type KeyboardSettings = { + enabled: boolean; + font: string; + size: number; + color: string; + backgroundColor: string; + backgroundOpacity: number; + position: KeyboardPosition; + fontWeight: number; + fadeDurationSecs: number; + displayDurationSecs: number; + groupingThresholdMs: number; + showModifiers: boolean; + showSpecialKeys: boolean; +}; + +export const defaultKeyboardSettings: KeyboardSettings = { + enabled: false, + font: "System Monospace", + size: 28, + color: "#FFFFFF", + backgroundColor: "#000000", + backgroundOpacity: 85, + position: "above-captions", + fontWeight: 500, + fadeDurationSecs: 0.15, + displayDurationSecs: 0.8, + groupingThresholdMs: 300, + showModifiers: true, + showSpecialKeys: true, +}; diff --git a/crates/editor/src/editor_instance.rs b/crates/editor/src/editor_instance.rs index bf42055aa6..58443a89bf 100644 --- a/crates/editor/src/editor_instance.rs +++ b/crates/editor/src/editor_instance.rs @@ -201,6 +201,7 @@ impl EditorInstance { scene_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), + keyboard_segments: Vec::new(), }); if let Err(e) = project.write(&recording_meta.project_path) { @@ -594,6 +595,7 @@ pub struct SegmentMedia { pub audio: Option>, pub system_audio: Option>, pub cursor: Arc, + pub keyboard: Arc, pub decoders: RecordingSegmentDecoders, } @@ -651,6 +653,20 @@ pub async fn create_segments( audio, system_audio: None, cursor, + keyboard: Arc::new({ + let kb_path = s.display.path.parent().map(|d| d.join("keyboard.json")); + kb_path + .and_then(|p| { + let full = recording_meta.path(&p); + full.exists() + .then(|| { + cap_project::keyboard::KeyboardEvents::load_from_file(&full) + .ok() + }) + .flatten() + }) + .unwrap_or_default() + }), decoders, }]) } @@ -693,10 +709,13 @@ pub async fn create_segments( .await .map_err(|e| format!("MultipleSegments {i} / {e}"))?; + let keyboard = Arc::new(s.keyboard_events(recording_meta)); + segments.push(SegmentMedia { audio, system_audio, cursor, + keyboard, decoders, }); } diff --git a/crates/export/src/gif.rs b/crates/export/src/gif.rs index f25dc52531..54e38507a6 100644 --- a/crates/export/src/gif.rs +++ b/crates/export/src/gif.rs @@ -122,6 +122,7 @@ impl GifExportSettings { .iter() .map(|s| RenderSegment { cursor: s.cursor.clone(), + keyboard: s.keyboard.clone(), decoders: s.decoders.clone(), }) .collect(), diff --git a/crates/export/src/mp4.rs b/crates/export/src/mp4.rs index 4cd014069e..fcc5b78f42 100644 --- a/crates/export/src/mp4.rs +++ b/crates/export/src/mp4.rs @@ -323,6 +323,7 @@ impl Mp4ExportSettings { .iter() .map(|s| RenderSegment { cursor: s.cursor.clone(), + keyboard: s.keyboard.clone(), decoders: s.decoders.clone(), }) .collect(), diff --git a/crates/project/src/configuration.rs b/crates/project/src/configuration.rs index f2f49fbea0..be0bae51c1 100644 --- a/crates/project/src/configuration.rs +++ b/crates/project/src/configuration.rs @@ -788,6 +788,8 @@ pub struct TimelineConfiguration { pub mask_segments: Vec, #[serde(default)] pub text_segments: Vec, + #[serde(default)] + pub keyboard_segments: Vec, } impl TimelineConfiguration { @@ -934,6 +936,50 @@ pub struct CaptionsData { pub settings: CaptionSettings, } +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase", default)] +pub struct KeyboardSettings { + pub enabled: bool, + pub font: String, + pub size: u32, + pub color: String, + pub background_color: String, + pub background_opacity: u32, + pub position: String, + pub font_weight: u32, + pub fade_duration_secs: f32, + pub display_duration_secs: f32, + pub grouping_threshold_ms: f64, + pub show_modifiers: bool, + pub show_special_keys: bool, +} + +impl Default for KeyboardSettings { + fn default() -> Self { + Self { + enabled: false, + font: "System Monospace".to_string(), + size: 28, + color: "#FFFFFF".to_string(), + background_color: "#000000".to_string(), + background_opacity: 85, + position: "above-captions".to_string(), + font_weight: 500, + fade_duration_secs: 0.15, + display_duration_secs: 0.8, + grouping_threshold_ms: 300.0, + show_modifiers: true, + show_special_keys: true, + } + } +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct KeyboardData { + pub settings: KeyboardSettings, +} + #[derive(Type, Serialize, Deserialize, Clone, Copy, Debug, Default)] pub struct ClipOffsets { #[serde(default)] @@ -1083,6 +1129,7 @@ pub struct ProjectConfiguration { pub hotkeys: HotkeysConfiguration, pub timeline: Option, pub captions: Option, + pub keyboard: Option, pub clips: Vec, pub annotations: Vec, #[serde(skip_serializing)] diff --git a/crates/project/src/keyboard.rs b/crates/project/src/keyboard.rs new file mode 100644 index 0000000000..9ad0c89a81 --- /dev/null +++ b/crates/project/src/keyboard.rs @@ -0,0 +1,590 @@ +use serde::{Deserialize, Serialize}; +use specta::Type; +use std::fs::File; +use std::path::Path; + +#[derive(Serialize, Deserialize, Clone, Type, Debug, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct KeyPressEvent { + pub key: String, + pub key_code: String, + pub time_ms: f64, + pub down: bool, +} + +impl PartialOrd for KeyPressEvent { + fn partial_cmp(&self, other: &Self) -> Option { + self.time_ms.partial_cmp(&other.time_ms) + } +} + +#[derive(Default, Serialize, Deserialize, Debug, Clone, Type)] +#[serde(rename_all = "camelCase")] +pub struct KeyboardEvents { + pub presses: Vec, +} + +impl KeyboardEvents { + pub fn load_from_file(path: &Path) -> Result { + let file = + File::open(path).map_err(|e| format!("Failed to open keyboard events file: {e}"))?; + serde_json::from_reader(file).map_err(|e| format!("Failed to parse keyboard events: {e}")) + } +} + +const MODIFIER_KEYS: &[&str] = &[ + "LShift", "RShift", "LControl", "RControl", "LAlt", "RAlt", "LMeta", "RMeta", "Meta", "Command", +]; + +const SPECIAL_KEY_SYMBOLS: &[(&str, &str)] = &[ + ("Enter", "⏎"), + ("Return", "⏎"), + ("Tab", "⇥"), + ("Backspace", "⌫"), + ("Delete", "⌦"), + ("Escape", "⎋"), + ("Space", "␣"), + ("Up", "↑"), + ("Down", "↓"), + ("Left", "←"), + ("Right", "→"), + ("Home", "⇱"), + ("End", "⇲"), + ("PageUp", "⇞"), + ("PageDown", "⇟"), +]; + +fn is_modifier_key(key: &str) -> bool { + MODIFIER_KEYS.contains(&key) +} + +fn is_shift_key(key: &str) -> bool { + matches!(key, "LShift" | "RShift") +} + +fn special_key_symbol(key: &str) -> Option<&'static str> { + SPECIAL_KEY_SYMBOLS + .iter() + .find(|&&(k, _)| k == key) + .map(|&(_, symbol)| symbol) +} + +fn display_char_for_key(key: &str) -> Option { + if key.len() == 1 { + return Some(key.to_string()); + } + + if let Some(symbol) = special_key_symbol(key) { + return Some(symbol.to_string()); + } + + if is_modifier_key(key) { + return None; + } + + None +} + +fn modifier_prefix(active_modifiers: &[String]) -> String { + let mut parts = Vec::new(); + + let has = |names: &[&str]| active_modifiers.iter().any(|m| names.contains(&m.as_str())); + + if has(&["LMeta", "RMeta", "Meta", "Command"]) { + parts.push("⌘"); + } + if has(&["LControl", "RControl"]) { + parts.push("⌃"); + } + if has(&["LAlt", "RAlt"]) { + parts.push("⌥"); + } + if has(&["LShift", "RShift"]) { + parts.push("⇧"); + } + + if parts.is_empty() { + String::new() + } else { + format!("{}+", parts.join("+")) + } +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct KeyPressDisplay { + pub key: String, + pub time_offset_ms: f64, +} + +#[derive(Type, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] +pub struct KeyboardTrackSegment { + pub id: String, + pub start: f64, + pub end: f64, + pub display_text: String, + #[serde(default)] + pub keys: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fade_duration_override: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub position_override: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color_override: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub background_color_override: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub font_size_override: Option, +} + +pub fn group_key_events( + events: &KeyboardEvents, + grouping_threshold_ms: f64, + display_duration_ms: f64, + show_modifiers: bool, + show_special_keys: bool, +) -> Vec { + let mut segments: Vec = Vec::new(); + + let mut down_events: Vec<&KeyPressEvent> = events.presses.iter().filter(|e| e.down).collect(); + down_events.sort_by(|a, b| { + a.time_ms + .partial_cmp(&b.time_ms) + .unwrap_or(std::cmp::Ordering::Equal) + }); + + if down_events.is_empty() { + return segments; + } + + let mut active_modifiers: Vec = Vec::new(); + let mut all_events_idx: usize = 0; + + let advance_modifiers_to = + |time_ms: f64, active: &mut Vec, idx: &mut usize, all_presses: &[KeyPressEvent]| { + while *idx < all_presses.len() && all_presses[*idx].time_ms <= time_ms { + let ev = &all_presses[*idx]; + if is_modifier_key(&ev.key) { + if ev.down { + if !active.contains(&ev.key) { + active.push(ev.key.clone()); + } + } else { + active.retain(|k| k != &ev.key); + } + } + *idx += 1; + } + }; + + let mut current_group_start: Option = None; + let mut current_display = String::new(); + let mut current_keys: Vec = Vec::new(); + let mut last_key_time: f64 = 0.0; + let mut segment_counter: u64 = 0; + + let min_segment_ms = 300.0_f64; + + let flush_group = |segments: &mut Vec, + counter: &mut u64, + start: f64, + end_key_time: f64, + display: &mut String, + keys: &mut Vec| { + if display.is_empty() { + return; + } + *counter += 1; + let end = (end_key_time + display_duration_ms) / 1000.0; + let start_secs = start / 1000.0; + let end_secs = end.max(start_secs + min_segment_ms / 1000.0); + segments.push(KeyboardTrackSegment { + id: format!("kb-{counter}"), + start: start_secs, + end: end_secs, + display_text: display.clone(), + keys: keys.clone(), + fade_duration_override: None, + position_override: None, + color_override: None, + background_color_override: None, + font_size_override: None, + }); + display.clear(); + keys.clear(); + }; + + for event in &down_events { + advance_modifiers_to( + event.time_ms, + &mut active_modifiers, + &mut all_events_idx, + &events.presses, + ); + + let is_modifier = is_modifier_key(&event.key); + let is_shift = is_shift_key(&event.key); + + if is_modifier && !show_modifiers { + continue; + } + + if is_modifier && !is_shift { + continue; + } + + let is_special = special_key_symbol(&event.key).is_some() && event.key != "Space"; + + if is_special && !show_special_keys && event.key != "Backspace" { + continue; + } + + if event.key == "Backspace" && !current_display.is_empty() { + current_display.pop(); + last_key_time = event.time_ms; + continue; + } + + let has_command_mod = active_modifiers.iter().any(|m| { + matches!( + m.as_str(), + "LMeta" | "RMeta" | "Meta" | "Command" | "LControl" | "RControl" + ) + }); + + if has_command_mod && show_modifiers { + flush_group( + &mut segments, + &mut segment_counter, + current_group_start.unwrap_or(event.time_ms), + last_key_time, + &mut current_display, + &mut current_keys, + ); + current_group_start = None; + + let prefix = modifier_prefix(&active_modifiers); + let key_display = display_char_for_key(&event.key).unwrap_or_else(|| event.key.clone()); + let has_shift = active_modifiers.iter().any(|m| is_shift_key(m)); + let key_display = if has_shift && key_display.len() == 1 { + key_display.to_uppercase() + } else { + key_display + }; + let combo = format!("{prefix}{key_display}"); + + if let Some(prev) = segments.last_mut() { + let prev_end_ms = prev.end * 1000.0; + if event.time_ms < prev_end_ms { + prev.display_text = format!("{} {}", prev.display_text, combo); + prev.end = (event.time_ms + display_duration_ms) / 1000.0; + prev.keys.push(KeyPressDisplay { + key: event.key.clone(), + time_offset_ms: event.time_ms - prev.start * 1000.0, + }); + last_key_time = event.time_ms; + continue; + } + } + + segment_counter += 1; + let start_secs = event.time_ms / 1000.0; + let end_secs = ((event.time_ms + display_duration_ms) / 1000.0) + .max(start_secs + min_segment_ms / 1000.0); + segments.push(KeyboardTrackSegment { + id: format!("kb-{segment_counter}"), + start: start_secs, + end: end_secs, + display_text: combo, + keys: vec![KeyPressDisplay { + key: event.key.clone(), + time_offset_ms: 0.0, + }], + fade_duration_override: None, + position_override: None, + color_override: None, + background_color_override: None, + font_size_override: None, + }); + + last_key_time = event.time_ms; + continue; + } + + let should_start_new_group = current_group_start.is_none() + || (event.time_ms - last_key_time) > grouping_threshold_ms; + + if should_start_new_group && current_group_start.is_some() { + flush_group( + &mut segments, + &mut segment_counter, + current_group_start.unwrap(), + last_key_time, + &mut current_display, + &mut current_keys, + ); + current_group_start = None; + } + + if let Some(display_char) = display_char_for_key(&event.key) { + if current_group_start.is_none() { + current_group_start = Some(event.time_ms); + } + + let offset = event.time_ms - current_group_start.unwrap(); + current_display.push_str(&display_char); + current_keys.push(KeyPressDisplay { + key: event.key.clone(), + time_offset_ms: offset, + }); + last_key_time = event.time_ms; + } + } + + if let Some(start) = current_group_start { + flush_group( + &mut segments, + &mut segment_counter, + start, + last_key_time, + &mut current_display, + &mut current_keys, + ); + } + + segments +} + +#[cfg(test)] +mod tests { + use super::*; + + fn key_down(key: &str, time_ms: f64) -> KeyPressEvent { + KeyPressEvent { + key: key.to_string(), + key_code: key.to_string(), + time_ms, + down: true, + } + } + + fn key_up(key: &str, time_ms: f64) -> KeyPressEvent { + KeyPressEvent { + key: key.to_string(), + key_code: key.to_string(), + time_ms, + down: false, + } + } + + #[test] + fn groups_rapid_typing_into_word() { + let events = KeyboardEvents { + presses: vec![ + key_down("h", 100.0), + key_up("h", 150.0), + key_down("e", 200.0), + key_up("e", 250.0), + key_down("l", 300.0), + key_up("l", 350.0), + key_down("l", 400.0), + key_up("l", 450.0), + key_down("o", 500.0), + key_up("o", 550.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0].display_text, "hello"); + assert_eq!(segments[0].keys.len(), 5); + } + + #[test] + fn splits_on_long_pause() { + let events = KeyboardEvents { + presses: vec![ + key_down("h", 100.0), + key_up("h", 150.0), + key_down("i", 200.0), + key_up("i", 250.0), + key_down("b", 1000.0), + key_up("b", 1050.0), + key_down("y", 1100.0), + key_up("y", 1150.0), + key_down("e", 1200.0), + key_up("e", 1250.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert_eq!(segments.len(), 2); + assert_eq!(segments[0].display_text, "hi"); + assert_eq!(segments[1].display_text, "bye"); + } + + #[test] + fn backspace_removes_last_char() { + let events = KeyboardEvents { + presses: vec![ + key_down("h", 100.0), + key_up("h", 150.0), + key_down("e", 200.0), + key_up("e", 250.0), + key_down("Backspace", 300.0), + key_up("Backspace", 350.0), + key_down("a", 400.0), + key_up("a", 450.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0].display_text, "ha"); + } + + #[test] + fn empty_events_returns_empty() { + let events = KeyboardEvents { presses: vec![] }; + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert!(segments.is_empty()); + } + + #[test] + fn special_keys_show_symbols() { + let events = KeyboardEvents { + presses: vec![key_down("Enter", 100.0), key_up("Enter", 150.0)], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0].display_text, "⏎"); + } + + #[test] + fn modifier_combo_cmd_c() { + let events = KeyboardEvents { + presses: vec![ + key_down("LMeta", 100.0), + key_down("c", 150.0), + key_up("c", 200.0), + key_up("LMeta", 250.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert_eq!(segments.len(), 1); + assert!(segments[0].display_text.contains('⌘')); + assert!(segments[0].display_text.contains('c')); + } + + #[test] + fn shift_capitalizes_without_new_segment() { + let events = KeyboardEvents { + presses: vec![ + key_down("h", 100.0), + key_up("h", 150.0), + key_down("LShift", 200.0), + key_down("e", 250.0), + key_up("e", 300.0), + key_up("LShift", 350.0), + key_down("l", 400.0), + key_up("l", 450.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, true); + assert!(segments.len() <= 2); + } + + #[test] + fn show_modifiers_false_hides_modifier_keys() { + let events = KeyboardEvents { + presses: vec![ + key_down("LMeta", 100.0), + key_down("c", 150.0), + key_up("c", 200.0), + key_up("LMeta", 250.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, false, true); + for seg in &segments { + assert!(!seg.display_text.contains('⌘')); + } + } + + #[test] + fn show_special_keys_false_hides_special() { + let events = KeyboardEvents { + presses: vec![ + key_down("h", 100.0), + key_up("h", 150.0), + key_down("Enter", 200.0), + key_up("Enter", 250.0), + key_down("i", 400.0), + key_up("i", 450.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, false); + for seg in &segments { + assert!(!seg.display_text.contains('⏎')); + } + } + + #[test] + fn rapid_hotkeys_merge_into_single_segment() { + let events = KeyboardEvents { + presses: vec![ + key_down("LMeta", 100.0), + key_down("c", 150.0), + key_up("c", 200.0), + key_down("v", 300.0), + key_up("v", 350.0), + key_down("z", 450.0), + key_up("z", 500.0), + key_up("LMeta", 550.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 800.0, true, true); + assert_eq!(segments.len(), 1); + assert!(segments[0].display_text.contains('c')); + assert!(segments[0].display_text.contains('v')); + assert!(segments[0].display_text.contains('z')); + } + + #[test] + fn segments_have_minimum_duration() { + let events = KeyboardEvents { + presses: vec![key_down("a", 100.0), key_up("a", 110.0)], + }; + + let segments = group_key_events(&events, 300.0, 50.0, true, true); + assert_eq!(segments.len(), 1); + let duration = segments[0].end - segments[0].start; + assert!(duration >= 0.3); + } + + #[test] + fn backspace_works_with_show_special_keys_false() { + let events = KeyboardEvents { + presses: vec![ + key_down("h", 100.0), + key_up("h", 150.0), + key_down("e", 200.0), + key_up("e", 250.0), + key_down("Backspace", 300.0), + key_up("Backspace", 350.0), + key_down("a", 400.0), + key_up("a", 450.0), + ], + }; + + let segments = group_key_events(&events, 300.0, 500.0, true, false); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0].display_text, "ha"); + } +} diff --git a/crates/project/src/lib.rs b/crates/project/src/lib.rs index dd239ba255..5d4d5d2363 100644 --- a/crates/project/src/lib.rs +++ b/crates/project/src/lib.rs @@ -1,5 +1,6 @@ mod configuration; pub mod cursor; +pub mod keyboard; mod meta; pub use configuration::*; diff --git a/crates/project/src/meta.rs b/crates/project/src/meta.rs index 05248b5df4..e2f8c8e285 100644 --- a/crates/project/src/meta.rs +++ b/crates/project/src/meta.rs @@ -11,7 +11,7 @@ use tracing::{debug, info, warn}; use crate::{ CaptionsData, CursorEvents, CursorImage, ProjectConfiguration, XY, - cursor::SHORT_CURSOR_SHAPE_DEBOUNCE_MS, + cursor::SHORT_CURSOR_SHAPE_DEBOUNCE_MS, keyboard::KeyboardEvents, }; #[derive(Debug, Clone, Serialize, Deserialize, Type)] @@ -361,6 +361,9 @@ pub struct MultipleSegment { #[serde(default, skip_serializing_if = "Option::is_none")] #[specta(type = Option)] pub cursor: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[specta(type = Option)] + pub keyboard: Option, } impl MultipleSegment { @@ -378,7 +381,7 @@ impl MultipleSegment { let mut data = match CursorEvents::load_from_file(&full_path) { Ok(data) => data, Err(e) => { - eprintln!("Failed to load cursor data: {e}"); + tracing::warn!("Failed to load cursor data: {e}"); return CursorEvents::default(); } }; @@ -395,6 +398,29 @@ impl MultipleSegment { data } + pub fn keyboard_events(&self, meta: &RecordingMeta) -> KeyboardEvents { + let keyboard_path = self.keyboard.clone().or_else(|| { + let display_dir = self.display.path.parent()?; + let fallback = display_dir.join("keyboard.json"); + let full = meta.path(&fallback); + full.exists().then_some(fallback) + }); + + let Some(keyboard_path) = keyboard_path else { + return KeyboardEvents::default(); + }; + + let full_path = meta.path(&keyboard_path); + + match KeyboardEvents::load_from_file(&full_path) { + Ok(data) => data, + Err(e) => { + tracing::warn!("Failed to load keyboard data: {e}"); + KeyboardEvents::default() + } + } + } + pub fn latest_start_time(&self) -> Option { let mut value = self.display.start_time?; diff --git a/crates/recording/src/cursor.rs b/crates/recording/src/cursor.rs index 796093d035..1d32785f53 100644 --- a/crates/recording/src/cursor.rs +++ b/crates/recording/src/cursor.rs @@ -1,6 +1,9 @@ use cap_cursor_capture::CursorCropBounds; use cap_cursor_info::CursorShape; -use cap_project::{CursorClickEvent, CursorEvents, CursorMoveEvent, XY}; +use cap_project::{ + CursorClickEvent, CursorEvents, CursorMoveEvent, XY, + keyboard::{KeyPressEvent, KeyboardEvents}, +}; use cap_timestamp::Timestamps; use futures::{FutureExt, future::Shared}; use std::{ @@ -23,11 +26,11 @@ pub type Cursors = HashMap; #[derive(Clone)] pub struct CursorActorResponse { - // pub cursor_images: HashMap>, pub cursors: Cursors, pub next_cursor_id: u32, pub moves: Vec, pub clicks: Vec, + pub keyboard_presses: Vec, } pub struct CursorActor { @@ -59,6 +62,178 @@ fn flush_cursor_data(output_path: &Path, moves: &[CursorMoveEvent], clicks: &[Cu } } +fn flush_keyboard_data(output_path: &Path, presses: &[KeyPressEvent]) { + let events = KeyboardEvents { + presses: presses.to_vec(), + }; + if let Ok(json) = serde_json::to_string_pretty(&events) + && let Err(e) = std::fs::write(output_path, json) + { + tracing::error!( + "Failed to write keyboard data to {}: {}", + output_path.display(), + e + ); + } +} + +fn keycode_to_string(key: &device_query::Keycode) -> (String, String) { + use device_query::Keycode; + let (display, code) = match key { + Keycode::Key0 => ("0", "Key0"), + Keycode::Key1 => ("1", "Key1"), + Keycode::Key2 => ("2", "Key2"), + Keycode::Key3 => ("3", "Key3"), + Keycode::Key4 => ("4", "Key4"), + Keycode::Key5 => ("5", "Key5"), + Keycode::Key6 => ("6", "Key6"), + Keycode::Key7 => ("7", "Key7"), + Keycode::Key8 => ("8", "Key8"), + Keycode::Key9 => ("9", "Key9"), + Keycode::A => ("a", "A"), + Keycode::B => ("b", "B"), + Keycode::C => ("c", "C"), + Keycode::D => ("d", "D"), + Keycode::E => ("e", "E"), + Keycode::F => ("f", "F"), + Keycode::G => ("g", "G"), + Keycode::H => ("h", "H"), + Keycode::I => ("i", "I"), + Keycode::J => ("j", "J"), + Keycode::K => ("k", "K"), + Keycode::L => ("l", "L"), + Keycode::M => ("m", "M"), + Keycode::N => ("n", "N"), + Keycode::O => ("o", "O"), + Keycode::P => ("p", "P"), + Keycode::Q => ("q", "Q"), + Keycode::R => ("r", "R"), + Keycode::S => ("s", "S"), + Keycode::T => ("t", "T"), + Keycode::U => ("u", "U"), + Keycode::V => ("v", "V"), + Keycode::W => ("w", "W"), + Keycode::X => ("x", "X"), + Keycode::Y => ("y", "Y"), + Keycode::Z => ("z", "Z"), + Keycode::F1 => ("F1", "F1"), + Keycode::F2 => ("F2", "F2"), + Keycode::F3 => ("F3", "F3"), + Keycode::F4 => ("F4", "F4"), + Keycode::F5 => ("F5", "F5"), + Keycode::F6 => ("F6", "F6"), + Keycode::F7 => ("F7", "F7"), + Keycode::F8 => ("F8", "F8"), + Keycode::F9 => ("F9", "F9"), + Keycode::F10 => ("F10", "F10"), + Keycode::F11 => ("F11", "F11"), + Keycode::F12 => ("F12", "F12"), + Keycode::Escape => ("Escape", "Escape"), + Keycode::Space => ("Space", "Space"), + Keycode::LControl => ("LControl", "LControl"), + Keycode::RControl => ("RControl", "RControl"), + Keycode::LShift => ("LShift", "LShift"), + Keycode::RShift => ("RShift", "RShift"), + Keycode::LAlt => ("LAlt", "LAlt"), + Keycode::RAlt => ("RAlt", "RAlt"), + Keycode::LMeta => ("Meta", "LMeta"), + Keycode::RMeta => ("Meta", "RMeta"), + Keycode::Enter => ("Enter", "Enter"), + Keycode::Up => ("Up", "Up"), + Keycode::Down => ("Down", "Down"), + Keycode::Left => ("Left", "Left"), + Keycode::Right => ("Right", "Right"), + Keycode::Backspace => ("Backspace", "Backspace"), + Keycode::CapsLock => ("CapsLock", "CapsLock"), + Keycode::Tab => ("Tab", "Tab"), + Keycode::Home => ("Home", "Home"), + Keycode::End => ("End", "End"), + Keycode::PageUp => ("PageUp", "PageUp"), + Keycode::PageDown => ("PageDown", "PageDown"), + Keycode::Insert => ("Insert", "Insert"), + Keycode::Delete => ("Delete", "Delete"), + Keycode::Numpad0 => ("0", "Numpad0"), + Keycode::Numpad1 => ("1", "Numpad1"), + Keycode::Numpad2 => ("2", "Numpad2"), + Keycode::Numpad3 => ("3", "Numpad3"), + Keycode::Numpad4 => ("4", "Numpad4"), + Keycode::Numpad5 => ("5", "Numpad5"), + Keycode::Numpad6 => ("6", "Numpad6"), + Keycode::Numpad7 => ("7", "Numpad7"), + Keycode::Numpad8 => ("8", "Numpad8"), + Keycode::Numpad9 => ("9", "Numpad9"), + Keycode::NumpadSubtract => ("-", "NumpadSubtract"), + Keycode::NumpadAdd => ("+", "NumpadAdd"), + Keycode::NumpadDivide => ("/", "NumpadDivide"), + Keycode::NumpadMultiply => ("*", "NumpadMultiply"), + Keycode::Grave => ("`", "Grave"), + Keycode::Minus => ("-", "Minus"), + Keycode::Equal => ("=", "Equal"), + Keycode::LeftBracket => ("[", "LeftBracket"), + Keycode::RightBracket => ("]", "RightBracket"), + Keycode::BackSlash => ("\\", "BackSlash"), + Keycode::Semicolon => (";", "Semicolon"), + Keycode::Apostrophe => ("'", "Apostrophe"), + Keycode::Comma => (",", "Comma"), + Keycode::Dot => (".", "Dot"), + Keycode::Slash => ("/", "Slash"), + _ => { + let s = format!("{key:?}"); + return (s.clone(), s); + } + }; + (display.to_string(), code.to_string()) +} + +const SHIFT_SYMBOL_MAP: &[(char, char)] = &[ + ('1', '!'), + ('2', '@'), + ('3', '#'), + ('4', '$'), + ('5', '%'), + ('6', '^'), + ('7', '&'), + ('8', '*'), + ('9', '('), + ('0', ')'), + ('-', '_'), + ('=', '+'), + ('[', '{'), + (']', '}'), + ('\\', '|'), + (';', ':'), + ('\'', '"'), + (',', '<'), + ('.', '>'), + ('/', '?'), + ('`', '~'), +]; + +fn apply_case_to_display(display: &str, shift_active: bool, caps_lock_active: bool) -> String { + if display.len() != 1 { + return display.to_string(); + } + let ch = display.chars().next().unwrap(); + + if ch.is_ascii_lowercase() { + let uppercase = shift_active ^ caps_lock_active; + return if uppercase { + ch.to_ascii_uppercase().to_string() + } else { + display.to_string() + }; + } + + if shift_active + && let Some(&(_, shifted)) = SHIFT_SYMBOL_MAP.iter().find(|&&(from, _)| from == ch) + { + return shifted.to_string(); + } + + display.to_string() +} + +#[allow(clippy::too_many_arguments)] #[tracing::instrument(name = "cursor", skip_all)] pub fn spawn_cursor_recorder( crop_bounds: CursorCropBounds, @@ -68,6 +243,7 @@ pub fn spawn_cursor_recorder( next_cursor_id: u32, start_time: Timestamps, output_path: Option, + keyboard_output_path: Option, ) -> CursorActor { use cap_utils::spawn_actor; use device_query::{DeviceQuery, DeviceState}; @@ -83,6 +259,7 @@ pub fn spawn_cursor_recorder( spawn_actor(async move { let device_state = DeviceState::new(); let mut last_mouse_state = device_state.get_mouse(); + let mut last_keys: Vec = device_state.get_keys(); let mut last_position = cap_cursor_capture::RawCursorPosition::get(); @@ -93,11 +270,13 @@ pub fn spawn_cursor_recorder( next_cursor_id, moves: vec![], clicks: vec![], + keyboard_presses: vec![], }; let mut last_flush = Instant::now(); let flush_interval = Duration::from_secs(CURSOR_FLUSH_INTERVAL_SECS); let mut last_cursor_id: Option = None; + let mut caps_lock_active = false; loop { let sleep = tokio::time::sleep(Duration::from_millis(16)); @@ -203,10 +382,58 @@ pub fn spawn_cursor_recorder( last_mouse_state = mouse_state; + let current_keys = device_state.get_keys(); + let shift_active = current_keys.iter().any(|k| { + matches!( + k, + device_query::Keycode::LShift | device_query::Keycode::RShift + ) + }); + + let caps_just_pressed = current_keys + .iter() + .any(|k| matches!(k, device_query::Keycode::CapsLock)) + && !last_keys + .iter() + .any(|k| matches!(k, device_query::Keycode::CapsLock)); + if caps_just_pressed { + caps_lock_active = !caps_lock_active; + } + + for key in ¤t_keys { + if !last_keys.contains(key) { + let (display, code) = keycode_to_string(key); + let display = apply_case_to_display(&display, shift_active, caps_lock_active); + response.keyboard_presses.push(KeyPressEvent { + key: display, + key_code: code, + time_ms: elapsed, + down: true, + }); + } + } + + for key in &last_keys { + if !current_keys.contains(key) { + let (display, code) = keycode_to_string(key); + response.keyboard_presses.push(KeyPressEvent { + key: display, + key_code: code, + time_ms: elapsed, + down: false, + }); + } + } + + last_keys = current_keys; + if let Some(ref path) = output_path && last_flush.elapsed() >= flush_interval { flush_cursor_data(path, &response.moves, &response.clicks); + if let Some(ref kb_path) = keyboard_output_path { + flush_keyboard_data(kb_path, &response.keyboard_presses); + } last_flush = Instant::now(); } } @@ -217,6 +444,10 @@ pub fn spawn_cursor_recorder( flush_cursor_data(path, &response.moves, &response.clicks); } + if let Some(ref kb_path) = keyboard_output_path { + flush_keyboard_data(kb_path, &response.keyboard_presses); + } + let _ = tx.send(response); }); diff --git a/crates/recording/src/recovery.rs b/crates/recording/src/recovery.rs index 08371848c0..56848fadbf 100644 --- a/crates/recording/src/recovery.rs +++ b/crates/recording/src/recovery.rs @@ -33,6 +33,7 @@ pub struct RecoverableSegment { pub mic_fragments: Option>, pub system_audio_fragments: Option>, pub cursor_path: Option, + pub keyboard_path: Option, } #[derive(Debug, Clone)] @@ -214,6 +215,7 @@ impl RecoveryManager { } let cursor_path = Self::probe_cursor(&segment_path.join("cursor.json")); + let keyboard_path = Self::probe_cursor(&segment_path.join("keyboard.json")); recoverable_segments.push(RecoverableSegment { index, @@ -224,6 +226,7 @@ impl RecoveryManager { mic_fragments, system_audio_fragments, cursor_path, + keyboard_path, }); } @@ -884,6 +887,13 @@ impl RecoveryManager { } else { None }, + keyboard: if seg.keyboard_path.is_some() { + Some(RelativePathBuf::from(format!( + "{segment_base}/keyboard.json" + ))) + } else { + None + }, } }) .collect(); @@ -952,6 +962,7 @@ impl RecoveryManager { scene_segments: Vec::new(), mask_segments: Vec::new(), text_segments: Vec::new(), + keyboard_segments: Vec::new(), }); config diff --git a/crates/recording/src/studio_recording.rs b/crates/recording/src/studio_recording.rs index 4d575f6782..601b065a9f 100644 --- a/crates/recording/src/studio_recording.rs +++ b/crates/recording/src/studio_recording.rs @@ -100,6 +100,15 @@ impl Actor { })?, )?; + if !res.keyboard_presses.is_empty() { + std::fs::write( + &cursor.keyboard_output_path, + serde_json::to_string_pretty(&cap_project::keyboard::KeyboardEvents { + presses: res.keyboard_presses, + })?, + )?; + } + (res.cursors, res.next_cursor_id) } else { (Default::default(), 0) @@ -419,6 +428,7 @@ impl Pipeline { struct CursorPipeline { output_path: PathBuf, + keyboard_output_path: PathBuf, actor: CursorActor, } @@ -790,6 +800,12 @@ async fn stop_recording( .cursor .as_ref() .map(|cursor| make_relative(&cursor.output_path)), + keyboard: s + .pipeline + .cursor + .as_ref() + .filter(|cursor| cursor.keyboard_output_path.exists()) + .map(|cursor| make_relative(&cursor.keyboard_output_path)), } }) .collect::>() @@ -1226,11 +1242,17 @@ async fn create_segment_pipeline( .ok_or(CreateSegmentPipelineError::NoBounds)?; let cursor_output_path = dir.join("cursor.json"); + let keyboard_output_path = dir.join("keyboard.json"); let incremental_output = if fragmented { Some(cursor_output_path.clone()) } else { None }; + let keyboard_incremental_output = if fragmented { + Some(keyboard_output_path.clone()) + } else { + None + }; let cursor_display = cursor_display.ok_or(CreateSegmentPipelineError::NoDisplay)?; @@ -1242,10 +1264,12 @@ async fn create_segment_pipeline( next_cursors_id, start_time, incremental_output, + keyboard_incremental_output, ); Ok::<_, CreateSegmentPipelineError>(CursorPipeline { output_path: cursor_output_path, + keyboard_output_path, actor: cursor, }) }) diff --git a/crates/rendering/src/layers/keyboard.rs b/crates/rendering/src/layers/keyboard.rs new file mode 100644 index 0000000000..c1ae5e792e --- /dev/null +++ b/crates/rendering/src/layers/keyboard.rs @@ -0,0 +1,590 @@ +use bytemuck::{Pod, Zeroable}; +use cap_project::XY; +use glyphon::{ + Attrs, Buffer, Cache, Color, Family, FontSystem, Metrics, Resolution, Shaping, SwashCache, + TextArea, TextAtlas, TextBounds, TextRenderer, Viewport, Weight, +}; +use std::str::FromStr; +use wgpu::{Device, Queue, include_wgsl, util::DeviceExt}; + +use crate::{DecodedSegmentFrames, ProjectUniforms, RenderVideoConstants, parse_color_component}; + +const BOUNCE_OFFSET_PIXELS: f64 = 6.0; +const PADDING_RATIO: f32 = 0.45; +const CORNER_RADIUS_RATIO: f32 = 0.5; +const Y_FACTOR_TOP: f32 = 0.08; +const Y_FACTOR_ABOVE_CAPTIONS: f32 = 0.75; +const Y_FACTOR_BOTTOM: f32 = 0.85; + +#[repr(C)] +#[derive(Copy, Clone, Pod, Zeroable, Debug)] +struct KeyboardBackgroundUniforms { + rect: [f32; 4], + color: [f32; 4], + radius: f32, + _padding: [f32; 3], + _padding2: [f32; 4], +} + +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub enum KeyboardPosition { + #[default] + AboveCaptions, + TopLeft, + TopCenter, + TopRight, + BottomLeft, + BottomCenter, + BottomRight, +} + +impl FromStr for KeyboardPosition { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "top-left" => Ok(Self::TopLeft), + "top-center" | "top" => Ok(Self::TopCenter), + "top-right" => Ok(Self::TopRight), + "bottom-left" => Ok(Self::BottomLeft), + "bottom-right" => Ok(Self::BottomRight), + "bottom-center" | "bottom" => Ok(Self::BottomCenter), + "above-captions" => Ok(Self::AboveCaptions), + _ => Err(()), + } + } +} + +impl KeyboardPosition { + fn y_factor(&self) -> f32 { + match self { + Self::TopLeft | Self::TopCenter | Self::TopRight => Y_FACTOR_TOP, + Self::AboveCaptions => Y_FACTOR_ABOVE_CAPTIONS, + Self::BottomLeft | Self::BottomCenter | Self::BottomRight => Y_FACTOR_BOTTOM, + } + } +} + +pub struct KeyboardLayer { + font_system: FontSystem, + swash_cache: SwashCache, + text_atlas: TextAtlas, + text_renderer: TextRenderer, + text_buffer: Buffer, + viewport: Viewport, + background_pipeline: wgpu::RenderPipeline, + background_bind_group: wgpu::BindGroup, + background_uniform_buffer: wgpu::Buffer, + background_scissor: Option<[u32; 4]>, + output_size: (u32, u32), + has_content: bool, +} + +impl KeyboardLayer { + pub fn new(device: &Device, queue: &Queue) -> Self { + let font_system = FontSystem::new(); + let swash_cache = SwashCache::new(); + let cache = Cache::new(device); + let viewport = Viewport::new(device, &cache); + let mut text_atlas = TextAtlas::new(device, queue, &cache, wgpu::TextureFormat::Rgba8Unorm); + let text_renderer = TextRenderer::new( + &mut text_atlas, + device, + wgpu::MultisampleState::default(), + None, + ); + + let metrics = Metrics::new(28.0, 28.0 * 1.2); + let text_buffer = Buffer::new_empty(metrics); + + let background_uniforms = KeyboardBackgroundUniforms { + rect: [0.0; 4], + color: [0.0; 4], + radius: 0.0, + _padding: [0.0; 3], + _padding2: [0.0; 4], + }; + + let background_uniform_buffer = + device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("Keyboard Background Uniform Buffer"), + contents: bytemuck::bytes_of(&background_uniforms), + usage: wgpu::BufferUsages::UNIFORM | wgpu::BufferUsages::COPY_DST, + }); + + let background_bind_group_layout = + device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: Some("Keyboard Background Bind Group Layout"), + entries: &[wgpu::BindGroupLayoutEntry { + binding: 0, + visibility: wgpu::ShaderStages::FRAGMENT, + ty: wgpu::BindingType::Buffer { + ty: wgpu::BufferBindingType::Uniform, + has_dynamic_offset: false, + min_binding_size: None, + }, + count: None, + }], + }); + + let background_bind_group = device.create_bind_group(&wgpu::BindGroupDescriptor { + label: Some("Keyboard Background Bind Group"), + layout: &background_bind_group_layout, + entries: &[wgpu::BindGroupEntry { + binding: 0, + resource: background_uniform_buffer.as_entire_binding(), + }], + }); + + let background_shader = + device.create_shader_module(include_wgsl!("../shaders/caption_bg.wgsl")); + + let background_pipeline_layout = + device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { + label: Some("Keyboard Background Pipeline Layout"), + bind_group_layouts: &[&background_bind_group_layout], + push_constant_ranges: &[], + }); + + let background_pipeline = device.create_render_pipeline(&wgpu::RenderPipelineDescriptor { + label: Some("Keyboard Background Pipeline"), + layout: Some(&background_pipeline_layout), + vertex: wgpu::VertexState { + module: &background_shader, + entry_point: Some("vs_main"), + buffers: &[], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }, + fragment: Some(wgpu::FragmentState { + module: &background_shader, + entry_point: Some("fs_main"), + targets: &[Some(wgpu::ColorTargetState { + format: wgpu::TextureFormat::Rgba8Unorm, + blend: Some(wgpu::BlendState::ALPHA_BLENDING), + write_mask: wgpu::ColorWrites::ALL, + })], + compilation_options: wgpu::PipelineCompilationOptions::default(), + }), + primitive: wgpu::PrimitiveState { + topology: wgpu::PrimitiveTopology::TriangleList, + ..Default::default() + }, + depth_stencil: None, + multisample: wgpu::MultisampleState::default(), + multiview: None, + cache: None, + }); + + Self { + font_system, + swash_cache, + text_atlas, + text_renderer, + text_buffer, + viewport, + background_pipeline, + background_bind_group, + background_uniform_buffer, + background_scissor: None, + output_size: (0, 0), + has_content: false, + } + } + + pub fn prepare( + &mut self, + uniforms: &ProjectUniforms, + _segment_frames: &DecodedSegmentFrames, + output_size: XY, + constants: &RenderVideoConstants, + ) { + self.has_content = false; + self.background_scissor = None; + self.output_size = (output_size.x, output_size.y); + + let Some(keyboard_data) = &uniforms.project.keyboard else { + return; + }; + + if !keyboard_data.settings.enabled { + return; + } + + let timeline = match &uniforms.project.timeline { + Some(t) => t, + None => return, + }; + + if timeline.keyboard_segments.is_empty() { + return; + } + + let current_time = uniforms.frame_number as f64 / uniforms.frame_rate as f64; + let settings = &keyboard_data.settings; + + let active_segment = find_active_keyboard_segment( + current_time, + &timeline.keyboard_segments, + settings.fade_duration_secs, + ); + + let Some(active) = active_segment else { + return; + }; + + let segment_fade = active + .segment + .fade_duration_override + .unwrap_or(settings.fade_duration_secs) as f64; + + let visible_text = build_visible_text(active.segment, current_time); + + if visible_text.is_empty() { + return; + } + + let fade_opacity = calculate_fade( + current_time, + active.segment.start, + active.segment.end, + segment_fade, + ); + + if fade_opacity <= 0.0 { + return; + } + + let bounce_offset = calculate_keyboard_bounce( + current_time, + active.segment.start, + active.segment.end, + segment_fade, + ); + + let (width, height) = (output_size.x, output_size.y); + let device = &constants.device; + let queue = &constants.queue; + + let position = active + .segment + .position_override + .as_deref() + .unwrap_or(&settings.position) + .parse::() + .unwrap_or_default(); + + let margin = width as f32 * 0.05; + + let color_hex = active + .segment + .color_override + .as_deref() + .unwrap_or(&settings.color); + let text_color = [ + parse_color_component(color_hex, 0), + parse_color_component(color_hex, 1), + parse_color_component(color_hex, 2), + ]; + + let bg_color_hex = active + .segment + .background_color_override + .as_deref() + .unwrap_or(&settings.background_color); + let background_color_rgb = [ + parse_color_component(bg_color_hex, 0), + parse_color_component(bg_color_hex, 1), + parse_color_component(bg_color_hex, 2), + ]; + + let background_alpha = + ((settings.background_opacity as f32 / 100.0) * fade_opacity).clamp(0.0, 1.0); + + let font_size_base = active.segment.font_size_override.unwrap_or(settings.size) as f32; + let font_size = font_size_base * (height as f32 / 1080.0); + let metrics = Metrics::new(font_size, font_size * 1.2); + + let mut updated_buffer = Buffer::new(&mut self.font_system, metrics); + let wrap_width = (width as f32 - margin * 2.0).max(font_size); + updated_buffer.set_size(&mut self.font_system, Some(wrap_width), None); + + let font_family = match settings.font.as_str() { + "System Serif" => Family::Serif, + "System Monospace" => Family::Monospace, + _ => Family::SansSerif, + }; + + let weight = if settings.font_weight >= 700 { + Weight::BOLD + } else if settings.font_weight >= 500 { + Weight::MEDIUM + } else { + Weight::NORMAL + }; + + let text_alpha = fade_opacity.clamp(0.0, 1.0); + let color = Color::rgba( + (text_color[0] * 255.0) as u8, + (text_color[1] * 255.0) as u8, + (text_color[2] * 255.0) as u8, + (text_alpha * 255.0) as u8, + ); + + let attrs = Attrs::new().family(font_family).weight(weight).color(color); + updated_buffer.set_text( + &mut self.font_system, + &visible_text, + &attrs, + Shaping::Advanced, + ); + + let mut layout_width: f32 = 0.0; + let mut layout_height: f32 = 0.0; + for run in glyphon::cosmic_text::LayoutRunIter::new(&updated_buffer) { + layout_width = layout_width.max(run.line_w); + layout_height = layout_height.max(run.line_top + run.line_height); + } + + if layout_height == 0.0 { + layout_height = font_size * 1.2; + layout_width = layout_width.max(font_size); + } + + let available_width = (width as f32 - margin * 2.0).max(1.0); + let padding = font_size * PADDING_RATIO; + let corner_radius = font_size * CORNER_RADIUS_RATIO; + let text_width = layout_width.min(available_width); + let text_height = layout_height; + let box_width = (text_width + padding * 2.0).min(available_width).max(1.0); + let box_height = (text_height + padding * 2.0).min(height as f32).max(1.0); + + let background_left = ((width as f32 - box_width) / 2.0).max(0.0); + + let center_y = height as f32 * position.y_factor(); + let base_background_top = + (center_y - box_height / 2.0).clamp(0.0, (height as f32 - box_height).max(0.0)); + let background_top = (base_background_top + bounce_offset as f32) + .clamp(0.0, (height as f32 - box_height).max(0.0)); + + let text_left = background_left + padding; + let text_top = background_top + padding; + + let bounds = TextBounds { + left: (text_left - 2.0).floor() as i32, + top: (text_top - 2.0).floor() as i32, + right: (text_left + text_width + 2.0).ceil() as i32, + bottom: (text_top + text_height + 2.0).ceil() as i32, + }; + + self.text_buffer = updated_buffer; + self.viewport.update(queue, Resolution { width, height }); + + let text_areas = vec![TextArea { + buffer: &self.text_buffer, + left: text_left, + top: text_top, + scale: 1.0, + bounds, + default_color: color, + custom_glyphs: &[], + }]; + + match self.text_renderer.prepare( + device, + queue, + &mut self.font_system, + &mut self.text_atlas, + &self.viewport, + text_areas, + &mut self.swash_cache, + ) { + Ok(_) => {} + Err(e) => tracing::warn!("Error preparing keyboard text: {e:?}"), + } + + let rect = KeyboardBackgroundUniforms { + rect: [ + background_left.max(0.0), + background_top.max(0.0), + box_width, + box_height, + ], + color: [ + background_color_rgb[0], + background_color_rgb[1], + background_color_rgb[2], + background_alpha, + ], + radius: corner_radius.min(box_width / 2.0).min(box_height / 2.0), + _padding: [0.0; 3], + _padding2: [0.0; 4], + }; + + queue.write_buffer( + &self.background_uniform_buffer, + 0, + bytemuck::bytes_of(&rect), + ); + + let scissor_padding = 4.0; + let scissor_x = (background_left - scissor_padding).max(0.0).floor() as u32; + let scissor_y = (background_top - scissor_padding).max(0.0).floor() as u32; + let max_width = width.saturating_sub(scissor_x); + let max_height = height.saturating_sub(scissor_y); + + if max_width == 0 || max_height == 0 { + self.has_content = false; + return; + } + + let scissor_width = (box_width + scissor_padding * 2.0) + .ceil() + .max(1.0) + .min(max_width as f32) as u32; + let scissor_height = (box_height + scissor_padding * 2.0) + .ceil() + .max(1.0) + .min(max_height as f32) as u32; + + if scissor_width == 0 || scissor_height == 0 { + self.has_content = false; + return; + } + + self.background_scissor = Some([scissor_x, scissor_y, scissor_width, scissor_height]); + self.has_content = true; + } + + pub fn has_content(&self) -> bool { + self.has_content + } + + pub fn render<'a>(&'a self, pass: &mut wgpu::RenderPass<'a>) { + if !self.has_content { + return; + } + + if let Some([x, y, width, height]) = self.background_scissor { + pass.set_scissor_rect(x, y, width, height); + pass.set_pipeline(&self.background_pipeline); + pass.set_bind_group(0, &self.background_bind_group, &[]); + pass.draw(0..6, 0..1); + } else if self.output_size.0 > 0 && self.output_size.1 > 0 { + pass.set_scissor_rect(0, 0, self.output_size.0, self.output_size.1); + } + + match self + .text_renderer + .render(&self.text_atlas, &self.viewport, pass) + { + Ok(_) => {} + Err(e) => tracing::warn!("Error rendering keyboard text: {e:?}"), + } + + if self.output_size.0 > 0 && self.output_size.1 > 0 { + pass.set_scissor_rect(0, 0, self.output_size.0, self.output_size.1); + } + } +} + +struct ActiveKeyboardSegment<'a> { + segment: &'a cap_project::keyboard::KeyboardTrackSegment, +} + +fn find_active_keyboard_segment<'a>( + time: f64, + segments: &'a [cap_project::keyboard::KeyboardTrackSegment], + default_fade_duration: f32, +) -> Option> { + for segment in segments { + if time >= segment.start && time < segment.end { + return Some(ActiveKeyboardSegment { segment }); + } + } + + for segment in segments { + let fade = segment + .fade_duration_override + .unwrap_or(default_fade_duration) as f64; + if time >= segment.end && time < segment.end + fade { + return Some(ActiveKeyboardSegment { segment }); + } + } + + None +} + +fn build_visible_text( + segment: &cap_project::keyboard::KeyboardTrackSegment, + current_time: f64, +) -> String { + if segment.keys.is_empty() { + return segment.display_text.clone(); + } + + let chars: Vec = segment.display_text.chars().collect(); + if segment.keys.len() != chars.len() { + return segment.display_text.clone(); + } + + let time_offset_from_start = (current_time - segment.start) * 1000.0; + + let mut visible = String::new(); + + for (i, key) in segment.keys.iter().enumerate() { + if time_offset_from_start >= key.time_offset_ms && i < chars.len() { + visible.push(chars[i]); + } + } + + if visible.is_empty() && !segment.display_text.is_empty() { + visible.push(chars[0]); + } + + visible +} + +fn calculate_fade(current_time: f64, start: f64, end: f64, fade_duration: f64) -> f32 { + if fade_duration <= 0.0 { + if current_time >= start && current_time < end { + return 1.0; + } + return 0.0; + } + + let time_from_start = current_time - start; + let time_to_end = end - current_time; + + let fade_in = (time_from_start / fade_duration).clamp(0.0, 1.0) as f32; + + let fade_out = if time_to_end >= 0.0 { + 1.0 + } else { + let past_end = -time_to_end; + (1.0 - past_end / fade_duration).clamp(0.0, 1.0) as f32 + }; + + fade_in.min(fade_out) +} + +fn calculate_keyboard_bounce(current_time: f64, start: f64, end: f64, fade_duration: f64) -> f64 { + if fade_duration <= 0.0 { + return 0.0; + } + + let time_from_start = current_time - start; + let time_to_end = end - current_time; + + let fade_in_progress = (time_from_start / fade_duration).clamp(0.0, 1.0); + let fade_out_progress = (time_to_end / fade_duration).clamp(0.0, 1.0); + + if fade_in_progress < 1.0 { + let ease = 1.0 - fade_in_progress; + -(ease * ease) * BOUNCE_OFFSET_PIXELS + } else if fade_out_progress < 1.0 { + let ease = 1.0 - fade_out_progress; + (ease * ease) * BOUNCE_OFFSET_PIXELS + } else { + 0.0 + } +} diff --git a/crates/rendering/src/layers/mod.rs b/crates/rendering/src/layers/mod.rs index 536fbb3bf6..7e727b27dc 100644 --- a/crates/rendering/src/layers/mod.rs +++ b/crates/rendering/src/layers/mod.rs @@ -4,6 +4,7 @@ mod camera; mod captions; mod cursor; mod display; +mod keyboard; mod mask; mod text; @@ -13,5 +14,6 @@ pub use camera::*; pub use captions::*; pub use cursor::*; pub use display::*; +pub use keyboard::*; pub use mask::*; pub use text::*; diff --git a/crates/rendering/src/lib.rs b/crates/rendering/src/lib.rs index 410a01c52a..113d64a4de 100644 --- a/crates/rendering/src/lib.rs +++ b/crates/rendering/src/lib.rs @@ -11,7 +11,7 @@ use frame_pipeline::{RenderSession, finish_encoder, finish_encoder_nv12, flush_p use futures::future::OptionFuture; use layers::{ Background, BackgroundLayer, BlurLayer, CameraLayer, CaptionsLayer, CursorLayer, DisplayLayer, - MaskLayer, TextLayer, + KeyboardLayer, MaskLayer, TextLayer, }; use specta::Type; use spring_mass_damper::SpringMassDamperSimulationConfig; @@ -317,6 +317,7 @@ pub enum RenderingError { pub struct RenderSegment { pub cursor: Arc, + pub keyboard: Arc, pub decoders: RecordingSegmentDecoders, } @@ -2522,6 +2523,7 @@ pub struct RendererLayers { mask: MaskLayer, text: TextLayer, captions: CaptionsLayer, + keyboard: KeyboardLayer, } impl RendererLayers { @@ -2561,6 +2563,7 @@ impl RendererLayers { mask: MaskLayer::new(device), text: TextLayer::new(device, queue), captions: CaptionsLayer::new(device, queue), + keyboard: KeyboardLayer::new(device, queue), } } @@ -2662,6 +2665,13 @@ impl RendererLayers { constants, ); + self.keyboard.prepare( + uniforms, + segment_frames, + XY::new(uniforms.output_size.0, uniforms.output_size.1), + constants, + ); + Ok(()) } @@ -2743,6 +2753,13 @@ impl RendererLayers { constants, ); + self.keyboard.prepare( + uniforms, + segment_frames, + XY::new(uniforms.output_size.0, uniforms.output_size.1), + constants, + ); + Ok(()) } @@ -2834,6 +2851,11 @@ impl RendererLayers { let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); self.captions.render(&mut pass); } + + if self.keyboard.has_content() { + let mut pass = render_pass!(session.current_texture_view(), wgpu::LoadOp::Load); + self.keyboard.render(&mut pass); + } } } diff --git a/crates/rendering/src/main.rs b/crates/rendering/src/main.rs index efbbc39ddf..b4d0d9bdcc 100644 --- a/crates/rendering/src/main.rs +++ b/crates/rendering/src/main.rs @@ -108,6 +108,7 @@ async fn main() -> Result<()> { vec![RenderSegment { cursor: Arc::new(Default::default()), + keyboard: Arc::new(Default::default()), decoders, }] } @@ -130,8 +131,13 @@ async fn main() -> Result<()> { })?; let cursor = Arc::new(s.cursor_events(&recording_meta)); + let keyboard = Arc::new(s.keyboard_events(&recording_meta)); - segments.push(RenderSegment { cursor, decoders }); + segments.push(RenderSegment { + cursor, + keyboard, + decoders, + }); } segments }